diff --git a/components/engine/.github/ISSUE_TEMPLATE.md b/components/engine/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000..c347950212 --- /dev/null +++ b/components/engine/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,51 @@ + + +Output of `docker version`: + +``` +(paste your output here) +``` + + +Output of `docker info`: + +``` +(paste your output here) +``` + +Provide additional environment details (AWS, VirtualBox, physical, etc.): + + + +List the steps to reproduce the issue: +1. +2. +3. + + +Describe the results you received: + + +Describe the results you expected: + + +Provide additional info you think is important: diff --git a/components/engine/.github/PULL_REQUEST_TEMPLATE.md b/components/engine/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..fee0c7864c --- /dev/null +++ b/components/engine/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,23 @@ + + +Please provide the following information: + +- What did you do? + +- How did you do it? + +- How do I see it or verify it? + +- A picture of a cute animal (not mandatory but encouraged) + diff --git a/components/engine/CHANGELOG.md b/components/engine/CHANGELOG.md index 2d19c2094a..c27e96bf36 100644 --- a/components/engine/CHANGELOG.md +++ b/components/engine/CHANGELOG.md @@ -5,6 +5,73 @@ information on the list of deprecated flags and APIs please have a look at https://docs.docker.com/misc/deprecated/ where target removal dates can also be found. +## 1.10.2 (2016-02-22) + +### Runtime + +- Prevent systemd from deleting containers' cgroups when its configuration is reloaded [#20518](https://github.com/docker/docker/pull/20518) +- Fix SELinux issues by disregarding `--read-only` when mounting `/dev/mqueue` [#20333](https://github.com/docker/docker/pull/20333) +- Fix chown permissions used during `docker cp` when userns is used [#20446](https://github.com/docker/docker/pull/20446) +- Fix configuration loading issue with all booleans defaulting to `true` [#20471](https://github.com/docker/docker/pull/20471) +- Fix occasional panic with `docker logs -f` [#20522](https://github.com/docker/docker/pull/20522) + +### Distribution + +- Keep layer reference if deletion failed to avoid a badly inconsistent state [#20513](https://github.com/docker/docker/pull/20513) +- Handle gracefully a corner case when canceling migration [#20372](https://github.com/docker/docker/pull/20372) +- Fix docker import on compressed data [#20367](https://github.com/docker/docker/pull/20367) +- Fix tar-split files corruption during migration that later cause docker push and docker save to fail [#20458](https://github.com/docker/docker/pull/20458) + +### Networking + +- Fix daemon crash if embedded DNS is sent garbage [#20510](https://github.com/docker/docker/pull/20510) + +### Volumes + +- Fix issue with multiple volume references with same name [#20381](https://github.com/docker/docker/pull/20381) + +### Security + +- Fix potential cache corruption and delegation conflict issues [#20523](https://github.com/docker/docker/pull/20523) + +## 1.10.1 (2016-02-11) + +### Runtime + +* Do not stop daemon on migration hard failure [#20156](https://github.com/docker/docker/pull/20156) +- Fix various issues with migration to content-addressable images [#20058](https://github.com/docker/docker/pull/20058) +- Fix ZFS permission bug with user namespaces [#20045](https://github.com/docker/docker/pull/20045) +- Do not leak /dev/mqueue from the host to all containers, keep it container-specific [#19876](https://github.com/docker/docker/pull/19876) [#20133](https://github.com/docker/docker/pull/20133) +- Fix `docker ps --filter before=...` to not show stopped containers without providing `-a` flag [#20135](https://github.com/docker/docker/pull/20135) + +### Security + +- Fix issue preventing docker events to work properly with authorization plugin [#20002](https://github.com/docker/docker/pull/20002) + +### Distribution + +* Add additional verifications and prevent from uploading invalid data to registries [#20164](https://github.com/docker/docker/pull/20164) +- Fix regression preventing uppercase characters in image reference hostname [#20175](https://github.com/docker/docker/pull/20175) + +### Networking + +- Fix embedded DNS for user-defined networks in the presence of firewalld [#20060](https://github.com/docker/docker/pull/20060) +- Fix issue where removing a network during shutdown left Docker inoperable [#20181](https://github.com/docker/docker/issues/20181) [#20235](https://github.com/docker/docker/issues/20235) +- Embedded DNS is now able to return compressed results [#20181](https://github.com/docker/docker/issues/20181) +- Fix port-mapping issue with `userland-proxy=false` [#20181](https://github.com/docker/docker/issues/20181) + +### Logging + +- Fix bug where tcp+tls protocol would be rejected [#20109](https://github.com/docker/docker/pull/20109) + +### Volumes + +- Fix issue whereby older volume drivers would not receive volume options [#19983](https://github.com/docker/docker/pull/19983) + +### Misc + +- Remove TasksMax from Docker systemd service [#20167](https://github.com/docker/docker/pull/20167) + ## 1.10.0 (2016-02-04) **IMPORTANT**: Docker 1.10 uses a new content-addressable storage for images and layers. @@ -1771,7 +1838,7 @@ With the ongoing changes to the networking and execution subsystems of docker te + Containers can expose public UDP ports (eg, '-p 123/udp') + Optionally specify an exact public port (eg. '-p 80:4500') * 'docker login' supports additional options -- Dont save a container`s hostname when committing an image. +- Don't save a container`s hostname when committing an image. #### Registry diff --git a/components/engine/CONTRIBUTING.md b/components/engine/CONTRIBUTING.md index e499e1a9d6..6b875d6901 100644 --- a/components/engine/CONTRIBUTING.md +++ b/components/engine/CONTRIBUTING.md @@ -154,6 +154,8 @@ However, there might be a way to implement that feature *on top of* Docker. The docker-dev group is for contributors and other people contributing to the Docker project. + You can join them without an google account by sending an email to e.g. "docker-user+subscribe@googlegroups.com". + After receiving the join-request message, you can simply reply to that to confirm the subscribtion. diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 45486db4ec..a3430d58bc 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -23,14 +23,16 @@ # the case. Therefore, you don't have to disable it anymore. # -FROM ubuntu:trusty +FROM debian:jessie # add zfs ppa -RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys E871F18B51E0147C77796AC81196BA81F6B0FC61 +RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys E871F18B51E0147C77796AC81196BA81F6B0FC61 \ + || apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys E871F18B51E0147C77796AC81196BA81F6B0FC61 RUN echo deb http://ppa.launchpad.net/zfs-native/stable/ubuntu trusty main > /etc/apt/sources.list.d/zfs.list # add llvm repo -RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 6084F3CF814B57C1CF12EFD515CF4D18AF4F7421 +RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 6084F3CF814B57C1CF12EFD515CF4D18AF4F7421 \ + || apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 6084F3CF814B57C1CF12EFD515CF4D18AF4F7421 RUN echo deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty main > /etc/apt/sources.list.d/llvm.list # Packaged dependencies @@ -56,12 +58,13 @@ RUN apt-get update && apt-get install -y \ libsystemd-journal-dev \ libtool \ mercurial \ + net-tools \ pkg-config \ python-dev \ python-mock \ python-pip \ python-websocket \ - s3cmd=1.1.0* \ + s3cmd=1.5.0* \ ubuntu-zfs \ xfsprogs \ libzfs-dev \ @@ -88,9 +91,11 @@ RUN cd /usr/local/lvm2 \ # Configure the container for OSX cross compilation ENV OSX_SDK MacOSX10.11.sdk +ENV OSX_CROSS_COMMIT 8aa9b71a394905e6c5f4b59e2b97b87a004658a4 RUN set -x \ && export OSXCROSS_PATH="/osxcross" \ - && git clone --depth 1 https://github.com/tpoechtrager/osxcross.git $OSXCROSS_PATH \ + && git clone https://github.com/tpoechtrager/osxcross.git $OSXCROSS_PATH \ + && ( cd $OSXCROSS_PATH && git checkout -q $OSX_CROSS_COMMIT) \ && curl -sSL https://s3.dockerproject.org/darwin/${OSX_SDK}.tar.xz -o "${OSXCROSS_PATH}/tarballs/${OSX_SDK}.tar.xz" \ && UNATTENDED=yes OSX_VERSION_MIN=10.6 ${OSXCROSS_PATH}/build.sh ENV PATH /osxcross/target/bin:$PATH @@ -114,7 +119,7 @@ RUN set -x \ # IMPORTANT: If the version of Go is updated, the Windows to Linux CI machines # will need updating, to avoid errors. Ping #docker-maintainers on IRC # with a heads-up. -ENV GO_VERSION 1.5.3 +ENV GO_VERSION 1.6 RUN curl -fsSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" \ | tar -xzC /usr/local ENV PATH /go/bin:/usr/local/go/bin:$PATH @@ -166,7 +171,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Install notary server -ENV NOTARY_VERSION docker-v1.10-5 +ENV NOTARY_VERSION v0.2.0 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone https://github.com/docker/notary.git "$GOPATH/src/github.com/docker/notary" \ diff --git a/components/engine/Dockerfile.aarch64 b/components/engine/Dockerfile.aarch64 index ee8c889e77..fa8fde9ac7 100644 --- a/components/engine/Dockerfile.aarch64 +++ b/components/engine/Dockerfile.aarch64 @@ -11,19 +11,11 @@ # # Run the test suite: # docker run --privileged docker hack/make.sh test # -# # Publish a release: -# docker run --privileged \ -# -e AWS_S3_BUCKET=baz \ -# -e AWS_ACCESS_KEY=foo \ -# -e AWS_SECRET_KEY=bar \ -# -e GPG_PASSPHRASE=gloubiboulga \ -# docker hack/release.sh -# # Note: AppArmor used to mess with privileged mode, but this is no longer # the case. Therefore, you don't have to disable it anymore. # -FROM aarch64/ubuntu:trusty +FROM aarch64/ubuntu:wily # Packaged dependencies RUN apt-get update && apt-get install -y \ @@ -45,15 +37,16 @@ RUN apt-get update && apt-get install -y \ libc6-dev \ libcap-dev \ libsqlite3-dev \ - libsystemd-journal-dev \ + libsystemd-dev \ mercurial \ + net-tools \ parallel \ pkg-config \ python-dev \ python-mock \ python-pip \ python-websocket \ - s3cmd=1.1.0* \ + gccgo \ --no-install-recommends # Install armhf loader to use armv6 binaries on armv8 @@ -103,14 +96,11 @@ RUN set -x \ # We don't have official binary tarballs for ARM64, eigher for Go or bootstrap, # so we use the official armv6 released binaries as a GOROOT_BOOTSTRAP, and # build Go from source code. -ENV BOOT_STRAP_VERSION 1.6beta1 ENV GO_VERSION 1.5.3 -RUN mkdir -p /usr/src/go-bootstrap \ - && curl -fsSL https://storage.googleapis.com/golang/go${BOOT_STRAP_VERSION}.linux-arm6.tar.gz | tar -v -C /usr/src/go-bootstrap -xz --strip-components=1 \ - && mkdir /usr/src/go \ - && curl -fsSL https://storage.googleapis.com/golang/go${GO_VERSION}.src.tar.gz | tar -v -C /usr/src/go -xz --strip-components=1 \ +RUN mkdir /usr/src/go && curl -fsSL https://storage.googleapis.com/golang/go${GO_VERSION}.src.tar.gz | tar -v -C /usr/src/go -xz --strip-components=1 \ && cd /usr/src/go/src \ - && GOOS=linux GOARCH=arm64 GOROOT_BOOTSTRAP=/usr/src/go-bootstrap ./make.bash + && GOOS=linux GOARCH=arm64 GOROOT_BOOTSTRAP="$(go env GOROOT)" ./make.bash + ENV PATH /usr/src/go/bin:$PATH ENV GOPATH /go:/go/src/github.com/docker/docker/vendor @@ -127,7 +117,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Install notary server -ENV NOTARY_VERSION docker-v1.10-5 +ENV NOTARY_VERSION v0.2.0 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone https://github.com/docker/notary.git "$GOPATH/src/github.com/docker/notary" \ diff --git a/components/engine/Dockerfile.armhf b/components/engine/Dockerfile.armhf index 8b48cf9261..fd6f8721fa 100644 --- a/components/engine/Dockerfile.armhf +++ b/components/engine/Dockerfile.armhf @@ -11,19 +11,11 @@ # # Run the test suite: # docker run --privileged docker hack/make.sh test # -# # Publish a release: -# docker run --privileged \ -# -e AWS_S3_BUCKET=baz \ -# -e AWS_ACCESS_KEY=foo \ -# -e AWS_SECRET_KEY=bar \ -# -e GPG_PASSPHRASE=gloubiboulga \ -# docker hack/release.sh -# # Note: AppArmor used to mess with privileged mode, but this is no longer # the case. Therefore, you don't have to disable it anymore. # -FROM armhf/ubuntu:trusty +FROM armhf/debian:jessie # Packaged dependencies RUN apt-get update && apt-get install -y \ @@ -143,7 +135,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Install notary server -ENV NOTARY_VERSION docker-v1.10-5 +ENV NOTARY_VERSION v0.2.0 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone https://github.com/docker/notary.git "$GOPATH/src/github.com/docker/notary" \ diff --git a/components/engine/Dockerfile.gccgo b/components/engine/Dockerfile.gccgo index 96f92f6145..c01f5dd895 100644 --- a/components/engine/Dockerfile.gccgo +++ b/components/engine/Dockerfile.gccgo @@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y \ libcap-dev \ libsqlite3-dev \ mercurial \ + net-tools \ parallel \ python-dev \ python-mock \ diff --git a/components/engine/Dockerfile.ppc64le b/components/engine/Dockerfile.ppc64le index d4a7e6f1a6..fc1d929f48 100644 --- a/components/engine/Dockerfile.ppc64le +++ b/components/engine/Dockerfile.ppc64le @@ -11,14 +11,6 @@ # # Run the test suite: # docker run --privileged docker hack/make.sh test # -# # Publish a release: -# docker run --privileged \ -# -e AWS_S3_BUCKET=baz \ -# -e AWS_ACCESS_KEY=foo \ -# -e AWS_SECRET_KEY=bar \ -# -e GPG_PASSPHRASE=gloubiboulga \ -# docker hack/release.sh -# # Note: AppArmor used to mess with privileged mode, but this is no longer # the case. Therefore, you don't have to disable it anymore. # @@ -82,7 +74,21 @@ RUN cd /usr/local/lvm2 \ # TODO install Go, using gccgo as GOROOT_BOOTSTRAP (Go 1.5+ supports ppc64le properly) # possibly a ppc64le/golang image? -ENV PATH /go/bin:$PATH +## BUILD GOLANG 1.6 +ENV GO_VERSION 1.6 +ENV GO_DOWNLOAD_URL https://golang.org/dl/go${GO_VERSION}.src.tar.gz +ENV GO_DOWNLOAD_SHA256 a96cce8ce43a9bf9b2a4c7d470bc7ee0cb00410da815980681c8353218dcf146 +ENV GOROOT_BOOTSTRAP /usr/local + +RUN curl -fsSL "$GO_DOWNLOAD_URL" -o golang.tar.gz \ + && echo "$GO_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \ + && tar -C /usr/src -xzf golang.tar.gz \ + && rm golang.tar.gz \ + && cd /usr/src/go/src && ./make.bash 2>&1 + +ENV GOROOT_BOOTSTRAP /usr/src/ + +ENV PATH /usr/src/go/bin/:/go/bin:$PATH ENV GOPATH /go:/go/src/github.com/docker/docker/vendor # This has been commented out and kept as reference because we don't support compiling with older Go anymore. @@ -90,7 +96,7 @@ ENV GOPATH /go:/go/src/github.com/docker/docker/vendor # RUN curl -sSL https://storage.googleapis.com/golang/go${GOFMT_VERSION}.$(go env GOOS)-$(go env GOARCH).tar.gz | tar -C /go/bin -xz --strip-components=2 go/bin/gofmt # TODO update this sha when we upgrade to Go 1.5+ -ENV GO_TOOLS_COMMIT 069d2f3bcb68257b627205f0486d6cc69a231ff9 +ENV GO_TOOLS_COMMIT d02228d1857b9f49cd0252788516ff5584266eb6 # Grab Go's cover tool for dead-simple code coverage testing # Grab Go's vet tool for examining go code to find suspicious constructs # and help prevent errors that the compiler might not catch @@ -99,7 +105,7 @@ RUN git clone https://github.com/golang/tools.git /go/src/golang.org/x/tools \ && go install -v golang.org/x/tools/cmd/cover \ && go install -v golang.org/x/tools/cmd/vet # Grab Go's lint tool -ENV GO_LINT_COMMIT f42f5c1c440621302702cb0741e9d2ca547ae80f +ENV GO_LINT_COMMIT 32a87160691b3c96046c0c678fe57c5bef761456 RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint \ && (cd /go/src/github.com/golang/lint && git checkout -q $GO_LINT_COMMIT) \ && go install -v github.com/golang/lint/golint @@ -121,16 +127,16 @@ RUN set -x \ go build -o /usr/local/bin/registry-v2-schema1 github.com/docker/distribution/cmd/registry \ && rm -rf "$GOPATH" -# TODO update this when we upgrade to Go 1.5.1+ + # Install notary server -#ENV NOTARY_VERSION docker-v1.10-5 -#RUN set -x \ -# && export GOPATH="$(mktemp -d)" \ -# && git clone https://github.com/docker/notary.git "$GOPATH/src/github.com/docker/notary" \ -# && (cd "$GOPATH/src/github.com/docker/notary" && git checkout -q "$NOTARY_VERSION") \ -# && GOPATH="$GOPATH/src/github.com/docker/notary/Godeps/_workspace:$GOPATH" \ -# go build -o /usr/local/bin/notary-server github.com/docker/notary/cmd/notary-server \ -# && rm -rf "$GOPATH" +ENV NOTARY_VERSION v0.2.0 +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone https://github.com/docker/notary.git "$GOPATH/src/github.com/docker/notary" \ + && (cd "$GOPATH/src/github.com/docker/notary" && git checkout -q "$NOTARY_VERSION") \ + && GOPATH="$GOPATH/src/github.com/docker/notary/Godeps/_workspace:$GOPATH" \ + go build -o /usr/local/bin/notary-server github.com/docker/notary/cmd/notary-server \ + && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests ENV DOCKER_PY_COMMIT e2878cbcc3a7eef99917adc1be252800b0e41ece diff --git a/components/engine/Dockerfile.s390x b/components/engine/Dockerfile.s390x index 90b7fb38aa..f46e1e0c9b 100644 --- a/components/engine/Dockerfile.s390x +++ b/components/engine/Dockerfile.s390x @@ -11,14 +11,6 @@ # # Run the test suite: # docker run --privileged docker hack/make.sh test # -# # Publish a release: -# docker run --privileged \ -# -e AWS_S3_BUCKET=baz \ -# -e AWS_ACCESS_KEY=foo \ -# -e AWS_SECRET_KEY=bar \ -# -e GPG_PASSPHRASE=gloubiboulga \ -# docker hack/release.sh -# # Note: AppArmor used to mess with privileged mode, but this is no longer # the case. Therefore, you don't have to disable it anymore. # @@ -116,7 +108,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Install notary server -ENV NOTARY_VERSION docker-v1.10-5 +ENV NOTARY_VERSION v0.2.0 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone https://github.com/docker/notary.git "$GOPATH/src/github.com/docker/notary" \ diff --git a/components/engine/Dockerfile.windows b/components/engine/Dockerfile.windows index 691bd6910f..78db5a3b34 100755 --- a/components/engine/Dockerfile.windows +++ b/components/engine/Dockerfile.windows @@ -19,14 +19,7 @@ # Important notes: # --------------- # -# Multiple commands in a single powershell RUN command are deliberately not done. This is -# because PS doesn't have a concept quite like set -e in bash. It would be possible to use -# try-catch script blocks, but that would make this file unreadable. The problem is that -# if there are two commands eg "RUN powershell -command fail; succeed", as far as docker -# would be concerned, the return code from the overall RUN is succeed. This doesn't apply to -# RUN which uses cmd as the command interpreter such as "RUN fail; succeed". -# -# 'sleep 5' is a deliberate workaround for a current problem on containers in Windows +# 'Start-Sleep' is a deliberate workaround for a current problem on containers in Windows # Server 2016. It ensures that the network is up and available for when the command is # network related. This bug is being tracked internally at Microsoft and exists in TP4. # Generally sleep 1 or 2 is probably enough, but making it 5 to make the build file @@ -39,55 +32,70 @@ # Don't try to use a volume for passing the source through. The cygwin posix utilities will # balk at reparse points. Again, see the example at the top of this file on how use a volume # to get the built binary out of the container. +# +# The steps are minimised dramatically to improve performance (TP4 is slow on commit) FROM windowsservercore # Environment variable notes: -# - GOLANG_VERSION should be updated to be consistent with the Linux dockerfile. +# - GOLANG_VERSION must consistent with 'Dockerfile' used by Linux'. # - FROM_DOCKERFILE is used for detection of building within a container. -ENV GOLANG_VERSION=1.5.3 \ - GIT_VERSION=2.7.0 \ +ENV GOLANG_VERSION=1.6 \ + GIT_LOCATION=https://github.com/git-for-windows/git/releases/download/v2.7.2.windows.1/Git-2.7.2-64-bit.exe \ RSRC_COMMIT=ba14da1f827188454a4591717fff29999010887f \ GOPATH=C:/go;C:/go/src/github.com/docker/docker/vendor \ FROM_DOCKERFILE=1 -# Make sure we're in temp for the downloads -WORKDIR c:/windows/temp - -# Download everything else we need to install -# We want a 64-bit make.exe, not 16 or 32-bit. This was hard to find, so documenting the links -# - http://sourceforge.net/p/mingw-w64/wiki2/Make/ --> -# - http://sourceforge.net/projects/mingw-w64/files/External%20binary%20packages%20%28Win64%20hosted%29/ --> -# - http://sourceforge.net/projects/mingw-w64/files/External binary packages %28Win64 hosted%29/make/ -RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile make.zip http://downloads.sourceforge.net/project/mingw-w64/External%20binary%20packages%20%28Win64%20hosted%29/make/make-3.82.90-20111115.zip -RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile gcc.zip http://downloads.sourceforge.net/project/tdm-gcc/TDM-GCC%205%20series/5.1.0-tdm64-1/gcc-5.1.0-tdm64-1-core.zip -RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile runtime.zip http://downloads.sourceforge.net/project/tdm-gcc/MinGW-w64%20runtime/GCC%205%20series/mingw64runtime-v4-git20150618-gcc5-tdm64-1.zip -RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile binutils.zip http://downloads.sourceforge.net/project/tdm-gcc/GNU%20binutils/binutils-2.25-tdm64-1.zip -RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile 7zsetup.exe http://www.7-zip.org/a/7z1514-x64.exe -RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile lzma.7z http://www.7-zip.org/a/lzma1514.7z -RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile gitsetup.exe https://github.com/git-for-windows/git/releases/download/v%GIT_VERSION%.windows.1/Git-%GIT_VERSION%-64-bit.exe -RUN powershell -command sleep 5; Invoke-WebRequest -UserAgent 'DockerCI' -outfile go.msi https://storage.googleapis.com/golang/go%GOLANG_VERSION%.windows-amd64.msi - -# Path -RUN setx /M Path "c:\git\cmd;c:\git\bin;c:\git\usr\bin;%Path%;c:\gcc\bin;c:\7zip" - -# Install and expand the bits we downloaded. -# Note: The git, 7z and go.msi installers execute asynchronously. -RUN powershell -command start-process .\gitsetup.exe -ArgumentList '/VERYSILENT /SUPPRESSMSGBOXES /CLOSEAPPLICATIONS /DIR=c:\git' -Wait -RUN powershell -command start-process .\7zsetup -ArgumentList '/S /D=c:/7zip' -Wait -RUN powershell -command start-process .\go.msi -ArgumentList '/quiet' -Wait -RUN powershell -command Expand-Archive gcc.zip \gcc -Force -RUN powershell -command Expand-Archive runtime.zip \gcc -Force -RUN powershell -command Expand-Archive binutils.zip \gcc -Force -RUN powershell -command 7z e lzma.7z bin/lzma.exe -RUN powershell -command 7z x make.zip make-3.82.90-20111115/bin_amd64/make.exe -RUN powershell -command mv make-3.82.90-20111115/bin_amd64/make.exe /gcc/bin/ - -# RSRC for manifest and icon -RUN powershell -command sleep 5 ; git clone https://github.com/akavel/rsrc.git c:\go\src\github.com\akavel\rsrc -RUN cd c:/go/src/github.com/akavel/rsrc && git checkout -q %RSRC_COMMIT% && go install -v - -# Prepare for building WORKDIR c:/ + +# Everything downloaded/installed in one go (better performance, esp on TP4) +RUN \ + setx /M Path "c:\git\cmd;c:\git\bin;c:\git\usr\bin;%Path%;c:\gcc\bin;c:\go\bin" && \ + setx GOROOT "c:\go" && \ + powershell -command \ + $ErrorActionPreference = 'Stop'; \ + Start-Sleep -Seconds 5; \ + Function Download-File([string] $source, [string] $target) { \ + $wc = New-Object net.webclient; $wc.Downloadfile($source, $target) \ + } \ + \ + Write-Host INFO: Downloading git...; \ + Download-File %GIT_LOCATION% gitsetup.exe; \ + \ + Write-Host INFO: Downloading go...; \ + Download-File https://storage.googleapis.com/golang/go%GOLANG_VERSION%.windows-amd64.msi go.msi; \ + \ + Write-Host INFO: Downloading compiler 1 of 3...; \ + Download-File https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/gcc.zip gcc.zip; \ + \ + Write-Host INFO: Downloading compiler 2 of 3...; \ + Download-File https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/runtime.zip runtime.zip; \ + \ + Write-Host INFO: Downloading compiler 3 of 3...; \ + Download-File https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/binutils.zip binutils.zip; \ + \ + Write-Host INFO: Installing git...; \ + Start-Process gitsetup.exe -ArgumentList '/VERYSILENT /SUPPRESSMSGBOXES /CLOSEAPPLICATIONS /DIR=c:\git\' -Wait; \ + \ + Write-Host INFO: Installing go..."; \ + Start-Process msiexec -ArgumentList '-i go.msi -quiet' -Wait; \ + \ + Write-Host INFO: Unzipping compiler...; \ + c:\git\usr\bin\unzip.exe -q -o gcc.zip -d /c/gcc; \ + c:\git\usr\bin\unzip.exe -q -o runtime.zip -d /c/gcc; \ + c:\git\usr\bin\unzip.exe -q -o binutils.zip -d /c/gcc"; \ + \ + Write-Host INFO: Removing interim files; \ + Remove-Item *.zip; \ + Remove-Item go.msi; \ + Remove-Item gitsetup.exe; \ + \ + Write-Host INFO: Cloning and installing RSRC; \ + c:\git\bin\git.exe clone https://github.com/akavel/rsrc.git c:\go\src\github.com\akavel\rsrc; \ + cd \go\src\github.com\akavel\rsrc; c:\git\bin\git.exe checkout -q %RSRC_COMMIT%; c:\go\bin\go.exe install -v; \ + \ + Write-Host INFO: Completed + +# Prepare for building COPY . /go/src/github.com/docker/docker diff --git a/components/engine/MAINTAINERS b/components/engine/MAINTAINERS index 43a2f8753d..99bbd1dc85 100644 --- a/components/engine/MAINTAINERS +++ b/components/engine/MAINTAINERS @@ -26,6 +26,7 @@ # the release process is clear and up-to-date. people = [ + "aaronlehmann", "calavera", "coolljt0725", "cpuguy83", @@ -111,6 +112,11 @@ # ADD YOURSELF HERE IN ALPHABETICAL ORDER + [people.aaronlehmann] + Name = "Aaron Lehmann" + Email = "aaron.lehmann@docker.com" + GitHub = "aaronlehmann" + [people.calavera] Name = "David Calavera" Email = "david.calavera@gmail.com" @@ -196,11 +202,6 @@ Email = "github@gone.nl" GitHub = "thaJeztah" - [people.theadactyl] - Name = "Thea Lamkin" - Email = "thea@docker.com" - GitHub = "theadactyl" - [people.tianon] Name = "Tianon Gravi" Email = "admwiggin@gmail.com" diff --git a/components/engine/Makefile b/components/engine/Makefile index ba45c4664e..b455c12f43 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -54,6 +54,10 @@ DOCKER_ENVS := \ BIND_DIR := $(if $(BINDDIR),$(BINDDIR),$(if $(DOCKER_HOST),,bundles)) DOCKER_MOUNT := $(if $(BIND_DIR),-v "$(CURDIR)/$(BIND_DIR):/go/src/github.com/docker/docker/$(BIND_DIR)") +# This allows the test suite to be able to run without worrying about the underlying fs used by the container running the daemon (e.g. aufs-on-aufs), so long as the host running the container is running a supported fs. +# The volume will be cleaned up when the container is removed due to `--rm`. +# Note that `BIND_DIR` will already be set to `bundles` if `DOCKER_HOST` is not set (see above BIND_DIR line), in such case this will do nothing since `DOCKER_MOUNT` will already be set. +DOCKER_MOUNT := $(if $(DOCKER_MOUNT),$(DOCKER_MOUNT),-v "/go/src/github.com/docker/docker/bundles") GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) DOCKER_IMAGE := docker-dev$(if $(GIT_BRANCH),:$(GIT_BRANCH)) @@ -80,6 +84,16 @@ binary: build $(DOCKER_RUN_DOCKER) hack/make.sh binary build: bundles +ifeq ($(DOCKER_OSARCH), linux/arm) + # A few libnetwork integration tests require that the kernel be + # configured with "dummy" network interface and has the module + # loaded. However, the dummy module is not available by default + # on arm images. This ensures that it's built and loaded. + echo "Syncing kernel modules" + oc-sync-kernel-modules + depmod + modprobe dummy +endif docker build ${DOCKER_BUILD_ARGS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . bundles: diff --git a/components/engine/README.md b/components/engine/README.md index e10fbce9df..aa9cc0ea81 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -216,7 +216,7 @@ We are always open to suggestions on process improvements, and are always lookin Internet Relay Chat (IRC)

- IRC a direct line to our most knowledgeable Docker users; we have + IRC is a direct line to our most knowledgeable Docker users; we have both the #docker and #docker-dev group on irc.freenode.net. IRC is a rich chat protocol but it can overwhelm new users. You can search @@ -234,6 +234,8 @@ We are always open to suggestions on process improvements, and are always lookin The docker-dev group is for contributors and other people contributing to the Docker project. + You can join them without an google account by sending an email to e.g. "docker-user+subscribe@googlegroups.com". + After receiving the join-request message, you can simply reply to that to confirm the subscribtion. diff --git a/components/engine/ROADMAP.md b/components/engine/ROADMAP.md index 4ec0bf0a05..514fdb7423 100644 --- a/components/engine/ROADMAP.md +++ b/components/engine/ROADMAP.md @@ -33,97 +33,58 @@ won't be accepting pull requests adding or removing items from this file. # 1. Features and refactoring -## 1.1 Security +## 1.1 Runtime improvements -Security is a top objective for the Docker Engine. The most notable items we intend to provide in -the near future are: +We recently introduced [`runC`](https://runc.io) as a standalone low-level tool for container +execution. The initial goal was to integrate runC as a replacement in the Engine for the traditional +default libcontainer `execdriver`, but the Engine internals were not ready for this. -- Trusted distribution of images: the effort is driven by the [distribution](https://github.com/docker/distribution) -group but will have significant impact on the Engine -- [User namespaces](https://github.com/docker/docker/pull/12648) -- [Seccomp support](https://github.com/docker/libcontainer/pull/613) +As runC continued evolving, and the OCI specification along with it, we created +[`containerd`](https://containerd.tools/), a daemon to control and monitor multiple `runC`. This is +the new target for Engine integration, as it can entirely replace the whole `execdriver` +architecture, and container monitoring along with it. -## 1.2 Plumbing project +Docker Engine will rely on a long-running `containerd` companion daemon for all container execution +related operations. This could open the door in the future for Engine restarts without interrupting +running containers. -We define a plumbing tool as a standalone piece of software usable and meaningful on its own. In -the current state of the Docker Engine, most subsystems provide independent functionalities (such -the builder, pushing and pulling images, running applications in a containerized environment, etc) -but all are coupled in a single binary. We want to offer the users to flexibility to use only the -pieces they need, and we will also gain in maintainability by splitting the project among multiple -repositories. +## 1.2 Plugins improvements -As it currently stands, the rough design outlines is to have: -- Low level plumbing tools, each dealing with one responsibility (e.g., [runC](https://runc.io)) -- Docker subsystems services, each exposing an elementary concept over an API, and relying on one or -multiple lower level plumbing tools for their implementation (e.g., network management) -- Docker Engine to expose higher level actions (e.g., create a container with volume `V` and network -`N`), while still providing pass-through access to the individual subsystems. +Docker Engine 1.7.0 introduced plugin support, initially for the use cases of volumes and networks +extensions. The plugin infrastructure was kept minimal as we were collecting use cases and real +world feedback before optimizing for any particular workflow. -The architectural details are still being worked on, but one thing we know for sure is that we need -to technically decouple the pieces. +In the future, we'd like plugins to become first class citizens, and encourage an ecosystem of +plugins. This implies in particular making it trivially easy to distribute plugins as containers +through any Registry instance, as well as solving the commonly heard pain points of plugins needing +to be treated as somewhat special (being active at all time, started before any other user +containers, and not as easily dismissed). -### 1.2.1 Runtime +## 1.3 Internal decoupling -A Runtime tool already exists today in the form of [runC](https://github.com/opencontainers/runc). -We intend to modify the Engine to directly call out to a binary implementing the Open Containers -Specification such as runC rather than relying on libcontainer to set the container runtime up. +A lot of work has been done in trying to decouple the Docker Engine's internals. In particular, the +API implementation has been refactored and ongoing work is happening to move the code to a separate +repository ([`docker/engine-api`](https://github.com/docker/engine-api)), and the Builder side of +the daemon is now [fully independent](https://github.com/docker/docker/tree/master/builder) while +still residing in the same repository. -This plan will deprecate the existing [`execdriver`](https://github.com/docker/docker/tree/master/daemon/execdriver) -as different runtime backends will be implemented as separated binaries instead of being compiled -into the Engine. +We are exploring ways to go further with that decoupling, capitalizing on the work introduced by the +runtime renovation and plugins improvement efforts. Indeed, the combination of `containerd` support +with the concept of "special" containers opens the door for bootstrapping more Engine internals +using the same facilities. -### 1.2.2 Builder +## 1.4 Cluster capable Engine -The Builder (i.e., the ability to build an image from a Dockerfile) is already nicely decoupled, -but would benefit from being entirely separated from the Engine, and rely on the standard Engine -API for its operations. +The community has been pushing for a more cluster capable Docker Engine, and a huge effort was spent +adding features such as multihost networking, and node discovery down at the Engine level. Yet, the +Engine is currently incapable of taking scheduling decisions alone, and continues relying on Swarm +for that. -### 1.2.3 Distribution - -Distribution already has a [dedicated repository](https://github.com/docker/distribution) which -holds the implementation for Registry v2 and client libraries. We could imagine going further by -having the Engine call out to a binary providing image distribution related functionalities. - -There are two short term goals related to image distribution. The first is stabilize and simplify -the push/pull code. Following that is the conversion to the more secure Registry V2 protocol. - -### 1.2.4 Networking - -Most of networking related code was already decoupled today in [libnetwork](https://github.com/docker/libnetwork). -As with other ingredients, we might want to take it a step further and make it a meaningful utility -that the Engine would call out to instead of a library. - -## 1.3 Plugins - -An initiative around plugins started with Docker 1.7.0, with the goal of allowing for out of -process extensibility of some Docker functionalities, starting with volumes and networking. The -approach is to provide specific extension points rather than generic hooking facilities. We also -deliberately keep the extensions API the simplest possible, expanding as we discover valid use -cases that cannot be implemented. - -At the time of writing: - -- Plugin support is merged as an experimental feature: real world use cases and user feedback will -help us refine the UX to make the feature more user friendly. -- There are no immediate plans to expand on the number of pluggable subsystems. -- Golang 1.5 might add language support for [plugins](https://docs.google.com/document/d/1nr-TQHw_er6GOQRsF6T43GGhFDelrAP0NqSS_00RgZQ) -which we consider supporting as an alternative to JSON/HTTP. - -## 1.4 Volume management - -Volumes are not a first class citizen in the Engine today: we would like better volume management, -similar to the way network are managed in the new [CNM](https://github.com/docker/docker/issues/9983). - -## 1.5 Better API implementation - -The current Engine API is insufficiently typed, versioned, and ultimately hard to maintain. We -also suffer from the lack of a common implementation with [Swarm](https://github.com/docker/swarm). - -## 1.6 Checkpoint/restore - -Support for checkpoint/restore was [merged](https://github.com/docker/libcontainer/pull/479) in -[libcontainer](https://github.com/docker/libcontainer) and made available through [runC](https://runc.io): -we intend to take advantage of it in the Engine. +We plan to complete this effort and make Engine fully cluster capable. Multiple instances of the +Docker Engine being already capable of discovering each other and establish overlay networking for +their container to communicate, the next step is for a given Engine to gain ability to dispatch work +to another node in the cluster. This will be introduced in a backward compatible way, such that a +`docker run` invocation on a particular node remains fully deterministic. # 2 Frozen features @@ -139,45 +100,41 @@ The Dockerfile syntax as we know it is simple, and has proven successful in supp definitive move, we temporarily won't accept more patches to the Dockerfile syntax for several reasons: -- Long term impact of syntax changes is a sensitive matter that require an amount of attention -the volume of Engine codebase and activity today doesn't allow us to provide. -- Allowing the Builder to be implemented as a separate utility consuming the Engine's API will -open the door for many possibilities, such as offering alternate syntaxes or DSL for existing -languages without cluttering the Engine's codebase. -- A standalone Builder will also offer the opportunity for a better dedicated group of maintainers -to own the Dockerfile syntax and decide collectively on the direction to give it. -- Our experience with official images tend to show that no new instruction or syntax expansion is -*strictly* necessary for the majority of use cases, and although we are aware many things are still -lacking for many, we cannot make it a priority yet for the above reasons. + - Long term impact of syntax changes is a sensitive matter that require an amount of attention the + volume of Engine codebase and activity today doesn't allow us to provide. + - Allowing the Builder to be implemented as a separate utility consuming the Engine's API will + open the door for many possibilities, such as offering alternate syntaxes or DSL for existing + languages without cluttering the Engine's codebase. + - A standalone Builder will also offer the opportunity for a better dedicated group of maintainers + to own the Dockerfile syntax and decide collectively on the direction to give it. + - Our experience with official images tend to show that no new instruction or syntax expansion is + *strictly* necessary for the majority of use cases, and although we are aware many things are + still lacking for many, we cannot make it a priority yet for the above reasons. Again, this is not about saying that the Dockerfile syntax is done, it's about making choices about what we want to do first! ## 2.3 Remote Registry Operations -A large amount of work is ongoing in the area of image distribution and -provenance. This includes moving to the V2 Registry API and heavily -refactoring the code that powers these features. The desired result is more -secure, reliable and easier to use image distribution. +A large amount of work is ongoing in the area of image distribution and provenance. This includes +moving to the V2 Registry API and heavily refactoring the code that powers these features. The +desired result is more secure, reliable and easier to use image distribution. -Part of the problem with this part of the code base is the lack of a stable -and flexible interface. If new features are added that access the registry -without solidifying these interfaces, achieving feature parity will continue -to be elusive. While we get a handle on this situation, we are imposing a -moratorium on new code that accesses the Registry API in commands that don't -already make remote calls. +Part of the problem with this part of the code base is the lack of a stable and flexible interface. +If new features are added that access the registry without solidifying these interfaces, achieving +feature parity will continue to be elusive. While we get a handle on this situation, we are imposing +a moratorium on new code that accesses the Registry API in commands that don't already make remote +calls. -Currently, only the following commands cause interaction with a remote -registry: +Currently, only the following commands cause interaction with a remote registry: -- push -- pull -- run -- build -- search -- login + - push + - pull + - run + - build + - search + - login -In the interest of stabilizing the registry access model during this ongoing -work, we are not accepting additions to other commands that will cause remote -interaction with the Registry API. This moratorium will lift when the goals of -the distribution project have been met. +In the interest of stabilizing the registry access model during this ongoing work, we are not +accepting additions to other commands that will cause remote interaction with the Registry API. This +moratorium will lift when the goals of the distribution project have been met. diff --git a/components/engine/api/client/attach.go b/components/engine/api/client/attach.go index efdad108ce..c7b1eae3f3 100644 --- a/components/engine/api/client/attach.go +++ b/components/engine/api/client/attach.go @@ -41,12 +41,6 @@ func (cli *DockerCli) CmdAttach(args ...string) error { return err } - if c.Config.Tty && cli.isTerminalOut { - if err := cli.monitorTtySize(cmd.Arg(0), false); err != nil { - logrus.Debugf("Error monitoring TTY size: %s", err) - } - } - if *detachKeys != "" { cli.configFile.DetachKeys = *detachKeys } @@ -82,6 +76,21 @@ func (cli *DockerCli) CmdAttach(args ...string) error { defer cli.restoreTerminal(in) } + if c.Config.Tty && cli.isTerminalOut { + height, width := cli.getTtySize() + // To handle the case where a user repeatedly attaches/detaches without resizing their + // terminal, the only way to get the shell prompt to display for attaches 2+ is to artificially + // resize it, then go back to normal. Without this, every attach after the first will + // require the user to manually resize or hit enter. + cli.resizeTtyTo(cmd.Arg(0), height+1, width+1, false) + + // After the above resizing occurs, the call to monitorTtySize below will handle resetting back + // to the actual size. + if err := cli.monitorTtySize(cmd.Arg(0), false); err != nil { + logrus.Debugf("Error monitoring TTY size: %s", err) + } + } + if err := cli.holdHijackedConnection(c.Config.Tty, in, cli.out, cli.err, resp); err != nil { return err } diff --git a/components/engine/api/client/build.go b/components/engine/api/client/build.go index e6a4749d39..eef716a99e 100644 --- a/components/engine/api/client/build.go +++ b/components/engine/api/client/build.go @@ -6,25 +6,20 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "os" - "os/exec" "path/filepath" "regexp" "runtime" - "strings" "golang.org/x/net/context" "github.com/docker/docker/api" + "github.com/docker/docker/builder" "github.com/docker/docker/builder/dockerignore" Cli "github.com/docker/docker/cli" "github.com/docker/docker/opts" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/fileutils" - "github.com/docker/docker/pkg/gitutils" - "github.com/docker/docker/pkg/httputils" - "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/jsonmessage" flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/progress" @@ -65,7 +60,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { flCgroupParent := cmd.String([]string{"-cgroup-parent"}, "", "Optional parent cgroup for the container") flBuildArg := opts.NewListOpts(runconfigopts.ValidateEnv) cmd.Var(&flBuildArg, []string{"-build-arg"}, "Set build-time variables") - isolation := cmd.String([]string{"-isolation"}, "", "Container isolation level") + isolation := cmd.String([]string{"-isolation"}, "", "Container isolation technology") ulimits := make(map[string]*units.Ulimit) flUlimits := runconfigopts.NewUlimitOpt(&ulimits) @@ -102,13 +97,13 @@ func (cli *DockerCli) CmdBuild(args ...string) error { switch { case specifiedContext == "-": - ctx, relDockerfile, err = getContextFromReader(cli.in, *dockerfileName) + ctx, relDockerfile, err = builder.GetContextFromReader(cli.in, *dockerfileName) case urlutil.IsGitURL(specifiedContext): - tempDir, relDockerfile, err = getContextFromGitURL(specifiedContext, *dockerfileName) + tempDir, relDockerfile, err = builder.GetContextFromGitURL(specifiedContext, *dockerfileName) case urlutil.IsURL(specifiedContext): - ctx, relDockerfile, err = getContextFromURL(progBuff, specifiedContext, *dockerfileName) + ctx, relDockerfile, err = builder.GetContextFromURL(progBuff, specifiedContext, *dockerfileName) default: - contextDir, relDockerfile, err = getContextFromLocalDir(specifiedContext, *dockerfileName) + contextDir, relDockerfile, err = builder.GetContextFromLocalDir(specifiedContext, *dockerfileName) } if err != nil { @@ -143,7 +138,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { } } - if err := validateContextDirectory(contextDir, excludes); err != nil { + if err := builder.ValidateContextDirectory(contextDir, excludes); err != nil { return fmt.Errorf("Error checking context: '%s'.", err) } @@ -223,7 +218,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { Remove: *rm, ForceRemove: *forceRm, PullParent: *pull, - IsolationLevel: container.IsolationLevel(*isolation), + Isolation: container.Isolation(*isolation), CPUSetCPUs: *flCPUSetCpus, CPUSetMems: *flCPUSetMems, CPUShares: *flCPUShares, @@ -234,7 +229,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { ShmSize: shmSize, Ulimits: flUlimits.GetList(), BuildArgs: runconfigopts.ConvertKVStringsToMap(flBuildArg.GetAll()), - AuthConfigs: cli.configFile.AuthConfigs, + AuthConfigs: cli.retrieveAuthConfigs(), } response, err := cli.client.ImageBuild(context.Background(), options) @@ -281,54 +276,6 @@ func (cli *DockerCli) CmdBuild(args ...string) error { return nil } -// validateContextDirectory checks if all the contents of the directory -// can be read and returns an error if some files can't be read -// symlinks which point to non-existing files don't trigger an error -func validateContextDirectory(srcPath string, excludes []string) error { - contextRoot, err := getContextRoot(srcPath) - if err != nil { - return err - } - return filepath.Walk(contextRoot, func(filePath string, f os.FileInfo, err error) error { - // skip this directory/file if it's not in the path, it won't get added to the context - if relFilePath, err := filepath.Rel(contextRoot, filePath); err != nil { - return err - } else if skip, err := fileutils.Matches(relFilePath, excludes); err != nil { - return err - } else if skip { - if f.IsDir() { - return filepath.SkipDir - } - return nil - } - - if err != nil { - if os.IsPermission(err) { - return fmt.Errorf("can't stat '%s'", filePath) - } - if os.IsNotExist(err) { - return nil - } - return err - } - - // skip checking if symlinks point to non-existing files, such symlinks can be useful - // also skip named pipes, because they hanging on open - if f.Mode()&(os.ModeSymlink|os.ModeNamedPipe) != 0 { - return nil - } - - if !f.IsDir() { - currentFile, err := os.Open(filePath) - if err != nil && os.IsPermission(err) { - return fmt.Errorf("no permission to read from '%s'", filePath) - } - currentFile.Close() - } - return nil - }) -} - // validateTag checks if the given image name can be resolved. func validateTag(rawRepo string) (string, error) { _, err := reference.ParseNamed(rawRepo) @@ -339,96 +286,6 @@ func validateTag(rawRepo string) (string, error) { return rawRepo, nil } -// isUNC returns true if the path is UNC (one starting \\). It always returns -// false on Linux. -func isUNC(path string) bool { - return runtime.GOOS == "windows" && strings.HasPrefix(path, `\\`) -} - -// getDockerfileRelPath uses the given context directory for a `docker build` -// and returns the absolute path to the context directory, the relative path of -// the dockerfile in that context directory, and a non-nil error on success. -func getDockerfileRelPath(givenContextDir, givenDockerfile string) (absContextDir, relDockerfile string, err error) { - if absContextDir, err = filepath.Abs(givenContextDir); err != nil { - return "", "", fmt.Errorf("unable to get absolute context directory: %v", err) - } - - // The context dir might be a symbolic link, so follow it to the actual - // target directory. - // - // FIXME. We use isUNC (always false on non-Windows platforms) to workaround - // an issue in golang. On Windows, EvalSymLinks does not work on UNC file - // paths (those starting with \\). This hack means that when using links - // on UNC paths, they will not be followed. - if !isUNC(absContextDir) { - absContextDir, err = filepath.EvalSymlinks(absContextDir) - if err != nil { - return "", "", fmt.Errorf("unable to evaluate symlinks in context path: %v", err) - } - } - - stat, err := os.Lstat(absContextDir) - if err != nil { - return "", "", fmt.Errorf("unable to stat context directory %q: %v", absContextDir, err) - } - - if !stat.IsDir() { - return "", "", fmt.Errorf("context must be a directory: %s", absContextDir) - } - - absDockerfile := givenDockerfile - if absDockerfile == "" { - // No -f/--file was specified so use the default relative to the - // context directory. - absDockerfile = filepath.Join(absContextDir, api.DefaultDockerfileName) - - // Just to be nice ;-) look for 'dockerfile' too but only - // use it if we found it, otherwise ignore this check - if _, err = os.Lstat(absDockerfile); os.IsNotExist(err) { - altPath := filepath.Join(absContextDir, strings.ToLower(api.DefaultDockerfileName)) - if _, err = os.Lstat(altPath); err == nil { - absDockerfile = altPath - } - } - } - - // If not already an absolute path, the Dockerfile path should be joined to - // the base directory. - if !filepath.IsAbs(absDockerfile) { - absDockerfile = filepath.Join(absContextDir, absDockerfile) - } - - // Evaluate symlinks in the path to the Dockerfile too. - // - // FIXME. We use isUNC (always false on non-Windows platforms) to workaround - // an issue in golang. On Windows, EvalSymLinks does not work on UNC file - // paths (those starting with \\). This hack means that when using links - // on UNC paths, they will not be followed. - if !isUNC(absDockerfile) { - absDockerfile, err = filepath.EvalSymlinks(absDockerfile) - if err != nil { - return "", "", fmt.Errorf("unable to evaluate symlinks in Dockerfile path: %v", err) - } - } - - if _, err := os.Lstat(absDockerfile); err != nil { - if os.IsNotExist(err) { - return "", "", fmt.Errorf("Cannot locate Dockerfile: %q", absDockerfile) - } - return "", "", fmt.Errorf("unable to stat Dockerfile: %v", err) - } - - if relDockerfile, err = filepath.Rel(absContextDir, absDockerfile); err != nil { - return "", "", fmt.Errorf("unable to get relative Dockerfile path: %v", err) - } - - if strings.HasPrefix(relDockerfile, ".."+string(filepath.Separator)) { - return "", "", fmt.Errorf("The Dockerfile (%s) must be within the build context (%s)", givenDockerfile, givenContextDir) - } - - return absContextDir, relDockerfile, nil -} - // writeToFile copies from the given reader and writes it to a file with the // given filename. func writeToFile(r io.Reader, filename string) error { @@ -445,107 +302,6 @@ func writeToFile(r io.Reader, filename string) error { return nil } -// getContextFromReader will read the contents of the given reader as either a -// Dockerfile or tar archive. Returns a tar archive used as a context and a -// path to the Dockerfile inside the tar. -func getContextFromReader(r io.ReadCloser, dockerfileName string) (out io.ReadCloser, relDockerfile string, err error) { - buf := bufio.NewReader(r) - - magic, err := buf.Peek(archive.HeaderSize) - if err != nil && err != io.EOF { - return nil, "", fmt.Errorf("failed to peek context header from STDIN: %v", err) - } - - if archive.IsArchive(magic) { - return ioutils.NewReadCloserWrapper(buf, func() error { return r.Close() }), dockerfileName, nil - } - - // Input should be read as a Dockerfile. - tmpDir, err := ioutil.TempDir("", "docker-build-context-") - if err != nil { - return nil, "", fmt.Errorf("unbale to create temporary context directory: %v", err) - } - - f, err := os.Create(filepath.Join(tmpDir, api.DefaultDockerfileName)) - if err != nil { - return nil, "", err - } - _, err = io.Copy(f, buf) - if err != nil { - f.Close() - return nil, "", err - } - - if err := f.Close(); err != nil { - return nil, "", err - } - if err := r.Close(); err != nil { - return nil, "", err - } - - tar, err := archive.Tar(tmpDir, archive.Uncompressed) - if err != nil { - return nil, "", err - } - - return ioutils.NewReadCloserWrapper(tar, func() error { - err := tar.Close() - os.RemoveAll(tmpDir) - return err - }), api.DefaultDockerfileName, nil - -} - -// getContextFromGitURL uses a Git URL as context for a `docker build`. The -// git repo is cloned into a temporary directory used as the context directory. -// Returns the absolute path to the temporary context directory, the relative -// path of the dockerfile in that context directory, and a non-nil error on -// success. -func getContextFromGitURL(gitURL, dockerfileName string) (absContextDir, relDockerfile string, err error) { - if _, err := exec.LookPath("git"); err != nil { - return "", "", fmt.Errorf("unable to find 'git': %v", err) - } - if absContextDir, err = gitutils.Clone(gitURL); err != nil { - return "", "", fmt.Errorf("unable to 'git clone' to temporary context directory: %v", err) - } - - return getDockerfileRelPath(absContextDir, dockerfileName) -} - -// getContextFromURL uses a remote URL as context for a `docker build`. The -// remote resource is downloaded as either a Dockerfile or a tar archive. -// Returns the tar archive used for the context and a path of the -// dockerfile inside the tar. -func getContextFromURL(out io.Writer, remoteURL, dockerfileName string) (io.ReadCloser, string, error) { - response, err := httputils.Download(remoteURL) - if err != nil { - return nil, "", fmt.Errorf("unable to download remote context %s: %v", remoteURL, err) - } - progressOutput := streamformatter.NewStreamFormatter().NewProgressOutput(out, true) - - // Pass the response body through a progress reader. - progReader := progress.NewProgressReader(response.Body, progressOutput, response.ContentLength, "", fmt.Sprintf("Downloading build context from remote url: %s", remoteURL)) - - return getContextFromReader(ioutils.NewReadCloserWrapper(progReader, func() error { return response.Body.Close() }), dockerfileName) -} - -// getContextFromLocalDir uses the given local directory as context for a -// `docker build`. Returns the absolute path to the local context directory, -// the relative path of the dockerfile in that context directory, and a non-nil -// error on success. -func getContextFromLocalDir(localDir, dockerfileName string) (absContextDir, relDockerfile string, err error) { - // When using a local context directory, when the Dockerfile is specified - // with the `-f/--file` option then it is considered relative to the - // current directory and not the context directory. - if dockerfileName != "" { - if dockerfileName, err = filepath.Abs(dockerfileName); err != nil { - return "", "", fmt.Errorf("unable to get absolute path to Dockerfile: %v", err) - } - } - - return getDockerfileRelPath(localDir, dockerfileName) -} - var dockerfileFromLinePattern = regexp.MustCompile(`(?i)^[\s]*FROM[ \f\r\t\v]+(?P[^ \f\r\t\v\n#]+)`) // resolvedTag records the repository, tag, and resolved digest reference diff --git a/components/engine/api/client/cli.go b/components/engine/api/client/cli.go index fd76fc9dbb..e49c5351d5 100644 --- a/components/engine/api/client/cli.go +++ b/components/engine/api/client/cli.go @@ -11,6 +11,7 @@ import ( "github.com/docker/docker/api" "github.com/docker/docker/cli" "github.com/docker/docker/cliconfig" + "github.com/docker/docker/cliconfig/credentials" "github.com/docker/docker/dockerversion" "github.com/docker/docker/opts" "github.com/docker/docker/pkg/term" @@ -125,6 +126,9 @@ func NewDockerCli(in io.ReadCloser, out, err io.Writer, clientFlags *cli.ClientF if e != nil { fmt.Fprintf(cli.err, "WARNING: Error loading config file:%v\n", e) } + if !configFile.ContainsAuth() { + credentials.DetectDefaultStore(configFile) + } cli.configFile = configFile host, err := getServerHost(clientFlags.Common.Hosts, clientFlags.Common.TLSOptions) diff --git a/components/engine/api/client/create.go b/components/engine/api/client/create.go index d0417322e9..0c19145463 100644 --- a/components/engine/api/client/create.go +++ b/components/engine/api/client/create.go @@ -42,7 +42,7 @@ func (cli *DockerCli) pullImageCustomOut(image string, out io.Writer) error { return err } - authConfig := cli.resolveAuthConfig(cli.configFile.AuthConfigs, repoInfo.Index) + authConfig := cli.resolveAuthConfig(repoInfo.Index) encodedAuth, err := encodeAuthToBase64(authConfig) if err != nil { return err diff --git a/components/engine/api/client/events.go b/components/engine/api/client/events.go index 54b88f054b..d2408c192e 100644 --- a/components/engine/api/client/events.go +++ b/components/engine/api/client/events.go @@ -6,10 +6,12 @@ import ( "io" "sort" "strings" + "sync" "time" "golang.org/x/net/context" + "github.com/Sirupsen/logrus" Cli "github.com/docker/docker/cli" "github.com/docker/docker/opts" "github.com/docker/docker/pkg/jsonlog" @@ -115,3 +117,30 @@ func printOutput(event eventtypes.Message, output io.Writer) { } fmt.Fprint(output, "\n") } + +type eventHandler struct { + handlers map[string]func(eventtypes.Message) + mu sync.Mutex +} + +func (w *eventHandler) Handle(action string, h func(eventtypes.Message)) { + w.mu.Lock() + w.handlers[action] = h + w.mu.Unlock() +} + +// Watch ranges over the passed in event chan and processes the events based on the +// handlers created for a given action. +// To stop watching, close the event chan. +func (w *eventHandler) Watch(c <-chan eventtypes.Message) { + for e := range c { + w.mu.Lock() + h, exists := w.handlers[e.Action] + w.mu.Unlock() + if !exists { + continue + } + logrus.Debugf("event handler: received event: %v", e) + go h(e) + } +} diff --git a/components/engine/api/client/formatter/custom.go b/components/engine/api/client/formatter/custom.go index 8a680705ca..9ac457a414 100644 --- a/components/engine/api/client/formatter/custom.go +++ b/components/engine/api/client/formatter/custom.go @@ -31,6 +31,7 @@ const ( repositoryHeader = "REPOSITORY" tagHeader = "TAG" digestHeader = "DIGEST" + mountsHeader = "MOUNTS" ) type containerContext struct { @@ -142,6 +143,20 @@ func (c *containerContext) Label(name string) string { return c.c.Labels[name] } +func (c *containerContext) Mounts() string { + c.addHeader(mountsHeader) + + var mounts []string + for _, m := range c.c.Mounts { + name := m.Name + if c.trunc { + name = stringutils.Truncate(name, 15) + } + mounts = append(mounts, name) + } + return strings.Join(mounts, ",") +} + type imageContext struct { baseSubContext trunc bool diff --git a/components/engine/api/client/formatter/custom_test.go b/components/engine/api/client/formatter/custom_test.go index 608622564a..6a21f2bcd4 100644 --- a/components/engine/api/client/formatter/custom_test.go +++ b/components/engine/api/client/formatter/custom_test.go @@ -12,7 +12,7 @@ import ( func TestContainerPsContext(t *testing.T) { containerID := stringid.GenerateRandomID() - unix := time.Now().Unix() + unix := time.Now().Add(-65 * time.Second).Unix() var ctx containerContext cases := []struct { @@ -55,7 +55,7 @@ func TestContainerPsContext(t *testing.T) { {types.Container{SizeRw: 10, SizeRootFs: 20}, true, "10 B (virtual 20 B)", sizeHeader, ctx.Size}, {types.Container{}, true, "", labelsHeader, ctx.Labels}, {types.Container{Labels: map[string]string{"cpu": "6", "storage": "ssd"}}, true, "cpu=6,storage=ssd", labelsHeader, ctx.Labels}, - {types.Container{Created: unix}, true, "Less than a second", runningForHeader, ctx.RunningFor}, + {types.Container{Created: unix}, true, "About a minute", runningForHeader, ctx.RunningFor}, } for _, c := range cases { diff --git a/components/engine/api/client/info.go b/components/engine/api/client/info.go index 42820cd6d3..2d02af3a58 100644 --- a/components/engine/api/client/info.go +++ b/components/engine/api/client/info.go @@ -50,6 +50,7 @@ func (cli *DockerCli) CmdInfo(args ...string) error { } ioutils.FprintfIfNotEmpty(cli.out, "Execution Driver: %s\n", info.ExecutionDriver) ioutils.FprintfIfNotEmpty(cli.out, "Logging Driver: %s\n", info.LoggingDriver) + ioutils.FprintfIfNotEmpty(cli.out, "Cgroup Driver: %s\n", info.CgroupDriver) fmt.Fprintf(cli.out, "Plugins: \n") fmt.Fprintf(cli.out, " Volume:") @@ -73,7 +74,7 @@ func (cli *DockerCli) CmdInfo(args ...string) error { fmt.Fprintf(cli.out, "Total Memory: %s\n", units.BytesSize(float64(info.MemTotal))) ioutils.FprintfIfNotEmpty(cli.out, "Name: %s\n", info.Name) ioutils.FprintfIfNotEmpty(cli.out, "ID: %s\n", info.ID) - + fmt.Fprintf(cli.out, "Docker Root Dir: %s\n", info.DockerRootDir) fmt.Fprintf(cli.out, "Debug mode (client): %v\n", utils.IsDebugEnabled()) fmt.Fprintf(cli.out, "Debug mode (server): %v\n", info.Debug) @@ -82,7 +83,6 @@ func (cli *DockerCli) CmdInfo(args ...string) error { fmt.Fprintf(cli.out, " Goroutines: %d\n", info.NGoroutines) fmt.Fprintf(cli.out, " System Time: %s\n", info.SystemTime) fmt.Fprintf(cli.out, " EventsListeners: %d\n", info.NEventsListener) - fmt.Fprintf(cli.out, " Docker Root Dir: %s\n", info.DockerRootDir) } ioutils.FprintfIfNotEmpty(cli.out, "Http Proxy: %s\n", info.HTTPProxy) @@ -105,6 +105,9 @@ func (cli *DockerCli) CmdInfo(args ...string) error { if !info.SwapLimit { fmt.Fprintln(cli.err, "WARNING: No swap limit support") } + if !info.KernelMemory { + fmt.Fprintln(cli.err, "WARNING: No kernel memory limit support") + } if !info.OomKillDisable { fmt.Fprintln(cli.err, "WARNING: No oom kill disable support") } diff --git a/components/engine/api/client/login.go b/components/engine/api/client/login.go index 18ce831911..6696feb8cf 100644 --- a/components/engine/api/client/login.go +++ b/components/engine/api/client/login.go @@ -9,13 +9,14 @@ import ( "strings" Cli "github.com/docker/docker/cli" + "github.com/docker/docker/cliconfig" + "github.com/docker/docker/cliconfig/credentials" flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/term" - "github.com/docker/engine-api/client" "github.com/docker/engine-api/types" ) -// CmdLogin logs in or registers a user to a Docker registry service. +// CmdLogin logs in a user to a Docker registry service. // // If no server is specified, the user will be logged into or registered to the registry's index server. // @@ -26,7 +27,9 @@ func (cli *DockerCli) CmdLogin(args ...string) error { flUser := cmd.String([]string{"u", "-username"}, "", "Username") flPassword := cmd.String([]string{"p", "-password"}, "", "Password") - flEmail := cmd.String([]string{"e", "-email"}, "", "Email") + + // Deprecated in 1.11: Should be removed in docker 1.13 + cmd.String([]string{"#e", "#-email"}, "", "Email") cmd.ParseFlags(args, true) @@ -36,32 +39,27 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } var serverAddress string + var isDefaultRegistry bool if len(cmd.Args()) > 0 { serverAddress = cmd.Arg(0) } else { serverAddress = cli.electAuthServer() + isDefaultRegistry = true } - authConfig, err := cli.configureAuth(*flUser, *flPassword, *flEmail, serverAddress) + authConfig, err := cli.configureAuth(*flUser, *flPassword, serverAddress, isDefaultRegistry) if err != nil { return err } response, err := cli.client.RegistryLogin(authConfig) if err != nil { - if client.IsErrUnauthorized(err) { - delete(cli.configFile.AuthConfigs, serverAddress) - if err2 := cli.configFile.Save(); err2 != nil { - fmt.Fprintf(cli.out, "WARNING: could not save config file: %v\n", err2) - } - } return err } - if err := cli.configFile.Save(); err != nil { - return fmt.Errorf("Error saving config file: %v", err) + if err := storeCredentials(cli.configFile, authConfig); err != nil { + return fmt.Errorf("Error saving credentials: %v", err) } - fmt.Fprintf(cli.out, "WARNING: login credentials saved in %s\n", cli.configFile.Filename()) if response.Status != "" { fmt.Fprintf(cli.out, "%s\n", response.Status) @@ -77,14 +75,19 @@ func (cli *DockerCli) promptWithDefault(prompt string, configDefault string) { } } -func (cli *DockerCli) configureAuth(flUser, flPassword, flEmail, serverAddress string) (types.AuthConfig, error) { - authconfig, ok := cli.configFile.AuthConfigs[serverAddress] - if !ok { - authconfig = types.AuthConfig{} +func (cli *DockerCli) configureAuth(flUser, flPassword, serverAddress string, isDefaultRegistry bool) (types.AuthConfig, error) { + authconfig, err := getCredentials(cli.configFile, serverAddress) + if err != nil { + return authconfig, err } + authconfig.Username = strings.TrimSpace(authconfig.Username) if flUser = strings.TrimSpace(flUser); flUser == "" { + if isDefaultRegistry { + // if this is a defauly registry (docker hub), then display the following message. + fmt.Fprintln(cli.out, "Login with your Docker ID to push and pull images from Docker Hub. If you don't have a Docker ID, head over to https://hub.docker.com to create one.") + } cli.promptWithDefault("Username", authconfig.Username) flUser = readInput(cli.in, cli.out) flUser = strings.TrimSpace(flUser) @@ -114,30 +117,10 @@ func (cli *DockerCli) configureAuth(flUser, flPassword, flEmail, serverAddress s } } - // Assume that a different username means they may not want to use - // the email from the config file, so prompt it - if flUser != authconfig.Username { - if flEmail == "" { - cli.promptWithDefault("Email", authconfig.Email) - flEmail = readInput(cli.in, cli.out) - if flEmail == "" { - flEmail = authconfig.Email - } - } - } else { - // However, if they don't override the username use the - // email from the cmd line if specified. IOW, allow - // then to change/override them. And if not specified, just - // use what's in the config file - if flEmail == "" { - flEmail = authconfig.Email - } - } authconfig.Username = flUser authconfig.Password = flPassword - authconfig.Email = flEmail authconfig.ServerAddress = serverAddress - cli.configFile.AuthConfigs[serverAddress] = authconfig + return authconfig, nil } @@ -150,3 +133,38 @@ func readInput(in io.Reader, out io.Writer) string { } return string(line) } + +// getCredentials loads the user credentials from a credentials store. +// The store is determined by the config file settings. +func getCredentials(c *cliconfig.ConfigFile, serverAddress string) (types.AuthConfig, error) { + s := loadCredentialsStore(c) + return s.Get(serverAddress) +} + +func getAllCredentials(c *cliconfig.ConfigFile) (map[string]types.AuthConfig, error) { + s := loadCredentialsStore(c) + return s.GetAll() +} + +// storeCredentials saves the user credentials in a credentials store. +// The store is determined by the config file settings. +func storeCredentials(c *cliconfig.ConfigFile, auth types.AuthConfig) error { + s := loadCredentialsStore(c) + return s.Store(auth) +} + +// eraseCredentials removes the user credentials from a credentials store. +// The store is determined by the config file settings. +func eraseCredentials(c *cliconfig.ConfigFile, serverAddress string) error { + s := loadCredentialsStore(c) + return s.Erase(serverAddress) +} + +// loadCredentialsStore initializes a new credentials store based +// in the settings provided in the configuration file. +func loadCredentialsStore(c *cliconfig.ConfigFile) credentials.Store { + if c.CredentialsStore != "" { + return credentials.NewNativeStore(c) + } + return credentials.NewFileStore(c) +} diff --git a/components/engine/api/client/logout.go b/components/engine/api/client/logout.go index f81eb8dd12..b5ff59ddd2 100644 --- a/components/engine/api/client/logout.go +++ b/components/engine/api/client/logout.go @@ -25,15 +25,16 @@ func (cli *DockerCli) CmdLogout(args ...string) error { serverAddress = cli.electAuthServer() } + // check if we're logged in based on the records in the config file + // which means it couldn't have user/pass cause they may be in the creds store if _, ok := cli.configFile.AuthConfigs[serverAddress]; !ok { fmt.Fprintf(cli.out, "Not logged in to %s\n", serverAddress) return nil } fmt.Fprintf(cli.out, "Remove login credentials for %s\n", serverAddress) - delete(cli.configFile.AuthConfigs, serverAddress) - if err := cli.configFile.Save(); err != nil { - return fmt.Errorf("Failed to save docker config: %v", err) + if err := eraseCredentials(cli.configFile, serverAddress); err != nil { + fmt.Fprintf(cli.out, "WARNING: could not erase credentials: %v\n", err) } return nil diff --git a/components/engine/api/client/network.go b/components/engine/api/client/network.go index 56adabc00b..f992af3da4 100644 --- a/components/engine/api/client/network.go +++ b/components/engine/api/client/network.go @@ -3,6 +3,7 @@ package client import ( "fmt" "net" + "sort" "strings" "text/tabwriter" @@ -50,6 +51,7 @@ func (cli *DockerCli) CmdNetworkCreate(args ...string) error { cmd.Var(flIpamOpt, []string{"-ipam-opt"}, "set IPAM driver specific options") flInternal := cmd.Bool([]string{"-internal"}, false, "restricts external access to the network") + flIPv6 := cmd.Bool([]string{"-ipv6"}, false, "enable IPv6 networking") cmd.Require(flag.Exact, 1) err := cmd.ParseFlags(args, true) @@ -77,6 +79,7 @@ func (cli *DockerCli) CmdNetworkCreate(args ...string) error { Options: flOpts.GetAll(), CheckDuplicate: true, Internal: *flInternal, + EnableIPv6: *flIPv6, } resp, err := cli.client.NetworkCreate(nc) @@ -192,7 +195,7 @@ func (cli *DockerCli) CmdNetworkLs(args ...string) error { if !*quiet { fmt.Fprintln(wr, "NETWORK ID\tNAME\tDRIVER") } - + sort.Sort(byNetworkName(networkResources)) for _, networkResource := range networkResources { ID := networkResource.ID netName := networkResource.Name @@ -214,6 +217,12 @@ func (cli *DockerCli) CmdNetworkLs(args ...string) error { return nil } +type byNetworkName []types.NetworkResource + +func (r byNetworkName) Len() int { return len(r) } +func (r byNetworkName) Swap(i, j int) { r[i], r[j] = r[j], r[i] } +func (r byNetworkName) Less(i, j int) bool { return r[i].Name < r[j].Name } + // CmdNetworkInspect inspects the network object for more details // // Usage: docker network inspect [OPTIONS] [NETWORK...] diff --git a/components/engine/api/client/pull.go b/components/engine/api/client/pull.go index cd15caaa4f..29d9677e95 100644 --- a/components/engine/api/client/pull.go +++ b/components/engine/api/client/pull.go @@ -56,7 +56,7 @@ func (cli *DockerCli) CmdPull(args ...string) error { return err } - authConfig := cli.resolveAuthConfig(cli.configFile.AuthConfigs, repoInfo.Index) + authConfig := cli.resolveAuthConfig(repoInfo.Index) requestPrivilege := cli.registryAuthenticationPrivilegedFunc(repoInfo.Index, "pull") if isTrusted() && !ref.HasDigest() { diff --git a/components/engine/api/client/push.go b/components/engine/api/client/push.go index f06a9892ef..29f26c4673 100644 --- a/components/engine/api/client/push.go +++ b/components/engine/api/client/push.go @@ -44,7 +44,7 @@ func (cli *DockerCli) CmdPush(args ...string) error { return err } // Resolve the Auth config relevant for this server - authConfig := cli.resolveAuthConfig(cli.configFile.AuthConfigs, repoInfo.Index) + authConfig := cli.resolveAuthConfig(repoInfo.Index) requestPrivilege := cli.registryAuthenticationPrivilegedFunc(repoInfo.Index, "push") if isTrusted() { diff --git a/components/engine/api/client/run.go b/components/engine/api/client/run.go index 223964d595..b65a0a1bbe 100644 --- a/components/engine/api/client/run.go +++ b/components/engine/api/client/run.go @@ -11,7 +11,6 @@ import ( "github.com/Sirupsen/logrus" Cli "github.com/docker/docker/cli" - derr "github.com/docker/docker/errors" "github.com/docker/docker/opts" "github.com/docker/docker/pkg/promise" "github.com/docker/docker/pkg/signal" @@ -21,6 +20,11 @@ import ( "github.com/docker/libnetwork/resolvconf/dns" ) +const ( + errCmdNotFound = "Container command not found or does not exist." + errCmdCouldNotBeInvoked = "Container command could not be invoked." +) + func (cid *cidFile) Close() error { cid.file.Close() @@ -46,20 +50,13 @@ func (cid *cidFile) Write(id string) error { // return 125 for generic docker daemon failures func runStartContainerErr(err error) error { trimmedErr := strings.Trim(err.Error(), "Error response from daemon: ") - statusError := Cli.StatusError{} - derrCmdNotFound := derr.ErrorCodeCmdNotFound.Message() - derrCouldNotInvoke := derr.ErrorCodeCmdCouldNotBeInvoked.Message() - derrNoSuchImage := derr.ErrorCodeNoSuchImageHash.Message() - derrNoSuchImageTag := derr.ErrorCodeNoSuchImageTag.Message() + statusError := Cli.StatusError{StatusCode: 125} + switch trimmedErr { - case derrCmdNotFound: + case errCmdNotFound: statusError = Cli.StatusError{StatusCode: 127} - case derrCouldNotInvoke: + case errCmdCouldNotBeInvoked: statusError = Cli.StatusError{StatusCode: 126} - case derrNoSuchImage, derrNoSuchImageTag: - statusError = Cli.StatusError{StatusCode: 125} - default: - statusError = Cli.StatusError{StatusCode: 125} } return statusError } diff --git a/components/engine/api/client/search.go b/components/engine/api/client/search.go index 2e1fcdafd3..c7196a74cf 100644 --- a/components/engine/api/client/search.go +++ b/components/engine/api/client/search.go @@ -36,7 +36,7 @@ func (cli *DockerCli) CmdSearch(args ...string) error { return err } - authConfig := cli.resolveAuthConfig(cli.configFile.AuthConfigs, indexInfo) + authConfig := cli.resolveAuthConfig(indexInfo) requestPrivilege := cli.registryAuthenticationPrivilegedFunc(indexInfo, "search") encodedAuth, err := encodeAuthToBase64(authConfig) diff --git a/components/engine/api/client/stats.go b/components/engine/api/client/stats.go index 0bfbe43f9d..3b9b8d8d48 100644 --- a/components/engine/api/client/stats.go +++ b/components/engine/api/client/stats.go @@ -1,10 +1,8 @@ package client import ( - "encoding/json" "fmt" "io" - "sort" "strings" "sync" "text/tabwriter" @@ -16,125 +14,8 @@ import ( "github.com/docker/engine-api/types" "github.com/docker/engine-api/types/events" "github.com/docker/engine-api/types/filters" - "github.com/docker/go-units" ) -type containerStats struct { - Name string - CPUPercentage float64 - Memory float64 - MemoryLimit float64 - MemoryPercentage float64 - NetworkRx float64 - NetworkTx float64 - BlockRead float64 - BlockWrite float64 - mu sync.RWMutex - err error -} - -type stats struct { - mu sync.Mutex - cs []*containerStats -} - -func (s *containerStats) Collect(cli *DockerCli, streamStats bool) { - responseBody, err := cli.client.ContainerStats(context.Background(), s.Name, streamStats) - if err != nil { - s.mu.Lock() - s.err = err - s.mu.Unlock() - return - } - defer responseBody.Close() - - var ( - previousCPU uint64 - previousSystem uint64 - dec = json.NewDecoder(responseBody) - u = make(chan error, 1) - ) - go func() { - for { - var v *types.StatsJSON - if err := dec.Decode(&v); err != nil { - u <- err - return - } - - var memPercent = 0.0 - var cpuPercent = 0.0 - - // MemoryStats.Limit will never be 0 unless the container is not running and we haven't - // got any data from cgroup - if v.MemoryStats.Limit != 0 { - memPercent = float64(v.MemoryStats.Usage) / float64(v.MemoryStats.Limit) * 100.0 - } - - previousCPU = v.PreCPUStats.CPUUsage.TotalUsage - previousSystem = v.PreCPUStats.SystemUsage - cpuPercent = calculateCPUPercent(previousCPU, previousSystem, v) - blkRead, blkWrite := calculateBlockIO(v.BlkioStats) - s.mu.Lock() - s.CPUPercentage = cpuPercent - s.Memory = float64(v.MemoryStats.Usage) - s.MemoryLimit = float64(v.MemoryStats.Limit) - s.MemoryPercentage = memPercent - s.NetworkRx, s.NetworkTx = calculateNetwork(v.Networks) - s.BlockRead = float64(blkRead) - s.BlockWrite = float64(blkWrite) - s.mu.Unlock() - u <- nil - if !streamStats { - return - } - } - }() - for { - select { - case <-time.After(2 * time.Second): - // zero out the values if we have not received an update within - // the specified duration. - s.mu.Lock() - s.CPUPercentage = 0 - s.Memory = 0 - s.MemoryPercentage = 0 - s.MemoryLimit = 0 - s.NetworkRx = 0 - s.NetworkTx = 0 - s.BlockRead = 0 - s.BlockWrite = 0 - s.mu.Unlock() - case err := <-u: - if err != nil { - s.mu.Lock() - s.err = err - s.mu.Unlock() - return - } - } - if !streamStats { - return - } - } -} - -func (s *containerStats) Display(w io.Writer) error { - s.mu.RLock() - defer s.mu.RUnlock() - if s.err != nil { - return s.err - } - fmt.Fprintf(w, "%s\t%.2f%%\t%s / %s\t%.2f%%\t%s / %s\t%s / %s\n", - s.Name, - s.CPUPercentage, - units.HumanSize(s.Memory), units.HumanSize(s.MemoryLimit), - s.MemoryPercentage, - units.HumanSize(s.NetworkRx), units.HumanSize(s.NetworkTx), - units.HumanSize(s.BlockRead), units.HumanSize(s.BlockWrite)) - return nil -} - // CmdStats displays a live stream of resource usage statistics for one or more containers. // // This shows real-time information on CPU usage, memory usage, and network I/O. @@ -149,125 +30,143 @@ func (cli *DockerCli) CmdStats(args ...string) error { names := cmd.Args() showAll := len(names) == 0 + closeChan := make(chan error) - if showAll { + // monitorContainerEvents watches for container creation and removal (only + // used when calling `docker stats` without arguments). + monitorContainerEvents := func(started chan<- struct{}, c chan events.Message) { + f := filters.NewArgs() + f.Add("type", "container") + options := types.EventsOptions{ + Filters: f, + } + resBody, err := cli.client.Events(context.Background(), options) + // Whether we successfully subscribed to events or not, we can now + // unblock the main goroutine. + close(started) + if err != nil { + closeChan <- err + return + } + defer resBody.Close() + + decodeEvents(resBody, func(event events.Message, err error) error { + if err != nil { + closeChan <- err + return nil + } + c <- event + return nil + }) + } + + // waitFirst is a WaitGroup to wait first stat data's reach for each container + waitFirst := &sync.WaitGroup{} + + cStats := stats{} + // getContainerList simulates creation event for all previously existing + // containers (only used when calling `docker stats` without arguments). + getContainerList := func() { options := types.ContainerListOptions{ All: *all, } cs, err := cli.client.ContainerList(options) if err != nil { - return err + closeChan <- err } - for _, c := range cs { - names = append(names, c.ID[:12]) + for _, container := range cs { + s := &containerStats{Name: container.ID[:12]} + if cStats.add(s) { + waitFirst.Add(1) + go s.Collect(cli.client, !*noStream, waitFirst) + } } } - if len(names) == 0 && !showAll { - return fmt.Errorf("No containers found") - } - sort.Strings(names) - var ( - cStats = stats{} - w = tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) - ) + if showAll { + // If no names were specified, start a long running goroutine which + // monitors container events. We make sure we're subscribed before + // retrieving the list of running containers to avoid a race where we + // would "miss" a creation. + started := make(chan struct{}) + eh := eventHandler{handlers: make(map[string]func(events.Message))} + eh.Handle("create", func(e events.Message) { + if *all { + s := &containerStats{Name: e.ID[:12]} + if cStats.add(s) { + waitFirst.Add(1) + go s.Collect(cli.client, !*noStream, waitFirst) + } + } + }) + + eh.Handle("start", func(e events.Message) { + s := &containerStats{Name: e.ID[:12]} + if cStats.add(s) { + waitFirst.Add(1) + go s.Collect(cli.client, !*noStream, waitFirst) + } + }) + + eh.Handle("die", func(e events.Message) { + if !*all { + cStats.remove(e.ID[:12]) + } + }) + + eventChan := make(chan events.Message) + go eh.Watch(eventChan) + go monitorContainerEvents(started, eventChan) + defer close(eventChan) + <-started + + // Start a short-lived goroutine to retrieve the initial list of + // containers. + getContainerList() + } else { + // Artificially send creation events for the containers we were asked to + // monitor (same code path than we use when monitoring all containers). + for _, name := range names { + s := &containerStats{Name: name} + if cStats.add(s) { + waitFirst.Add(1) + go s.Collect(cli.client, !*noStream, waitFirst) + } + } + + // We don't expect any asynchronous errors: closeChan can be closed. + close(closeChan) + + // Do a quick pause to detect any error with the provided list of + // container names. + time.Sleep(1500 * time.Millisecond) + var errs []string + cStats.mu.Lock() + for _, c := range cStats.cs { + c.mu.Lock() + if c.err != nil { + errs = append(errs, fmt.Sprintf("%s: %v", c.Name, c.err)) + } + c.mu.Unlock() + } + cStats.mu.Unlock() + if len(errs) > 0 { + return fmt.Errorf("%s", strings.Join(errs, ", ")) + } + } + + // before print to screen, make sure each container get at least one valid stat data + waitFirst.Wait() + + w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) printHeader := func() { if !*noStream { fmt.Fprint(cli.out, "\033[2J") fmt.Fprint(cli.out, "\033[H") } - io.WriteString(w, "CONTAINER\tCPU %\tMEM USAGE / LIMIT\tMEM %\tNET I/O\tBLOCK I/O\n") + io.WriteString(w, "CONTAINER\tCPU %\tMEM USAGE / LIMIT\tMEM %\tNET I/O\tBLOCK I/O\tPIDS\n") } - for _, n := range names { - s := &containerStats{Name: n} - // no need to lock here since only the main goroutine is running here - cStats.cs = append(cStats.cs, s) - go s.Collect(cli, !*noStream) - } - closeChan := make(chan error) - if showAll { - type watch struct { - cid string - event string - err error - } - getNewContainers := func(c chan<- watch) { - f := filters.NewArgs() - f.Add("type", "container") - options := types.EventsOptions{ - Filters: f, - } - resBody, err := cli.client.Events(context.Background(), options) - if err != nil { - c <- watch{err: err} - return - } - defer resBody.Close() - decodeEvents(resBody, func(event events.Message, err error) error { - if err != nil { - c <- watch{err: err} - return nil - } - - c <- watch{event.ID[:12], event.Action, nil} - return nil - }) - } - go func(stopChan chan<- error) { - cChan := make(chan watch) - go getNewContainers(cChan) - for { - c := <-cChan - if c.err != nil { - stopChan <- c.err - return - } - switch c.event { - case "create": - s := &containerStats{Name: c.cid} - cStats.mu.Lock() - cStats.cs = append(cStats.cs, s) - cStats.mu.Unlock() - go s.Collect(cli, !*noStream) - case "stop": - case "die": - if !*all { - var remove int - // cStats cannot be O(1) with a map cause ranging over it would cause - // containers in stats to move up and down in the list...:( - cStats.mu.Lock() - for i, s := range cStats.cs { - if s.Name == c.cid { - remove = i - break - } - } - cStats.cs = append(cStats.cs[:remove], cStats.cs[remove+1:]...) - cStats.mu.Unlock() - } - } - } - }(closeChan) - } else { - close(closeChan) - } - // do a quick pause so that any failed connections for containers that do not exist are able to be - // evicted before we display the initial or default values. - time.Sleep(1500 * time.Millisecond) - var errs []string - cStats.mu.Lock() - for _, c := range cStats.cs { - c.mu.Lock() - if c.err != nil { - errs = append(errs, fmt.Sprintf("%s: %v", c.Name, c.err)) - } - c.mu.Unlock() - } - cStats.mu.Unlock() - if len(errs) > 0 { - return fmt.Errorf("%s", strings.Join(errs, ", ")) - } for range time.Tick(500 * time.Millisecond) { printHeader() toRemove := []int{} @@ -307,40 +206,3 @@ func (cli *DockerCli) CmdStats(args ...string) error { } return nil } - -func calculateCPUPercent(previousCPU, previousSystem uint64, v *types.StatsJSON) float64 { - var ( - cpuPercent = 0.0 - // calculate the change for the cpu usage of the container in between readings - cpuDelta = float64(v.CPUStats.CPUUsage.TotalUsage) - float64(previousCPU) - // calculate the change for the entire system between readings - systemDelta = float64(v.CPUStats.SystemUsage) - float64(previousSystem) - ) - - if systemDelta > 0.0 && cpuDelta > 0.0 { - cpuPercent = (cpuDelta / systemDelta) * float64(len(v.CPUStats.CPUUsage.PercpuUsage)) * 100.0 - } - return cpuPercent -} - -func calculateBlockIO(blkio types.BlkioStats) (blkRead uint64, blkWrite uint64) { - for _, bioEntry := range blkio.IoServiceBytesRecursive { - switch strings.ToLower(bioEntry.Op) { - case "read": - blkRead = blkRead + bioEntry.Value - case "write": - blkWrite = blkWrite + bioEntry.Value - } - } - return -} - -func calculateNetwork(network map[string]types.NetworkStats) (float64, float64) { - var rx, tx float64 - - for _, v := range network { - rx += float64(v.RxBytes) - tx += float64(v.TxBytes) - } - return rx, tx -} diff --git a/components/engine/api/client/stats_helpers.go b/components/engine/api/client/stats_helpers.go new file mode 100644 index 0000000000..5c88b7424d --- /dev/null +++ b/components/engine/api/client/stats_helpers.go @@ -0,0 +1,217 @@ +package client + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "sync" + "time" + + "github.com/docker/engine-api/client" + "github.com/docker/engine-api/types" + "github.com/docker/go-units" + "golang.org/x/net/context" +) + +type containerStats struct { + Name string + CPUPercentage float64 + Memory float64 + MemoryLimit float64 + MemoryPercentage float64 + NetworkRx float64 + NetworkTx float64 + BlockRead float64 + BlockWrite float64 + PidsCurrent uint64 + mu sync.RWMutex + err error +} + +type stats struct { + mu sync.Mutex + cs []*containerStats +} + +func (s *stats) add(cs *containerStats) bool { + s.mu.Lock() + defer s.mu.Unlock() + if _, exists := s.isKnownContainer(cs.Name); !exists { + s.cs = append(s.cs, cs) + return true + } + return false +} + +func (s *stats) remove(id string) { + s.mu.Lock() + if i, exists := s.isKnownContainer(id); exists { + s.cs = append(s.cs[:i], s.cs[i+1:]...) + } + s.mu.Unlock() +} + +func (s *stats) isKnownContainer(cid string) (int, bool) { + for i, c := range s.cs { + if c.Name == cid { + return i, true + } + } + return -1, false +} + +func (s *containerStats) Collect(cli client.APIClient, streamStats bool, waitFirst *sync.WaitGroup) { + var ( + getFirst bool + previousCPU uint64 + previousSystem uint64 + u = make(chan error, 1) + ) + + defer func() { + // if error happens and we get nothing of stats, release wait group whatever + if !getFirst { + getFirst = true + waitFirst.Done() + } + }() + + responseBody, err := cli.ContainerStats(context.Background(), s.Name, streamStats) + if err != nil { + s.mu.Lock() + s.err = err + s.mu.Unlock() + return + } + defer responseBody.Close() + + dec := json.NewDecoder(responseBody) + go func() { + for { + var v *types.StatsJSON + if err := dec.Decode(&v); err != nil { + u <- err + return + } + + var memPercent = 0.0 + var cpuPercent = 0.0 + + // MemoryStats.Limit will never be 0 unless the container is not running and we haven't + // got any data from cgroup + if v.MemoryStats.Limit != 0 { + memPercent = float64(v.MemoryStats.Usage) / float64(v.MemoryStats.Limit) * 100.0 + } + + previousCPU = v.PreCPUStats.CPUUsage.TotalUsage + previousSystem = v.PreCPUStats.SystemUsage + cpuPercent = calculateCPUPercent(previousCPU, previousSystem, v) + blkRead, blkWrite := calculateBlockIO(v.BlkioStats) + s.mu.Lock() + s.CPUPercentage = cpuPercent + s.Memory = float64(v.MemoryStats.Usage) + s.MemoryLimit = float64(v.MemoryStats.Limit) + s.MemoryPercentage = memPercent + s.NetworkRx, s.NetworkTx = calculateNetwork(v.Networks) + s.BlockRead = float64(blkRead) + s.BlockWrite = float64(blkWrite) + s.mu.Unlock() + u <- nil + if !streamStats { + return + } + } + }() + for { + select { + case <-time.After(2 * time.Second): + // zero out the values if we have not received an update within + // the specified duration. + s.mu.Lock() + s.CPUPercentage = 0 + s.Memory = 0 + s.MemoryPercentage = 0 + s.MemoryLimit = 0 + s.NetworkRx = 0 + s.NetworkTx = 0 + s.BlockRead = 0 + s.BlockWrite = 0 + s.mu.Unlock() + // if this is the first stat you get, release WaitGroup + if !getFirst { + getFirst = true + waitFirst.Done() + } + case err := <-u: + if err != nil { + s.mu.Lock() + s.err = err + s.mu.Unlock() + return + } + // if this is the first stat you get, release WaitGroup + if !getFirst { + getFirst = true + waitFirst.Done() + } + } + if !streamStats { + return + } + } +} + +func (s *containerStats) Display(w io.Writer) error { + s.mu.RLock() + defer s.mu.RUnlock() + if s.err != nil { + return s.err + } + fmt.Fprintf(w, "%s\t%.2f%%\t%s / %s\t%.2f%%\t%s / %s\t%s / %s\t%d\n", + s.Name, + s.CPUPercentage, + units.HumanSize(s.Memory), units.HumanSize(s.MemoryLimit), + s.MemoryPercentage, + units.HumanSize(s.NetworkRx), units.HumanSize(s.NetworkTx), + units.HumanSize(s.BlockRead), units.HumanSize(s.BlockWrite), + s.PidsCurrent) + return nil +} + +func calculateCPUPercent(previousCPU, previousSystem uint64, v *types.StatsJSON) float64 { + var ( + cpuPercent = 0.0 + // calculate the change for the cpu usage of the container in between readings + cpuDelta = float64(v.CPUStats.CPUUsage.TotalUsage) - float64(previousCPU) + // calculate the change for the entire system between readings + systemDelta = float64(v.CPUStats.SystemUsage) - float64(previousSystem) + ) + + if systemDelta > 0.0 && cpuDelta > 0.0 { + cpuPercent = (cpuDelta / systemDelta) * float64(len(v.CPUStats.CPUUsage.PercpuUsage)) * 100.0 + } + return cpuPercent +} + +func calculateBlockIO(blkio types.BlkioStats) (blkRead uint64, blkWrite uint64) { + for _, bioEntry := range blkio.IoServiceBytesRecursive { + switch strings.ToLower(bioEntry.Op) { + case "read": + blkRead = blkRead + bioEntry.Value + case "write": + blkWrite = blkWrite + bioEntry.Value + } + } + return +} + +func calculateNetwork(network map[string]types.NetworkStats) (float64, float64) { + var rx, tx float64 + + for _, v := range network { + rx += float64(v.RxBytes) + tx += float64(v.TxBytes) + } + return rx, tx +} diff --git a/components/engine/api/client/stats_unit_test.go b/components/engine/api/client/stats_unit_test.go index ce1c3a1741..36081c5772 100644 --- a/components/engine/api/client/stats_unit_test.go +++ b/components/engine/api/client/stats_unit_test.go @@ -19,6 +19,7 @@ func TestDisplay(t *testing.T) { NetworkTx: 800 * 1024 * 1024, BlockRead: 100 * 1024 * 1024, BlockWrite: 800 * 1024 * 1024, + PidsCurrent: 1, mu: sync.RWMutex{}, } var b bytes.Buffer @@ -26,7 +27,7 @@ func TestDisplay(t *testing.T) { t.Fatalf("c.Display() gave error: %s", err) } got := b.String() - want := "app\t30.00%\t104.9 MB / 2.147 GB\t4.88%\t104.9 MB / 838.9 MB\t104.9 MB / 838.9 MB\n" + want := "app\t30.00%\t104.9 MB / 2.147 GB\t4.88%\t104.9 MB / 838.9 MB\t104.9 MB / 838.9 MB\t1\n" if got != want { t.Fatalf("c.Display() = %q, want %q", got, want) } diff --git a/components/engine/api/client/trust.go b/components/engine/api/client/trust.go index 753bcd6fc7..88cfb3244b 100644 --- a/components/engine/api/client/trust.go +++ b/components/engine/api/client/trust.go @@ -107,7 +107,10 @@ func (scs simpleCredentialStore) Basic(u *url.URL) (string, string) { return scs.auth.Username, scs.auth.Password } -func (cli *DockerCli) getNotaryRepository(repoInfo *registry.RepositoryInfo, authConfig types.AuthConfig) (*client.NotaryRepository, error) { +// getNotaryRepository returns a NotaryRepository which stores all the +// information needed to operate on a notary repository. +// It creates a HTTP transport providing authentication support. +func (cli *DockerCli) getNotaryRepository(repoInfo *registry.RepositoryInfo, authConfig types.AuthConfig, actions ...string) (*client.NotaryRepository, error) { server, err := trustServer(repoInfo.Index) if err != nil { return nil, err @@ -169,7 +172,7 @@ func (cli *DockerCli) getNotaryRepository(repoInfo *registry.RepositoryInfo, aut } creds := simpleCredentialStore{auth: authConfig} - tokenHandler := auth.NewTokenHandler(authTransport, creds, repoInfo.FullName(), "push", "pull") + tokenHandler := auth.NewTokenHandler(authTransport, creds, repoInfo.FullName(), actions...) basicHandler := auth.NewBasicHandler(creds) modifiers = append(modifiers, transport.RequestModifier(auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))) tr := transport.NewTransport(base, modifiers...) @@ -235,7 +238,7 @@ func (cli *DockerCli) trustedReference(ref reference.NamedTagged) (reference.Can } // Resolve the Auth config relevant for this server - authConfig := cli.resolveAuthConfig(cli.configFile.AuthConfigs, repoInfo.Index) + authConfig := cli.resolveAuthConfig(repoInfo.Index) notaryRepo, err := cli.getNotaryRepository(repoInfo, authConfig) if err != nil { @@ -302,7 +305,7 @@ func notaryError(repoName string, err error) error { func (cli *DockerCli) trustedPull(repoInfo *registry.RepositoryInfo, ref registry.Reference, authConfig types.AuthConfig, requestPrivilege apiclient.RequestPrivilegeFunc) error { var refs []target - notaryRepo, err := cli.getNotaryRepository(repoInfo, authConfig) + notaryRepo, err := cli.getNotaryRepository(repoInfo, authConfig, "pull") if err != nil { fmt.Fprintf(cli.out, "Error establishing connection to trust repository: %s\n", err) return err @@ -372,60 +375,74 @@ func (cli *DockerCli) trustedPush(repoInfo *registry.RepositoryInfo, tag string, defer responseBody.Close() - targets := []target{} + // If it is a trusted push we would like to find the target entry which match the + // tag provided in the function and then do an AddTarget later. + target := &client.Target{} + // Count the times of calling for handleTarget, + // if it is called more that once, that should be considered an error in a trusted push. + cnt := 0 handleTarget := func(aux *json.RawMessage) { + cnt++ + if cnt > 1 { + // handleTarget should only be called one. This will be treated as an error. + return + } + var pushResult distribution.PushResult err := json.Unmarshal(*aux, &pushResult) if err == nil && pushResult.Tag != "" && pushResult.Digest.Validate() == nil { - targets = append(targets, target{ - reference: registry.ParseReference(pushResult.Tag), - digest: pushResult.Digest, - size: int64(pushResult.Size), - }) + h, err := hex.DecodeString(pushResult.Digest.Hex()) + if err != nil { + target = nil + return + } + target.Name = registry.ParseReference(pushResult.Tag).String() + target.Hashes = data.Hashes{string(pushResult.Digest.Algorithm()): h} + target.Length = int64(pushResult.Size) } } - err = jsonmessage.DisplayJSONMessagesStream(responseBody, cli.out, cli.outFd, cli.isTerminalOut, handleTarget) - if err != nil { + // We want trust signatures to always take an explicit tag, + // otherwise it will act as an untrusted push. + if tag == "" { + if err = jsonmessage.DisplayJSONMessagesStream(responseBody, cli.out, cli.outFd, cli.isTerminalOut, nil); err != nil { + return err + } + fmt.Fprintln(cli.out, "No tag specified, skipping trust metadata push") + return nil + } + + if err = jsonmessage.DisplayJSONMessagesStream(responseBody, cli.out, cli.outFd, cli.isTerminalOut, handleTarget); err != nil { return err } - if tag == "" { - fmt.Fprintf(cli.out, "No tag specified, skipping trust metadata push\n") - return nil + if cnt > 1 { + return fmt.Errorf("internal error: only one call to handleTarget expected") } - if len(targets) == 0 { - fmt.Fprintf(cli.out, "No targets found, skipping trust metadata push\n") + + if target == nil { + fmt.Fprintln(cli.out, "No targets found, please provide a specific tag in order to sign it") return nil } - fmt.Fprintf(cli.out, "Signing and pushing trust metadata\n") + fmt.Fprintln(cli.out, "Signing and pushing trust metadata") - repo, err := cli.getNotaryRepository(repoInfo, authConfig) + repo, err := cli.getNotaryRepository(repoInfo, authConfig, "push", "pull") if err != nil { fmt.Fprintf(cli.out, "Error establishing connection to notary repository: %s\n", err) return err } - for _, target := range targets { - h, err := hex.DecodeString(target.digest.Hex()) - if err != nil { - return err - } - t := &client.Target{ - Name: target.reference.String(), - Hashes: data.Hashes{ - string(target.digest.Algorithm()): h, - }, - Length: int64(target.size), - } - if err := repo.AddTarget(t, releasesRole); err != nil { - return err - } + if err := repo.AddTarget(target, releasesRole); err != nil { + return err } err = repo.Publish() - if _, ok := err.(client.ErrRepoNotInitialized); !ok { + if err == nil { + fmt.Fprintf(cli.out, "Successfully signed %q:%s\n", repoInfo.FullName(), tag) + return nil + } else if _, ok := err.(client.ErrRepoNotInitialized); !ok { + fmt.Fprintf(cli.out, "Failed to sign %q:%s - %s\n", repoInfo.FullName(), tag, err.Error()) return notaryError(repoInfo.FullName(), err) } @@ -444,7 +461,8 @@ func (cli *DockerCli) trustedPush(repoInfo *registry.RepositoryInfo, tag string, rootKeyID = rootPublicKey.ID() } - if err := repo.Initialize(rootKeyID); err != nil { + // Initialize the notary repository with a remotely managed snapshot key + if err := repo.Initialize(rootKeyID, data.CanonicalSnapshotRole); err != nil { return notaryError(repoInfo.FullName(), err) } fmt.Fprintf(cli.out, "Finished initializing %q\n", repoInfo.FullName()) diff --git a/components/engine/api/client/update.go b/components/engine/api/client/update.go index b33ed476a6..0a576e7c73 100644 --- a/components/engine/api/client/update.go +++ b/components/engine/api/client/update.go @@ -6,6 +6,7 @@ import ( Cli "github.com/docker/docker/cli" flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/runconfig/opts" "github.com/docker/engine-api/types/container" "github.com/docker/go-units" ) @@ -25,6 +26,7 @@ func (cli *DockerCli) CmdUpdate(args ...string) error { flMemoryReservation := cmd.String([]string{"-memory-reservation"}, "", "Memory soft limit") flMemorySwap := cmd.String([]string{"-memory-swap"}, "", "Swap limit equal to memory plus swap: '-1' to enable unlimited swap") flKernelMemory := cmd.String([]string{"-kernel-memory"}, "", "Kernel memory limit") + flRestartPolicy := cmd.String([]string{"-restart"}, "", "Restart policy to apply when a container exits") cmd.Require(flag.Min, 1) cmd.ParseFlags(args, true) @@ -69,6 +71,14 @@ func (cli *DockerCli) CmdUpdate(args ...string) error { } } + var restartPolicy container.RestartPolicy + if *flRestartPolicy != "" { + restartPolicy, err = opts.ParseRestartPolicy(*flRestartPolicy) + if err != nil { + return err + } + } + resources := container.Resources{ BlkioWeight: *flBlkioWeight, CpusetCpus: *flCpusetCpus, @@ -83,7 +93,8 @@ func (cli *DockerCli) CmdUpdate(args ...string) error { } updateConfig := container.UpdateConfig{ - Resources: resources, + Resources: resources, + RestartPolicy: restartPolicy, } names := cmd.Args() diff --git a/components/engine/api/client/utils.go b/components/engine/api/client/utils.go index e87e593fe2..0ea3cb060c 100644 --- a/components/engine/api/client/utils.go +++ b/components/engine/api/client/utils.go @@ -10,7 +10,6 @@ import ( gosignal "os/signal" "path/filepath" "runtime" - "strings" "time" "github.com/Sirupsen/logrus" @@ -49,7 +48,7 @@ func (cli *DockerCli) registryAuthenticationPrivilegedFunc(index *registrytypes. return func() (string, error) { fmt.Fprintf(cli.out, "\nPlease login prior to %s:\n", cmdName) indexServer := registry.GetAuthConfigKey(index) - authConfig, err := cli.configureAuth("", "", "", indexServer) + authConfig, err := cli.configureAuth("", "", indexServer, false) if err != nil { return "", err } @@ -59,6 +58,10 @@ func (cli *DockerCli) registryAuthenticationPrivilegedFunc(index *registrytypes. func (cli *DockerCli) resizeTty(id string, isExec bool) { height, width := cli.getTtySize() + cli.resizeTtyTo(id, height, width, isExec) +} + +func (cli *DockerCli) resizeTtyTo(id string, height, width int, isExec bool) { if height == 0 && width == 0 { return } @@ -181,38 +184,17 @@ func copyToFile(outfile string, r io.Reader) error { // resolveAuthConfig is like registry.ResolveAuthConfig, but if using the // default index, it uses the default index name for the daemon's platform, // not the client's platform. -func (cli *DockerCli) resolveAuthConfig(authConfigs map[string]types.AuthConfig, index *registrytypes.IndexInfo) types.AuthConfig { +func (cli *DockerCli) resolveAuthConfig(index *registrytypes.IndexInfo) types.AuthConfig { configKey := index.Name if index.Official { configKey = cli.electAuthServer() } - // First try the happy case - if c, found := authConfigs[configKey]; found || index.Official { - return c - } + a, _ := getCredentials(cli.configFile, configKey) + return a +} - convertToHostname := func(url string) string { - stripped := url - if strings.HasPrefix(url, "http://") { - stripped = strings.Replace(url, "http://", "", 1) - } else if strings.HasPrefix(url, "https://") { - stripped = strings.Replace(url, "https://", "", 1) - } - - nameParts := strings.SplitN(stripped, "/", 2) - - return nameParts[0] - } - - // Maybe they have a legacy config file, we will iterate the keys converting - // them to the new format and testing - for registry, ac := range authConfigs { - if configKey == convertToHostname(registry) { - return ac - } - } - - // When all else fails, return an empty auth config - return types.AuthConfig{} +func (cli *DockerCli) retrieveAuthConfigs() map[string]types.AuthConfig { + acs, _ := getAllCredentials(cli.configFile) + return acs } diff --git a/components/engine/api/client/volume.go b/components/engine/api/client/volume.go index 284e30c3cb..8bb37951ee 100644 --- a/components/engine/api/client/volume.go +++ b/components/engine/api/client/volume.go @@ -2,6 +2,7 @@ package client import ( "fmt" + "sort" "text/tabwriter" Cli "github.com/docker/docker/cli" @@ -72,6 +73,7 @@ func (cli *DockerCli) CmdVolumeLs(args ...string) error { fmt.Fprintf(w, "\n") } + sort.Sort(byVolumeName(volumes.Volumes)) for _, vol := range volumes.Volumes { if *quiet { fmt.Fprintln(w, vol.Name) @@ -83,6 +85,14 @@ func (cli *DockerCli) CmdVolumeLs(args ...string) error { return nil } +type byVolumeName []*types.Volume + +func (r byVolumeName) Len() int { return len(r) } +func (r byVolumeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] } +func (r byVolumeName) Less(i, j int) bool { + return r[i].Name < r[j].Name +} + // CmdVolumeInspect displays low-level information on one or more volumes. // // Usage: docker volume inspect [OPTIONS] VOLUME [VOLUME...] diff --git a/components/engine/api/common.go b/components/engine/api/common.go index 51be1e27ed..63560c6dea 100644 --- a/components/engine/api/common.go +++ b/components/engine/api/common.go @@ -23,9 +23,6 @@ const ( // MinVersion represents Minimum REST API version supported MinVersion version.Version = "1.12" - // DefaultDockerfileName is the Default filename with Docker commands, read by docker build - DefaultDockerfileName string = "Dockerfile" - // NoBaseImageSpecifier is the symbol used by the FROM // command to specify that no base image is to be used. NoBaseImageSpecifier string = "scratch" diff --git a/components/engine/api/common_test.go b/components/engine/api/common_test.go index 4f36b45471..c214660cc4 100644 --- a/components/engine/api/common_test.go +++ b/components/engine/api/common_test.go @@ -310,7 +310,7 @@ func TestLoadOrCreateTrustKeyCreateKey(t *testing.T) { } // With the need to create the folder hierarchy as tmpKeyFie is in a path - // where some folder do not exists. + // where some folders do not exist. tmpKeyFile = filepath.Join(tmpKeyFolderPath, "folder/hierarchy/keyfile") if key, err := LoadOrCreateTrustKey(tmpKeyFile); err != nil || key == nil { diff --git a/components/engine/api/server/httputils/errors.go b/components/engine/api/server/httputils/errors.go new file mode 100644 index 0000000000..c6b0a6b2d8 --- /dev/null +++ b/components/engine/api/server/httputils/errors.go @@ -0,0 +1,69 @@ +package httputils + +import ( + "net/http" + "strings" + + "github.com/Sirupsen/logrus" +) + +// httpStatusError is an interface +// that errors with custom status codes +// implement to tell the api layer +// which response status to set. +type httpStatusError interface { + HTTPErrorStatusCode() int +} + +// inputValidationError is an interface +// that errors generated by invalid +// inputs can implement to tell the +// api layer to set a 400 status code +// in the response. +type inputValidationError interface { + IsValidationError() bool +} + +// WriteError decodes a specific docker error and sends it in the response. +func WriteError(w http.ResponseWriter, err error) { + if err == nil || w == nil { + logrus.WithFields(logrus.Fields{"error": err, "writer": w}).Error("unexpected HTTP error handling") + return + } + + var statusCode int + errMsg := err.Error() + + switch e := err.(type) { + case httpStatusError: + statusCode = e.HTTPErrorStatusCode() + case inputValidationError: + statusCode = http.StatusBadRequest + default: + // FIXME: this is brittle and should not be necessary, but we still need to identify if + // there are errors falling back into this logic. + // If we need to differentiate between different possible error types, + // we should create appropriate error types that implement the httpStatusError interface. + errStr := strings.ToLower(errMsg) + for keyword, status := range map[string]int{ + "not found": http.StatusNotFound, + "no such": http.StatusNotFound, + "bad parameter": http.StatusBadRequest, + "conflict": http.StatusConflict, + "impossible": http.StatusNotAcceptable, + "wrong login/password": http.StatusUnauthorized, + "hasn't been activated": http.StatusForbidden, + } { + if strings.Contains(errStr, keyword) { + statusCode = status + break + } + } + } + + if statusCode == 0 { + statusCode = http.StatusInternalServerError + } + + http.Error(w, errMsg, statusCode) +} diff --git a/components/engine/api/server/httputils/httputils.go b/components/engine/api/server/httputils/httputils.go index ecf26e2a14..c81256b145 100644 --- a/components/engine/api/server/httputils/httputils.go +++ b/components/engine/api/server/httputils/httputils.go @@ -9,8 +9,6 @@ import ( "golang.org/x/net/context" - "github.com/Sirupsen/logrus" - "github.com/docker/distribution/registry/api/errcode" "github.com/docker/docker/api" "github.com/docker/docker/pkg/version" ) @@ -19,7 +17,7 @@ import ( const APIVersionKey = "api-version" // APIFunc is an adapter to allow the use of ordinary functions as Docker API endpoints. -// Any function that has the appropriate signature can be register as a API endpoint (e.g. getVersion). +// Any function that has the appropriate signature can be registered as a API endpoint (e.g. getVersion). type APIFunc func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error // HijackConnection interrupts the http response writer to get the @@ -77,7 +75,7 @@ func ParseForm(r *http.Request) error { return nil } -// ParseMultipartForm ensure the request form is parsed, even with invalid content types. +// ParseMultipartForm ensures the request form is parsed, even with invalid content types. func ParseMultipartForm(r *http.Request) error { if err := r.ParseMultipartForm(4096); err != nil && !strings.HasPrefix(err.Error(), "mime:") { return err @@ -85,78 +83,6 @@ func ParseMultipartForm(r *http.Request) error { return nil } -// WriteError decodes a specific docker error and sends it in the response. -func WriteError(w http.ResponseWriter, err error) { - if err == nil || w == nil { - logrus.WithFields(logrus.Fields{"error": err, "writer": w}).Error("unexpected HTTP error handling") - return - } - - statusCode := http.StatusInternalServerError - errMsg := err.Error() - - // Based on the type of error we get we need to process things - // slightly differently to extract the error message. - // In the 'errcode.*' cases there are two different type of - // error that could be returned. errocode.ErrorCode is the base - // type of error object - it is just an 'int' that can then be - // used as the look-up key to find the message. errorcode.Error - // extends errorcode.Error by adding error-instance specific - // data, like 'details' or variable strings to be inserted into - // the message. - // - // Ideally, we should just be able to call err.Error() for all - // cases but the errcode package doesn't support that yet. - // - // Additionally, in both errcode cases, there might be an http - // status code associated with it, and if so use it. - switch err.(type) { - case errcode.ErrorCode: - daError, _ := err.(errcode.ErrorCode) - statusCode = daError.Descriptor().HTTPStatusCode - errMsg = daError.Message() - - case errcode.Error: - // For reference, if you're looking for a particular error - // then you can do something like : - // import ( derr "github.com/docker/docker/errors" ) - // if daError.ErrorCode() == derr.ErrorCodeNoSuchContainer { ... } - - daError, _ := err.(errcode.Error) - statusCode = daError.ErrorCode().Descriptor().HTTPStatusCode - errMsg = daError.Message - - default: - // This part of will be removed once we've - // converted everything over to use the errcode package - - // FIXME: this is brittle and should not be necessary. - // If we need to differentiate between different possible error types, - // we should create appropriate error types with clearly defined meaning - errStr := strings.ToLower(err.Error()) - for keyword, status := range map[string]int{ - "not found": http.StatusNotFound, - "no such": http.StatusNotFound, - "bad parameter": http.StatusBadRequest, - "conflict": http.StatusConflict, - "impossible": http.StatusNotAcceptable, - "wrong login/password": http.StatusUnauthorized, - "hasn't been activated": http.StatusForbidden, - } { - if strings.Contains(errStr, keyword) { - statusCode = status - break - } - } - } - - if statusCode == 0 { - statusCode = http.StatusInternalServerError - } - - http.Error(w, errMsg, statusCode) -} - // WriteJSON writes the value v to the http response stream as json with standard json encoding. func WriteJSON(w http.ResponseWriter, code int, v interface{}) error { w.Header().Set("Content-Type", "application/json") diff --git a/components/engine/api/server/middleware.go b/components/engine/api/server/middleware.go index 5b277bc1f3..2622bf1bbe 100644 --- a/components/engine/api/server/middleware.go +++ b/components/engine/api/server/middleware.go @@ -1,195 +1,41 @@ package server import ( - "bufio" - "encoding/json" - "io" - "net/http" - "runtime" - "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api" "github.com/docker/docker/api/server/httputils" + "github.com/docker/docker/api/server/middleware" "github.com/docker/docker/dockerversion" - "github.com/docker/docker/errors" "github.com/docker/docker/pkg/authorization" - "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/version" - "golang.org/x/net/context" ) -// middleware is an adapter to allow the use of ordinary functions as Docker API filters. -// Any function that has the appropriate signature can be register as a middleware. -type middleware func(handler httputils.APIFunc) httputils.APIFunc - -// debugRequestMiddleware dumps the request to logger -func debugRequestMiddleware(handler httputils.APIFunc) httputils.APIFunc { - return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - logrus.Debugf("%s %s", r.Method, r.RequestURI) - - if r.Method != "POST" { - return handler(ctx, w, r, vars) - } - if err := httputils.CheckForJSON(r); err != nil { - return handler(ctx, w, r, vars) - } - maxBodySize := 4096 // 4KB - if r.ContentLength > int64(maxBodySize) { - return handler(ctx, w, r, vars) - } - - body := r.Body - bufReader := bufio.NewReaderSize(body, maxBodySize) - r.Body = ioutils.NewReadCloserWrapper(bufReader, func() error { return body.Close() }) - - b, err := bufReader.Peek(maxBodySize) - if err != io.EOF { - // either there was an error reading, or the buffer is full (in which case the request is too large) - return handler(ctx, w, r, vars) - } - - var postForm map[string]interface{} - if err := json.Unmarshal(b, &postForm); err == nil { - if _, exists := postForm["password"]; exists { - postForm["password"] = "*****" - } - formStr, errMarshal := json.Marshal(postForm) - if errMarshal == nil { - logrus.Debugf("form data: %s", string(formStr)) - } else { - logrus.Debugf("form data: %q", postForm) - } - } - - return handler(ctx, w, r, vars) - } -} - -// authorizationMiddleware perform authorization on the request. -func (s *Server) authorizationMiddleware(handler httputils.APIFunc) httputils.APIFunc { - return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - // FIXME: fill when authN gets in - // User and UserAuthNMethod are taken from AuthN plugins - // Currently tracked in https://github.com/docker/docker/pull/13994 - user := "" - userAuthNMethod := "" - authCtx := authorization.NewCtx(s.authZPlugins, user, userAuthNMethod, r.Method, r.RequestURI) - - if err := authCtx.AuthZRequest(w, r); err != nil { - logrus.Errorf("AuthZRequest for %s %s returned error: %s", r.Method, r.RequestURI, err) - return err - } - - rw := authorization.NewResponseModifier(w) - - if err := handler(ctx, rw, r, vars); err != nil { - logrus.Errorf("Handler for %s %s returned error: %s", r.Method, r.RequestURI, err) - return err - } - - if err := authCtx.AuthZResponse(rw, r); err != nil { - logrus.Errorf("AuthZResponse for %s %s returned error: %s", r.Method, r.RequestURI, err) - return err - } - return nil - } -} - -// userAgentMiddleware checks the User-Agent header looking for a valid docker client spec. -func (s *Server) userAgentMiddleware(handler httputils.APIFunc) httputils.APIFunc { - return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if strings.Contains(r.Header.Get("User-Agent"), "Docker-Client/") { - dockerVersion := version.Version(s.cfg.Version) - - userAgent := strings.Split(r.Header.Get("User-Agent"), "/") - - // v1.20 onwards includes the GOOS of the client after the version - // such as Docker/1.7.0 (linux) - if len(userAgent) == 2 && strings.Contains(userAgent[1], " ") { - userAgent[1] = strings.Split(userAgent[1], " ")[0] - } - - if len(userAgent) == 2 && !dockerVersion.Equal(version.Version(userAgent[1])) { - logrus.Warnf("Warning: client and server don't have the same version (client: %s, server: %s)", userAgent[1], dockerVersion) - } - } - return handler(ctx, w, r, vars) - } -} - -// corsMiddleware sets the CORS header expectations in the server. -func (s *Server) corsMiddleware(handler httputils.APIFunc) httputils.APIFunc { - return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - // If "api-cors-header" is not given, but "api-enable-cors" is true, we set cors to "*" - // otherwise, all head values will be passed to HTTP handler - corsHeaders := s.cfg.CorsHeaders - if corsHeaders == "" && s.cfg.EnableCors { - corsHeaders = "*" - } - - if corsHeaders != "" { - writeCorsHeaders(w, r, corsHeaders) - } - return handler(ctx, w, r, vars) - } -} - -// versionMiddleware checks the api version requirements before passing the request to the server handler. -func versionMiddleware(handler httputils.APIFunc) httputils.APIFunc { - return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - apiVersion := version.Version(vars["version"]) - if apiVersion == "" { - apiVersion = api.DefaultVersion - } - - if apiVersion.GreaterThan(api.DefaultVersion) { - return errors.ErrorCodeNewerClientVersion.WithArgs(apiVersion, api.DefaultVersion) - } - if apiVersion.LessThan(api.MinVersion) { - return errors.ErrorCodeOldClientVersion.WithArgs(apiVersion, api.MinVersion) - } - - w.Header().Set("Server", "Docker/"+dockerversion.Version+" ("+runtime.GOOS+")") - ctx = context.WithValue(ctx, httputils.APIVersionKey, apiVersion) - return handler(ctx, w, r, vars) - } -} - // handleWithGlobalMiddlwares wraps the handler function for a request with // the server's global middlewares. The order of the middlewares is backwards, // meaning that the first in the list will be evaluated last. -// -// Example: handleWithGlobalMiddlewares(s.getContainersName) -// -// s.loggingMiddleware( -// s.userAgentMiddleware( -// s.corsMiddleware( -// versionMiddleware(s.getContainersName) -// ) -// ) -// ) -// ) func (s *Server) handleWithGlobalMiddlewares(handler httputils.APIFunc) httputils.APIFunc { - middlewares := []middleware{ - versionMiddleware, - s.corsMiddleware, - s.userAgentMiddleware, + next := handler + + handleVersion := middleware.NewVersionMiddleware(dockerversion.Version, api.DefaultVersion, api.MinVersion) + next = handleVersion(next) + + if s.cfg.EnableCors { + handleCORS := middleware.NewCORSMiddleware(s.cfg.CorsHeaders) + next = handleCORS(next) } + handleUserAgent := middleware.NewUserAgentMiddleware(s.cfg.Version) + next = handleUserAgent(next) + // Only want this on debug level if s.cfg.Logging && logrus.GetLevel() == logrus.DebugLevel { - middlewares = append(middlewares, debugRequestMiddleware) + next = middleware.DebugRequestMiddleware(next) } if len(s.cfg.AuthorizationPluginNames) > 0 { s.authZPlugins = authorization.NewPlugins(s.cfg.AuthorizationPluginNames) - middlewares = append(middlewares, s.authorizationMiddleware) + handleAuthorization := middleware.NewAuthorizationMiddleware(s.authZPlugins) + next = handleAuthorization(next) } - h := handler - for _, m := range middlewares { - h = m(h) - } - return h + return next } diff --git a/components/engine/api/server/middleware/authorization.go b/components/engine/api/server/middleware/authorization.go new file mode 100644 index 0000000000..cbfa99e7b3 --- /dev/null +++ b/components/engine/api/server/middleware/authorization.go @@ -0,0 +1,42 @@ +package middleware + +import ( + "net/http" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/api/server/httputils" + "github.com/docker/docker/pkg/authorization" + "golang.org/x/net/context" +) + +// NewAuthorizationMiddleware creates a new Authorization middleware. +func NewAuthorizationMiddleware(plugins []authorization.Plugin) Middleware { + return func(handler httputils.APIFunc) httputils.APIFunc { + return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + // FIXME: fill when authN gets in + // User and UserAuthNMethod are taken from AuthN plugins + // Currently tracked in https://github.com/docker/docker/pull/13994 + user := "" + userAuthNMethod := "" + authCtx := authorization.NewCtx(plugins, user, userAuthNMethod, r.Method, r.RequestURI) + + if err := authCtx.AuthZRequest(w, r); err != nil { + logrus.Errorf("AuthZRequest for %s %s returned error: %s", r.Method, r.RequestURI, err) + return err + } + + rw := authorization.NewResponseModifier(w) + + if err := handler(ctx, rw, r, vars); err != nil { + logrus.Errorf("Handler for %s %s returned error: %s", r.Method, r.RequestURI, err) + return err + } + + if err := authCtx.AuthZResponse(rw, r); err != nil { + logrus.Errorf("AuthZResponse for %s %s returned error: %s", r.Method, r.RequestURI, err) + return err + } + return nil + } + } +} diff --git a/components/engine/api/server/middleware/cors.go b/components/engine/api/server/middleware/cors.go new file mode 100644 index 0000000000..de21897d2c --- /dev/null +++ b/components/engine/api/server/middleware/cors.go @@ -0,0 +1,33 @@ +package middleware + +import ( + "net/http" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/api/server/httputils" + "golang.org/x/net/context" +) + +// NewCORSMiddleware creates a new CORS middleware. +func NewCORSMiddleware(defaultHeaders string) Middleware { + return func(handler httputils.APIFunc) httputils.APIFunc { + return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + // If "api-cors-header" is not given, but "api-enable-cors" is true, we set cors to "*" + // otherwise, all head values will be passed to HTTP handler + corsHeaders := defaultHeaders + if corsHeaders == "" { + corsHeaders = "*" + } + + writeCorsHeaders(w, r, corsHeaders) + return handler(ctx, w, r, vars) + } + } +} + +func writeCorsHeaders(w http.ResponseWriter, r *http.Request, corsHeaders string) { + logrus.Debugf("CORS header is enabled and set to: %s", corsHeaders) + w.Header().Add("Access-Control-Allow-Origin", corsHeaders) + w.Header().Add("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, X-Registry-Auth") + w.Header().Add("Access-Control-Allow-Methods", "HEAD, GET, POST, DELETE, PUT, OPTIONS") +} diff --git a/components/engine/api/server/middleware/debug.go b/components/engine/api/server/middleware/debug.go new file mode 100644 index 0000000000..be7056f6c6 --- /dev/null +++ b/components/engine/api/server/middleware/debug.go @@ -0,0 +1,56 @@ +package middleware + +import ( + "bufio" + "encoding/json" + "io" + "net/http" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/api/server/httputils" + "github.com/docker/docker/pkg/ioutils" + "golang.org/x/net/context" +) + +// DebugRequestMiddleware dumps the request to logger +func DebugRequestMiddleware(handler httputils.APIFunc) httputils.APIFunc { + return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + logrus.Debugf("Calling %s %s", r.Method, r.RequestURI) + + if r.Method != "POST" { + return handler(ctx, w, r, vars) + } + if err := httputils.CheckForJSON(r); err != nil { + return handler(ctx, w, r, vars) + } + maxBodySize := 4096 // 4KB + if r.ContentLength > int64(maxBodySize) { + return handler(ctx, w, r, vars) + } + + body := r.Body + bufReader := bufio.NewReaderSize(body, maxBodySize) + r.Body = ioutils.NewReadCloserWrapper(bufReader, func() error { return body.Close() }) + + b, err := bufReader.Peek(maxBodySize) + if err != io.EOF { + // either there was an error reading, or the buffer is full (in which case the request is too large) + return handler(ctx, w, r, vars) + } + + var postForm map[string]interface{} + if err := json.Unmarshal(b, &postForm); err == nil { + if _, exists := postForm["password"]; exists { + postForm["password"] = "*****" + } + formStr, errMarshal := json.Marshal(postForm) + if errMarshal == nil { + logrus.Debugf("form data: %s", string(formStr)) + } else { + logrus.Debugf("form data: %q", postForm) + } + } + + return handler(ctx, w, r, vars) + } +} diff --git a/components/engine/api/server/middleware/middleware.go b/components/engine/api/server/middleware/middleware.go new file mode 100644 index 0000000000..588331ae7e --- /dev/null +++ b/components/engine/api/server/middleware/middleware.go @@ -0,0 +1,7 @@ +package middleware + +import "github.com/docker/docker/api/server/httputils" + +// Middleware is an adapter to allow the use of ordinary functions as Docker API filters. +// Any function that has the appropriate signature can be registered as a middleware. +type Middleware func(handler httputils.APIFunc) httputils.APIFunc diff --git a/components/engine/api/server/middleware/user_agent.go b/components/engine/api/server/middleware/user_agent.go new file mode 100644 index 0000000000..be4e171cd2 --- /dev/null +++ b/components/engine/api/server/middleware/user_agent.go @@ -0,0 +1,35 @@ +package middleware + +import ( + "net/http" + "strings" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/api/server/httputils" + "github.com/docker/docker/pkg/version" + "golang.org/x/net/context" +) + +// NewUserAgentMiddleware creates a new UserAgent middleware. +func NewUserAgentMiddleware(versionCheck string) Middleware { + serverVersion := version.Version(versionCheck) + + return func(handler httputils.APIFunc) httputils.APIFunc { + return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if strings.Contains(r.Header.Get("User-Agent"), "Docker-Client/") { + userAgent := strings.Split(r.Header.Get("User-Agent"), "/") + + // v1.20 onwards includes the GOOS of the client after the version + // such as Docker/1.7.0 (linux) + if len(userAgent) == 2 && strings.Contains(userAgent[1], " ") { + userAgent[1] = strings.Split(userAgent[1], " ")[0] + } + + if len(userAgent) == 2 && !serverVersion.Equal(version.Version(userAgent[1])) { + logrus.Debugf("Client and server don't have the same version (client: %s, server: %s)", userAgent[1], serverVersion) + } + } + return handler(ctx, w, r, vars) + } + } +} diff --git a/components/engine/api/server/middleware/version.go b/components/engine/api/server/middleware/version.go new file mode 100644 index 0000000000..41d518bcbc --- /dev/null +++ b/components/engine/api/server/middleware/version.go @@ -0,0 +1,45 @@ +package middleware + +import ( + "fmt" + "net/http" + "runtime" + + "github.com/docker/docker/api/server/httputils" + "github.com/docker/docker/pkg/version" + "golang.org/x/net/context" +) + +type badRequestError struct { + error +} + +func (badRequestError) HTTPErrorStatusCode() int { + return http.StatusBadRequest +} + +// NewVersionMiddleware creates a new Version middleware. +func NewVersionMiddleware(versionCheck string, defaultVersion, minVersion version.Version) Middleware { + serverVersion := version.Version(versionCheck) + + return func(handler httputils.APIFunc) httputils.APIFunc { + return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + apiVersion := version.Version(vars["version"]) + if apiVersion == "" { + apiVersion = defaultVersion + } + + if apiVersion.GreaterThan(defaultVersion) { + return badRequestError{fmt.Errorf("client is newer than server (client API version: %s, server API version: %s)", apiVersion, defaultVersion)} + } + if apiVersion.LessThan(minVersion) { + return badRequestError{fmt.Errorf("client version %s is too old. Minimum supported API version is %s, please upgrade your client to a newer version", apiVersion, minVersion)} + } + + header := fmt.Sprintf("Docker/%s (%s)", serverVersion, runtime.GOOS) + w.Header().Set("Server", header) + ctx = context.WithValue(ctx, httputils.APIVersionKey, apiVersion) + return handler(ctx, w, r, vars) + } + } +} diff --git a/components/engine/api/server/middleware_test.go b/components/engine/api/server/middleware/version_test.go similarity index 61% rename from components/engine/api/server/middleware_test.go rename to components/engine/api/server/middleware/version_test.go index 4f48b20990..f60a98e518 100644 --- a/components/engine/api/server/middleware_test.go +++ b/components/engine/api/server/middleware/version_test.go @@ -1,13 +1,13 @@ -package server +package middleware import ( "net/http" "net/http/httptest" + "strings" "testing" - "github.com/docker/distribution/registry/api/errcode" "github.com/docker/docker/api/server/httputils" - "github.com/docker/docker/errors" + "github.com/docker/docker/pkg/version" "golang.org/x/net/context" ) @@ -19,7 +19,10 @@ func TestVersionMiddleware(t *testing.T) { return nil } - h := versionMiddleware(handler) + defaultVersion := version.Version("1.10.0") + minVersion := version.Version("1.2.0") + m := NewVersionMiddleware(defaultVersion.String(), defaultVersion, minVersion) + h := m(handler) req, _ := http.NewRequest("GET", "/containers/json", nil) resp := httptest.NewRecorder() @@ -37,7 +40,10 @@ func TestVersionMiddlewareWithErrors(t *testing.T) { return nil } - h := versionMiddleware(handler) + defaultVersion := version.Version("1.10.0") + minVersion := version.Version("1.2.0") + m := NewVersionMiddleware(defaultVersion.String(), defaultVersion, minVersion) + h := m(handler) req, _ := http.NewRequest("GET", "/containers/json", nil) resp := httptest.NewRecorder() @@ -45,13 +51,14 @@ func TestVersionMiddlewareWithErrors(t *testing.T) { vars := map[string]string{"version": "0.1"} err := h(ctx, resp, req, vars) - if derr, ok := err.(errcode.Error); !ok || derr.ErrorCode() != errors.ErrorCodeOldClientVersion { - t.Fatalf("Expected ErrorCodeOldClientVersion, got %v", err) + + if !strings.Contains(err.Error(), "client version 0.1 is too old. Minimum supported API version is 1.2.0") { + t.Fatalf("Expected too old client error, got %v", err) } vars["version"] = "100000" err = h(ctx, resp, req, vars) - if derr, ok := err.(errcode.Error); !ok || derr.ErrorCode() != errors.ErrorCodeNewerClientVersion { - t.Fatalf("Expected ErrorCodeNewerClientVersion, got %v", err) + if !strings.Contains(err.Error(), "client is newer than server") { + t.Fatalf("Expected client newer than server error, got %v", err) } } diff --git a/components/engine/api/server/profiler.go b/components/engine/api/server/profiler.go index 766462bd6a..3c0dfd08f2 100644 --- a/components/engine/api/server/profiler.go +++ b/components/engine/api/server/profiler.go @@ -9,8 +9,10 @@ import ( "github.com/gorilla/mux" ) -func profilerSetup(mainRouter *mux.Router, path string) { - var r = mainRouter.PathPrefix(path).Subrouter() +const debugPathPrefix = "/debug/" + +func profilerSetup(mainRouter *mux.Router) { + var r = mainRouter.PathPrefix(debugPathPrefix).Subrouter() r.HandleFunc("/vars", expVars) r.HandleFunc("/pprof/", pprof.Index) r.HandleFunc("/pprof/cmdline", pprof.Cmdline) diff --git a/components/engine/api/server/router/build/build_routes.go b/components/engine/api/server/router/build/build_routes.go index 152dfff328..0025c85a93 100644 --- a/components/engine/api/server/router/build/build_routes.go +++ b/components/engine/api/server/router/build/build_routes.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/base64" "encoding/json" - "errors" "fmt" "io" "net/http" @@ -17,7 +16,6 @@ import ( "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/streamformatter" - "github.com/docker/docker/utils" "github.com/docker/engine-api/types" "github.com/docker/engine-api/types/container" "github.com/docker/go-units" @@ -60,11 +58,11 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui options.ShmSize = shmSize } - if i := container.IsolationLevel(r.FormValue("isolation")); i != "" { - if !container.IsolationLevel.IsValid(i) { + if i := container.Isolation(r.FormValue("isolation")); i != "" { + if !container.Isolation.IsValid(i) { return nil, fmt.Errorf("Unsupported isolation: %q", i) } - options.IsolationLevel = i + options.Isolation = i } var buildUlimits = []*units.Ulimit{} @@ -117,7 +115,7 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r * if !output.Flushed() { return err } - _, err = w.Write(sf.FormatError(errors.New(utils.GetErrorMessage(err)))) + _, err = w.Write(sf.FormatError(err)) if err != nil { logrus.Warnf("could not write error response: %v", err) } @@ -159,6 +157,8 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r * buildOptions.Dockerfile = dockerfileName } + buildOptions.AuthConfigs = authConfigs + out = output if buildOptions.SuppressOutput { out = notVerboseBuffer diff --git a/components/engine/api/server/router/container/backend.go b/components/engine/api/server/router/container/backend.go index cc25f10919..bd8919759c 100644 --- a/components/engine/api/server/router/container/backend.go +++ b/components/engine/api/server/router/container/backend.go @@ -5,7 +5,6 @@ import ( "time" "github.com/docker/docker/api/types/backend" - "github.com/docker/docker/daemon/exec" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/version" "github.com/docker/engine-api/types" @@ -15,7 +14,7 @@ import ( // execBackend includes functions to implement to provide exec functionality. type execBackend interface { ContainerExecCreate(config *types.ExecConfig) (string, error) - ContainerExecInspect(id string) (*exec.Config, error) + ContainerExecInspect(id string) (*backend.ExecInspect, error) ContainerExecResize(name string, height, width int) error ContainerExecStart(name string, stdin io.ReadCloser, stdout io.Writer, stderr io.Writer) error ExecExists(name string) (bool, error) diff --git a/components/engine/api/server/router/container/container_routes.go b/components/engine/api/server/router/container/container_routes.go index 07d40168be..016e00f05b 100644 --- a/components/engine/api/server/router/container/container_routes.go +++ b/components/engine/api/server/router/container/container_routes.go @@ -11,15 +11,12 @@ import ( "time" "github.com/Sirupsen/logrus" - "github.com/docker/distribution/registry/api/errcode" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types/backend" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/signal" "github.com/docker/docker/pkg/term" "github.com/docker/docker/runconfig" - "github.com/docker/docker/utils" "github.com/docker/engine-api/types" "github.com/docker/engine-api/types/container" "github.com/docker/engine-api/types/filters" @@ -126,7 +123,7 @@ func (s *containerRouter) getContainersLogs(ctx context.Context, w http.Response // The client may be expecting all of the data we're sending to // be multiplexed, so send it through OutStream, which will // have been set up to handle that if needed. - fmt.Fprintf(logsConfig.OutStream, "Error running logs job: %s\n", utils.GetErrorMessage(err)) + fmt.Fprintf(logsConfig.OutStream, "Error running logs job: %v\n", err) default: return err } @@ -182,6 +179,10 @@ func (s *containerRouter) postContainersStop(ctx context.Context, w http.Respons return nil } +type errContainerIsRunning interface { + ContainerIsRunning() bool +} + func (s *containerRouter) postContainersKill(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if err := httputils.ParseForm(r); err != nil { return err @@ -199,15 +200,17 @@ func (s *containerRouter) postContainersKill(ctx context.Context, w http.Respons } if err := s.backend.ContainerKill(name, uint64(sig)); err != nil { - theErr, isDerr := err.(errcode.ErrorCoder) - isStopped := isDerr && theErr.ErrorCode() == derr.ErrorCodeNotRunning + var isStopped bool + if e, ok := err.(errContainerIsRunning); ok { + isStopped = !e.ContainerIsRunning() + } // Return error that's not caused because the container is stopped. // Return error if the container is not running and the api is >= 1.20 // to keep backwards compatibility. version := httputils.VersionFromContext(ctx) if version.GreaterThanOrEqualTo("1.20") || !isStopped { - return fmt.Errorf("Cannot kill container %s: %v", name, utils.GetErrorMessage(err)) + return fmt.Errorf("Cannot kill container %s: %v", name, err) } } @@ -322,7 +325,8 @@ func (s *containerRouter) postContainerUpdate(ctx context.Context, w http.Respon } hostConfig := &container.HostConfig{ - Resources: updateConfig.Resources, + Resources: updateConfig.Resources, + RestartPolicy: updateConfig.RestartPolicy, } name := vars["name"] @@ -429,7 +433,7 @@ func (s *containerRouter) postContainersAttach(ctx context.Context, w http.Respo hijacker, ok := w.(http.Hijacker) if !ok { - return derr.ErrorCodeNoHijackConnection.WithArgs(containerName) + return fmt.Errorf("error attaching to container %s, hijack connection missing", containerName) } setupStreams := func() (io.ReadCloser, io.Writer, io.Writer, error) { diff --git a/components/engine/api/server/router/container/exec.go b/components/engine/api/server/router/container/exec.go index caa5da061d..bc336f6039 100644 --- a/components/engine/api/server/router/container/exec.go +++ b/components/engine/api/server/router/container/exec.go @@ -10,7 +10,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/pkg/stdcopy" - "github.com/docker/docker/utils" "github.com/docker/engine-api/types" "golang.org/x/net/context" ) @@ -46,7 +45,7 @@ func (s *containerRouter) postContainerExecCreate(ctx context.Context, w http.Re // Register an instance of Exec in container. id, err := s.backend.ContainerExecCreate(execConfig) if err != nil { - logrus.Errorf("Error setting up exec command in container %s: %s", name, utils.GetErrorMessage(err)) + logrus.Errorf("Error setting up exec command in container %s: %v", name, err) return err } @@ -113,7 +112,7 @@ func (s *containerRouter) postContainerExecStart(ctx context.Context, w http.Res if execStartCheck.Detach { return err } - logrus.Errorf("Error running exec in container: %v\n", utils.GetErrorMessage(err)) + logrus.Errorf("Error running exec in container: %v\n", err) } return nil } diff --git a/components/engine/api/server/router/image/backend.go b/components/engine/api/server/router/image/backend.go index 73e8216025..8c76ef9260 100644 --- a/components/engine/api/server/router/image/backend.go +++ b/components/engine/api/server/router/image/backend.go @@ -20,7 +20,6 @@ type Backend interface { type containerBackend interface { Commit(name string, config *types.ContainerCommitConfig) (imageID string, err error) - Exists(containerName string) bool } type imageBackend interface { diff --git a/components/engine/api/server/router/image/image.go b/components/engine/api/server/router/image/image.go index e1b6bed9d7..d6a1297a97 100644 --- a/components/engine/api/server/router/image/image.go +++ b/components/engine/api/server/router/image/image.go @@ -4,14 +4,14 @@ import "github.com/docker/docker/api/server/router" // imageRouter is a router to talk with the image controller type imageRouter struct { - daemon Backend - routes []router.Route + backend Backend + routes []router.Route } // NewRouter initializes a new image router -func NewRouter(daemon Backend) router.Router { +func NewRouter(backend Backend) router.Router { r := &imageRouter{ - daemon: daemon, + backend: backend, } r.initRoutes() return r diff --git a/components/engine/api/server/router/image/image_routes.go b/components/engine/api/server/router/image/image_routes.go index c84317aec8..dade346925 100644 --- a/components/engine/api/server/router/image/image_routes.go +++ b/components/engine/api/server/router/image/image_routes.go @@ -14,7 +14,6 @@ import ( "github.com/docker/distribution/registry/api/errcode" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/builder/dockerfile" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/reference" @@ -49,10 +48,6 @@ func (s *imageRouter) postCommit(ctx context.Context, w http.ResponseWriter, r * c = &container.Config{} } - if !s.daemon.Exists(cname) { - return derr.ErrorCodeNoSuchContainer.WithArgs(cname) - } - newConfig, err := dockerfile.BuildFromConfig(c, r.Form["changes"]) if err != nil { return err @@ -68,7 +63,7 @@ func (s *imageRouter) postCommit(ctx context.Context, w http.ResponseWriter, r * MergeConfigs: true, } - imgID, err := s.daemon.Commit(cname, commitCfg) + imgID, err := s.backend.Commit(cname, commitCfg) if err != nil { return err } @@ -134,7 +129,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite } } - err = s.daemon.PullImage(ref, metaHeaders, authConfig, output) + err = s.backend.PullImage(ref, metaHeaders, authConfig, output) } } // Check the error from pulling an image to make sure the request @@ -175,7 +170,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite return err } - err = s.daemon.ImportImage(src, newRef, message, r.Body, output, newConfig) + err = s.backend.ImportImage(src, newRef, message, r.Body, output, newConfig) } if err != nil { if !output.Flushed() { @@ -233,7 +228,7 @@ func (s *imageRouter) postImagesPush(ctx context.Context, w http.ResponseWriter, w.Header().Set("Content-Type", "application/json") - if err := s.daemon.PushImage(ref, metaHeaders, authConfig, output); err != nil { + if err := s.backend.PushImage(ref, metaHeaders, authConfig, output); err != nil { if !output.Flushed() { return err } @@ -259,7 +254,7 @@ func (s *imageRouter) getImagesGet(ctx context.Context, w http.ResponseWriter, r names = r.Form["names"] } - if err := s.daemon.ExportImage(names, output); err != nil { + if err := s.backend.ExportImage(names, output); err != nil { if !output.Flushed() { return err } @@ -275,7 +270,7 @@ func (s *imageRouter) postImagesLoad(ctx context.Context, w http.ResponseWriter, } quiet := httputils.BoolValueOrDefault(r, "quiet", true) w.Header().Set("Content-Type", "application/json") - return s.daemon.LoadImage(r.Body, w, quiet) + return s.backend.LoadImage(r.Body, w, quiet) } func (s *imageRouter) deleteImages(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { @@ -292,7 +287,7 @@ func (s *imageRouter) deleteImages(ctx context.Context, w http.ResponseWriter, r force := httputils.BoolValue(r, "force") prune := !httputils.BoolValue(r, "noprune") - list, err := s.daemon.ImageDelete(name, force, prune) + list, err := s.backend.ImageDelete(name, force, prune) if err != nil { return err } @@ -301,7 +296,7 @@ func (s *imageRouter) deleteImages(ctx context.Context, w http.ResponseWriter, r } func (s *imageRouter) getImagesByName(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - imageInspect, err := s.daemon.LookupImage(vars["name"]) + imageInspect, err := s.backend.LookupImage(vars["name"]) if err != nil { return err } @@ -315,7 +310,7 @@ func (s *imageRouter) getImagesJSON(ctx context.Context, w http.ResponseWriter, } // FIXME: The filter parameter could just be a match filter - images, err := s.daemon.Images(r.Form.Get("filters"), r.Form.Get("filter"), httputils.BoolValue(r, "all")) + images, err := s.backend.Images(r.Form.Get("filters"), r.Form.Get("filter"), httputils.BoolValue(r, "all")) if err != nil { return err } @@ -325,7 +320,7 @@ func (s *imageRouter) getImagesJSON(ctx context.Context, w http.ResponseWriter, func (s *imageRouter) getImagesHistory(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { name := vars["name"] - history, err := s.daemon.ImageHistory(name) + history, err := s.backend.ImageHistory(name) if err != nil { return err } @@ -348,7 +343,7 @@ func (s *imageRouter) postImagesTag(ctx context.Context, w http.ResponseWriter, return err } } - if err := s.daemon.TagImage(newTag, vars["name"]); err != nil { + if err := s.backend.TagImage(newTag, vars["name"]); err != nil { return err } w.WriteHeader(http.StatusCreated) @@ -378,7 +373,7 @@ func (s *imageRouter) getImagesSearch(ctx context.Context, w http.ResponseWriter headers[k] = v } } - query, err := s.daemon.SearchRegistryForImages(r.Form.Get("term"), config, headers) + query, err := s.backend.SearchRegistryForImages(r.Form.Get("term"), config, headers) if err != nil { return err } diff --git a/components/engine/api/server/router/network/backend.go b/components/engine/api/server/router/network/backend.go index 113c497ce1..731f92e7de 100644 --- a/components/engine/api/server/router/network/backend.go +++ b/components/engine/api/server/router/network/backend.go @@ -8,13 +8,11 @@ import ( // Backend is all the methods that need to be implemented // to provide network specific functionality. type Backend interface { - NetworkControllerEnabled() bool - FindNetwork(idName string) (libnetwork.Network, error) GetNetworkByName(idName string) (libnetwork.Network, error) GetNetworksByID(partialID string) []libnetwork.Network GetAllNetworks() []libnetwork.Network - CreateNetwork(name, driver string, ipam network.IPAM, options map[string]string, internal bool) (libnetwork.Network, error) + CreateNetwork(name, driver string, ipam network.IPAM, options map[string]string, internal bool, enableIPv6 bool) (libnetwork.Network, error) ConnectContainerToNetwork(containerName, networkName string, endpointConfig *network.EndpointSettings) error DisconnectContainerFromNetwork(containerName string, network libnetwork.Network, force bool) error DeleteNetwork(name string) error diff --git a/components/engine/api/server/router/network/filter.go b/components/engine/api/server/router/network/filter.go index 31d8d0c521..f1648cc2ae 100644 --- a/components/engine/api/server/router/network/filter.go +++ b/components/engine/api/server/router/network/filter.go @@ -84,8 +84,8 @@ func filterNetworkByID(nws []libnetwork.Network, id string) (retNws []libnetwork return retNws, nil } -// filterAllNetworks filter network list according to user specified filter -// and return user chosen networks +// filterAllNetworks filters network list according to user specified filter +// and returns user chosen networks func filterNetworks(nws []libnetwork.Network, filter filters.Args) ([]libnetwork.Network, error) { // if filter is empty, return original network list if filter.Len() == 0 { diff --git a/components/engine/api/server/router/network/network.go b/components/engine/api/server/router/network/network.go index 59641bc03b..7c88089623 100644 --- a/components/engine/api/server/router/network/network.go +++ b/components/engine/api/server/router/network/network.go @@ -1,13 +1,6 @@ package network -import ( - "net/http" - - "github.com/docker/docker/api/server/httputils" - "github.com/docker/docker/api/server/router" - "github.com/docker/docker/errors" - "golang.org/x/net/context" -) +import "github.com/docker/docker/api/server/router" // networkRouter is a router to talk with the network controller type networkRouter struct { @@ -32,24 +25,13 @@ func (r *networkRouter) Routes() []router.Route { func (r *networkRouter) initRoutes() { r.routes = []router.Route{ // GET - router.NewGetRoute("/networks", r.controllerEnabledMiddleware(r.getNetworksList)), - router.NewGetRoute("/networks/{id:.*}", r.controllerEnabledMiddleware(r.getNetwork)), + router.NewGetRoute("/networks", r.getNetworksList), + router.NewGetRoute("/networks/{id:.*}", r.getNetwork), // POST - router.NewPostRoute("/networks/create", r.controllerEnabledMiddleware(r.postNetworkCreate)), - router.NewPostRoute("/networks/{id:.*}/connect", r.controllerEnabledMiddleware(r.postNetworkConnect)), - router.NewPostRoute("/networks/{id:.*}/disconnect", r.controllerEnabledMiddleware(r.postNetworkDisconnect)), + router.NewPostRoute("/networks/create", r.postNetworkCreate), + router.NewPostRoute("/networks/{id:.*}/connect", r.postNetworkConnect), + router.NewPostRoute("/networks/{id:.*}/disconnect", r.postNetworkDisconnect), // DELETE - router.NewDeleteRoute("/networks/{id:.*}", r.controllerEnabledMiddleware(r.deleteNetwork)), + router.NewDeleteRoute("/networks/{id:.*}", r.deleteNetwork), } } - -func (r *networkRouter) controllerEnabledMiddleware(handler httputils.APIFunc) httputils.APIFunc { - if r.backend.NetworkControllerEnabled() { - return handler - } - return networkControllerDisabled -} - -func networkControllerDisabled(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - return errors.ErrorNetworkControllerNotEnabled.WithArgs() -} diff --git a/components/engine/api/server/router/network/network_routes.go b/components/engine/api/server/router/network/network_routes.go index 25f0e83ca6..851a10cb89 100644 --- a/components/engine/api/server/router/network/network_routes.go +++ b/components/engine/api/server/router/network/network_routes.go @@ -91,7 +91,7 @@ func (n *networkRouter) postNetworkCreate(ctx context.Context, w http.ResponseWr warning = fmt.Sprintf("Network with name %s (id : %s) already exists", nw.Name(), nw.ID()) } - nw, err = n.backend.CreateNetwork(create.Name, create.Driver, create.IPAM, create.Options, create.Internal) + nw, err = n.backend.CreateNetwork(create.Name, create.Driver, create.IPAM, create.Options, create.Internal, create.EnableIPv6) if err != nil { return err } @@ -160,6 +160,8 @@ func buildNetworkResource(nw libnetwork.Network) *types.NetworkResource { r.ID = nw.ID() r.Scope = nw.Info().Scope() r.Driver = nw.Type() + r.EnableIPv6 = nw.Info().IPv6Enabled() + r.Internal = nw.Info().Internal() r.Options = nw.Info().DriverOptions() r.Containers = make(map[string]types.EndpointResource) buildIpamResources(r, nw) diff --git a/components/engine/api/server/router_swapper.go b/components/engine/api/server/router_swapper.go index b5f1d06d8d..1ecc7a7f39 100644 --- a/components/engine/api/server/router_swapper.go +++ b/components/engine/api/server/router_swapper.go @@ -7,7 +7,7 @@ import ( "github.com/gorilla/mux" ) -// routerSwapper is an http.Handler that allow you to swap +// routerSwapper is an http.Handler that allows you to swap // mux routers. type routerSwapper struct { mu sync.Mutex diff --git a/components/engine/api/server/server.go b/components/engine/api/server/server.go index 5e8337a7c2..1379b7372e 100644 --- a/components/engine/api/server/server.go +++ b/components/engine/api/server/server.go @@ -9,17 +9,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/server/router" - "github.com/docker/docker/api/server/router/build" - "github.com/docker/docker/api/server/router/container" - "github.com/docker/docker/api/server/router/image" - "github.com/docker/docker/api/server/router/network" - "github.com/docker/docker/api/server/router/system" - "github.com/docker/docker/api/server/router/volume" - "github.com/docker/docker/builder/dockerfile" - "github.com/docker/docker/daemon" "github.com/docker/docker/pkg/authorization" - "github.com/docker/docker/utils" - "github.com/docker/go-connections/sockets" "github.com/gorilla/mux" "golang.org/x/net/context" ) @@ -37,7 +27,6 @@ type Config struct { Version string SocketGroup string TLSConfig *tls.Config - Addrs []Addr } // Server contains instance details for the server @@ -49,27 +38,25 @@ type Server struct { routerSwapper *routerSwapper } -// Addr contains string representation of address and its protocol (tcp, unix...). -type Addr struct { - Proto string - Addr string -} - // New returns a new instance of the server based on the specified configuration. // It allocates resources which will be needed for ServeAPI(ports, unix-sockets). -func New(cfg *Config) (*Server, error) { - s := &Server{ +func New(cfg *Config) *Server { + return &Server{ cfg: cfg, } - for _, addr := range cfg.Addrs { - srv, err := s.newServer(addr.Proto, addr.Addr) - if err != nil { - return nil, err +} + +// Accept sets a listener the server accepts connections into. +func (s *Server) Accept(addr string, listeners ...net.Listener) { + for _, listener := range listeners { + httpServer := &HTTPServer{ + srv: &http.Server{ + Addr: addr, + }, + l: listener, } - logrus.Debugf("Server created for HTTP on %s (%s)", addr.Proto, addr.Addr) - s.servers = append(s.servers, srv...) + s.servers = append(s.servers, httpServer) } - return s, nil } // Close closes servers and thus stop receiving requests @@ -84,8 +71,6 @@ func (s *Server) Close() { // serveAPI loops through all initialized servers and spawns goroutine // with Server method for each. It sets createMux() as Handler also. func (s *Server) serveAPI() error { - s.initRouterSwapper() - var chErrors = make(chan error, len(s.servers)) for _, srv := range s.servers { srv.srv.Handler = s.routerSwapper @@ -127,31 +112,8 @@ func (s *HTTPServer) Close() error { return s.l.Close() } -func writeCorsHeaders(w http.ResponseWriter, r *http.Request, corsHeaders string) { - logrus.Debugf("CORS header is enabled and set to: %s", corsHeaders) - w.Header().Add("Access-Control-Allow-Origin", corsHeaders) - w.Header().Add("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, X-Registry-Auth") - w.Header().Add("Access-Control-Allow-Methods", "HEAD, GET, POST, DELETE, PUT, OPTIONS") -} - -func (s *Server) initTCPSocket(addr string) (l net.Listener, err error) { - if s.cfg.TLSConfig == nil || s.cfg.TLSConfig.ClientAuth != tls.RequireAndVerifyClientCert { - logrus.Warn("/!\\ DON'T BIND ON ANY IP ADDRESS WITHOUT setting -tlsverify IF YOU DON'T KNOW WHAT YOU'RE DOING /!\\") - } - if l, err = sockets.NewTCPSocket(addr, s.cfg.TLSConfig); err != nil { - return nil, err - } - if err := allocateDaemonPort(addr); err != nil { - return nil, err - } - return -} - func (s *Server) makeHTTPHandler(handler httputils.APIFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - // log the handler call - logrus.Debugf("Calling %s %s", r.Method, r.URL.Path) - // Define the context that we'll pass around to share info // like the docker-request-id. // @@ -168,33 +130,31 @@ func (s *Server) makeHTTPHandler(handler httputils.APIFunc) http.HandlerFunc { } if err := handlerFunc(ctx, w, r, vars); err != nil { - logrus.Errorf("Handler for %s %s returned error: %s", r.Method, r.URL.Path, utils.GetErrorMessage(err)) + logrus.Errorf("Handler for %s %s returned error: %v", r.Method, r.URL.Path, err) httputils.WriteError(w, err) } } } -// InitRouters initializes a list of routers for the server. -func (s *Server) InitRouters(d *daemon.Daemon) { - s.addRouter(container.NewRouter(d)) - s.addRouter(image.NewRouter(d)) - s.addRouter(network.NewRouter(d)) - s.addRouter(system.NewRouter(d)) - s.addRouter(volume.NewRouter(d)) - s.addRouter(build.NewRouter(dockerfile.NewBuildManager(d))) -} +// InitRouter initializes the list of routers for the server. +// This method also enables the Go profiler if enableProfiler is true. +func (s *Server) InitRouter(enableProfiler bool, routers ...router.Router) { + for _, r := range routers { + s.routers = append(s.routers, r) + } -// addRouter adds a new router to the server. -func (s *Server) addRouter(r router.Router) { - s.routers = append(s.routers, r) + m := s.createMux() + if enableProfiler { + profilerSetup(m) + } + s.routerSwapper = &routerSwapper{ + router: m, + } } // createMux initializes the main router the server uses. func (s *Server) createMux() *mux.Router { m := mux.NewRouter() - if utils.IsDebugEnabled() { - profilerSetup(m, "/debug/") - } logrus.Debugf("Registering routers") for _, apiRouter := range s.routers { @@ -222,23 +182,14 @@ func (s *Server) Wait(waitChan chan error) { waitChan <- nil } -func (s *Server) initRouterSwapper() { - s.routerSwapper = &routerSwapper{ - router: s.createMux(), - } +// DisableProfiler reloads the server mux without adding the profiler routes. +func (s *Server) DisableProfiler() { + s.routerSwapper.Swap(s.createMux()) } -// Reload reads configuration changes and modifies the -// server according to those changes. -// Currently, only the --debug configuration is taken into account. -func (s *Server) Reload(config *daemon.Config) { - debugEnabled := utils.IsDebugEnabled() - switch { - case debugEnabled && !config.Debug: // disable debug - utils.DisableDebug() - s.routerSwapper.Swap(s.createMux()) - case config.Debug && !debugEnabled: // enable debug - utils.EnableDebug() - s.routerSwapper.Swap(s.createMux()) - } +// EnableProfiler reloads the server mux adding the profiler routes. +func (s *Server) EnableProfiler() { + m := s.createMux() + profilerSetup(m) + s.routerSwapper.Swap(m) } diff --git a/components/engine/api/types/backend/backend.go b/components/engine/api/types/backend/backend.go index c871a148c4..ffe9b709e1 100644 --- a/components/engine/api/types/backend/backend.go +++ b/components/engine/api/types/backend/backend.go @@ -42,3 +42,28 @@ type ContainerStatsConfig struct { Stop <-chan bool Version string } + +// ExecInspect holds information about a running process started +// with docker exec. +type ExecInspect struct { + ID string + Running bool + ExitCode *int + ProcessConfig *ExecProcessConfig + OpenStdin bool + OpenStderr bool + OpenStdout bool + CanRemove bool + ContainerID string + DetachKeys []byte +} + +// ExecProcessConfig holds information about the exec process +// running on the host. +type ExecProcessConfig struct { + Tty bool `json:"tty"` + Entrypoint string `json:"entrypoint"` + Arguments []string `json:"arguments"` + Privileged *bool `json:"privileged,omitempty"` + User string `json:"user,omitempty"` +} diff --git a/components/engine/builder/builder.go b/components/engine/builder/builder.go index e20893f18e..43586a1e4f 100644 --- a/components/engine/builder/builder.go +++ b/components/engine/builder/builder.go @@ -14,6 +14,11 @@ import ( "github.com/docker/engine-api/types/container" ) +const ( + // DefaultDockerfileName is the Default filename with Docker commands, read by docker build + DefaultDockerfileName string = "Dockerfile" +) + // Context represents a file system tree. type Context interface { // Close allows to signal that the filesystem tree won't be used anymore. @@ -141,7 +146,7 @@ type Image interface { // ImageCache abstracts an image cache store. // (parent image, child runconfig) -> child image type ImageCache interface { - // GetCachedImage returns a reference to a cached image whose parent equals `parent` + // GetCachedImageOnBuild returns a reference to a cached image whose parent equals `parent` // and runconfig equals `cfg`. A cache miss is expected to return an empty ID and a nil error. GetCachedImageOnBuild(parentID string, cfg *container.Config) (imageID string, err error) } diff --git a/components/engine/builder/context.go b/components/engine/builder/context.go new file mode 100644 index 0000000000..53a90f1be2 --- /dev/null +++ b/components/engine/builder/context.go @@ -0,0 +1,260 @@ +package builder + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/fileutils" + "github.com/docker/docker/pkg/gitutils" + "github.com/docker/docker/pkg/httputils" + "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/progress" + "github.com/docker/docker/pkg/streamformatter" +) + +// ValidateContextDirectory checks if all the contents of the directory +// can be read and returns an error if some files can't be read +// symlinks which point to non-existing files don't trigger an error +func ValidateContextDirectory(srcPath string, excludes []string) error { + contextRoot, err := getContextRoot(srcPath) + if err != nil { + return err + } + return filepath.Walk(contextRoot, func(filePath string, f os.FileInfo, err error) error { + // skip this directory/file if it's not in the path, it won't get added to the context + if relFilePath, err := filepath.Rel(contextRoot, filePath); err != nil { + return err + } else if skip, err := fileutils.Matches(relFilePath, excludes); err != nil { + return err + } else if skip { + if f.IsDir() { + return filepath.SkipDir + } + return nil + } + + if err != nil { + if os.IsPermission(err) { + return fmt.Errorf("can't stat '%s'", filePath) + } + if os.IsNotExist(err) { + return nil + } + return err + } + + // skip checking if symlinks point to non-existing files, such symlinks can be useful + // also skip named pipes, because they hanging on open + if f.Mode()&(os.ModeSymlink|os.ModeNamedPipe) != 0 { + return nil + } + + if !f.IsDir() { + currentFile, err := os.Open(filePath) + if err != nil && os.IsPermission(err) { + return fmt.Errorf("no permission to read from '%s'", filePath) + } + currentFile.Close() + } + return nil + }) +} + +// GetContextFromReader will read the contents of the given reader as either a +// Dockerfile or tar archive. Returns a tar archive used as a context and a +// path to the Dockerfile inside the tar. +func GetContextFromReader(r io.ReadCloser, dockerfileName string) (out io.ReadCloser, relDockerfile string, err error) { + buf := bufio.NewReader(r) + + magic, err := buf.Peek(archive.HeaderSize) + if err != nil && err != io.EOF { + return nil, "", fmt.Errorf("failed to peek context header from STDIN: %v", err) + } + + if archive.IsArchive(magic) { + return ioutils.NewReadCloserWrapper(buf, func() error { return r.Close() }), dockerfileName, nil + } + + // Input should be read as a Dockerfile. + tmpDir, err := ioutil.TempDir("", "docker-build-context-") + if err != nil { + return nil, "", fmt.Errorf("unbale to create temporary context directory: %v", err) + } + + f, err := os.Create(filepath.Join(tmpDir, DefaultDockerfileName)) + if err != nil { + return nil, "", err + } + _, err = io.Copy(f, buf) + if err != nil { + f.Close() + return nil, "", err + } + + if err := f.Close(); err != nil { + return nil, "", err + } + if err := r.Close(); err != nil { + return nil, "", err + } + + tar, err := archive.Tar(tmpDir, archive.Uncompressed) + if err != nil { + return nil, "", err + } + + return ioutils.NewReadCloserWrapper(tar, func() error { + err := tar.Close() + os.RemoveAll(tmpDir) + return err + }), DefaultDockerfileName, nil + +} + +// GetContextFromGitURL uses a Git URL as context for a `docker build`. The +// git repo is cloned into a temporary directory used as the context directory. +// Returns the absolute path to the temporary context directory, the relative +// path of the dockerfile in that context directory, and a non-nil error on +// success. +func GetContextFromGitURL(gitURL, dockerfileName string) (absContextDir, relDockerfile string, err error) { + if _, err := exec.LookPath("git"); err != nil { + return "", "", fmt.Errorf("unable to find 'git': %v", err) + } + if absContextDir, err = gitutils.Clone(gitURL); err != nil { + return "", "", fmt.Errorf("unable to 'git clone' to temporary context directory: %v", err) + } + + return getDockerfileRelPath(absContextDir, dockerfileName) +} + +// GetContextFromURL uses a remote URL as context for a `docker build`. The +// remote resource is downloaded as either a Dockerfile or a tar archive. +// Returns the tar archive used for the context and a path of the +// dockerfile inside the tar. +func GetContextFromURL(out io.Writer, remoteURL, dockerfileName string) (io.ReadCloser, string, error) { + response, err := httputils.Download(remoteURL) + if err != nil { + return nil, "", fmt.Errorf("unable to download remote context %s: %v", remoteURL, err) + } + progressOutput := streamformatter.NewStreamFormatter().NewProgressOutput(out, true) + + // Pass the response body through a progress reader. + progReader := progress.NewProgressReader(response.Body, progressOutput, response.ContentLength, "", fmt.Sprintf("Downloading build context from remote url: %s", remoteURL)) + + return GetContextFromReader(ioutils.NewReadCloserWrapper(progReader, func() error { return response.Body.Close() }), dockerfileName) +} + +// GetContextFromLocalDir uses the given local directory as context for a +// `docker build`. Returns the absolute path to the local context directory, +// the relative path of the dockerfile in that context directory, and a non-nil +// error on success. +func GetContextFromLocalDir(localDir, dockerfileName string) (absContextDir, relDockerfile string, err error) { + // When using a local context directory, when the Dockerfile is specified + // with the `-f/--file` option then it is considered relative to the + // current directory and not the context directory. + if dockerfileName != "" { + if dockerfileName, err = filepath.Abs(dockerfileName); err != nil { + return "", "", fmt.Errorf("unable to get absolute path to Dockerfile: %v", err) + } + } + + return getDockerfileRelPath(localDir, dockerfileName) +} + +// getDockerfileRelPath uses the given context directory for a `docker build` +// and returns the absolute path to the context directory, the relative path of +// the dockerfile in that context directory, and a non-nil error on success. +func getDockerfileRelPath(givenContextDir, givenDockerfile string) (absContextDir, relDockerfile string, err error) { + if absContextDir, err = filepath.Abs(givenContextDir); err != nil { + return "", "", fmt.Errorf("unable to get absolute context directory: %v", err) + } + + // The context dir might be a symbolic link, so follow it to the actual + // target directory. + // + // FIXME. We use isUNC (always false on non-Windows platforms) to workaround + // an issue in golang. On Windows, EvalSymLinks does not work on UNC file + // paths (those starting with \\). This hack means that when using links + // on UNC paths, they will not be followed. + if !isUNC(absContextDir) { + absContextDir, err = filepath.EvalSymlinks(absContextDir) + if err != nil { + return "", "", fmt.Errorf("unable to evaluate symlinks in context path: %v", err) + } + } + + stat, err := os.Lstat(absContextDir) + if err != nil { + return "", "", fmt.Errorf("unable to stat context directory %q: %v", absContextDir, err) + } + + if !stat.IsDir() { + return "", "", fmt.Errorf("context must be a directory: %s", absContextDir) + } + + absDockerfile := givenDockerfile + if absDockerfile == "" { + // No -f/--file was specified so use the default relative to the + // context directory. + absDockerfile = filepath.Join(absContextDir, DefaultDockerfileName) + + // Just to be nice ;-) look for 'dockerfile' too but only + // use it if we found it, otherwise ignore this check + if _, err = os.Lstat(absDockerfile); os.IsNotExist(err) { + altPath := filepath.Join(absContextDir, strings.ToLower(DefaultDockerfileName)) + if _, err = os.Lstat(altPath); err == nil { + absDockerfile = altPath + } + } + } + + // If not already an absolute path, the Dockerfile path should be joined to + // the base directory. + if !filepath.IsAbs(absDockerfile) { + absDockerfile = filepath.Join(absContextDir, absDockerfile) + } + + // Evaluate symlinks in the path to the Dockerfile too. + // + // FIXME. We use isUNC (always false on non-Windows platforms) to workaround + // an issue in golang. On Windows, EvalSymLinks does not work on UNC file + // paths (those starting with \\). This hack means that when using links + // on UNC paths, they will not be followed. + if !isUNC(absDockerfile) { + absDockerfile, err = filepath.EvalSymlinks(absDockerfile) + if err != nil { + return "", "", fmt.Errorf("unable to evaluate symlinks in Dockerfile path: %v", err) + } + } + + if _, err := os.Lstat(absDockerfile); err != nil { + if os.IsNotExist(err) { + return "", "", fmt.Errorf("Cannot locate Dockerfile: %q", absDockerfile) + } + return "", "", fmt.Errorf("unable to stat Dockerfile: %v", err) + } + + if relDockerfile, err = filepath.Rel(absContextDir, absDockerfile); err != nil { + return "", "", fmt.Errorf("unable to get relative Dockerfile path: %v", err) + } + + if strings.HasPrefix(relDockerfile, ".."+string(filepath.Separator)) { + return "", "", fmt.Errorf("The Dockerfile (%s) must be within the build context (%s)", givenDockerfile, givenContextDir) + } + + return absContextDir, relDockerfile, nil +} + +// isUNC returns true if the path is UNC (one starting \\). It always returns +// false on Linux. +func isUNC(path string) bool { + return runtime.GOOS == "windows" && strings.HasPrefix(path, `\\`) +} diff --git a/components/engine/api/client/utils_unix.go b/components/engine/builder/context_unix.go similarity index 90% rename from components/engine/api/client/utils_unix.go rename to components/engine/builder/context_unix.go index ff10ddde9e..d1f72e0573 100644 --- a/components/engine/api/client/utils_unix.go +++ b/components/engine/builder/context_unix.go @@ -1,6 +1,6 @@ // +build !windows -package client +package builder import ( "path/filepath" diff --git a/components/engine/api/client/utils_windows.go b/components/engine/builder/context_windows.go similarity index 94% rename from components/engine/api/client/utils_windows.go rename to components/engine/builder/context_windows.go index 09c33dadd8..b8ba2ba231 100644 --- a/components/engine/api/client/utils_windows.go +++ b/components/engine/builder/context_windows.go @@ -1,6 +1,6 @@ // +build windows -package client +package builder import ( "path/filepath" diff --git a/components/engine/builder/dockerfile/dispatchers.go b/components/engine/builder/dockerfile/dispatchers.go index f800acbe8e..cc2d177ccd 100644 --- a/components/engine/builder/dockerfile/dispatchers.go +++ b/components/engine/builder/dockerfile/dispatchers.go @@ -19,7 +19,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/api" "github.com/docker/docker/builder" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/signal" "github.com/docker/docker/pkg/system" runconfigopts "github.com/docker/docker/runconfig/opts" @@ -40,12 +39,12 @@ func nullDispatch(b *Builder, args []string, attributes map[string]bool, origina // func env(b *Builder, args []string, attributes map[string]bool, original string) error { if len(args) == 0 { - return derr.ErrorCodeAtLeastOneArg.WithArgs("ENV") + return errAtLeastOneArgument("ENV") } if len(args)%2 != 0 { // should never get here, but just in case - return derr.ErrorCodeTooManyArgs.WithArgs("ENV") + return errTooManyArguments("ENV") } if err := b.flags.Parse(); err != nil { @@ -99,7 +98,7 @@ func env(b *Builder, args []string, attributes map[string]bool, original string) // Sets the maintainer metadata. func maintainer(b *Builder, args []string, attributes map[string]bool, original string) error { if len(args) != 1 { - return derr.ErrorCodeExactlyOneArg.WithArgs("MAINTAINER") + return errExactlyOneArgument("MAINTAINER") } if err := b.flags.Parse(); err != nil { @@ -116,11 +115,11 @@ func maintainer(b *Builder, args []string, attributes map[string]bool, original // func label(b *Builder, args []string, attributes map[string]bool, original string) error { if len(args) == 0 { - return derr.ErrorCodeAtLeastOneArg.WithArgs("LABEL") + return errAtLeastOneArgument("LABEL") } if len(args)%2 != 0 { // should never get here, but just in case - return derr.ErrorCodeTooManyArgs.WithArgs("LABEL") + return errTooManyArguments("LABEL") } if err := b.flags.Parse(); err != nil { @@ -152,7 +151,7 @@ func label(b *Builder, args []string, attributes map[string]bool, original strin // func add(b *Builder, args []string, attributes map[string]bool, original string) error { if len(args) < 2 { - return derr.ErrorCodeAtLeastTwoArgs.WithArgs("ADD") + return errAtLeastOneArgument("ADD") } if err := b.flags.Parse(); err != nil { @@ -168,7 +167,7 @@ func add(b *Builder, args []string, attributes map[string]bool, original string) // func dispatchCopy(b *Builder, args []string, attributes map[string]bool, original string) error { if len(args) < 2 { - return derr.ErrorCodeAtLeastTwoArgs.WithArgs("COPY") + return errAtLeastOneArgument("COPY") } if err := b.flags.Parse(); err != nil { @@ -184,7 +183,7 @@ func dispatchCopy(b *Builder, args []string, attributes map[string]bool, origina // func from(b *Builder, args []string, attributes map[string]bool, original string) error { if len(args) != 1 { - return derr.ErrorCodeExactlyOneArg.WithArgs("FROM") + return errExactlyOneArgument("FROM") } if err := b.flags.Parse(); err != nil { @@ -233,7 +232,7 @@ func from(b *Builder, args []string, attributes map[string]bool, original string // func onbuild(b *Builder, args []string, attributes map[string]bool, original string) error { if len(args) == 0 { - return derr.ErrorCodeAtLeastOneArg.WithArgs("ONBUILD") + return errAtLeastOneArgument("ONBUILD") } if err := b.flags.Parse(); err != nil { @@ -243,9 +242,9 @@ func onbuild(b *Builder, args []string, attributes map[string]bool, original str triggerInstruction := strings.ToUpper(strings.TrimSpace(args[0])) switch triggerInstruction { case "ONBUILD": - return derr.ErrorCodeChainOnBuild + return fmt.Errorf("Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed") case "MAINTAINER", "FROM": - return derr.ErrorCodeBadOnBuildCmd.WithArgs(triggerInstruction) + return fmt.Errorf("%s isn't allowed as an ONBUILD trigger", triggerInstruction) } original = regexp.MustCompile(`(?i)^\s*ONBUILD\s*`).ReplaceAllString(original, "") @@ -260,7 +259,7 @@ func onbuild(b *Builder, args []string, attributes map[string]bool, original str // func workdir(b *Builder, args []string, attributes map[string]bool, original string) error { if len(args) != 1 { - return derr.ErrorCodeExactlyOneArg.WithArgs("WORKDIR") + return errExactlyOneArgument("WORKDIR") } if err := b.flags.Parse(); err != nil { @@ -293,7 +292,7 @@ func workdir(b *Builder, args []string, attributes map[string]bool, original str // func run(b *Builder, args []string, attributes map[string]bool, original string) error { if b.image == "" && !b.noBaseImage { - return derr.ErrorCodeMissingFrom + return fmt.Errorf("Please provide a source image with `from` prior to run") } if err := b.flags.Parse(); err != nil { @@ -311,20 +310,20 @@ func run(b *Builder, args []string, attributes map[string]bool, original string) } config := &container.Config{ - Cmd: strslice.New(args...), + Cmd: strslice.StrSlice(args), Image: b.image, } // stash the cmd cmd := b.runConfig.Cmd - if b.runConfig.Entrypoint.Len() == 0 && b.runConfig.Cmd.Len() == 0 { + if len(b.runConfig.Entrypoint) == 0 && len(b.runConfig.Cmd) == 0 { b.runConfig.Cmd = config.Cmd } // stash the config environment env := b.runConfig.Env - defer func(cmd *strslice.StrSlice) { b.runConfig.Cmd = cmd }(cmd) + defer func(cmd strslice.StrSlice) { b.runConfig.Cmd = cmd }(cmd) defer func(env []string) { b.runConfig.Env = env }(env) // derive the net build-time environment for this run. We let config @@ -367,7 +366,7 @@ func run(b *Builder, args []string, attributes map[string]bool, original string) if len(cmdBuildEnv) > 0 { sort.Strings(cmdBuildEnv) tmpEnv := append([]string{fmt.Sprintf("|%d", len(cmdBuildEnv))}, cmdBuildEnv...) - saveCmd = strslice.New(append(tmpEnv, saveCmd.Slice()...)...) + saveCmd = strslice.StrSlice(append(tmpEnv, saveCmd...)) } b.runConfig.Cmd = saveCmd @@ -425,7 +424,7 @@ func cmd(b *Builder, args []string, attributes map[string]bool, original string) } } - b.runConfig.Cmd = strslice.New(cmdSlice...) + b.runConfig.Cmd = strslice.StrSlice(cmdSlice) if err := b.commit("", b.runConfig.Cmd, fmt.Sprintf("CMD %q", cmdSlice)); err != nil { return err @@ -456,16 +455,16 @@ func entrypoint(b *Builder, args []string, attributes map[string]bool, original switch { case attributes["json"]: // ENTRYPOINT ["echo", "hi"] - b.runConfig.Entrypoint = strslice.New(parsed...) + b.runConfig.Entrypoint = strslice.StrSlice(parsed) case len(parsed) == 0: // ENTRYPOINT [] b.runConfig.Entrypoint = nil default: // ENTRYPOINT echo hi if runtime.GOOS != "windows" { - b.runConfig.Entrypoint = strslice.New("/bin/sh", "-c", parsed[0]) + b.runConfig.Entrypoint = strslice.StrSlice{"/bin/sh", "-c", parsed[0]} } else { - b.runConfig.Entrypoint = strslice.New("cmd", "/S", "/C", parsed[0]) + b.runConfig.Entrypoint = strslice.StrSlice{"cmd", "/S", "/C", parsed[0]} } } @@ -491,7 +490,7 @@ func expose(b *Builder, args []string, attributes map[string]bool, original stri portsTab := args if len(args) == 0 { - return derr.ErrorCodeAtLeastOneArg.WithArgs("EXPOSE") + return errAtLeastOneArgument("EXPOSE") } if err := b.flags.Parse(); err != nil { @@ -530,7 +529,7 @@ func expose(b *Builder, args []string, attributes map[string]bool, original stri // func user(b *Builder, args []string, attributes map[string]bool, original string) error { if len(args) != 1 { - return derr.ErrorCodeExactlyOneArg.WithArgs("USER") + return errExactlyOneArgument("USER") } if err := b.flags.Parse(); err != nil { @@ -547,7 +546,7 @@ func user(b *Builder, args []string, attributes map[string]bool, original string // func volume(b *Builder, args []string, attributes map[string]bool, original string) error { if len(args) == 0 { - return derr.ErrorCodeAtLeastOneArg.WithArgs("VOLUME") + return errAtLeastOneArgument("VOLUME") } if err := b.flags.Parse(); err != nil { @@ -560,7 +559,7 @@ func volume(b *Builder, args []string, attributes map[string]bool, original stri for _, v := range args { v = strings.TrimSpace(v) if v == "" { - return derr.ErrorCodeVolumeEmpty + return fmt.Errorf("Volume specified can not be an empty string") } b.runConfig.Volumes[v] = struct{}{} } @@ -631,3 +630,15 @@ func arg(b *Builder, args []string, attributes map[string]bool, original string) return b.commit("", b.runConfig.Cmd, fmt.Sprintf("ARG %s", arg)) } + +func errAtLeastOneArgument(command string) error { + return fmt.Errorf("%s requires at least one argument", command) +} + +func errExactlyOneArgument(command string) error { + return fmt.Errorf("%s requires exactly one argument", command) +} + +func errTooManyArguments(command string) error { + return fmt.Errorf("Bad input to %s, too many arguments", command) +} diff --git a/components/engine/builder/dockerfile/internals.go b/components/engine/builder/dockerfile/internals.go index bd352ae036..ff3e1a25d2 100644 --- a/components/engine/builder/dockerfile/internals.go +++ b/components/engine/builder/dockerfile/internals.go @@ -19,7 +19,6 @@ import ( "time" "github.com/Sirupsen/logrus" - "github.com/docker/docker/api" "github.com/docker/docker/builder" "github.com/docker/docker/builder/dockerfile/parser" "github.com/docker/docker/pkg/archive" @@ -38,7 +37,7 @@ import ( "github.com/docker/engine-api/types/strslice" ) -func (b *Builder) commit(id string, autoCmd *strslice.StrSlice, comment string) error { +func (b *Builder) commit(id string, autoCmd strslice.StrSlice, comment string) error { if b.disableCommit { return nil } @@ -49,11 +48,11 @@ func (b *Builder) commit(id string, autoCmd *strslice.StrSlice, comment string) if id == "" { cmd := b.runConfig.Cmd if runtime.GOOS != "windows" { - b.runConfig.Cmd = strslice.New("/bin/sh", "-c", "#(nop) "+comment) + b.runConfig.Cmd = strslice.StrSlice{"/bin/sh", "-c", "#(nop) " + comment} } else { - b.runConfig.Cmd = strslice.New("cmd", "/S /C", "REM (nop) "+comment) + b.runConfig.Cmd = strslice.StrSlice{"cmd", "/S /C", "REM (nop) " + comment} } - defer func(cmd *strslice.StrSlice) { b.runConfig.Cmd = cmd }(cmd) + defer func(cmd strslice.StrSlice) { b.runConfig.Cmd = cmd }(cmd) hit, err := b.probeCache() if err != nil { @@ -172,11 +171,11 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalD cmd := b.runConfig.Cmd if runtime.GOOS != "windows" { - b.runConfig.Cmd = strslice.New("/bin/sh", "-c", fmt.Sprintf("#(nop) %s %s in %s", cmdName, srcHash, dest)) + b.runConfig.Cmd = strslice.StrSlice{"/bin/sh", "-c", fmt.Sprintf("#(nop) %s %s in %s", cmdName, srcHash, dest)} } else { - b.runConfig.Cmd = strslice.New("cmd", "/S", "/C", fmt.Sprintf("REM (nop) %s %s in %s", cmdName, srcHash, dest)) + b.runConfig.Cmd = strslice.StrSlice{"cmd", "/S", "/C", fmt.Sprintf("REM (nop) %s %s in %s", cmdName, srcHash, dest)} } - defer func(cmd *strslice.StrSlice) { b.runConfig.Cmd = cmd }(cmd) + defer func(cmd strslice.StrSlice) { b.runConfig.Cmd = cmd }(cmd) if hit, err := b.probeCache(); err != nil { return err @@ -506,7 +505,7 @@ func (b *Builder) create() (string, error) { // TODO: why not embed a hostconfig in builder? hostConfig := &container.HostConfig{ - Isolation: b.options.IsolationLevel, + Isolation: b.options.Isolation, ShmSize: b.options.ShmSize, Resources: resources, } @@ -528,9 +527,9 @@ func (b *Builder) create() (string, error) { b.tmpContainers[c.ID] = struct{}{} fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(c.ID)) - if config.Cmd.Len() > 0 { + if len(config.Cmd) > 0 { // override the entry point that may have been picked up from the base image - if err := b.docker.ContainerUpdateCmdOnBuild(c.ID, config.Cmd.Slice()); err != nil { + if err := b.docker.ContainerUpdateCmdOnBuild(c.ID, config.Cmd); err != nil { return "", err } } @@ -568,7 +567,7 @@ func (b *Builder) run(cID string) (err error) { if ret, _ := b.docker.ContainerWait(cID, -1); ret != 0 { // TODO: change error type, because jsonmessage.JSONError assumes HTTP return &jsonmessage.JSONError{ - Message: fmt.Sprintf("The command '%s' returned a non-zero code: %d", b.runConfig.Cmd.ToString(), ret), + Message: fmt.Sprintf("The command '%s' returned a non-zero code: %d", strings.Join(b.runConfig.Cmd, " "), ret), Code: ret, } } @@ -604,7 +603,7 @@ func (b *Builder) readDockerfile() error { // that then look for 'dockerfile'. If neither are found then default // back to 'Dockerfile' and use that in the error message. if b.options.Dockerfile == "" { - b.options.Dockerfile = api.DefaultDockerfileName + b.options.Dockerfile = builder.DefaultDockerfileName if _, _, err := b.context.Stat(b.options.Dockerfile); os.IsNotExist(err) { lowercase := strings.ToLower(b.options.Dockerfile) if _, _, err := b.context.Stat(lowercase); err == nil { diff --git a/components/engine/builder/dockerfile/parser/line_parsers.go b/components/engine/builder/dockerfile/parser/line_parsers.go index b8792708d5..8cfd39bb2f 100644 --- a/components/engine/builder/dockerfile/parser/line_parsers.go +++ b/components/engine/builder/dockerfile/parser/line_parsers.go @@ -71,7 +71,7 @@ func parseWords(rest string) []string { if unicode.IsSpace(ch) { // skip spaces continue } - phase = inWord // found it, fall thru + phase = inWord // found it, fall through } if (phase == inWord || phase == inQuote) && (pos == len(rest)) { if blankOK || len(word) > 0 { diff --git a/components/engine/builder/dockerfile/parser/utils.go b/components/engine/builder/dockerfile/parser/utils.go index 352d7a7e1c..b21eb62ae0 100644 --- a/components/engine/builder/dockerfile/parser/utils.go +++ b/components/engine/builder/dockerfile/parser/utils.go @@ -118,7 +118,7 @@ func extractBuilderFlags(line string) (string, []string, error) { return line[pos:], words, nil } - phase = inWord // found someting with "--", fall thru + phase = inWord // found someting with "--", fall through } if (phase == inWord || phase == inQuote) && (pos == len(line)) { if word != "--" && (blankOK || len(word) > 0) { diff --git a/components/engine/builder/remote.go b/components/engine/builder/remote.go index 3ab5923109..12f34c7b60 100644 --- a/components/engine/builder/remote.go +++ b/components/engine/builder/remote.go @@ -8,7 +8,6 @@ import ( "io/ioutil" "regexp" - "github.com/docker/docker/api" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/urlutil" @@ -87,7 +86,7 @@ func DetectContextFromRemoteURL(r io.ReadCloser, remoteURL string, createProgres // dockerfileName is set to signal that the remote was interpreted as a single Dockerfile, in which case the caller // should use dockerfileName as the new name for the Dockerfile, irrespective of any other user input. - dockerfileName = api.DefaultDockerfileName + dockerfileName = DefaultDockerfileName // TODO: return a context without tarsum return archive.Generate(dockerfileName, string(dockerfile)) diff --git a/components/engine/cli/common.go b/components/engine/cli/common.go index 1ece1fb616..df6a6ec115 100644 --- a/components/engine/cli/common.go +++ b/components/engine/cli/common.go @@ -19,7 +19,7 @@ type CommonFlags struct { TrustKey string } -// Command is the struct contains command name and description +// Command is the struct containing the command name and description type Command struct { Name string Description string @@ -42,7 +42,7 @@ var dockerCommands = []Command{ {"inspect", "Return low-level information on a container or image"}, {"kill", "Kill a running container"}, {"load", "Load an image from a tar archive or STDIN"}, - {"login", "Register or log in to a Docker registry"}, + {"login", "Log in to a Docker registry"}, {"logout", "Log out from a Docker registry"}, {"logs", "Fetch the logs of a container"}, {"network", "Manage Docker networks"}, @@ -64,7 +64,7 @@ var dockerCommands = []Command{ {"tag", "Tag an image into a repository"}, {"top", "Display the running processes of a container"}, {"unpause", "Unpause all processes within a container"}, - {"update", "Update resources of one or more containers"}, + {"update", "Update configuration of one or more containers"}, {"version", "Show the Docker version information"}, {"volume", "Manage Docker volumes"}, {"wait", "Block until a container stops, then print its exit code"}, diff --git a/components/engine/cliconfig/config.go b/components/engine/cliconfig/config.go index f5b2be8a40..54e0ea387d 100644 --- a/components/engine/cliconfig/config.go +++ b/components/engine/cliconfig/config.go @@ -17,6 +17,7 @@ import ( const ( // ConfigFileName is the name of config file ConfigFileName = "config.json" + configFileDir = ".docker" oldConfigfile = ".dockercfg" // This constant is only used for really old config files when the @@ -31,7 +32,7 @@ var ( func init() { if configDir == "" { - configDir = filepath.Join(homedir.Get(), ".docker") + configDir = filepath.Join(homedir.Get(), configFileDir) } } @@ -47,12 +48,13 @@ func SetConfigDir(dir string) { // ConfigFile ~/.docker/config.json file info type ConfigFile struct { - AuthConfigs map[string]types.AuthConfig `json:"auths"` - HTTPHeaders map[string]string `json:"HttpHeaders,omitempty"` - PsFormat string `json:"psFormat,omitempty"` - ImagesFormat string `json:"imagesFormat,omitempty"` - DetachKeys string `json:"detachKeys,omitempty"` - filename string // Note: not serialized - for internal use only + AuthConfigs map[string]types.AuthConfig `json:"auths"` + HTTPHeaders map[string]string `json:"HttpHeaders,omitempty"` + PsFormat string `json:"psFormat,omitempty"` + ImagesFormat string `json:"imagesFormat,omitempty"` + DetachKeys string `json:"detachKeys,omitempty"` + CredentialsStore string `json:"credsStore,omitempty"` + filename string // Note: not serialized - for internal use only } // NewConfigFile initializes an empty configuration file for the given filename 'fn' @@ -86,11 +88,6 @@ func (configFile *ConfigFile) LegacyLoadFromReader(configData io.Reader) error { if err != nil { return err } - origEmail := strings.Split(arr[1], " = ") - if len(origEmail) != 2 { - return fmt.Errorf("Invalid Auth config file") - } - authConfig.Email = origEmail[1] authConfig.ServerAddress = defaultIndexserver configFile.AuthConfigs[defaultIndexserver] = authConfig } else { @@ -126,6 +123,13 @@ func (configFile *ConfigFile) LoadFromReader(configData io.Reader) error { return nil } +// ContainsAuth returns whether there is authentication configured +// in this file or not. +func (configFile *ConfigFile) ContainsAuth() bool { + return configFile.CredentialsStore != "" || + (configFile.AuthConfigs != nil && len(configFile.AuthConfigs) > 0) +} + // LegacyLoadFromReader is a convenience function that creates a ConfigFile object from // a non-nested reader func LegacyLoadFromReader(configData io.Reader) (*ConfigFile, error) { @@ -249,6 +253,10 @@ func (configFile *ConfigFile) Filename() string { // encodeAuth creates a base64 encoded string to containing authorization information func encodeAuth(authConfig *types.AuthConfig) string { + if authConfig.Username == "" && authConfig.Password == "" { + return "" + } + authStr := authConfig.Username + ":" + authConfig.Password msg := []byte(authStr) encoded := make([]byte, base64.StdEncoding.EncodedLen(len(msg))) @@ -258,6 +266,10 @@ func encodeAuth(authConfig *types.AuthConfig) string { // decodeAuth decodes a base64 encoded string and returns username and password func decodeAuth(authStr string) (string, string, error) { + if authStr == "" { + return "", "", nil + } + decLen := base64.StdEncoding.DecodedLen(len(authStr)) decoded := make([]byte, decLen) authByte := []byte(authStr) diff --git a/components/engine/cliconfig/config_test.go b/components/engine/cliconfig/config_test.go index 17f9fd673b..5ea6f4071c 100644 --- a/components/engine/cliconfig/config_test.go +++ b/components/engine/cliconfig/config_test.go @@ -111,12 +111,9 @@ func TestOldInvalidsAuth(t *testing.T) { invalids := map[string]string{ `username = test`: "The Auth config file is empty", `username -password -email`: "Invalid Auth config file", +password`: "Invalid Auth config file", `username = test email`: "Invalid auth configuration file", - `username = am9lam9lOmhlbGxv -email`: "Invalid Auth config file", } tmpHome, err := ioutil.TempDir("", "config-test") @@ -164,7 +161,7 @@ func TestOldValidAuth(t *testing.T) { fn := filepath.Join(tmpHome, oldConfigfile) js := `username = am9lam9lOmhlbGxv -email = user@example.com` + email = user@example.com` if err := ioutil.WriteFile(fn, []byte(js), 0600); err != nil { t.Fatal(err) } @@ -176,15 +173,23 @@ email = user@example.com` // defaultIndexserver is https://index.docker.io/v1/ ac := config.AuthConfigs["https://index.docker.io/v1/"] - if ac.Email != "user@example.com" || ac.Username != "joejoe" || ac.Password != "hello" { + if ac.Username != "joejoe" || ac.Password != "hello" { t.Fatalf("Missing data from parsing:\n%q", config) } // Now save it and make sure it shows up in new form configStr := saveConfigAndValidateNewFormat(t, config, tmpHome) - if !strings.Contains(configStr, "user@example.com") { - t.Fatalf("Should have save in new form: %s", configStr) + expConfStr := `{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "am9lam9lOmhlbGxv" + } + } +}` + + if configStr != expConfStr { + t.Fatalf("Should have save in new form: \n%s\n not \n%s", configStr, expConfStr) } } @@ -239,15 +244,24 @@ func TestOldJson(t *testing.T) { } ac := config.AuthConfigs["https://index.docker.io/v1/"] - if ac.Email != "user@example.com" || ac.Username != "joejoe" || ac.Password != "hello" { + if ac.Username != "joejoe" || ac.Password != "hello" { t.Fatalf("Missing data from parsing:\n%q", config) } // Now save it and make sure it shows up in new form configStr := saveConfigAndValidateNewFormat(t, config, tmpHome) - if !strings.Contains(configStr, "user@example.com") { - t.Fatalf("Should have save in new form: %s", configStr) + expConfStr := `{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "am9lam9lOmhlbGxv", + "email": "user@example.com" + } + } +}` + + if configStr != expConfStr { + t.Fatalf("Should have save in new form: \n'%s'\n not \n'%s'\n", configStr, expConfStr) } } @@ -259,7 +273,7 @@ func TestNewJson(t *testing.T) { defer os.RemoveAll(tmpHome) fn := filepath.Join(tmpHome, ConfigFileName) - js := ` { "auths": { "https://index.docker.io/v1/": { "auth": "am9lam9lOmhlbGxv", "email": "user@example.com" } } }` + js := ` { "auths": { "https://index.docker.io/v1/": { "auth": "am9lam9lOmhlbGxv" } } }` if err := ioutil.WriteFile(fn, []byte(js), 0600); err != nil { t.Fatal(err) } @@ -270,15 +284,62 @@ func TestNewJson(t *testing.T) { } ac := config.AuthConfigs["https://index.docker.io/v1/"] - if ac.Email != "user@example.com" || ac.Username != "joejoe" || ac.Password != "hello" { + if ac.Username != "joejoe" || ac.Password != "hello" { t.Fatalf("Missing data from parsing:\n%q", config) } // Now save it and make sure it shows up in new form configStr := saveConfigAndValidateNewFormat(t, config, tmpHome) - if !strings.Contains(configStr, "user@example.com") { - t.Fatalf("Should have save in new form: %s", configStr) + expConfStr := `{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "am9lam9lOmhlbGxv" + } + } +}` + + if configStr != expConfStr { + t.Fatalf("Should have save in new form: \n%s\n not \n%s", configStr, expConfStr) + } +} + +func TestNewJsonNoEmail(t *testing.T) { + tmpHome, err := ioutil.TempDir("", "config-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpHome) + + fn := filepath.Join(tmpHome, ConfigFileName) + js := ` { "auths": { "https://index.docker.io/v1/": { "auth": "am9lam9lOmhlbGxv" } } }` + if err := ioutil.WriteFile(fn, []byte(js), 0600); err != nil { + t.Fatal(err) + } + + config, err := Load(tmpHome) + if err != nil { + t.Fatalf("Failed loading on empty json file: %q", err) + } + + ac := config.AuthConfigs["https://index.docker.io/v1/"] + if ac.Username != "joejoe" || ac.Password != "hello" { + t.Fatalf("Missing data from parsing:\n%q", config) + } + + // Now save it and make sure it shows up in new form + configStr := saveConfigAndValidateNewFormat(t, config, tmpHome) + + expConfStr := `{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "am9lam9lOmhlbGxv" + } + } +}` + + if configStr != expConfStr { + t.Fatalf("Should have save in new form: \n%s\n not \n%s", configStr, expConfStr) } } @@ -366,7 +427,7 @@ func TestJsonReaderNoFile(t *testing.T) { } ac := config.AuthConfigs["https://index.docker.io/v1/"] - if ac.Email != "user@example.com" || ac.Username != "joejoe" || ac.Password != "hello" { + if ac.Username != "joejoe" || ac.Password != "hello" { t.Fatalf("Missing data from parsing:\n%q", config) } @@ -381,7 +442,7 @@ func TestOldJsonReaderNoFile(t *testing.T) { } ac := config.AuthConfigs["https://index.docker.io/v1/"] - if ac.Email != "user@example.com" || ac.Username != "joejoe" || ac.Password != "hello" { + if ac.Username != "joejoe" || ac.Password != "hello" { t.Fatalf("Missing data from parsing:\n%q", config) } } @@ -404,7 +465,7 @@ func TestJsonWithPsFormatNoFile(t *testing.T) { func TestJsonSaveWithNoFile(t *testing.T) { js := `{ - "auths": { "https://index.docker.io/v1/": { "auth": "am9lam9lOmhlbGxv", "email": "user@example.com" } }, + "auths": { "https://index.docker.io/v1/": { "auth": "am9lam9lOmhlbGxv" } }, "psFormat": "table {{.ID}}\\t{{.Label \"com.docker.label.cpu\"}}" }` config, err := LoadFromReader(strings.NewReader(js)) @@ -426,9 +487,16 @@ func TestJsonSaveWithNoFile(t *testing.T) { t.Fatalf("Failed saving to file: %q", err) } buf, err := ioutil.ReadFile(filepath.Join(tmpHome, ConfigFileName)) - if !strings.Contains(string(buf), `"auths":`) || - !strings.Contains(string(buf), "user@example.com") { - t.Fatalf("Should have save in new form: %s", string(buf)) + expConfStr := `{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "am9lam9lOmhlbGxv" + } + }, + "psFormat": "table {{.ID}}\\t{{.Label \"com.docker.label.cpu\"}}" +}` + if string(buf) != expConfStr { + t.Fatalf("Should have save in new form: \n%s\nnot \n%s", string(buf), expConfStr) } } @@ -454,14 +522,23 @@ func TestLegacyJsonSaveWithNoFile(t *testing.T) { t.Fatalf("Failed saving to file: %q", err) } buf, err := ioutil.ReadFile(filepath.Join(tmpHome, ConfigFileName)) - if !strings.Contains(string(buf), `"auths":`) || - !strings.Contains(string(buf), "user@example.com") { - t.Fatalf("Should have save in new form: %s", string(buf)) + + expConfStr := `{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "am9lam9lOmhlbGxv", + "email": "user@example.com" + } + } +}` + + if string(buf) != expConfStr { + t.Fatalf("Should have save in new form: \n%s\n not \n%s", string(buf), expConfStr) } } func TestEncodeAuth(t *testing.T) { - newAuthConfig := &types.AuthConfig{Username: "ken", Password: "test", Email: "test@example.com"} + newAuthConfig := &types.AuthConfig{Username: "ken", Password: "test"} authStr := encodeAuth(newAuthConfig) decAuthConfig := &types.AuthConfig{} var err error diff --git a/components/engine/cliconfig/credentials/credentials.go b/components/engine/cliconfig/credentials/credentials.go new file mode 100644 index 0000000000..510cf8cf0e --- /dev/null +++ b/components/engine/cliconfig/credentials/credentials.go @@ -0,0 +1,17 @@ +package credentials + +import ( + "github.com/docker/engine-api/types" +) + +// Store is the interface that any credentials store must implement. +type Store interface { + // Erase removes credentials from the store for a given server. + Erase(serverAddress string) error + // Get retrieves credentials from the store for a given server. + Get(serverAddress string) (types.AuthConfig, error) + // GetAll retrieves all the credentials from the store. + GetAll() (map[string]types.AuthConfig, error) + // Store saves credentials in the store. + Store(authConfig types.AuthConfig) error +} diff --git a/components/engine/cliconfig/credentials/default_store.go b/components/engine/cliconfig/credentials/default_store.go new file mode 100644 index 0000000000..b5fc47ccb3 --- /dev/null +++ b/components/engine/cliconfig/credentials/default_store.go @@ -0,0 +1,22 @@ +package credentials + +import ( + "os/exec" + + "github.com/docker/docker/cliconfig" +) + +// DetectDefaultStore sets the default credentials store +// if the host includes the default store helper program. +func DetectDefaultStore(c *cliconfig.ConfigFile) { + if c.CredentialsStore != "" { + // user defined + return + } + + if defaultCredentialsStore != "" { + if _, err := exec.LookPath(remoteCredentialsPrefix + defaultCredentialsStore); err == nil { + c.CredentialsStore = defaultCredentialsStore + } + } +} diff --git a/components/engine/cliconfig/credentials/default_store_darwin.go b/components/engine/cliconfig/credentials/default_store_darwin.go new file mode 100644 index 0000000000..63e8ed4010 --- /dev/null +++ b/components/engine/cliconfig/credentials/default_store_darwin.go @@ -0,0 +1,3 @@ +package credentials + +const defaultCredentialsStore = "osxkeychain" diff --git a/components/engine/cliconfig/credentials/default_store_linux.go b/components/engine/cliconfig/credentials/default_store_linux.go new file mode 100644 index 0000000000..864c540f6c --- /dev/null +++ b/components/engine/cliconfig/credentials/default_store_linux.go @@ -0,0 +1,3 @@ +package credentials + +const defaultCredentialsStore = "secretservice" diff --git a/components/engine/cliconfig/credentials/default_store_unsupported.go b/components/engine/cliconfig/credentials/default_store_unsupported.go new file mode 100644 index 0000000000..519ef53dcd --- /dev/null +++ b/components/engine/cliconfig/credentials/default_store_unsupported.go @@ -0,0 +1,5 @@ +// +build !windows,!darwin,!linux + +package credentials + +const defaultCredentialsStore = "" diff --git a/components/engine/cliconfig/credentials/default_store_windows.go b/components/engine/cliconfig/credentials/default_store_windows.go new file mode 100644 index 0000000000..fb6a9745cf --- /dev/null +++ b/components/engine/cliconfig/credentials/default_store_windows.go @@ -0,0 +1,3 @@ +package credentials + +const defaultCredentialsStore = "wincred" diff --git a/components/engine/cliconfig/credentials/file_store.go b/components/engine/cliconfig/credentials/file_store.go new file mode 100644 index 0000000000..8e7edd624a --- /dev/null +++ b/components/engine/cliconfig/credentials/file_store.go @@ -0,0 +1,67 @@ +package credentials + +import ( + "strings" + + "github.com/docker/docker/cliconfig" + "github.com/docker/engine-api/types" +) + +// fileStore implements a credentials store using +// the docker configuration file to keep the credentials in plain text. +type fileStore struct { + file *cliconfig.ConfigFile +} + +// NewFileStore creates a new file credentials store. +func NewFileStore(file *cliconfig.ConfigFile) Store { + return &fileStore{ + file: file, + } +} + +// Erase removes the given credentials from the file store. +func (c *fileStore) Erase(serverAddress string) error { + delete(c.file.AuthConfigs, serverAddress) + return c.file.Save() +} + +// Get retrieves credentials for a specific server from the file store. +func (c *fileStore) Get(serverAddress string) (types.AuthConfig, error) { + authConfig, ok := c.file.AuthConfigs[serverAddress] + if !ok { + // Maybe they have a legacy config file, we will iterate the keys converting + // them to the new format and testing + for registry, ac := range c.file.AuthConfigs { + if serverAddress == convertToHostname(registry) { + return ac, nil + } + } + + authConfig = types.AuthConfig{} + } + return authConfig, nil +} + +func (c *fileStore) GetAll() (map[string]types.AuthConfig, error) { + return c.file.AuthConfigs, nil +} + +// Store saves the given credentials in the file store. +func (c *fileStore) Store(authConfig types.AuthConfig) error { + c.file.AuthConfigs[authConfig.ServerAddress] = authConfig + return c.file.Save() +} + +func convertToHostname(url string) string { + stripped := url + if strings.HasPrefix(url, "http://") { + stripped = strings.Replace(url, "http://", "", 1) + } else if strings.HasPrefix(url, "https://") { + stripped = strings.Replace(url, "https://", "", 1) + } + + nameParts := strings.SplitN(stripped, "/", 2) + + return nameParts[0] +} diff --git a/components/engine/cliconfig/credentials/file_store_test.go b/components/engine/cliconfig/credentials/file_store_test.go new file mode 100644 index 0000000000..668b6f097d --- /dev/null +++ b/components/engine/cliconfig/credentials/file_store_test.go @@ -0,0 +1,138 @@ +package credentials + +import ( + "io/ioutil" + "testing" + + "github.com/docker/docker/cliconfig" + "github.com/docker/engine-api/types" +) + +func newConfigFile(auths map[string]types.AuthConfig) *cliconfig.ConfigFile { + tmp, _ := ioutil.TempFile("", "docker-test") + name := tmp.Name() + tmp.Close() + + c := cliconfig.NewConfigFile(name) + c.AuthConfigs = auths + return c +} + +func TestFileStoreAddCredentials(t *testing.T) { + f := newConfigFile(make(map[string]types.AuthConfig)) + + s := NewFileStore(f) + err := s.Store(types.AuthConfig{ + Auth: "super_secret_token", + Email: "foo@example.com", + ServerAddress: "https://example.com", + }) + + if err != nil { + t.Fatal(err) + } + + if len(f.AuthConfigs) != 1 { + t.Fatalf("expected 1 auth config, got %d", len(f.AuthConfigs)) + } + + a, ok := f.AuthConfigs["https://example.com"] + if !ok { + t.Fatalf("expected auth for https://example.com, got %v", f.AuthConfigs) + } + if a.Auth != "super_secret_token" { + t.Fatalf("expected auth `super_secret_token`, got %s", a.Auth) + } + if a.Email != "foo@example.com" { + t.Fatalf("expected email `foo@example.com`, got %s", a.Email) + } +} + +func TestFileStoreGet(t *testing.T) { + f := newConfigFile(map[string]types.AuthConfig{ + "https://example.com": { + Auth: "super_secret_token", + Email: "foo@example.com", + ServerAddress: "https://example.com", + }, + }) + + s := NewFileStore(f) + a, err := s.Get("https://example.com") + if err != nil { + t.Fatal(err) + } + if a.Auth != "super_secret_token" { + t.Fatalf("expected auth `super_secret_token`, got %s", a.Auth) + } + if a.Email != "foo@example.com" { + t.Fatalf("expected email `foo@example.com`, got %s", a.Email) + } +} + +func TestFileStoreGetAll(t *testing.T) { + s1 := "https://example.com" + s2 := "https://example2.com" + f := newConfigFile(map[string]types.AuthConfig{ + s1: { + Auth: "super_secret_token", + Email: "foo@example.com", + ServerAddress: "https://example.com", + }, + s2: { + Auth: "super_secret_token2", + Email: "foo@example2.com", + ServerAddress: "https://example2.com", + }, + }) + + s := NewFileStore(f) + as, err := s.GetAll() + if err != nil { + t.Fatal(err) + } + if len(as) != 2 { + t.Fatalf("wanted 2, got %d", len(as)) + } + if as[s1].Auth != "super_secret_token" { + t.Fatalf("expected auth `super_secret_token`, got %s", as[s1].Auth) + } + if as[s1].Email != "foo@example.com" { + t.Fatalf("expected email `foo@example.com`, got %s", as[s1].Email) + } + if as[s2].Auth != "super_secret_token2" { + t.Fatalf("expected auth `super_secret_token2`, got %s", as[s2].Auth) + } + if as[s2].Email != "foo@example2.com" { + t.Fatalf("expected email `foo@example2.com`, got %s", as[s2].Email) + } +} + +func TestFileStoreErase(t *testing.T) { + f := newConfigFile(map[string]types.AuthConfig{ + "https://example.com": { + Auth: "super_secret_token", + Email: "foo@example.com", + ServerAddress: "https://example.com", + }, + }) + + s := NewFileStore(f) + err := s.Erase("https://example.com") + if err != nil { + t.Fatal(err) + } + + // file store never returns errors, check that the auth config is empty + a, err := s.Get("https://example.com") + if err != nil { + t.Fatal(err) + } + + if a.Auth != "" { + t.Fatalf("expected empty auth token, got %s", a.Auth) + } + if a.Email != "" { + t.Fatalf("expected empty email, got %s", a.Email) + } +} diff --git a/components/engine/cliconfig/credentials/native_store.go b/components/engine/cliconfig/credentials/native_store.go new file mode 100644 index 0000000000..2da041d4b2 --- /dev/null +++ b/components/engine/cliconfig/credentials/native_store.go @@ -0,0 +1,180 @@ +package credentials + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/cliconfig" + "github.com/docker/engine-api/types" +) + +const remoteCredentialsPrefix = "docker-credential-" + +// Standarize the not found error, so every helper returns +// the same message and docker can handle it properly. +var errCredentialsNotFound = errors.New("credentials not found in native keychain") + +// command is an interface that remote executed commands implement. +type command interface { + Output() ([]byte, error) + Input(in io.Reader) +} + +// credentialsRequest holds information shared between docker and a remote credential store. +type credentialsRequest struct { + ServerURL string + Username string + Password string +} + +// credentialsGetResponse is the information serialized from a remote store +// when the plugin sends requests to get the user credentials. +type credentialsGetResponse struct { + Username string + Password string +} + +// nativeStore implements a credentials store +// using native keychain to keep credentials secure. +// It piggybacks into a file store to keep users' emails. +type nativeStore struct { + commandFn func(args ...string) command + fileStore Store +} + +// NewNativeStore creates a new native store that +// uses a remote helper program to manage credentials. +func NewNativeStore(file *cliconfig.ConfigFile) Store { + return &nativeStore{ + commandFn: shellCommandFn(file.CredentialsStore), + fileStore: NewFileStore(file), + } +} + +// Erase removes the given credentials from the native store. +func (c *nativeStore) Erase(serverAddress string) error { + if err := c.eraseCredentialsFromStore(serverAddress); err != nil { + return err + } + + // Fallback to plain text store to remove email + return c.fileStore.Erase(serverAddress) +} + +// Get retrieves credentials for a specific server from the native store. +func (c *nativeStore) Get(serverAddress string) (types.AuthConfig, error) { + // load user email if it exist or an empty auth config. + auth, _ := c.fileStore.Get(serverAddress) + + creds, err := c.getCredentialsFromStore(serverAddress) + if err != nil { + return auth, err + } + auth.Username = creds.Username + auth.Password = creds.Password + + return auth, nil +} + +// GetAll retrieves all the credentials from the native store. +func (c *nativeStore) GetAll() (map[string]types.AuthConfig, error) { + auths, _ := c.fileStore.GetAll() + + for s, ac := range auths { + creds, _ := c.getCredentialsFromStore(s) + ac.Username = creds.Username + ac.Password = creds.Password + auths[s] = ac + } + + return auths, nil +} + +// Store saves the given credentials in the file store. +func (c *nativeStore) Store(authConfig types.AuthConfig) error { + if err := c.storeCredentialsInStore(authConfig); err != nil { + return err + } + authConfig.Username = "" + authConfig.Password = "" + + // Fallback to old credential in plain text to save only the email + return c.fileStore.Store(authConfig) +} + +// storeCredentialsInStore executes the command to store the credentials in the native store. +func (c *nativeStore) storeCredentialsInStore(config types.AuthConfig) error { + cmd := c.commandFn("store") + creds := &credentialsRequest{ + ServerURL: config.ServerAddress, + Username: config.Username, + Password: config.Password, + } + + buffer := new(bytes.Buffer) + if err := json.NewEncoder(buffer).Encode(creds); err != nil { + return err + } + cmd.Input(buffer) + + out, err := cmd.Output() + if err != nil { + t := strings.TrimSpace(string(out)) + logrus.Debugf("error adding credentials - err: %v, out: `%s`", err, t) + return fmt.Errorf(t) + } + + return nil +} + +// getCredentialsFromStore executes the command to get the credentials from the native store. +func (c *nativeStore) getCredentialsFromStore(serverAddress string) (types.AuthConfig, error) { + var ret types.AuthConfig + + cmd := c.commandFn("get") + cmd.Input(strings.NewReader(serverAddress)) + + out, err := cmd.Output() + if err != nil { + t := strings.TrimSpace(string(out)) + + // do not return an error if the credentials are not + // in the keyckain. Let docker ask for new credentials. + if t == errCredentialsNotFound.Error() { + return ret, nil + } + + logrus.Debugf("error getting credentials - err: %v, out: `%s`", err, t) + return ret, fmt.Errorf(t) + } + + var resp credentialsGetResponse + if err := json.NewDecoder(bytes.NewReader(out)).Decode(&resp); err != nil { + return ret, err + } + + ret.Username = resp.Username + ret.Password = resp.Password + ret.ServerAddress = serverAddress + return ret, nil +} + +// eraseCredentialsFromStore executes the command to remove the server redentails from the native store. +func (c *nativeStore) eraseCredentialsFromStore(serverURL string) error { + cmd := c.commandFn("erase") + cmd.Input(strings.NewReader(serverURL)) + + out, err := cmd.Output() + if err != nil { + t := strings.TrimSpace(string(out)) + logrus.Debugf("error erasing credentials - err: %v, out: `%s`", err, t) + return fmt.Errorf(t) + } + + return nil +} diff --git a/components/engine/cliconfig/credentials/native_store_test.go b/components/engine/cliconfig/credentials/native_store_test.go new file mode 100644 index 0000000000..454fd0bd91 --- /dev/null +++ b/components/engine/cliconfig/credentials/native_store_test.go @@ -0,0 +1,309 @@ +package credentials + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "strings" + "testing" + + "github.com/docker/engine-api/types" +) + +const ( + validServerAddress = "https://index.docker.io/v1" + validServerAddress2 = "https://example.com:5002" + invalidServerAddress = "https://foobar.example.com" + missingCredsAddress = "https://missing.docker.io/v1" +) + +var errCommandExited = fmt.Errorf("exited 1") + +// mockCommand simulates interactions between the docker client and a remote +// credentials helper. +// Unit tests inject this mocked command into the remote to control execution. +type mockCommand struct { + arg string + input io.Reader +} + +// Output returns responses from the remote credentials helper. +// It mocks those reponses based in the input in the mock. +func (m *mockCommand) Output() ([]byte, error) { + in, err := ioutil.ReadAll(m.input) + if err != nil { + return nil, err + } + inS := string(in) + + switch m.arg { + case "erase": + switch inS { + case validServerAddress: + return nil, nil + default: + return []byte("error erasing credentials"), errCommandExited + } + case "get": + switch inS { + case validServerAddress, validServerAddress2: + return []byte(`{"Username": "foo", "Password": "bar"}`), nil + case missingCredsAddress: + return []byte(errCredentialsNotFound.Error()), errCommandExited + case invalidServerAddress: + return []byte("error getting credentials"), errCommandExited + } + case "store": + var c credentialsRequest + err := json.NewDecoder(strings.NewReader(inS)).Decode(&c) + if err != nil { + return []byte("error storing credentials"), errCommandExited + } + switch c.ServerURL { + case validServerAddress: + return nil, nil + default: + return []byte("error storing credentials"), errCommandExited + } + } + + return []byte(fmt.Sprintf("unknown argument %q with %q", m.arg, inS)), errCommandExited +} + +// Input sets the input to send to a remote credentials helper. +func (m *mockCommand) Input(in io.Reader) { + m.input = in +} + +func mockCommandFn(args ...string) command { + return &mockCommand{ + arg: args[0], + } +} + +func TestNativeStoreAddCredentials(t *testing.T) { + f := newConfigFile(make(map[string]types.AuthConfig)) + f.CredentialsStore = "mock" + + s := &nativeStore{ + commandFn: mockCommandFn, + fileStore: NewFileStore(f), + } + err := s.Store(types.AuthConfig{ + Username: "foo", + Password: "bar", + Email: "foo@example.com", + ServerAddress: validServerAddress, + }) + + if err != nil { + t.Fatal(err) + } + + if len(f.AuthConfigs) != 1 { + t.Fatalf("expected 1 auth config, got %d", len(f.AuthConfigs)) + } + + a, ok := f.AuthConfigs[validServerAddress] + if !ok { + t.Fatalf("expected auth for %s, got %v", validServerAddress, f.AuthConfigs) + } + if a.Auth != "" { + t.Fatalf("expected auth to be empty, got %s", a.Auth) + } + if a.Username != "" { + t.Fatalf("expected username to be empty, got %s", a.Username) + } + if a.Password != "" { + t.Fatalf("expected password to be empty, got %s", a.Password) + } + if a.Email != "foo@example.com" { + t.Fatalf("expected email `foo@example.com`, got %s", a.Email) + } +} + +func TestNativeStoreAddInvalidCredentials(t *testing.T) { + f := newConfigFile(make(map[string]types.AuthConfig)) + f.CredentialsStore = "mock" + + s := &nativeStore{ + commandFn: mockCommandFn, + fileStore: NewFileStore(f), + } + err := s.Store(types.AuthConfig{ + Username: "foo", + Password: "bar", + Email: "foo@example.com", + ServerAddress: invalidServerAddress, + }) + + if err == nil { + t.Fatal("expected error, got nil") + } + + if err.Error() != "error storing credentials" { + t.Fatalf("expected `error storing credentials`, got %v", err) + } + + if len(f.AuthConfigs) != 0 { + t.Fatalf("expected 0 auth config, got %d", len(f.AuthConfigs)) + } +} + +func TestNativeStoreGet(t *testing.T) { + f := newConfigFile(map[string]types.AuthConfig{ + validServerAddress: { + Email: "foo@example.com", + }, + }) + f.CredentialsStore = "mock" + + s := &nativeStore{ + commandFn: mockCommandFn, + fileStore: NewFileStore(f), + } + a, err := s.Get(validServerAddress) + if err != nil { + t.Fatal(err) + } + + if a.Username != "foo" { + t.Fatalf("expected username `foo`, got %s", a.Username) + } + if a.Password != "bar" { + t.Fatalf("expected password `bar`, got %s", a.Password) + } + if a.Email != "foo@example.com" { + t.Fatalf("expected email `foo@example.com`, got %s", a.Email) + } +} + +func TestNativeStoreGetAll(t *testing.T) { + f := newConfigFile(map[string]types.AuthConfig{ + validServerAddress: { + Email: "foo@example.com", + }, + validServerAddress2: { + Email: "foo@example2.com", + }, + }) + f.CredentialsStore = "mock" + + s := &nativeStore{ + commandFn: mockCommandFn, + fileStore: NewFileStore(f), + } + as, err := s.GetAll() + if err != nil { + t.Fatal(err) + } + + if len(as) != 2 { + t.Fatalf("wanted 2, got %d", len(as)) + } + + if as[validServerAddress].Username != "foo" { + t.Fatalf("expected username `foo` for %s, got %s", validServerAddress, as[validServerAddress].Username) + } + if as[validServerAddress].Password != "bar" { + t.Fatalf("expected password `bar` for %s, got %s", validServerAddress, as[validServerAddress].Password) + } + if as[validServerAddress].Email != "foo@example.com" { + t.Fatalf("expected email `foo@example.com` for %s, got %s", validServerAddress, as[validServerAddress].Email) + } + if as[validServerAddress2].Username != "foo" { + t.Fatalf("expected username `foo` for %s, got %s", validServerAddress2, as[validServerAddress2].Username) + } + if as[validServerAddress2].Password != "bar" { + t.Fatalf("expected password `bar` for %s, got %s", validServerAddress2, as[validServerAddress2].Password) + } + if as[validServerAddress2].Email != "foo@example2.com" { + t.Fatalf("expected email `foo@example2.com` for %s, got %s", validServerAddress2, as[validServerAddress2].Email) + } +} + +func TestNativeStoreGetMissingCredentials(t *testing.T) { + f := newConfigFile(map[string]types.AuthConfig{ + validServerAddress: { + Email: "foo@example.com", + }, + }) + f.CredentialsStore = "mock" + + s := &nativeStore{ + commandFn: mockCommandFn, + fileStore: NewFileStore(f), + } + _, err := s.Get(missingCredsAddress) + if err != nil { + // missing credentials do not produce an error + t.Fatal(err) + } +} + +func TestNativeStoreGetInvalidAddress(t *testing.T) { + f := newConfigFile(map[string]types.AuthConfig{ + validServerAddress: { + Email: "foo@example.com", + }, + }) + f.CredentialsStore = "mock" + + s := &nativeStore{ + commandFn: mockCommandFn, + fileStore: NewFileStore(f), + } + _, err := s.Get(invalidServerAddress) + if err == nil { + t.Fatal("expected error, got nil") + } + + if err.Error() != "error getting credentials" { + t.Fatalf("expected `error getting credentials`, got %v", err) + } +} + +func TestNativeStoreErase(t *testing.T) { + f := newConfigFile(map[string]types.AuthConfig{ + validServerAddress: { + Email: "foo@example.com", + }, + }) + f.CredentialsStore = "mock" + + s := &nativeStore{ + commandFn: mockCommandFn, + fileStore: NewFileStore(f), + } + err := s.Erase(validServerAddress) + if err != nil { + t.Fatal(err) + } + + if len(f.AuthConfigs) != 0 { + t.Fatalf("expected 0 auth configs, got %d", len(f.AuthConfigs)) + } +} + +func TestNativeStoreEraseInvalidAddress(t *testing.T) { + f := newConfigFile(map[string]types.AuthConfig{ + validServerAddress: { + Email: "foo@example.com", + }, + }) + f.CredentialsStore = "mock" + + s := &nativeStore{ + commandFn: mockCommandFn, + fileStore: NewFileStore(f), + } + err := s.Erase(invalidServerAddress) + if err == nil { + t.Fatal("expected error, got nil") + } + + if err.Error() != "error erasing credentials" { + t.Fatalf("expected `error erasing credentials`, got %v", err) + } +} diff --git a/components/engine/cliconfig/credentials/shell_command.go b/components/engine/cliconfig/credentials/shell_command.go new file mode 100644 index 0000000000..fa481b195d --- /dev/null +++ b/components/engine/cliconfig/credentials/shell_command.go @@ -0,0 +1,28 @@ +package credentials + +import ( + "io" + "os/exec" +) + +func shellCommandFn(storeName string) func(args ...string) command { + name := remoteCredentialsPrefix + storeName + return func(args ...string) command { + return &shell{cmd: exec.Command(name, args...)} + } +} + +// shell invokes shell commands to talk with a remote credentials helper. +type shell struct { + cmd *exec.Cmd +} + +// Output returns responses from the remote credentials helper. +func (s *shell) Output() ([]byte, error) { + return s.cmd.Output() +} + +// Input sets the input to send to a remote credentials helper. +func (s *shell) Input(in io.Reader) { + s.cmd.Stdin = in +} diff --git a/components/engine/container/container.go b/components/engine/container/container.go index c92e3de4d1..42ce6db8f4 100644 --- a/components/engine/container/container.go +++ b/components/engine/container/container.go @@ -16,13 +16,12 @@ import ( "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/jsonfilelog" "github.com/docker/docker/daemon/network" - derr "github.com/docker/docker/errors" "github.com/docker/docker/image" "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/promise" "github.com/docker/docker/pkg/signal" "github.com/docker/docker/pkg/symlink" - "github.com/docker/docker/pkg/system" "github.com/docker/docker/runconfig" "github.com/docker/docker/volume" containertypes "github.com/docker/engine-api/types/container" @@ -185,10 +184,17 @@ func (container *Container) WriteHostConfig() error { } // SetupWorkingDirectory sets up the container's working directory as set in container.Config.WorkingDir -func (container *Container) SetupWorkingDirectory() error { +func (container *Container) SetupWorkingDirectory(rootUID, rootGID int) error { if container.Config.WorkingDir == "" { return nil } + + // If can't mount container FS at this point (eg Hyper-V Containers on + // Windows) bail out now with no action. + if !container.canMountFS() { + return nil + } + container.Config.WorkingDir = filepath.Clean(container.Config.WorkingDir) pth, err := container.GetResourcePath(container.Config.WorkingDir) @@ -196,10 +202,10 @@ func (container *Container) SetupWorkingDirectory() error { return err } - if err := system.MkdirAll(pth, 0755); err != nil { + if err := idtools.MkdirAllNewAs(pth, 0755, rootUID, rootGID); err != nil { pthInfo, err2 := os.Stat(pth) if err2 == nil && pthInfo != nil && !pthInfo.IsDir() { - return derr.ErrorCodeNotADir.WithArgs(container.Config.WorkingDir) + return fmt.Errorf("Cannot mkdir: %s is not a directory", container.Config.WorkingDir) } return err @@ -277,37 +283,17 @@ func (container *Container) ConfigPath() (string, error) { return container.GetRootResourcePath(configFileName) } -func validateID(id string) error { - if id == "" { - return derr.ErrorCodeEmptyID - } - return nil -} - // Returns true if the container exposes a certain port func (container *Container) exposes(p nat.Port) bool { _, exists := container.Config.ExposedPorts[p] return exists } -// GetLogConfig returns the log configuration for the container. -func (container *Container) GetLogConfig(defaultConfig containertypes.LogConfig) containertypes.LogConfig { - cfg := container.HostConfig.LogConfig - if cfg.Type != "" || len(cfg.Config) > 0 { // container has log driver configured - if cfg.Type == "" { - cfg.Type = jsonfilelog.Name - } - return cfg - } - // Use daemon's default log config for containers - return defaultConfig -} - // StartLogger starts a new logger driver for the container. func (container *Container) StartLogger(cfg containertypes.LogConfig) (logger.Logger, error) { c, err := logger.GetLogDriver(cfg.Type) if err != nil { - return nil, derr.ErrorCodeLoggingFactory.WithArgs(err) + return nil, fmt.Errorf("Failed to get logging factory: %v", err) } ctx := logger.Context{ Config: cfg.Config, @@ -594,3 +580,20 @@ func (container *Container) InitDNSHostConfig() { container.HostConfig.DNSOptions = make([]string, 0) } } + +// UpdateMonitor updates monitor configure for running container +func (container *Container) UpdateMonitor(restartPolicy containertypes.RestartPolicy) { + monitor := container.monitor + // No need to update monitor if container hasn't got one + // monitor will be generated correctly according to container + if monitor == nil { + return + } + + monitor.mux.Lock() + // to check whether restart policy has changed. + if restartPolicy.Name != "" && !monitor.restartPolicy.IsSame(&restartPolicy) { + monitor.restartPolicy = restartPolicy + } + monitor.mux.Unlock() +} diff --git a/components/engine/container/container_unix.go b/components/engine/container/container_unix.go index 3d85e32e48..3cffb8a1d0 100644 --- a/components/engine/container/container_unix.go +++ b/components/engine/container/container_unix.go @@ -14,7 +14,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/execdriver" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/symlink" "github.com/docker/docker/pkg/system" @@ -34,6 +33,11 @@ import ( // DefaultSHMSize is the default size (64MB) of the SHM which will be mounted in the container const DefaultSHMSize int64 = 67108864 +var ( + errInvalidEndpoint = fmt.Errorf("invalid endpoint while building port map info") + errInvalidNetwork = fmt.Errorf("invalid network settings while building port map info") +) + // Container holds the fields specific to unixen implementations. // See CommonContainer for standard fields common to all containers. type Container struct { @@ -46,6 +50,7 @@ type Container struct { ShmPath string ResolvConfPath string SeccompProfile string + NoNewPrivileges bool } // CreateDaemonEnvironment returns the list of all environment variables given the list of @@ -116,12 +121,12 @@ func (container *Container) GetEndpointInNetwork(n libnetwork.Network) (libnetwo func (container *Container) buildPortMapInfo(ep libnetwork.Endpoint) error { if ep == nil { - return derr.ErrorCodeEmptyEndpoint + return errInvalidEndpoint } networkSettings := container.NetworkSettings if networkSettings == nil { - return derr.ErrorCodeEmptyNetwork + return errInvalidNetwork } if len(networkSettings.Ports) == 0 { @@ -151,7 +156,7 @@ func getEndpointPortMapInfo(ep libnetwork.Endpoint) (nat.PortMap, error) { for _, tp := range exposedPorts { natPort, err := nat.NewPort(tp.Proto.String(), strconv.Itoa(int(tp.Port))) if err != nil { - return pm, derr.ErrorCodeParsingPort.WithArgs(tp.Port, err) + return pm, fmt.Errorf("Error parsing Port value(%v):%v", tp.Port, err) } pm[natPort] = nil } @@ -195,12 +200,12 @@ func getSandboxPortMapInfo(sb libnetwork.Sandbox) nat.PortMap { // BuildEndpointInfo sets endpoint-related fields on container.NetworkSettings based on the provided network and endpoint. func (container *Container) BuildEndpointInfo(n libnetwork.Network, ep libnetwork.Endpoint) error { if ep == nil { - return derr.ErrorCodeEmptyEndpoint + return errInvalidEndpoint } networkSettings := container.NetworkSettings if networkSettings == nil { - return derr.ErrorCodeEmptyNetwork + return errInvalidNetwork } epInfo := ep.Info() @@ -285,7 +290,6 @@ func (container *Container) BuildJoinOptions(n libnetwork.Network) ([]libnetwork // BuildCreateEndpointOptions builds endpoint options from a given network. func (container *Container) BuildCreateEndpointOptions(n libnetwork.Network, epConfig *network.EndpointSettings, sb libnetwork.Sandbox) ([]libnetwork.EndpointOption, error) { var ( - portSpecs = make(nat.PortSet) bindings = make(nat.PortMap) pbList []types.PortBinding exposeList []types.TransportPort @@ -338,10 +342,6 @@ func (container *Container) BuildCreateEndpointOptions(n libnetwork.Network, epC return createOptions, nil } - if container.Config.ExposedPorts != nil { - portSpecs = container.Config.ExposedPorts - } - if container.HostConfig.PortBindings != nil { for p, b := range container.HostConfig.PortBindings { bindings[p] = []nat.PortBinding{} @@ -354,6 +354,7 @@ func (container *Container) BuildCreateEndpointOptions(n libnetwork.Network, epC } } + portSpecs := container.Config.ExposedPorts ports := make([]nat.Port, len(portSpecs)) var i int for p := range portSpecs { @@ -377,7 +378,7 @@ func (container *Container) BuildCreateEndpointOptions(n libnetwork.Network, epC portStart, portEnd, err = newP.Range() } if err != nil { - return nil, derr.ErrorCodeHostPort.WithArgs(binding[i].HostPort, err) + return nil, fmt.Errorf("Error parsing HostPort value(%s):%v", binding[i].HostPort, err) } pbCopy.HostPort = uint16(portStart) pbCopy.HostPortEnd = uint16(portEnd) @@ -498,11 +499,6 @@ func (container *Container) ShmResourcePath() (string, error) { return container.GetRootResourcePath("shm") } -// MqueueResourcePath returns path to mqueue -func (container *Container) MqueueResourcePath() (string, error) { - return container.GetRootResourcePath("mqueue") -} - // HasMountFor checks if path is a mountpoint func (container *Container) HasMountFor(path string) bool { _, exists := container.MountPoints[path] @@ -564,10 +560,11 @@ func updateCommand(c *execdriver.Command, resources containertypes.Resources) { c.Resources.KernelMemory = resources.KernelMemory } -// UpdateContainer updates resources of a container. +// UpdateContainer updates configuration of a container. func (container *Container) UpdateContainer(hostConfig *containertypes.HostConfig) error { container.Lock() + // update resources of container resources := hostConfig.Resources cResources := &container.HostConfig.Resources if resources.BlkioWeight != 0 { @@ -600,6 +597,11 @@ func (container *Container) UpdateContainer(hostConfig *containertypes.HostConfi if resources.KernelMemory != 0 { cResources.KernelMemory = resources.KernelMemory } + + // update HostConfig of container + if hostConfig.RestartPolicy.Name != "" { + container.HostConfig.RestartPolicy = hostConfig.RestartPolicy + } container.Unlock() // If container is not running, update hostConfig struct is enough, @@ -722,3 +724,9 @@ func (container *Container) TmpfsMounts() []execdriver.Mount { func cleanResourcePath(path string) string { return filepath.Join(string(os.PathSeparator), path) } + +// canMountFS determines if the file system for the container +// can be mounted locally. A no-op on non-Windows platforms +func (container *Container) canMountFS() bool { + return true +} diff --git a/components/engine/container/container_windows.go b/components/engine/container/container_windows.go index 61a8994244..18c7e0b0ff 100644 --- a/components/engine/container/container_windows.go +++ b/components/engine/container/container_windows.go @@ -3,12 +3,13 @@ package container import ( + "fmt" "os" "path/filepath" "github.com/docker/docker/daemon/execdriver" "github.com/docker/docker/volume" - "github.com/docker/engine-api/types/container" + containertypes "github.com/docker/engine-api/types/container" ) // Container holds fields specific to the Windows implementation. See @@ -45,8 +46,22 @@ func (container *Container) TmpfsMounts() []execdriver.Mount { return nil } -// UpdateContainer updates resources of a container -func (container *Container) UpdateContainer(hostConfig *container.HostConfig) error { +// UpdateContainer updates configuration of a container +func (container *Container) UpdateContainer(hostConfig *containertypes.HostConfig) error { + container.Lock() + defer container.Unlock() + resources := hostConfig.Resources + if resources.BlkioWeight != 0 || resources.CPUShares != 0 || + resources.CPUPeriod != 0 || resources.CPUQuota != 0 || + resources.CpusetCpus != "" || resources.CpusetMems != "" || + resources.Memory != 0 || resources.MemorySwap != 0 || + resources.MemoryReservation != 0 || resources.KernelMemory != 0 { + return fmt.Errorf("Resource updating isn't supported on Windows") + } + // update HostConfig of container + if hostConfig.RestartPolicy.Name != "" { + container.HostConfig.RestartPolicy = hostConfig.RestartPolicy + } return nil } @@ -68,3 +83,10 @@ func cleanResourcePath(path string) string { } return filepath.Join(string(os.PathSeparator), path) } + +// canMountFS determines if the file system for the container +// can be mounted locally. In the case of Windows, this is not possible +// for Hyper-V containers during WORKDIR execution for example. +func (container *Container) canMountFS() bool { + return !containertypes.Isolation.IsHyperV(container.HostConfig.Isolation) +} diff --git a/components/engine/container/monitor.go b/components/engine/container/monitor.go index a292d8ce3f..afea01fcc9 100644 --- a/components/engine/container/monitor.go +++ b/components/engine/container/monitor.go @@ -1,6 +1,7 @@ package container import ( + "fmt" "io" "os/exec" "strings" @@ -10,10 +11,8 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/execdriver" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/promise" "github.com/docker/docker/pkg/stringid" - "github.com/docker/docker/utils" "github.com/docker/engine-api/types/container" ) @@ -79,11 +78,11 @@ type containerMonitor struct { // StartMonitor initializes a containerMonitor for this container with the provided supervisor and restart policy // and starts the container's process. -func (container *Container) StartMonitor(s supervisor, policy container.RestartPolicy) error { +func (container *Container) StartMonitor(s supervisor) error { container.monitor = &containerMonitor{ supervisor: s, container: container, - restartPolicy: policy, + restartPolicy: container.HostConfig.RestartPolicy, timeIncrement: defaultTimeIncrement, stopChan: make(chan struct{}), startSignal: make(chan struct{}), @@ -126,9 +125,6 @@ func (m *containerMonitor) Close() error { // Cleanup networking and mounts m.supervisor.Cleanup(m.container) - // FIXME: here is race condition between two RUN instructions in Dockerfile - // because they share same runconfig and change image. Must be fixed - // in builder/builder.go if err := m.container.ToDisk(); err != nil { logrus.Errorf("Error dumping container %s state to disk: %s", m.container.ID, err) @@ -190,7 +186,7 @@ func (m *containerMonitor) start() error { if m.container.RestartCount == 0 { m.container.ExitCode = 127 m.resetContainer(false) - return derr.ErrorCodeCmdNotFound + return fmt.Errorf("Container command not found or does not exist.") } } // set to 126 for container cmd can't be invoked errors @@ -198,7 +194,7 @@ func (m *containerMonitor) start() error { if m.container.RestartCount == 0 { m.container.ExitCode = 126 m.resetContainer(false) - return derr.ErrorCodeCmdCouldNotBeInvoked + return fmt.Errorf("Container command could not be invoked.") } } @@ -206,7 +202,7 @@ func (m *containerMonitor) start() error { m.container.ExitCode = -1 m.resetContainer(false) - return derr.ErrorCodeCantStart.WithArgs(m.container.ID, utils.GetErrorMessage(err)) + return fmt.Errorf("Cannot start container %s: %v", m.container.ID, err) } logrus.Errorf("Error running container: %s", err) diff --git a/components/engine/container/state.go b/components/engine/container/state.go index 4a923aa968..7173c7632f 100644 --- a/components/engine/container/state.go +++ b/components/engine/container/state.go @@ -6,7 +6,6 @@ import ( "time" "github.com/docker/docker/daemon/execdriver" - derr "github.com/docker/docker/errors" "github.com/docker/go-units" ) @@ -113,17 +112,17 @@ func wait(waitChan <-chan struct{}, timeout time.Duration) error { } select { case <-time.After(timeout): - return derr.ErrorCodeTimedOut.WithArgs(timeout) + return fmt.Errorf("Timed out: %v", timeout) case <-waitChan: return nil } } -// waitRunning waits until state is running. If state is already +// WaitRunning waits until state is running. If state is already // running it returns immediately. If you want wait forever you must // supply negative timeout. Returns pid, that was passed to // SetRunning. -func (s *State) waitRunning(timeout time.Duration) (int, error) { +func (s *State) WaitRunning(timeout time.Duration) (int, error) { s.Lock() if s.Running { pid := s.Pid @@ -256,14 +255,15 @@ func (s *State) IsRestarting() bool { } // SetRemovalInProgress sets the container state as being removed. -func (s *State) SetRemovalInProgress() error { +// It returns true if the container was already in that state. +func (s *State) SetRemovalInProgress() bool { s.Lock() defer s.Unlock() if s.RemovalInProgress { - return derr.ErrorCodeAlreadyRemoving + return true } s.RemovalInProgress = true - return nil + return false } // ResetRemovalInProgress make the RemovalInProgress state to false. diff --git a/components/engine/container/state_test.go b/components/engine/container/state_test.go index 00c45f1324..75028168d4 100644 --- a/components/engine/container/state_test.go +++ b/components/engine/container/state_test.go @@ -14,7 +14,7 @@ func TestStateRunStop(t *testing.T) { started := make(chan struct{}) var pid int64 go func() { - runPid, _ := s.waitRunning(-1 * time.Second) + runPid, _ := s.WaitRunning(-1 * time.Second) atomic.StoreInt64(&pid, int64(runPid)) close(started) }() @@ -41,8 +41,8 @@ func TestStateRunStop(t *testing.T) { if runPid != i+100 { t.Fatalf("Pid %v, expected %v", runPid, i+100) } - if pid, err := s.waitRunning(-1 * time.Second); err != nil || pid != i+100 { - t.Fatalf("waitRunning returned pid: %v, err: %v, expected pid: %v, err: %v", pid, err, i+100, nil) + if pid, err := s.WaitRunning(-1 * time.Second); err != nil || pid != i+100 { + t.Fatalf("WaitRunning returned pid: %v, err: %v, expected pid: %v, err: %v", pid, err, i+100, nil) } stopped := make(chan struct{}) @@ -82,7 +82,7 @@ func TestStateTimeoutWait(t *testing.T) { s := NewState() started := make(chan struct{}) go func() { - s.waitRunning(100 * time.Millisecond) + s.WaitRunning(100 * time.Millisecond) close(started) }() select { @@ -98,7 +98,7 @@ func TestStateTimeoutWait(t *testing.T) { stopped := make(chan struct{}) go func() { - s.waitRunning(100 * time.Millisecond) + s.WaitRunning(100 * time.Millisecond) close(stopped) }() select { diff --git a/components/engine/contrib/README b/components/engine/contrib/README.md similarity index 100% rename from components/engine/contrib/README rename to components/engine/contrib/README.md diff --git a/components/engine/contrib/apparmor/main.go b/components/engine/contrib/apparmor/main.go index 25f6e8c480..f4a2978b86 100644 --- a/components/engine/contrib/apparmor/main.go +++ b/components/engine/contrib/apparmor/main.go @@ -11,8 +11,7 @@ import ( ) type profileData struct { - MajorVersion int - MinorVersion int + Version int } func main() { @@ -23,13 +22,12 @@ func main() { // parse the arg apparmorProfilePath := os.Args[1] - majorVersion, minorVersion, err := aaparser.GetVersion() + version, err := aaparser.GetVersion() if err != nil { log.Fatal(err) } data := profileData{ - MajorVersion: majorVersion, - MinorVersion: minorVersion, + Version: version, } fmt.Printf("apparmor_parser is of version %+v\n", data) diff --git a/components/engine/contrib/apparmor/template.go b/components/engine/contrib/apparmor/template.go index ea9c706d11..e5e1c8bed6 100644 --- a/components/engine/contrib/apparmor/template.go +++ b/components/engine/contrib/apparmor/template.go @@ -20,11 +20,11 @@ profile /usr/bin/docker (attach_disconnected, complain) { umount, pivot_root, -{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}} +{{if ge .Version 209000}} signal (receive) peer=@{profile_name}, signal (receive) peer=unconfined, signal (send), -{{end}}{{end}} +{{end}} network, capability, owner /** rw, @@ -46,12 +46,12 @@ profile /usr/bin/docker (attach_disconnected, complain) { /etc/ld.so.cache r, /etc/passwd r, -{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}} +{{if ge .Version 209000}} ptrace peer=@{profile_name}, ptrace (read) peer=docker-default, deny ptrace (trace) peer=docker-default, deny ptrace peer=/usr/bin/docker///bin/ps, -{{end}}{{end}} +{{end}} /usr/lib/** rm, /lib/** rm, @@ -72,11 +72,11 @@ profile /usr/bin/docker (attach_disconnected, complain) { /sbin/zfs rCx, /sbin/apparmor_parser rCx, -{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}} +{{if ge .Version 209000}} # Transitions change_profile -> docker-*, change_profile -> unconfined, -{{end}}{{end}} +{{end}} profile /bin/cat (complain) { /etc/ld.so.cache r, @@ -98,10 +98,10 @@ profile /usr/bin/docker (attach_disconnected, complain) { /dev/null rw, /bin/ps mr, -{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}} +{{if ge .Version 209000}} # We don't need ptrace so we'll deny and ignore the error. deny ptrace (read, trace), -{{end}}{{end}} +{{end}} # Quiet dac_override denials deny capability dac_override, @@ -119,15 +119,15 @@ profile /usr/bin/docker (attach_disconnected, complain) { /proc/tty/drivers r, } profile /sbin/iptables (complain) { -{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}} +{{if ge .Version 209000}} signal (receive) peer=/usr/bin/docker, -{{end}}{{end}} +{{end}} capability net_admin, } profile /sbin/auplink flags=(attach_disconnected, complain) { -{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}} +{{if ge .Version 209000}} signal (receive) peer=/usr/bin/docker, -{{end}}{{end}} +{{end}} capability sys_admin, capability dac_override, @@ -146,9 +146,9 @@ profile /usr/bin/docker (attach_disconnected, complain) { /proc/[0-9]*/mounts rw, } profile /sbin/modprobe /bin/kmod (complain) { -{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}} +{{if ge .Version 209000}} signal (receive) peer=/usr/bin/docker, -{{end}}{{end}} +{{end}} capability sys_module, /etc/ld.so.cache r, /lib/** rm, @@ -162,9 +162,9 @@ profile /usr/bin/docker (attach_disconnected, complain) { } # xz works via pipes, so we do not need access to the filesystem. profile /usr/bin/xz (complain) { -{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}} +{{if ge .Version 209000}} signal (receive) peer=/usr/bin/docker, -{{end}}{{end}} +{{end}} /etc/ld.so.cache r, /lib/** rm, /usr/bin/xz rm, diff --git a/components/engine/contrib/check-config.sh b/components/engine/contrib/check-config.sh index 3a043658e7..bcc90d4a1f 100755 --- a/components/engine/contrib/check-config.sh +++ b/components/engine/contrib/check-config.sh @@ -115,6 +115,17 @@ check_device() { fi } +check_distro_userns() { + source /etc/os-release 2>/dev/null || /bin/true + if [[ "${ID}" =~ ^(centos|rhel)$ && "${VERSION_ID}" =~ ^7 ]]; then + # this is a CentOS7 or RHEL7 system + grep -q "user_namespace.enable=1" /proc/cmdline || { + # no user namespace support enabled + wrap_bad " (RHEL7/CentOS7" "User namespaces disabled; add 'user_namespace.enable=1' to boot command line)" + } + fi +} + if [ ! -e "$CONFIG" ]; then wrap_warning "warning: $CONFIG does not exist, searching other paths for kernel config ..." for tryConfig in "${possibleConfigs[@]}"; do @@ -171,6 +182,7 @@ flags=( NAMESPACES {NET,PID,IPC,UTS}_NS DEVPTS_MULTIPLE_INSTANCES CGROUPS CGROUP_CPUACCT CGROUP_DEVICE CGROUP_FREEZER CGROUP_SCHED CPUSETS MEMCG + KEYS MACVLAN VETH BRIDGE BRIDGE_NETFILTER NF_NAT_IPV4 IP_NF_FILTER IP_NF_TARGET_MASQUERADE NETFILTER_XT_MATCH_{ADDRTYPE,CONNTRACK} @@ -185,10 +197,14 @@ echo echo 'Optional Features:' { check_flags USER_NS + check_distro_userns } { check_flags SECCOMP } +{ + check_flags CGROUP_PIDS +} { check_flags MEMCG_KMEM MEMCG_SWAP MEMCG_SWAP_ENABLED if is_set MEMCG_SWAP && ! is_set MEMCG_SWAP_ENABLED; then diff --git a/components/engine/contrib/completion/bash/docker b/components/engine/contrib/completion/bash/docker index 7588e33355..343e1dc717 100644 --- a/components/engine/contrib/completion/bash/docker +++ b/components/engine/contrib/completion/bash/docker @@ -395,7 +395,9 @@ __docker_complete_isolation() { __docker_complete_log_drivers() { COMPREPLY=( $( compgen -W " awslogs + etwlogs fluentd + gcplogs gelf journald json-file @@ -409,13 +411,14 @@ __docker_complete_log_options() { # see docs/reference/logging/index.md local awslogs_options="awslogs-region awslogs-group awslogs-stream" local fluentd_options="env fluentd-address labels tag" + local gcplogs_options="env gcp-log-cmd gcp-project labels" local gelf_options="env gelf-address labels tag" local journald_options="env labels tag" local json_file_options="env labels max-file max-size" local syslog_options="syslog-address syslog-tls-ca-cert syslog-tls-cert syslog-tls-key syslog-tls-skip-verify syslog-facility tag" local splunk_options="env labels splunk-caname splunk-capath splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url tag" - local all_options="$fluentd_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options" + local all_options="$fluentd_options $gcplogs_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options" case $(__docker_value_of_option --log-driver) in '') @@ -427,6 +430,9 @@ __docker_complete_log_options() { fluentd) COMPREPLY=( $( compgen -W "$fluentd_options" -S = -- "$cur" ) ) ;; + gcplogs) + COMPREPLY=( $( compgen -W "$gcplogs_options" -S = -- "$cur" ) ) + ;; gelf) COMPREPLY=( $( compgen -W "$gelf_options" -S = -- "$cur" ) ) ;; @@ -515,6 +521,22 @@ __docker_complete_log_levels() { COMPREPLY=( $( compgen -W "debug info warn error fatal" -- "$cur" ) ) } +__docker_complete_restart() { + case "$prev" in + --restart) + case "$cur" in + on-failure:*) + ;; + *) + COMPREPLY=( $( compgen -W "always no on-failure on-failure: unless-stopped" -- "$cur") ) + ;; + esac + return + ;; + esac + return 1 +} + # a selection of the available signals that is most likely of interest in the # context of docker containers. __docker_complete_signals() { @@ -794,7 +816,7 @@ _docker_daemon() { return ;; esac - + local key=$(__docker_map_key_of_current_option '--storage-opt') case "$key" in dm.@(blkdiscard|override_udev_sync_check|use_deferred_@(removal|deletion))) @@ -1188,14 +1210,14 @@ _docker_load() { _docker_login() { case "$prev" in - --email|-e|--password|-p|--username|-u) + --password|-p|--username|-u) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "--email -e --help --password -p --username -u" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --password -p --username -u" -- "$cur" ) ) ;; esac } @@ -1275,7 +1297,7 @@ _docker_network_connect() { _docker_network_create() { case "$prev" in - --aux-address|--gateway|--ip-range|--ipam-opt|--opt|-o|--subnet) + --aux-address|--gateway|--internal|--ip-range|--ipam-opt|--ipv6|--opt|-o|--subnet) return ;; --ipam-driver) @@ -1294,7 +1316,7 @@ _docker_network_create() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--aux-address --driver -d --gateway --help --internal --ip-range --ipam-driver --ipam-opt --opt -o --subnet" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--aux-address --driver -d --gateway --help --internal --ip-range --ipam-driver --ipam-opt --ipv6 --opt -o --subnet" -- "$cur" ) ) ;; esac } @@ -1615,6 +1637,7 @@ _docker_run() { --net-alias --oom-score-adj --pid + --pids-limit --publish -p --restart --security-opt @@ -1657,6 +1680,7 @@ _docker_run() { __docker_complete_log_driver_options && return + __docker_complete_restart && return case "$prev" in --add-host) @@ -1754,16 +1778,6 @@ _docker_run() { esac return ;; - --restart) - case "$cur" in - on-failure:*) - ;; - *) - COMPREPLY=( $( compgen -W "always no on-failure on-failure: unless-stopped" -- "$cur") ) - ;; - esac - return - ;; --security-opt) case "$cur" in label:*:*) @@ -1938,6 +1952,7 @@ _docker_update() { --memory -m --memory-reservation --memory-swap + --restart " local boolean_options=" @@ -1946,6 +1961,8 @@ _docker_update() { local all_options="$options_with_args $boolean_options" + __docker_complete_restart && return + case "$prev" in $(__docker_to_extglob "$options_with_args") ) return diff --git a/components/engine/contrib/completion/fish/docker.fish b/components/engine/contrib/completion/fish/docker.fish index 17af1c0171..1e734c7d3a 100644 --- a/components/engine/contrib/completion/fish/docker.fish +++ b/components/engine/contrib/completion/fish/docker.fish @@ -221,8 +221,7 @@ complete -c docker -A -f -n '__fish_seen_subcommand_from load' -l help -d 'Print complete -c docker -A -f -n '__fish_seen_subcommand_from load' -s i -l input -d 'Read from a tar archive file, instead of STDIN' # login -complete -c docker -f -n '__fish_docker_no_subcommand' -a login -d 'Register or log in to a Docker registry server' -complete -c docker -A -f -n '__fish_seen_subcommand_from login' -s e -l email -d 'Email' +complete -c docker -f -n '__fish_docker_no_subcommand' -a login -d 'Log in to a Docker registry server' complete -c docker -A -f -n '__fish_seen_subcommand_from login' -l help -d 'Print usage' complete -c docker -A -f -n '__fish_seen_subcommand_from login' -s p -l password -d 'Password' complete -c docker -A -f -n '__fish_seen_subcommand_from login' -s u -l username -d 'Username' @@ -290,6 +289,7 @@ complete -c docker -A -f -n '__fish_seen_subcommand_from rm' -l help -d 'Print u complete -c docker -A -f -n '__fish_seen_subcommand_from rm' -s l -l link -d 'Remove the specified link and not the underlying container' complete -c docker -A -f -n '__fish_seen_subcommand_from rm' -s v -l volumes -d 'Remove the volumes associated with the container' complete -c docker -A -f -n '__fish_seen_subcommand_from rm' -a '(__fish_print_docker_containers stopped)' -d "Container" +complete -c docker -A -f -n '__fish_seen_subcommand_from rm' -s f -l force -a '(__fish_print_docker_containers all)' -d "Container" # rmi complete -c docker -f -n '__fish_docker_no_subcommand' -a rmi -d 'Remove one or more images' @@ -398,5 +398,3 @@ complete -c docker -f -n '__fish_docker_no_subcommand' -a version -d 'Show the D complete -c docker -f -n '__fish_docker_no_subcommand' -a wait -d 'Block until a container stops, then print its exit code' complete -c docker -A -f -n '__fish_seen_subcommand_from wait' -l help -d 'Print usage' complete -c docker -A -f -n '__fish_seen_subcommand_from wait' -a '(__fish_print_docker_containers running)' -d "Container" - - diff --git a/components/engine/contrib/completion/zsh/_docker b/components/engine/contrib/completion/zsh/_docker index b77435a156..0f2a361a6b 100644 --- a/components/engine/contrib/completion/zsh/_docker +++ b/components/engine/contrib/completion/zsh/_docker @@ -201,6 +201,7 @@ __docker_get_log_options() { awslogs_options=("awslogs-region" "awslogs-group" "awslogs-stream") fluentd_options=("env" "fluentd-address" "labels" "tag") + gcplogs_options=("env" "gcp-log-cmd" "gcp-project" "labels") gelf_options=("env" "gelf-address" "labels" "tag") journald_options=("env" "labels") json_file_options=("env" "labels" "max-file" "max-size") @@ -209,6 +210,7 @@ __docker_get_log_options() { [[ $log_driver = (awslogs|all) ]] && _describe -t awslogs-options "awslogs options" awslogs_options "$@" && ret=0 [[ $log_driver = (fluentd|all) ]] && _describe -t fluentd-options "fluentd options" fluentd_options "$@" && ret=0 + [[ $log_driver = (gcplogs|all) ]] && _describe -t gcplogs-options "gcplogs options" gcplogs_options "$@" && ret=0 [[ $log_driver = (gelf|all) ]] && _describe -t gelf-options "gelf options" gelf_options "$@" && ret=0 [[ $log_driver = (journald|all) ]] && _describe -t journald-options "journald options" journald_options "$@" && ret=0 [[ $log_driver = (json-file|all) ]] && _describe -t json-file-options "json-file options" json_file_options "$@" && ret=0 @@ -325,14 +327,15 @@ __docker_network_subcommand() { (create) _arguments $(__docker_arguments) -A '-*' \ $opts_help \ - "($help)*--aux-address[Auxiliary ipv4 or ipv6 addresses used by network driver]:key=IP: " \ + "($help)*--aux-address[Auxiliary IPv4 or IPv6 addresses used by network driver]:key=IP: " \ "($help -d --driver)"{-d=,--driver=}"[Driver to manage the Network]:driver:(null host bridge overlay)" \ - "($help)*--gateway=[ipv4 or ipv6 Gateway for the master subnet]:IP: " \ + "($help)*--gateway=[IPv4 or IPv6 Gateway for the master subnet]:IP: " \ "($help)--internal[Restricts external access to the network]" \ "($help)*--ip-range=[Allocate container ip from a sub-range]:IP/mask: " \ "($help)--ipam-driver=[IP Address Management Driver]:driver:(default)" \ - "($help)*--ipam-opt=[Set custom IPAM plugin options]:opt=value: " \ - "($help)*"{-o=,--opt=}"[Set driver specific options]:opt=value: " \ + "($help)*--ipam-opt=[Custom IPAM plugin options]:opt=value: " \ + "($help)--ipv6[Enable IPv6 networking]" \ + "($help)*"{-o=,--opt=}"[Driver specific options]:opt=value: " \ "($help)*--subnet=[Subnet in CIDR format that represents a network segment]:IP/mask: " \ "($help -)1:Network Name: " && ret=0 ;; @@ -421,9 +424,9 @@ __docker_volume_subcommand() { (create) _arguments $(__docker_arguments) \ $opts_help \ - "($help -d --driver)"{-d=,--driver=}"[Specify volume driver name]:Driver name:(local)" \ - "($help)--name=[Specify volume name]" \ - "($help)*"{-o=,--opt=}"[Set driver specific options]:Driver option: " && ret=0 + "($help -d --driver)"{-d=,--driver=}"[Volume driver name]:Driver name:(local)" \ + "($help)--name=[Volume name]" \ + "($help)*"{-o=,--opt=}"[Driver specific options]:Driver option: " && ret=0 ;; (inspect) _arguments $(__docker_arguments) \ @@ -483,8 +486,8 @@ __docker_subcommand() { opts_help=("(: -)--help[Print usage]") opts_build_create_run=( "($help)--cgroup-parent=[Parent cgroup for the container]:cgroup: " - "($help)--isolation=[]:isolation:(default hyperv process)" - "($help)*--shm-size=[Size of '/dev/shm'. The format is ''. Default is '64m'.]:shm size: " + "($help)--isolation=[Container isolation technology]:isolation:(default hyperv process)" + "($help)*--shm-size=[Size of '/dev/shm' (format is '')]:shm size: " "($help)*--ulimit=[ulimit options]:ulimit: " ) opts_build_create_run_update=( @@ -508,10 +511,10 @@ __docker_subcommand() { "($help)*--device-read-iops=[Limit the read rate (IO per second) from a device]:device:IO rate: " "($help)*--device-write-bps=[Limit the write rate (bytes per second) to a device]:device:IO rate: " "($help)*--device-write-iops=[Limit the write rate (IO per second) to a device]:device:IO rate: " - "($help)*--dns=[Set custom DNS servers]:DNS server: " - "($help)*--dns-opt=[Set custom DNS options]:DNS option: " - "($help)*--dns-search=[Set custom DNS search domains]:DNS domains: " - "($help)*"{-e=,--env=}"[Set environment variables]:environment variable: " + "($help)*--dns=[Custom DNS servers]:DNS server: " + "($help)*--dns-opt=[Custom DNS options]:DNS option: " + "($help)*--dns-search=[Custom DNS search domains]:DNS domains: " + "($help)*"{-e=,--env=}"[Environment variables]:environment variable: " "($help)--entrypoint=[Overwrite the default entrypoint of the image]:entry point: " "($help)*--env-file=[Read environment variables from a file]:environment file:_files" "($help)*--expose=[Expose a port from the container without publishing it]: " @@ -522,7 +525,7 @@ __docker_subcommand() { "($help)--ip6=[Container IPv6 address]:IPv6: " "($help)--ipc=[IPC namespace to use]:IPC namespace: " "($help)*--link=[Add link to another container]:link:->link" - "($help)*"{-l=,--label=}"[Set meta data on a container]:label: " + "($help)*"{-l=,--label=}"[Container metadata]:label: " "($help)--log-driver=[Default driver for container logs]:Logging driver:(json-file syslog journald gelf fluentd awslogs splunk none)" "($help)*--log-opt=[Log driver specific options]:log driver options:__docker_log_options" "($help)--mac-address=[Container MAC address]:MAC address: " @@ -531,6 +534,7 @@ __docker_subcommand() { "($help)*--net-alias=[Add network-scoped alias for the container]:alias: " "($help)--oom-kill-disable[Disable OOM Killer]" "($help)--oom-score-adj[Tune the host's OOM preferences for containers (accepts -1000 to 1000)]" + "($help)--pids-limit[Tune container pids limit (set -1 for unlimited)]" "($help -P --publish-all)"{-P,--publish-all}"[Publish all exposed ports]" "($help)*"{-p=,--publish=}"[Expose a container's port to the host]:port:_ports" "($help)--pid=[PID namespace to use]:PID: " @@ -548,11 +552,11 @@ __docker_subcommand() { ) opts_create_run_update=( "($help)--blkio-weight=[Block IO (relative weight), between 10 and 1000]:Block IO weight:(10 100 500 1000)" - "($help)--kernel-memory=[Kernel memory limit in bytes.]:Memory limit: " + "($help)--kernel-memory=[Kernel memory limit in bytes]:Memory limit: " "($help)--memory-reservation=[Memory soft limit]:Memory limit: " ) opts_attach_exec_run_start=( - "($help)--detach-keys=[Specify the escape key sequence used to detach a container]:sequence:__docker_complete_detach_keys" + "($help)--detach-keys=[Escape key sequence used to detach a container]:sequence:__docker_complete_detach_keys" ) case "$words[1]" in @@ -569,7 +573,7 @@ __docker_subcommand() { $opts_help \ $opts_build_create_run \ $opts_build_create_run_update \ - "($help)*--build-arg[Set build-time variables]:=: " \ + "($help)*--build-arg[Build-time variables]:=: " \ "($help -f --file)"{-f=,--file=}"[Name of the Dockerfile]:Dockerfile:_files" \ "($help)--force-rm[Always remove intermediate containers]" \ "($help)--no-cache[Do not use cache when building the image]" \ @@ -592,7 +596,7 @@ __docker_subcommand() { (cp) _arguments $(__docker_arguments) \ $opts_help \ - "($help -L --follow-link)"{-L,--follow-link}"[Always follow symbol link in SRC_PATH]" \ + "($help -L --follow-link)"{-L,--follow-link}"[Always follow symbol link]" \ "($help -)1:container:->container" \ "($help -)2:hostpath:_files" && ret=0 case $state in @@ -630,23 +634,23 @@ __docker_subcommand() { (daemon) _arguments $(__docker_arguments) \ $opts_help \ - "($help)--api-cors-header=[Set CORS headers in the remote API]:CORS headers: " \ - "($help)*--authorization-plugin=[Set authorization plugins to load]" \ + "($help)--api-cors-header=[CORS headers in the remote API]:CORS headers: " \ + "($help)*--authorization-plugin=[Authorization plugins to load]" \ "($help -b --bridge)"{-b=,--bridge=}"[Attach containers to a network bridge]:bridge:_net_interfaces" \ - "($help)--bip=[Specify network bridge IP]" \ - "($help)--cgroup-parent=[Set parent cgroup for all containers]:cgroup: " \ + "($help)--bip=[Network bridge IP]:IP address: " \ + "($help)--cgroup-parent=[Parent cgroup for all containers]:cgroup: " \ "($help -D --debug)"{-D,--debug}"[Enable debug mode]" \ "($help)--default-gateway[Container default gateway IPv4 address]:IPv4 address: " \ "($help)--default-gateway-v6[Container default gateway IPv6 address]:IPv6 address: " \ "($help)--cluster-store=[URL of the distributed storage backend]:Cluster Store:->cluster-store" \ "($help)--cluster-advertise=[Address of the daemon instance to advertise]:Instance to advertise (host\:port): " \ - "($help)*--cluster-store-opt=[Set cluster options]:Cluster options:->cluster-store-options" \ + "($help)*--cluster-store-opt=[Cluster options]:Cluster options:->cluster-store-options" \ "($help)*--dns=[DNS server to use]:DNS: " \ "($help)*--dns-search=[DNS search domains to use]:DNS search: " \ "($help)*--dns-opt=[DNS options to use]:DNS option: " \ - "($help)*--default-ulimit=[Set default ulimit settings for containers]:ulimit: " \ + "($help)*--default-ulimit=[Default ulimit settings for containers]:ulimit: " \ "($help)--disable-legacy-registry[Do not contact legacy registries]" \ - "($help)*--exec-opt=[Set exec driver options]:exec driver options: " \ + "($help)*--exec-opt=[Exec driver options]:exec driver options: " \ "($help)--exec-root=[Root of the Docker execdriver]:path:_directories" \ "($help)--fixed-cidr=[IPv4 subnet for fixed IPs]:IPv4 subnet: " \ "($help)--fixed-cidr-v6=[IPv6 subnet for fixed IPs]:IPv6 subnet: " \ @@ -660,17 +664,17 @@ __docker_subcommand() { "($help)--ip-masq[Enable IP masquerading]" \ "($help)--iptables[Enable addition of iptables rules]" \ "($help)--ipv6[Enable IPv6 networking]" \ - "($help -l --log-level)"{-l=,--log-level=}"[Set the logging level]:level:(debug info warn error fatal)" \ - "($help)*--label=[Set key=value labels to the daemon]:label: " \ + "($help -l --log-level)"{-l=,--log-level=}"[Logging level]:level:(debug info warn error fatal)" \ + "($help)*--label=[Key=value labels]:label: " \ "($help)--log-driver=[Default driver for container logs]:Logging driver:(json-file syslog journald gelf fluentd awslogs splunk none)" \ "($help)*--log-opt=[Log driver specific options]:log driver options:__docker_log_options" \ - "($help)--mtu=[Set the containers network MTU]:mtu:(0 576 1420 1500 9000)" \ + "($help)--mtu=[Network MTU]:mtu:(0 576 1420 1500 9000)" \ "($help -p --pidfile)"{-p=,--pidfile=}"[Path to use for daemon PID file]:PID file:_files" \ "($help)--raw-logs[Full timestamps without ANSI coloring]" \ "($help)*--registry-mirror=[Preferred Docker registry mirror]:registry mirror: " \ "($help -s --storage-driver)"{-s=,--storage-driver=}"[Storage driver to use]:driver:(aufs devicemapper btrfs zfs overlay)" \ "($help)--selinux-enabled[Enable selinux support]" \ - "($help)*--storage-opt=[Set storage driver options]:storage driver options: " \ + "($help)*--storage-opt=[Storage driver options]:storage driver options: " \ "($help)--tls[Use TLS]" \ "($help)--tlscacert=[Trust certs signed only by this CA]:PEM file:_files -g "*.(pem|crt)"" \ "($help)--tlscert=[Path to TLS certificate file]:PEM file:_files -g "*.(pem|crt)"" \ @@ -768,7 +772,7 @@ __docker_subcommand() { _arguments $(__docker_arguments) \ $opts_help \ "($help)*"{-c=,--change=}"[Apply Dockerfile instruction to the created image]:Dockerfile:_files" \ - "($help -m --message)"{-m=,--message=}"[Set commit message for imported image]:message: " \ + "($help -m --message)"{-m=,--message=}"[Commit message for imported image]:message: " \ "($help -):URL:(- http:// file://)" \ "($help -): :__docker_repositories_with_tags" && ret=0 ;; @@ -811,7 +815,6 @@ __docker_subcommand() { (login) _arguments $(__docker_arguments) \ $opts_help \ - "($help -e --email)"{-e=,--email=}"[Email]:email: " \ "($help -p --password)"{-p=,--password=}"[Password]:password: " \ "($help -u --user)"{-u=,--user=}"[Username]:username: " \ "($help -)1:server: " && ret=0 @@ -1048,7 +1051,7 @@ _docker() { "($help)--config[Location of client config files]:path:_directories" \ "($help -D --debug)"{-D,--debug}"[Enable debug mode]" \ "($help -H --host)"{-H=,--host=}"[tcp://host:port to bind/connect to]:host: " \ - "($help -l --log-level)"{-l=,--log-level=}"[Set the logging level]:level:(debug info warn error fatal)" \ + "($help -l --log-level)"{-l=,--log-level=}"[Logging level]:level:(debug info warn error fatal)" \ "($help)--tls[Use TLS]" \ "($help)--tlscacert=[Trust certs signed only by this CA]:PEM file:_files -g "*.(pem|crt)"" \ "($help)--tlscert=[Path to TLS certificate file]:PEM file:_files -g "*.(pem|crt)"" \ diff --git a/components/engine/contrib/download-frozen-image-v2.sh b/components/engine/contrib/download-frozen-image-v2.sh index 81b047561e..111e3fa2ba 100755 --- a/components/engine/contrib/download-frozen-image-v2.sh +++ b/components/engine/contrib/download-frozen-image-v2.sh @@ -95,6 +95,7 @@ while [ $# -gt 0 ]; do echo "skipping existing ${imageId:0:12}" continue fi + token="$(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$image:pull" | jq --raw-output .token)" curl -SL --progress -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$image/blobs/$imageLayer" -o "$dir/$imageId/layer.tar" # -C - done echo diff --git a/components/engine/contrib/init/systemd/docker.service b/components/engine/contrib/init/systemd/docker.service index 6015b7441f..75cb68c8b3 100644 --- a/components/engine/contrib/init/systemd/docker.service +++ b/components/engine/contrib/init/systemd/docker.service @@ -6,12 +6,17 @@ Requires=docker.socket [Service] Type=notify +# the default is not to use systemd for cgroups because the delegate issues still +# exists and systemd currently does not support the cgroup feature set required +# for containers run by docker ExecStart=/usr/bin/docker daemon -H fd:// MountFlags=slave LimitNOFILE=1048576 LimitNPROC=1048576 LimitCORE=infinity TimeoutStartSec=0 +# set delegate yes so that systemd does not reset the cgroups of docker containers +Delegate=yes [Install] WantedBy=multi-user.target diff --git a/components/engine/contrib/nnp-test/Dockerfile b/components/engine/contrib/nnp-test/Dockerfile new file mode 100644 index 0000000000..026d86954f --- /dev/null +++ b/components/engine/contrib/nnp-test/Dockerfile @@ -0,0 +1,9 @@ +FROM buildpack-deps:jessie + +COPY . /usr/src/ + +WORKDIR /usr/src/ + +RUN gcc -g -Wall -static nnp-test.c -o /usr/bin/nnp-test + +RUN chmod +s /usr/bin/nnp-test diff --git a/components/engine/contrib/nnp-test/nnp-test.c b/components/engine/contrib/nnp-test/nnp-test.c new file mode 100644 index 0000000000..b767da7e1a --- /dev/null +++ b/components/engine/contrib/nnp-test/nnp-test.c @@ -0,0 +1,10 @@ +#include +#include +#include + +int main(int argc, char *argv[]) +{ + printf("EUID=%d\n", geteuid()); + return 0; +} + diff --git a/components/engine/contrib/report-issue.sh b/components/engine/contrib/report-issue.sh old mode 100644 new mode 100755 diff --git a/components/engine/daemon/archive.go b/components/engine/daemon/archive.go index 5d0e9e0b52..5cf6985210 100644 --- a/components/engine/daemon/archive.go +++ b/components/engine/daemon/archive.go @@ -250,13 +250,13 @@ func (daemon *Daemon) containerExtractToDir(container *container.Container, path return ErrRootFSReadOnly } + uid, gid := daemon.GetRemappedUIDGID() options := &archive.TarOptions{ - ChownOpts: &archive.TarChownOptions{ - UID: 0, GID: 0, // TODO: use config.User? Remap to userns root? - }, NoOverwriteDirNonDir: noOverwriteDirNonDir, + ChownOpts: &archive.TarChownOptions{ + UID: uid, GID: gid, // TODO: should all ownership be set to root (either real or remapped)? + }, } - if err := chrootarchive.Untar(content, resolvedPath, options); err != nil { return err } diff --git a/components/engine/daemon/attach.go b/components/engine/daemon/attach.go index 1beedbf138..79e9cd51da 100644 --- a/components/engine/daemon/attach.go +++ b/components/engine/daemon/attach.go @@ -9,7 +9,7 @@ import ( "github.com/docker/docker/api/types/backend" "github.com/docker/docker/container" "github.com/docker/docker/daemon/logger" - derr "github.com/docker/docker/errors" + "github.com/docker/docker/errors" "github.com/docker/docker/pkg/stdcopy" ) @@ -17,10 +17,11 @@ import ( func (daemon *Daemon) ContainerAttach(prefixOrName string, c *backend.ContainerAttachConfig) error { container, err := daemon.GetContainer(prefixOrName) if err != nil { - return derr.ErrorCodeNoSuchContainer.WithArgs(prefixOrName) + return err } if container.IsPaused() { - return derr.ErrorCodePausedContainer.WithArgs(prefixOrName) + err := fmt.Errorf("Container %s is paused. Unpause the container before attach", prefixOrName) + return errors.NewRequestConflictError(err) } inStream, outStream, errStream, err := c.GetStreams() diff --git a/components/engine/daemon/commit.go b/components/engine/daemon/commit.go index 6f2b38d52d..7bc7b6f25d 100644 --- a/components/engine/daemon/commit.go +++ b/components/engine/daemon/commit.go @@ -70,8 +70,8 @@ func merge(userConf, imageConf *containertypes.Config) error { userConf.Labels = imageConf.Labels } - if userConf.Entrypoint.Len() == 0 { - if userConf.Cmd.Len() == 0 { + if len(userConf.Entrypoint) == 0 { + if len(userConf.Cmd) == 0 { userConf.Cmd = imageConf.Cmd } @@ -89,6 +89,10 @@ func merge(userConf, imageConf *containertypes.Config) error { userConf.Volumes[k] = v } } + + if userConf.StopSignal == "" { + userConf.StopSignal = imageConf.StopSignal + } return nil } @@ -147,7 +151,7 @@ func (daemon *Daemon) Commit(name string, c *types.ContainerCommitConfig) (strin h := image.History{ Author: c.Author, Created: time.Now().UTC(), - CreatedBy: strings.Join(container.Config.Cmd.Slice(), " "), + CreatedBy: strings.Join(container.Config.Cmd, " "), Comment: c.Comment, EmptyLayer: true, } diff --git a/components/engine/daemon/config.go b/components/engine/daemon/config.go index fb46f26667..50e814ba4e 100644 --- a/components/engine/daemon/config.go +++ b/components/engine/daemon/config.go @@ -154,14 +154,20 @@ func parseClusterAdvertiseSettings(clusterStore, clusterAdvertise string) (strin } // ReloadConfiguration reads the configuration in the host and reloads the daemon and server. -func ReloadConfiguration(configFile string, flags *flag.FlagSet, reload func(*Config)) { +func ReloadConfiguration(configFile string, flags *flag.FlagSet, reload func(*Config)) error { logrus.Infof("Got signal to reload configuration, reloading from: %s", configFile) newConfig, err := getConflictFreeConfiguration(configFile, flags) if err != nil { - logrus.Error(err) - } else { - reload(newConfig) + return err } + reload(newConfig) + return nil +} + +// boolValue is an interface that boolean value flags implement +// to tell the command line how to make -name equivalent to -name=true. +type boolValue interface { + IsBoolFlag() bool } // MergeDaemonConfigurations reads a configuration file, @@ -206,6 +212,36 @@ func getConflictFreeConfiguration(configFile string, flags *flag.FlagSet) (*Conf return nil, err } + // Override flag values to make sure the values set in the config file with nullable values, like `false`, + // are not overriden by default truthy values from the flags that were not explicitly set. + // See https://github.com/docker/docker/issues/20289 for an example. + // + // TODO: Rewrite configuration logic to avoid same issue with other nullable values, like numbers. + namedOptions := make(map[string]interface{}) + for key, value := range configSet { + f := flags.Lookup("-" + key) + if f == nil { // ignore named flags that don't match + namedOptions[key] = value + continue + } + + if _, ok := f.Value.(boolValue); ok { + f.Value.Set(fmt.Sprintf("%v", value)) + } + } + if len(namedOptions) > 0 { + // set also default for mergeVal flags that are boolValue at the same time. + flags.VisitAll(func(f *flag.Flag) { + if opt, named := f.Value.(opts.NamedOption); named { + v, set := namedOptions[opt.Name()] + _, boolean := f.Value.(boolValue) + if set && boolean { + f.Value.Set(fmt.Sprintf("%v", v)) + } + } + }) + } + config.valuesSet = configSet } @@ -245,14 +281,16 @@ func findConfigurationConflicts(config map[string]interface{}, flags *flag.FlagS // 2. Discard values that implement NamedOption. // Their configuration name differs from their flag name, like `labels` and `label`. - unknownNamedConflicts := func(f *flag.Flag) { - if namedOption, ok := f.Value.(opts.NamedOption); ok { - if _, valid := unknownKeys[namedOption.Name()]; valid { - delete(unknownKeys, namedOption.Name()) + if len(unknownKeys) > 0 { + unknownNamedConflicts := func(f *flag.Flag) { + if namedOption, ok := f.Value.(opts.NamedOption); ok { + if _, valid := unknownKeys[namedOption.Name()]; valid { + delete(unknownKeys, namedOption.Name()) + } } } + flags.VisitAll(unknownNamedConflicts) } - flags.VisitAll(unknownNamedConflicts) if len(unknownKeys) > 0 { var unknown []string diff --git a/components/engine/daemon/container_operations_unix.go b/components/engine/daemon/container_operations_unix.go index 4db5b4d62f..e93f2dad4c 100644 --- a/components/engine/daemon/container_operations_unix.go +++ b/components/engine/daemon/container_operations_unix.go @@ -4,6 +4,7 @@ package daemon import ( "fmt" + "net" "os" "path" "path/filepath" @@ -17,7 +18,7 @@ import ( "github.com/docker/docker/daemon/execdriver" "github.com/docker/docker/daemon/links" "github.com/docker/docker/daemon/network" - derr "github.com/docker/docker/errors" + "github.com/docker/docker/errors" "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/mount" @@ -25,10 +26,12 @@ import ( "github.com/docker/docker/runconfig" containertypes "github.com/docker/engine-api/types/container" networktypes "github.com/docker/engine-api/types/network" + "github.com/docker/go-connections/nat" "github.com/docker/go-units" "github.com/docker/libnetwork" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/options" + "github.com/docker/libnetwork/types" "github.com/opencontainers/runc/libcontainer/configs" "github.com/opencontainers/runc/libcontainer/devices" "github.com/opencontainers/runc/libcontainer/label" @@ -45,7 +48,7 @@ func (daemon *Daemon) setupLinkedContainers(container *container.Container) ([]s for linkAlias, child := range children { if !child.IsRunning() { - return nil, derr.ErrorCodeLinkNotRunning.WithArgs(child.Name, linkAlias) + return nil, fmt.Errorf("Cannot link to a non running container: %s AS %s", child.Name, linkAlias) } childBridgeSettings := child.NetworkSettings.Networks["bridge"] @@ -198,6 +201,7 @@ func (daemon *Daemon) populateCommand(c *container.Container, env []string) erro BlkioThrottleWriteBpsDevice: writeBpsDevice, BlkioThrottleReadIOpsDevice: readIOpsDevice, BlkioThrottleWriteIOpsDevice: writeIOpsDevice, + PidsLimit: c.HostConfig.PidsLimit, MemorySwappiness: -1, } @@ -257,8 +261,8 @@ func (daemon *Daemon) populateCommand(c *container.Container, env []string) erro AllowedDevices: allowedDevices, AppArmorProfile: c.AppArmorProfile, AutoCreatedDevices: autoCreatedDevices, - CapAdd: c.HostConfig.CapAdd.Slice(), - CapDrop: c.HostConfig.CapDrop.Slice(), + CapAdd: c.HostConfig.CapAdd, + CapDrop: c.HostConfig.CapDrop, CgroupParent: defaultCgroupParent, GIDMapping: gidMap, GroupAdd: c.HostConfig.GroupAdd, @@ -270,6 +274,7 @@ func (daemon *Daemon) populateCommand(c *container.Container, env []string) erro SeccompProfile: c.SeccompProfile, UIDMapping: uidMap, UTS: uts, + NoNewPrivileges: c.NoNewPrivileges, } if c.HostConfig.CgroupParent != "" { c.Command.CgroupParent = c.HostConfig.CgroupParent @@ -318,6 +323,9 @@ func (daemon *Daemon) buildSandboxOptions(container *container.Container, n libn dns []string dnsSearch []string dnsOptions []string + bindings = make(nat.PortMap) + pbList []types.PortBinding + exposeList []types.TransportPort ) sboxOptions = append(sboxOptions, libnetwork.OptionHostname(container.Config.Hostname), @@ -392,6 +400,59 @@ func (daemon *Daemon) buildSandboxOptions(container *container.Container, n libn sboxOptions = append(sboxOptions, libnetwork.OptionExtraHost(parts[0], parts[1])) } + if container.HostConfig.PortBindings != nil { + for p, b := range container.HostConfig.PortBindings { + bindings[p] = []nat.PortBinding{} + for _, bb := range b { + bindings[p] = append(bindings[p], nat.PortBinding{ + HostIP: bb.HostIP, + HostPort: bb.HostPort, + }) + } + } + } + + portSpecs := container.Config.ExposedPorts + ports := make([]nat.Port, len(portSpecs)) + var i int + for p := range portSpecs { + ports[i] = p + i++ + } + nat.SortPortMap(ports, bindings) + for _, port := range ports { + expose := types.TransportPort{} + expose.Proto = types.ParseProtocol(port.Proto()) + expose.Port = uint16(port.Int()) + exposeList = append(exposeList, expose) + + pb := types.PortBinding{Port: expose.Port, Proto: expose.Proto} + binding := bindings[port] + for i := 0; i < len(binding); i++ { + pbCopy := pb.GetCopy() + newP, err := nat.NewPort(nat.SplitProtoPort(binding[i].HostPort)) + var portStart, portEnd int + if err == nil { + portStart, portEnd, err = newP.Range() + } + if err != nil { + return nil, fmt.Errorf("Error parsing HostPort value(%s):%v", binding[i].HostPort, err) + } + pbCopy.HostPort = uint16(portStart) + pbCopy.HostPortEnd = uint16(portEnd) + pbCopy.HostIP = net.ParseIP(binding[i].HostIP) + pbList = append(pbList, pbCopy) + } + + if container.HostConfig.PublishAllPorts && len(binding) == 0 { + pbList = append(pbList, pb) + } + } + + sboxOptions = append(sboxOptions, + libnetwork.OptionPortMapping(pbList), + libnetwork.OptionExposedPorts(exposeList)) + // Link feature is supported only for the default bridge network. // return if this call to build join options is not for default bridge network if n.Name() != "bridge" { @@ -509,7 +570,7 @@ func (daemon *Daemon) updateNetwork(container *container.Container) error { sb, err := ctrl.SandboxByID(sid) if err != nil { - return derr.ErrorCodeNoSandbox.WithArgs(sid, err) + return fmt.Errorf("error locating sandbox id %s: %v", sid, err) } // Find if container is connected to the default bridge network @@ -532,11 +593,11 @@ func (daemon *Daemon) updateNetwork(container *container.Container) error { options, err := daemon.buildSandboxOptions(container, n) if err != nil { - return derr.ErrorCodeNetworkUpdate.WithArgs(err) + return fmt.Errorf("Update network failed: %v", err) } if err := sb.Refresh(options...); err != nil { - return derr.ErrorCodeNetworkRefresh.WithArgs(sid, err) + return fmt.Errorf("Update network failed: Failure in refresh sandbox %s: %v", sid, err) } return nil @@ -730,7 +791,7 @@ func (daemon *Daemon) updateNetworkConfig(container *container.Container, idOrNa func (daemon *Daemon) ConnectToNetwork(container *container.Container, idOrName string, endpointConfig *networktypes.EndpointSettings) error { if !container.Running { if container.RemovalInProgress || container.Dead { - return derr.ErrorCodeRemovalContainer.WithArgs(container.ID) + return errRemovalContainer(container.ID) } if _, err := daemon.updateNetworkConfig(container, idOrName, endpointConfig, true); err != nil { return err @@ -810,7 +871,7 @@ func (daemon *Daemon) connectToNetwork(container *container.Container, idOrName } if err := container.UpdateJoinInfo(n, ep); err != nil { - return derr.ErrorCodeJoinInfo.WithArgs(err) + return fmt.Errorf("Updating join info failed: %v", err) } daemon.LogNetworkEventWithAttributes(n, "connect", map[string]string{"container": container.ID}) @@ -833,7 +894,7 @@ func (daemon *Daemon) DisconnectFromNetwork(container *container.Container, n li } if !container.Running { if container.RemovalInProgress || container.Dead { - return derr.ErrorCodeRemovalContainer.WithArgs(container.ID) + return errRemovalContainer(container.ID) } if _, ok := container.NetworkSettings.Networks[n.Name()]; ok { delete(container.NetworkSettings.Networks, n.Name()) @@ -950,7 +1011,7 @@ func (daemon *Daemon) setNetworkNamespaceKey(containerID string, pid int) error search := libnetwork.SandboxContainerWalker(&sandbox, containerID) daemon.netController.WalkSandboxes(search) if sandbox == nil { - return derr.ErrorCodeNoSandbox.WithArgs(containerID, "no sandbox found") + return fmt.Errorf("error locating sandbox id %s: no sandbox found", containerID) } return sandbox.SetKey(path) @@ -963,10 +1024,10 @@ func (daemon *Daemon) getIpcContainer(container *container.Container) (*containe return nil, err } if !c.IsRunning() { - return nil, derr.ErrorCodeIPCRunning.WithArgs(containerID) + return nil, fmt.Errorf("cannot join IPC of a non running container: %s", containerID) } if c.IsRestarting() { - return nil, derr.ErrorCodeContainerRestarting.WithArgs(containerID) + return nil, errContainerIsRestarting(container.ID) } return c, nil } @@ -977,13 +1038,14 @@ func (daemon *Daemon) getNetworkedContainer(containerID, connectedContainerID st return nil, err } if containerID == nc.ID { - return nil, derr.ErrorCodeJoinSelf + return nil, fmt.Errorf("cannot join own network") } if !nc.IsRunning() { - return nil, derr.ErrorCodeJoinRunning.WithArgs(connectedContainerID) + err := fmt.Errorf("cannot join network of a non running container: %s", connectedContainerID) + return nil, errors.NewRequestConflictError(err) } if nc.IsRestarting() { - return nil, derr.ErrorCodeContainerRestarting.WithArgs(connectedContainerID) + return nil, errContainerIsRestarting(connectedContainerID) } return nc, nil } @@ -1098,7 +1160,9 @@ func killProcessDirectly(container *container.Container) error { if err != syscall.ESRCH { return err } - logrus.Debugf("Cannot kill process (pid=%d) with signal 9: no such process.", pid) + e := errNoSuchProcess{pid, 9} + logrus.Debug(e) + return e } } } @@ -1106,7 +1170,16 @@ func killProcessDirectly(container *container.Container) error { } func getDevicesFromPath(deviceMapping containertypes.DeviceMapping) (devs []*configs.Device, err error) { - device, err := devices.DeviceFromPath(deviceMapping.PathOnHost, deviceMapping.CgroupPermissions) + resolvedPathOnHost := deviceMapping.PathOnHost + + // check if it is a symbolic link + if src, e := os.Lstat(deviceMapping.PathOnHost); e == nil && src.Mode()&os.ModeSymlink == os.ModeSymlink { + if linkedPathOnHost, e := os.Readlink(deviceMapping.PathOnHost); e == nil { + resolvedPathOnHost = linkedPathOnHost + } + } + + device, err := devices.DeviceFromPath(resolvedPathOnHost, deviceMapping.CgroupPermissions) // if there was no error, return the device if err == nil { device.Path = deviceMapping.PathInContainer @@ -1118,10 +1191,10 @@ func getDevicesFromPath(deviceMapping containertypes.DeviceMapping) (devs []*con if err == devices.ErrNotADevice { // check if it is a directory - if src, e := os.Stat(deviceMapping.PathOnHost); e == nil && src.IsDir() { + if src, e := os.Stat(resolvedPathOnHost); e == nil && src.IsDir() { // mount the internal devices recursively - filepath.Walk(deviceMapping.PathOnHost, func(dpath string, f os.FileInfo, e error) error { + filepath.Walk(resolvedPathOnHost, func(dpath string, f os.FileInfo, e error) error { childDevice, e := devices.DeviceFromPath(dpath, deviceMapping.CgroupPermissions) if e != nil { // ignore the device @@ -1129,7 +1202,7 @@ func getDevicesFromPath(deviceMapping containertypes.DeviceMapping) (devs []*con } // add the device to userSpecified devices - childDevice.Path = strings.Replace(dpath, deviceMapping.PathOnHost, deviceMapping.PathInContainer, 1) + childDevice.Path = strings.Replace(dpath, resolvedPathOnHost, deviceMapping.PathInContainer, 1) devs = append(devs, childDevice) return nil @@ -1141,7 +1214,7 @@ func getDevicesFromPath(deviceMapping containertypes.DeviceMapping) (devs []*con return devs, nil } - return devs, derr.ErrorCodeDeviceInfo.WithArgs(deviceMapping.PathOnHost, err) + return devs, fmt.Errorf("error gathering device information while adding custom device %q: %s", deviceMapping.PathOnHost, err) } func mergeDevices(defaultDevices, userDevices []*configs.Device) []*configs.Device { @@ -1172,3 +1245,7 @@ func isLinkable(child *container.Container) bool { _, ok := child.NetworkSettings.Networks["bridge"] return ok } + +func errRemovalContainer(containerID string) error { + return fmt.Errorf("Container %s is marked for removal and cannot be connected or disconnected to the network", containerID) +} diff --git a/components/engine/daemon/container_operations_windows.go b/components/engine/daemon/container_operations_windows.go index b812bfc5f4..02173d09f1 100644 --- a/components/engine/daemon/container_operations_windows.go +++ b/components/engine/daemon/container_operations_windows.go @@ -3,12 +3,12 @@ package daemon import ( + "fmt" "strings" "github.com/docker/docker/container" "github.com/docker/docker/daemon/execdriver" "github.com/docker/docker/daemon/execdriver/windows" - derr "github.com/docker/docker/errors" "github.com/docker/docker/layer" networktypes "github.com/docker/engine-api/types/network" "github.com/docker/libnetwork" @@ -64,7 +64,7 @@ func (daemon *Daemon) populateCommand(c *container.Container, env []string) erro } } default: - return derr.ErrorCodeInvalidNetworkMode.WithArgs(c.HostConfig.NetworkMode) + return fmt.Errorf("invalid network mode: %s", c.HostConfig.NetworkMode) } // TODO Windows. More resource controls to be implemented later. @@ -88,7 +88,7 @@ func (daemon *Daemon) populateCommand(c *container.Container, env []string) erro var layerPaths []string img, err := daemon.imageStore.Get(c.ImageID) if err != nil { - return derr.ErrorCodeGetGraph.WithArgs(c.ImageID, err) + return fmt.Errorf("Failed to graph.Get on ImageID %s - %s", c.ImageID, err) } if img.RootFS != nil && img.RootFS.Type == "layers+base" { @@ -97,7 +97,7 @@ func (daemon *Daemon) populateCommand(c *container.Container, env []string) erro img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i] path, err := layer.GetLayerPath(daemon.layerStore, img.RootFS.ChainID()) if err != nil { - return derr.ErrorCodeGetLayer.WithArgs(err) + return fmt.Errorf("Failed to get layer path from graphdriver %s for ImageID %s - %s", daemon.layerStore, img.RootFS.ChainID(), err) } // Reverse order, expecting parent most first layerPaths = append([]string{path}, layerPaths...) @@ -106,7 +106,7 @@ func (daemon *Daemon) populateCommand(c *container.Container, env []string) erro m, err := c.RWLayer.Metadata() if err != nil { - return derr.ErrorCodeGetLayerMetadata.WithArgs(err) + return fmt.Errorf("Failed to get layer metadata - %s", err) } layerFolder := m["dir"] diff --git a/components/engine/daemon/create.go b/components/engine/daemon/create.go index 166af3bf73..425c4344bb 100644 --- a/components/engine/daemon/create.go +++ b/components/engine/daemon/create.go @@ -1,9 +1,10 @@ package daemon import ( + "fmt" + "github.com/Sirupsen/logrus" "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/idtools" @@ -18,10 +19,10 @@ import ( // ContainerCreate creates a container. func (daemon *Daemon) ContainerCreate(params types.ContainerCreateConfig) (types.ContainerCreateResponse, error) { if params.Config == nil { - return types.ContainerCreateResponse{}, derr.ErrorCodeEmptyConfig + return types.ContainerCreateResponse{}, fmt.Errorf("Config cannot be empty in order to create a container") } - warnings, err := daemon.verifyContainerSettings(params.HostConfig, params.Config) + warnings, err := daemon.verifyContainerSettings(params.HostConfig, params.Config, false) if err != nil { return types.ContainerCreateResponse{Warnings: warnings}, err } @@ -174,7 +175,7 @@ func (daemon *Daemon) VolumeCreate(name, driverName string, opts map[string]stri v, err := daemon.volumes.Create(name, driverName, opts) if err != nil { if volumestore.IsNameConflict(err) { - return nil, derr.ErrorVolumeNameTaken.WithArgs(name) + return nil, fmt.Errorf("A volume named %s already exists. Choose a different volume name.", name) } return nil, err } diff --git a/components/engine/daemon/create_unix.go b/components/engine/daemon/create_unix.go index 8eca648deb..583ca13b76 100644 --- a/components/engine/daemon/create_unix.go +++ b/components/engine/daemon/create_unix.go @@ -3,12 +3,12 @@ package daemon import ( + "fmt" "os" "path/filepath" "github.com/Sirupsen/logrus" "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/stringid" containertypes "github.com/docker/engine-api/types/container" "github.com/opencontainers/runc/libcontainer/label" @@ -21,7 +21,8 @@ func (daemon *Daemon) createContainerPlatformSpecificSettings(container *contain } defer daemon.Unmount(container) - if err := container.SetupWorkingDirectory(); err != nil { + rootUID, rootGID := daemon.GetRemappedUIDGID() + if err := container.SetupWorkingDirectory(rootUID, rootGID); err != nil { return err } @@ -41,7 +42,7 @@ func (daemon *Daemon) createContainerPlatformSpecificSettings(container *contain stat, err := os.Stat(path) if err == nil && !stat.IsDir() { - return derr.ErrorCodeMountOverFile.WithArgs(path) + return fmt.Errorf("cannot mount volume over existing file, file exists %s", path) } v, err := daemon.volumes.CreateWithRef(name, hostConfig.VolumeDriver, container.ID, nil) diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go index 33c3f10caf..892e83dac0 100644 --- a/components/engine/daemon/daemon.go +++ b/components/engine/daemon/daemon.go @@ -6,7 +6,6 @@ package daemon import ( - "errors" "fmt" "io" "io/ioutil" @@ -15,6 +14,7 @@ import ( "path" "path/filepath" "runtime" + "strings" "sync" "syscall" "time" @@ -28,6 +28,7 @@ import ( "github.com/docker/docker/daemon/exec" "github.com/docker/docker/daemon/execdriver" "github.com/docker/docker/daemon/execdriver/execdrivers" + "github.com/docker/docker/errors" "github.com/docker/engine-api/types" containertypes "github.com/docker/engine-api/types/container" eventtypes "github.com/docker/engine-api/types/events" @@ -43,7 +44,6 @@ import ( dmetadata "github.com/docker/docker/distribution/metadata" "github.com/docker/docker/distribution/xfer" "github.com/docker/docker/dockerversion" - derr "github.com/docker/docker/errors" "github.com/docker/docker/image" "github.com/docker/docker/image/tarexport" "github.com/docker/docker/layer" @@ -90,7 +90,7 @@ var ( validContainerNameChars = utils.RestrictedNameChars validContainerNamePattern = utils.RestrictedNamePattern - errSystemNotSupported = errors.New("The Docker daemon is not supported on this platform.") + errSystemNotSupported = fmt.Errorf("The Docker daemon is not supported on this platform.") ) // ErrImageDoesNotExist is error returned when no image can be found for a reference. @@ -157,7 +157,8 @@ func (daemon *Daemon) GetContainer(prefixOrName string) (*container.Container, e if indexError != nil { // When truncindex defines an error type, use that instead if indexError == truncindex.ErrNotExist { - return nil, derr.ErrorCodeNoSuchContainer.WithArgs(prefixOrName) + err := fmt.Errorf("No such container: %s", prefixOrName) + return nil, errors.NewRequestNotFoundError(err) } return nil, indexError } @@ -411,7 +412,7 @@ func (daemon *Daemon) mergeAndVerifyConfig(config *containertypes.Config, img *i return err } } - if config.Entrypoint.Len() == 0 && config.Cmd.Len() == 0 { + if len(config.Entrypoint) == 0 && len(config.Cmd) == 0 { return fmt.Errorf("No command specified") } return nil @@ -494,13 +495,11 @@ func (daemon *Daemon) generateHostname(id string, config *containertypes.Config) } } -func (daemon *Daemon) getEntrypointAndArgs(configEntrypoint *strslice.StrSlice, configCmd *strslice.StrSlice) (string, []string) { - cmdSlice := configCmd.Slice() - if configEntrypoint.Len() != 0 { - eSlice := configEntrypoint.Slice() - return eSlice[0], append(eSlice[1:], cmdSlice...) +func (daemon *Daemon) getEntrypointAndArgs(configEntrypoint strslice.StrSlice, configCmd strslice.StrSlice) (string, []string) { + if len(configEntrypoint) != 0 { + return configEntrypoint[0], append(configEntrypoint[1:], configCmd...) } - return cmdSlice[0], cmdSlice[1:] + return configCmd[0], configCmd[1:] } func (daemon *Daemon) newContainer(name string, config *containertypes.Config, imgID image.ID) (*container.Container, error) { @@ -760,7 +759,7 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo migrationStart := time.Now() if err := v1.Migrate(config.Root, graphDriver, d.layerStore, d.imageStore, referenceStore, distributionMetadataStore); err != nil { - return nil, err + logrus.Errorf("Graph migration failed: %q. Your old graph data was found to be too inconsistent for upgrading to content-addressable storage. Some of the old data was probably not upgraded. We recommend starting over with a clean storage directory if possible.", err) } logrus.Infof("Graph migration to content-addressability took %.2f seconds", time.Since(migrationStart).Seconds()) @@ -1211,7 +1210,7 @@ func (daemon *Daemon) ImageHistory(name string) ([]*types.ImageHistory, error) { if !h.EmptyLayer { if len(img.RootFS.DiffIDs) <= layerCounter { - return nil, errors.New("too many non-empty layers in History section") + return nil, fmt.Errorf("too many non-empty layers in History section") } rootFS.Append(img.RootFS.DiffIDs[layerCounter]) @@ -1450,7 +1449,7 @@ func setDefaultMtu(config *Config) { // verifyContainerSettings performs validation of the hostconfig and config // structures. -func (daemon *Daemon) verifyContainerSettings(hostConfig *containertypes.HostConfig, config *containertypes.Config) ([]string, error) { +func (daemon *Daemon) verifyContainerSettings(hostConfig *containertypes.HostConfig, config *containertypes.Config, update bool) ([]string, error) { // First perform verification of settings common across all platforms. if config != nil { @@ -1473,6 +1472,11 @@ func (daemon *Daemon) verifyContainerSettings(hostConfig *containertypes.HostCon return nil, nil } + logCfg := daemon.getLogConfig(hostConfig.LogConfig) + if err := logger.ValidateLogOpts(logCfg.Type, logCfg.Config); err != nil { + return nil, err + } + for port := range hostConfig.PortBindings { _, portStr := nat.SplitProtoPort(string(port)) if _, err := nat.ParsePort(portStr); err != nil { @@ -1487,7 +1491,7 @@ func (daemon *Daemon) verifyContainerSettings(hostConfig *containertypes.HostCon } // Now do platform-specific verification - return verifyPlatformContainerSettings(daemon, hostConfig, config) + return verifyPlatformContainerSettings(daemon, hostConfig, config, update) } // Checks if the client set configurations for more than one network while creating a container @@ -1499,7 +1503,8 @@ func (daemon *Daemon) verifyNetworkingConfig(nwConfig *networktypes.NetworkingCo for k := range nwConfig.EndpointsConfig { l = append(l, k) } - return derr.ErrorCodeMultipleNetworkConnect.WithArgs(fmt.Sprintf("%v", l)) + err := fmt.Errorf("Container cannot be connected to network endpoints: %s", strings.Join(l, ", ")) + return errors.NewBadRequestError(err) } func configureVolumes(config *Config, rootUID, rootGID int) (*store.VolumeStore, error) { @@ -1598,28 +1603,41 @@ func (daemon *Daemon) initDiscovery(config *Config) error { // daemon according to those changes. // This are the settings that Reload changes: // - Daemon labels. +// - Cluster discovery (reconfigure and restart). func (daemon *Daemon) Reload(config *Config) error { daemon.configStore.reloadLock.Lock() - daemon.configStore.Labels = config.Labels - daemon.configStore.reloadLock.Unlock() - - return nil + defer daemon.configStore.reloadLock.Unlock() + if config.IsValueSet("label") { + daemon.configStore.Labels = config.Labels + } + if config.IsValueSet("debug") { + daemon.configStore.Debug = config.Debug + } + return daemon.reloadClusterDiscovery(config) } func (daemon *Daemon) reloadClusterDiscovery(config *Config) error { - newAdvertise, err := parseClusterAdvertiseSettings(config.ClusterStore, config.ClusterAdvertise) - if err != nil && err != errDiscoveryDisabled { - return err + var err error + newAdvertise := daemon.configStore.ClusterAdvertise + newClusterStore := daemon.configStore.ClusterStore + if config.IsValueSet("cluster-advertise") { + if config.IsValueSet("cluster-store") { + newClusterStore = config.ClusterStore + } + newAdvertise, err = parseClusterAdvertiseSettings(newClusterStore, config.ClusterAdvertise) + if err != nil && err != errDiscoveryDisabled { + return err + } } // check discovery modifications - if !modifiedDiscoverySettings(daemon.configStore, newAdvertise, config.ClusterStore, config.ClusterOpts) { + if !modifiedDiscoverySettings(daemon.configStore, newAdvertise, newClusterStore, config.ClusterOpts) { return nil } // enable discovery for the first time if it was not previously enabled if daemon.discoveryWatcher == nil { - discoveryWatcher, err := initDiscovery(config.ClusterStore, newAdvertise, config.ClusterOpts) + discoveryWatcher, err := initDiscovery(newClusterStore, newAdvertise, config.ClusterOpts) if err != nil { return fmt.Errorf("discovery initialization failed (%v)", err) } @@ -1636,10 +1654,23 @@ func (daemon *Daemon) reloadClusterDiscovery(config *Config) error { } } - daemon.configStore.ClusterStore = config.ClusterStore + daemon.configStore.ClusterStore = newClusterStore daemon.configStore.ClusterOpts = config.ClusterOpts daemon.configStore.ClusterAdvertise = newAdvertise + if daemon.netController == nil { + return nil + } + netOptions, err := daemon.networkOptions(daemon.configStore) + if err != nil { + logrus.Warnf("Failed to reload configuration with network controller: %v", err) + return nil + } + err = daemon.netController.ReloadConfiguration(netOptions...) + if err != nil { + logrus.Warnf("Failed to reload configuration with network controller: %v", err) + } + return nil } @@ -1658,7 +1689,7 @@ func convertLnNetworkStats(name string, stats *lntypes.InterfaceStatistics) *lib func validateID(id string) error { if id == "" { - return derr.ErrorCodeEmptyID + return fmt.Errorf("Invalid empty id") } return nil } diff --git a/components/engine/daemon/daemon_test.go b/components/engine/daemon/daemon_test.go index 4a78bf2875..1c34d3ae4c 100644 --- a/components/engine/daemon/daemon_test.go +++ b/components/engine/daemon/daemon_test.go @@ -315,9 +315,12 @@ func TestDaemonReloadLabels(t *testing.T) { }, } + valuesSets := make(map[string]interface{}) + valuesSets["label"] = "foo:baz" newConfig := &Config{ CommonConfig: CommonConfig{ - Labels: []string{"foo:baz"}, + Labels: []string{"foo:baz"}, + valuesSet: valuesSets, }, } @@ -328,6 +331,35 @@ func TestDaemonReloadLabels(t *testing.T) { } } +func TestDaemonReloadNotAffectOthers(t *testing.T) { + daemon := &Daemon{} + daemon.configStore = &Config{ + CommonConfig: CommonConfig{ + Labels: []string{"foo:bar"}, + Debug: true, + }, + } + + valuesSets := make(map[string]interface{}) + valuesSets["label"] = "foo:baz" + newConfig := &Config{ + CommonConfig: CommonConfig{ + Labels: []string{"foo:baz"}, + valuesSet: valuesSets, + }, + } + + daemon.Reload(newConfig) + label := daemon.configStore.Labels[0] + if label != "foo:baz" { + t.Fatalf("Expected daemon label `foo:baz`, got %s", label) + } + debug := daemon.configStore.Debug + if !debug { + t.Fatalf("Expected debug 'enabled', got 'disabled'") + } +} + func TestDaemonDiscoveryReload(t *testing.T) { daemon := &Daemon{} daemon.configStore = &Config{ @@ -360,10 +392,14 @@ func TestDaemonDiscoveryReload(t *testing.T) { t.Fatal(e) } + valuesSets := make(map[string]interface{}) + valuesSets["cluster-store"] = "memory://127.0.0.1:2222" + valuesSets["cluster-advertise"] = "127.0.0.1:5555" newConfig := &Config{ CommonConfig: CommonConfig{ ClusterStore: "memory://127.0.0.1:2222", ClusterAdvertise: "127.0.0.1:5555", + valuesSet: valuesSets, }, } @@ -371,7 +407,7 @@ func TestDaemonDiscoveryReload(t *testing.T) { &discovery.Entry{Host: "127.0.0.1", Port: "5555"}, } - if err := daemon.reloadClusterDiscovery(newConfig); err != nil { + if err := daemon.Reload(newConfig); err != nil { t.Fatal(err) } ch, errCh = daemon.discoveryWatcher.Watch(stopCh) @@ -392,10 +428,14 @@ func TestDaemonDiscoveryReloadFromEmptyDiscovery(t *testing.T) { daemon := &Daemon{} daemon.configStore = &Config{} + valuesSet := make(map[string]interface{}) + valuesSet["cluster-store"] = "memory://127.0.0.1:2222" + valuesSet["cluster-advertise"] = "127.0.0.1:5555" newConfig := &Config{ CommonConfig: CommonConfig{ ClusterStore: "memory://127.0.0.1:2222", ClusterAdvertise: "127.0.0.1:5555", + valuesSet: valuesSet, }, } @@ -403,7 +443,7 @@ func TestDaemonDiscoveryReloadFromEmptyDiscovery(t *testing.T) { &discovery.Entry{Host: "127.0.0.1", Port: "5555"}, } - if err := daemon.reloadClusterDiscovery(newConfig); err != nil { + if err := daemon.Reload(newConfig); err != nil { t.Fatal(err) } stopCh := make(chan struct{}) @@ -421,3 +461,42 @@ func TestDaemonDiscoveryReloadFromEmptyDiscovery(t *testing.T) { t.Fatal(e) } } + +func TestDaemonDiscoveryReloadOnlyClusterAdvertise(t *testing.T) { + daemon := &Daemon{} + daemon.configStore = &Config{ + CommonConfig: CommonConfig{ + ClusterStore: "memory://127.0.0.1", + }, + } + valuesSets := make(map[string]interface{}) + valuesSets["cluster-advertise"] = "127.0.0.1:5555" + newConfig := &Config{ + CommonConfig: CommonConfig{ + ClusterAdvertise: "127.0.0.1:5555", + valuesSet: valuesSets, + }, + } + expected := discovery.Entries{ + &discovery.Entry{Host: "127.0.0.1", Port: "5555"}, + } + + if err := daemon.Reload(newConfig); err != nil { + t.Fatal(err) + } + stopCh := make(chan struct{}) + defer close(stopCh) + ch, errCh := daemon.discoveryWatcher.Watch(stopCh) + + select { + case <-time.After(1 * time.Second): + t.Fatal("failed to get discovery advertisements in time") + case e := <-ch: + if !reflect.DeepEqual(e, expected) { + t.Fatalf("expected %v, got %v\n", expected, e) + } + case e := <-errCh: + t.Fatal(e) + } + +} diff --git a/components/engine/daemon/daemon_unix.go b/components/engine/daemon/daemon_unix.go index faa6c0ee68..21923fb0b2 100644 --- a/components/engine/daemon/daemon_unix.go +++ b/components/engine/daemon/daemon_unix.go @@ -16,7 +16,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/idtools" @@ -76,17 +75,23 @@ func parseSecurityOpt(container *container.Container, config *containertypes.Hos for _, opt := range config.SecurityOpt { con := strings.SplitN(opt, ":", 2) if len(con) == 1 { - return fmt.Errorf("Invalid --security-opt: %q", opt) - } - switch con[0] { - case "label": - labelOpts = append(labelOpts, con[1]) - case "apparmor": - container.AppArmorProfile = con[1] - case "seccomp": - container.SeccompProfile = con[1] - default: - return fmt.Errorf("Invalid --security-opt: %q", opt) + switch con[0] { + case "no-new-privileges": + container.NoNewPrivileges = true + default: + return fmt.Errorf("Invalid --security-opt 1: %q", opt) + } + } else { + switch con[0] { + case "label": + labelOpts = append(labelOpts, con[1]) + case "apparmor": + container.AppArmorProfile = con[1] + case "seccomp": + container.SeccompProfile = con[1] + default: + return fmt.Errorf("Invalid --security-opt 2: %q", opt) + } } } @@ -221,7 +226,7 @@ func (daemon *Daemon) adaptContainerSettings(hostConfig *containertypes.HostConf return nil } -func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysinfo.SysInfo) ([]string, error) { +func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysinfo.SysInfo, update bool) ([]string, error) { warnings := []string{} // memory subsystem checks and adjustments @@ -242,7 +247,7 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi if resources.Memory > 0 && resources.MemorySwap > 0 && resources.MemorySwap < resources.Memory { return warnings, fmt.Errorf("Minimum memoryswap limit should be larger than memory limit, see usage.") } - if resources.Memory == 0 && resources.MemorySwap > 0 { + if resources.Memory == 0 && resources.MemorySwap > 0 && !update { return warnings, fmt.Errorf("You should always set the Memory limit when using Memoryswap limit, see usage.") } if resources.MemorySwappiness != nil && *resources.MemorySwappiness != -1 && !sysInfo.MemorySwappiness { @@ -286,6 +291,12 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi resources.OomKillDisable = nil } + if resources.PidsLimit != 0 && !sysInfo.PidsLimit { + warnings = append(warnings, "Your kernel does not support pids limit capabilities, pids limit discarded.") + logrus.Warnf("Your kernel does not support pids limit capabilities, pids limit discarded.") + resources.PidsLimit = 0 + } + // cpu subsystem checks and adjustments if resources.CPUShares > 0 && !sysInfo.CPUShares { warnings = append(warnings, "Your kernel does not support CPU shares. Shares discarded.") @@ -312,17 +323,17 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi } cpusAvailable, err := sysInfo.IsCpusetCpusAvailable(resources.CpusetCpus) if err != nil { - return warnings, derr.ErrorCodeInvalidCpusetCpus.WithArgs(resources.CpusetCpus) + return warnings, fmt.Errorf("Invalid value %s for cpuset cpus.", resources.CpusetCpus) } if !cpusAvailable { - return warnings, derr.ErrorCodeNotAvailableCpusetCpus.WithArgs(resources.CpusetCpus, sysInfo.Cpus) + return warnings, fmt.Errorf("Requested CPUs are not available - requested %s, available: %s.", resources.CpusetCpus, sysInfo.Cpus) } memsAvailable, err := sysInfo.IsCpusetMemsAvailable(resources.CpusetMems) if err != nil { - return warnings, derr.ErrorCodeInvalidCpusetMems.WithArgs(resources.CpusetMems) + return warnings, fmt.Errorf("Invalid value %s for cpuset mems.", resources.CpusetMems) } if !memsAvailable { - return warnings, derr.ErrorCodeNotAvailableCpusetMems.WithArgs(resources.CpusetMems, sysInfo.Mems) + return warnings, fmt.Errorf("Requested memory nodes are not available - requested %s, available: %s.", resources.CpusetMems, sysInfo.Mems) } // blkio subsystem checks and adjustments @@ -363,6 +374,14 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi return warnings, nil } +func (daemon *Daemon) getCgroupDriver() string { + cgroupDriver := "cgroupfs" + if daemon.usingSystemd() { + cgroupDriver = "systemd" + } + return cgroupDriver +} + func usingSystemd(config *Config) bool { for _, option := range config.ExecOptions { key, val, err := parsers.ParseKeyValueOpt(option) @@ -383,7 +402,7 @@ func (daemon *Daemon) usingSystemd() bool { // verifyPlatformContainerSettings performs platform-specific validation of the // hostconfig and config structures. -func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes.HostConfig, config *containertypes.Config) ([]string, error) { +func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes.HostConfig, config *containertypes.Config, update bool) ([]string, error) { warnings := []string{} sysInfo := sysinfo.New(true) @@ -392,7 +411,7 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes. return warnings, err } - w, err := verifyContainerResources(&hostConfig.Resources, sysInfo) + w, err := verifyContainerResources(&hostConfig.Resources, sysInfo, update) if err != nil { return warnings, err } @@ -694,10 +713,8 @@ func initBridgeDriver(controller libnetwork.NetworkController, config *Config) e } // Initialize default network on "bridge" with the same name _, err = controller.NewNetwork("bridge", "bridge", - libnetwork.NetworkOptionGeneric(options.Generic{ - netlabel.GenericData: netOption, - netlabel.EnableIPv6: config.bridgeConfig.EnableIPv6, - }), + libnetwork.NetworkOptionEnableIPv6(config.bridgeConfig.EnableIPv6), + libnetwork.NetworkOptionDriverOpts(netOption), libnetwork.NetworkOptionIpam("default", "", v4Conf, v6Conf, nil), libnetwork.NetworkOptionDeferIPv6Alloc(deferIPv6Alloc)) if err != nil { diff --git a/components/engine/daemon/daemon_unix_test.go b/components/engine/daemon/daemon_unix_test.go index 62f870fbd8..26eb93fcd3 100644 --- a/components/engine/daemon/daemon_unix_test.go +++ b/components/engine/daemon/daemon_unix_test.go @@ -142,7 +142,7 @@ func TestNetworkOptions(t *testing.T) { } if _, err := daemon.networkOptions(dconfigCorrect); err != nil { - t.Fatalf("Expect networkOptions sucess, got error: %v", err) + t.Fatalf("Expect networkOptions success, got error: %v", err) } dconfigWrong := &Config{ diff --git a/components/engine/daemon/daemon_windows.go b/components/engine/daemon/daemon_windows.go index 2491caf28d..b9aadc6b00 100644 --- a/components/engine/daemon/daemon_windows.go +++ b/components/engine/daemon/daemon_windows.go @@ -22,6 +22,7 @@ import ( "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/system" "github.com/docker/libnetwork" + nwconfig "github.com/docker/libnetwork/config" blkiodev "github.com/opencontainers/runc/libcontainer/configs" ) @@ -64,6 +65,10 @@ func checkKernel() error { return nil } +func (daemon *Daemon) getCgroupDriver() string { + return "" +} + // adaptContainerSettings is called during container creation to modify any // settings necessary in the HostConfig structure. func (daemon *Daemon) adaptContainerSettings(hostConfig *containertypes.HostConfig, adjustCPUShares bool) error { @@ -84,7 +89,7 @@ func (daemon *Daemon) adaptContainerSettings(hostConfig *containertypes.HostConf // verifyPlatformContainerSettings performs platform-specific validation of the // hostconfig and config structures. -func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes.HostConfig, config *containertypes.Config) ([]string, error) { +func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes.HostConfig, config *containertypes.Config, update bool) ([]string, error) { return nil, nil } @@ -251,3 +256,7 @@ func restoreCustomImage(is image.Store, ls layer.Store, rs reference.Store) erro } return nil } + +func (daemon *Daemon) networkOptions(dconfig *Config) ([]nwconfig.Option, error) { + return nil, fmt.Errorf("Network controller config reload not aavailable on Windows yet") +} diff --git a/components/engine/daemon/delete.go b/components/engine/daemon/delete.go index 6c56fd7ba1..75af4c01a1 100644 --- a/components/engine/daemon/delete.go +++ b/components/engine/daemon/delete.go @@ -8,7 +8,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" + "github.com/docker/docker/errors" "github.com/docker/docker/layer" volumestore "github.com/docker/docker/volume/store" "github.com/docker/engine-api/types" @@ -25,12 +25,8 @@ func (daemon *Daemon) ContainerRm(name string, config *types.ContainerRmConfig) } // Container state RemovalInProgress should be used to avoid races. - if err = container.SetRemovalInProgress(); err != nil { - if err == derr.ErrorCodeAlreadyRemoving { - // do not fail when the removal is in progress started by other request. - return nil - } - return derr.ErrorCodeRmState.WithArgs(container.ID, err) + if inProgress := container.SetRemovalInProgress(); inProgress { + return nil } defer container.ResetRemovalInProgress() @@ -84,10 +80,11 @@ func (daemon *Daemon) rmLink(container *container.Container, name string) error func (daemon *Daemon) cleanupContainer(container *container.Container, forceRemove bool) (err error) { if container.IsRunning() { if !forceRemove { - return derr.ErrorCodeRmRunning.WithArgs(container.ID) + err := fmt.Errorf("You cannot remove a running container %s. Stop the container before attempting removal or use -f", container.ID) + return errors.NewRequestConflictError(err) } if err := daemon.Kill(container); err != nil { - return derr.ErrorCodeRmFailed.WithArgs(container.ID, err) + return fmt.Errorf("Could not kill running container %s, cannot remove - %v", container.ID, err) } } @@ -123,17 +120,17 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, forceRemo }() if err = os.RemoveAll(container.Root); err != nil { - return derr.ErrorCodeRmFS.WithArgs(container.ID, err) + return fmt.Errorf("Unable to remove filesystem for %v: %v", container.ID, err) } metadata, err := daemon.layerStore.ReleaseRWLayer(container.RWLayer) layer.LogReleaseMetadata(metadata) if err != nil && err != layer.ErrMountDoesNotExist { - return derr.ErrorCodeRmDriverFS.WithArgs(daemon.GraphDriverName(), container.ID, err) + return fmt.Errorf("Driver %s failed to remove root filesystem %s: %s", daemon.GraphDriverName(), container.ID, err) } if err = daemon.execDriver.Clean(container.ID); err != nil { - return derr.ErrorCodeRmExecDriver.WithArgs(container.ID, err) + return fmt.Errorf("Unable to remove execdriver data for %s: %s", container.ID, err) } return nil } @@ -149,9 +146,10 @@ func (daemon *Daemon) VolumeRm(name string) error { if err := daemon.volumes.Remove(v); err != nil { if volumestore.IsInUse(err) { - return derr.ErrorCodeRmVolumeInUse.WithArgs(err) + err := fmt.Errorf("Unable to remove volume, volume still in use: %v", err) + return errors.NewRequestConflictError(err) } - return derr.ErrorCodeRmVolume.WithArgs(name, err) + return fmt.Errorf("Error while removing volume %s: %v", name, err) } daemon.LogVolumeEvent(v.Name(), "destroy", map[string]string{"driver": v.DriverName()}) return nil diff --git a/components/engine/daemon/delete_test.go b/components/engine/daemon/delete_test.go index 0d39b4d68f..adce2eb8c5 100644 --- a/components/engine/daemon/delete_test.go +++ b/components/engine/daemon/delete_test.go @@ -32,9 +32,7 @@ func TestContainerDoubleDelete(t *testing.T) { daemon.containers.Add(container.ID, container) // Mark the container as having a delete in progress - if err := container.SetRemovalInProgress(); err != nil { - t.Fatal(err) - } + container.SetRemovalInProgress() // Try to remove the container when it's start is removalInProgress. // It should ignore the container and not return an error. diff --git a/components/engine/daemon/errors.go b/components/engine/daemon/errors.go index 8fff66b9ff..131c9a1e22 100644 --- a/components/engine/daemon/errors.go +++ b/components/engine/daemon/errors.go @@ -1,26 +1,57 @@ package daemon import ( + "fmt" "strings" - derr "github.com/docker/docker/errors" + "github.com/docker/docker/errors" "github.com/docker/docker/reference" ) func (d *Daemon) imageNotExistToErrcode(err error) error { if dne, isDNE := err.(ErrImageDoesNotExist); isDNE { if strings.Contains(dne.RefOrID, "@") { - return derr.ErrorCodeNoSuchImageHash.WithArgs(dne.RefOrID) + e := fmt.Errorf("No such image: %s", dne.RefOrID) + return errors.NewRequestNotFoundError(e) } tag := reference.DefaultTag ref, err := reference.ParseNamed(dne.RefOrID) if err != nil { - return derr.ErrorCodeNoSuchImageTag.WithArgs(dne.RefOrID, tag) + e := fmt.Errorf("No such image: %s:%s", dne.RefOrID, tag) + return errors.NewRequestNotFoundError(e) } if tagged, isTagged := ref.(reference.NamedTagged); isTagged { tag = tagged.Tag() } - return derr.ErrorCodeNoSuchImageTag.WithArgs(ref.Name(), tag) + e := fmt.Errorf("No such image: %s:%s", ref.Name(), tag) + return errors.NewRequestNotFoundError(e) } return err } + +type errNotRunning struct { + containerID string +} + +func (e errNotRunning) Error() string { + return fmt.Sprintf("Container %s is not running", e.containerID) +} + +func (e errNotRunning) ContainerIsRunning() bool { + return false +} + +func errContainerIsRestarting(containerID string) error { + err := fmt.Errorf("Container %s is restarting, wait until the container is running", containerID) + return errors.NewRequestConflictError(err) +} + +func errExecNotFound(id string) error { + err := fmt.Errorf("No such exec instance '%s' found in daemon", id) + return errors.NewRequestNotFoundError(err) +} + +func errExecPaused(id string) error { + err := fmt.Errorf("Container %s is paused, unpause the container before exec", id) + return errors.NewRequestConflictError(err) +} diff --git a/components/engine/daemon/events/events.go b/components/engine/daemon/events/events.go index a8cd66fd99..ac1c98cd46 100644 --- a/components/engine/daemon/events/events.go +++ b/components/engine/daemon/events/events.go @@ -50,33 +50,23 @@ func (e *Events) Subscribe() ([]eventtypes.Message, chan interface{}, func()) { // of interface{}, so you need type assertion). func (e *Events) SubscribeTopic(since, sinceNano int64, ef *Filter) ([]eventtypes.Message, chan interface{}) { e.mu.Lock() - defer e.mu.Unlock() - var buffered []eventtypes.Message - topic := func(m interface{}) bool { - return ef.Include(m.(eventtypes.Message)) + var topic func(m interface{}) bool + if ef != nil && ef.filter.Len() > 0 { + topic = func(m interface{}) bool { return ef.Include(m.(eventtypes.Message)) } } - if since != -1 { - for i := len(e.events) - 1; i >= 0; i-- { - ev := e.events[i] - if ev.Time < since || ((ev.Time == since) && (ev.TimeNano < sinceNano)) { - break - } - if ef.filter.Len() == 0 || topic(ev) { - buffered = append([]eventtypes.Message{ev}, buffered...) - } - } - } + buffered := e.loadBufferedEvents(since, sinceNano, topic) var ch chan interface{} - if ef.filter.Len() > 0 { + if topic != nil { ch = e.pub.SubscribeTopic(topic) } else { // Subscribe to all events if there are no filters ch = e.pub.Subscribe() } + e.mu.Unlock() return buffered, ch } @@ -124,3 +114,29 @@ func (e *Events) Log(action, eventType string, actor eventtypes.Actor) { func (e *Events) SubscribersCount() int { return e.pub.Len() } + +// loadBufferedEvents iterates over the cached events in the buffer +// and returns those that were emitted before a specific date. +// The date is splitted in two values: +// - the `since` argument is a date timestamp without nanoseconds, or -1 to return an empty slice. +// - the `sinceNano` argument is the nanoseconds offset from the timestamp. +// It uses `time.Unix(seconds, nanoseconds)` to generate a valid date with those two first arguments. +// It filters those buffered messages with a topic function if it's not nil, otherwise it adds all messages. +func (e *Events) loadBufferedEvents(since, sinceNano int64, topic func(interface{}) bool) []eventtypes.Message { + var buffered []eventtypes.Message + if since == -1 { + return buffered + } + + sinceNanoUnix := time.Unix(since, sinceNano).UnixNano() + for i := len(e.events) - 1; i >= 0; i-- { + ev := e.events[i] + if ev.TimeNano < sinceNanoUnix { + break + } + if topic == nil || topic(ev) { + buffered = append([]eventtypes.Message{ev}, buffered...) + } + } + return buffered +} diff --git a/components/engine/daemon/events/events_test.go b/components/engine/daemon/events/events_test.go index fc3b84bb85..5fd577b992 100644 --- a/components/engine/daemon/events/events_test.go +++ b/components/engine/daemon/events/events_test.go @@ -5,7 +5,9 @@ import ( "testing" "time" + "github.com/docker/docker/daemon/events/testutils" "github.com/docker/engine-api/types/events" + timetypes "github.com/docker/engine-api/types/time" ) func TestEventsLog(t *testing.T) { @@ -150,3 +152,45 @@ func TestLogEvents(t *testing.T) { t.Fatalf("Last action is %s, must be action_89", lastC.Status) } } + +// https://github.com/docker/docker/issues/20999 +// Fixtures: +// +//2016-03-07T17:28:03.022433271+02:00 container die 0b863f2a26c18557fc6cdadda007c459f9ec81b874780808138aea78a3595079 (image=ubuntu, name=small_hoover) +//2016-03-07T17:28:03.091719377+02:00 network disconnect 19c5ed41acb798f26b751e0035cd7821741ab79e2bbd59a66b5fd8abf954eaa0 (type=bridge, container=0b863f2a26c18557fc6cdadda007c459f9ec81b874780808138aea78a3595079, name=bridge) +//2016-03-07T17:28:03.129014751+02:00 container destroy 0b863f2a26c18557fc6cdadda007c459f9ec81b874780808138aea78a3595079 (image=ubuntu, name=small_hoover) +func TestLoadBufferedEvents(t *testing.T) { + now := time.Now() + f, err := timetypes.GetTimestamp("2016-03-07T17:28:03.100000000+02:00", now) + if err != nil { + t.Fatal(err) + } + since, sinceNano, err := timetypes.ParseTimestamps(f, -1) + if err != nil { + t.Fatal(err) + } + + m1, err := eventstestutils.Scan("2016-03-07T17:28:03.022433271+02:00 container die 0b863f2a26c18557fc6cdadda007c459f9ec81b874780808138aea78a3595079 (image=ubuntu, name=small_hoover)") + if err != nil { + t.Fatal(err) + } + m2, err := eventstestutils.Scan("2016-03-07T17:28:03.091719377+02:00 network disconnect 19c5ed41acb798f26b751e0035cd7821741ab79e2bbd59a66b5fd8abf954eaa0 (type=bridge, container=0b863f2a26c18557fc6cdadda007c459f9ec81b874780808138aea78a3595079, name=bridge)") + if err != nil { + t.Fatal(err) + } + m3, err := eventstestutils.Scan("2016-03-07T17:28:03.129014751+02:00 container destroy 0b863f2a26c18557fc6cdadda007c459f9ec81b874780808138aea78a3595079 (image=ubuntu, name=small_hoover)") + if err != nil { + t.Fatal(err) + } + + buffered := []events.Message{*m1, *m2, *m3} + + events := &Events{ + events: buffered, + } + + out := events.loadBufferedEvents(since, sinceNano, nil) + if len(out) != 1 { + t.Fatalf("expected 1 message, got %d: %v", len(out), out) + } +} diff --git a/components/engine/daemon/events/testutils/testutils.go b/components/engine/daemon/events/testutils/testutils.go new file mode 100644 index 0000000000..c84418a9e7 --- /dev/null +++ b/components/engine/daemon/events/testutils/testutils.go @@ -0,0 +1,76 @@ +package eventstestutils + +import ( + "fmt" + "regexp" + "strings" + "time" + + "github.com/docker/engine-api/types/events" + timetypes "github.com/docker/engine-api/types/time" +) + +var ( + reTimestamp = `(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{9}(:?(:?(:?-|\+)\d{2}:\d{2})|Z))` + reEventType = `(?P\w+)` + reAction = `(?P\w+)` + reID = `(?P[^\s]+)` + reAttributes = `(\s\((?P[^\)]+)\))?` + reString = fmt.Sprintf(`\A%s\s%s\s%s\s%s%s\z`, reTimestamp, reEventType, reAction, reID, reAttributes) + + // eventCliRegexp is a regular expression that matches all possible event outputs in the cli + eventCliRegexp = regexp.MustCompile(reString) +) + +// ScanMap turns an event string like the default ones formatted in the cli output +// and turns it into map. +func ScanMap(text string) map[string]string { + matches := eventCliRegexp.FindAllStringSubmatch(text, -1) + md := map[string]string{} + if len(matches) == 0 { + return md + } + + names := eventCliRegexp.SubexpNames() + for i, n := range matches[0] { + md[names[i]] = n + } + return md +} + +// Scan turns an event string like the default ones formatted in the cli output +// and turns it into an event message. +func Scan(text string) (*events.Message, error) { + md := ScanMap(text) + if len(md) == 0 { + return nil, fmt.Errorf("text is not an event: %s", text) + } + + f, err := timetypes.GetTimestamp(md["timestamp"], time.Now()) + if err != nil { + return nil, err + } + + t, tn, err := timetypes.ParseTimestamps(f, -1) + if err != nil { + return nil, err + } + + attrs := make(map[string]string) + for _, a := range strings.SplitN(md["attributes"], ", ", -1) { + kv := strings.SplitN(a, "=", 2) + attrs[kv[0]] = kv[1] + } + + tu := time.Unix(t, tn) + return &events.Message{ + Time: t, + TimeNano: tu.UnixNano(), + Type: md["eventType"], + Action: md["action"], + Actor: events.Actor{ + ID: md["id"], + Attributes: attrs, + }, + }, nil +} diff --git a/components/engine/daemon/exec.go b/components/engine/daemon/exec.go index 1151a77c7b..56798a5979 100644 --- a/components/engine/daemon/exec.go +++ b/components/engine/daemon/exec.go @@ -1,6 +1,7 @@ package daemon import ( + "fmt" "io" "strings" "time" @@ -9,7 +10,7 @@ import ( "github.com/docker/docker/container" "github.com/docker/docker/daemon/exec" "github.com/docker/docker/daemon/execdriver" - derr "github.com/docker/docker/errors" + "github.com/docker/docker/errors" "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/promise" "github.com/docker/docker/pkg/term" @@ -47,19 +48,19 @@ func (d *Daemon) getExecConfig(name string) (*exec.Config, error) { if ec != nil { if container := d.containers.Get(ec.ContainerID); container != nil { if !container.IsRunning() { - return nil, derr.ErrorCodeContainerNotRunning.WithArgs(container.ID, container.State.String()) + return nil, fmt.Errorf("Container %s is not running: %s", container.ID, container.State.String()) } if container.IsPaused() { - return nil, derr.ErrorCodeExecPaused.WithArgs(container.ID) + return nil, errExecPaused(container.ID) } if container.IsRestarting() { - return nil, derr.ErrorCodeContainerRestarting.WithArgs(container.ID) + return nil, errContainerIsRestarting(container.ID) } return ec, nil } } - return nil, derr.ErrorCodeNoExecID.WithArgs(name) + return nil, errExecNotFound(name) } func (d *Daemon) unregisterExecCommand(container *container.Container, execConfig *exec.Config) { @@ -74,13 +75,13 @@ func (d *Daemon) getActiveContainer(name string) (*container.Container, error) { } if !container.IsRunning() { - return nil, derr.ErrorCodeNotRunning.WithArgs(name) + return nil, errNotRunning{container.ID} } if container.IsPaused() { - return nil, derr.ErrorCodeExecPaused.WithArgs(name) + return nil, errExecPaused(name) } if container.IsRestarting() { - return nil, derr.ErrorCodeContainerRestarting.WithArgs(name) + return nil, errContainerIsRestarting(container.ID) } return container, nil } @@ -92,8 +93,8 @@ func (d *Daemon) ContainerExecCreate(config *types.ExecConfig) (string, error) { return "", err } - cmd := strslice.New(config.Cmd...) - entrypoint, args := d.getEntrypointAndArgs(strslice.New(), cmd) + cmd := strslice.StrSlice(config.Cmd) + entrypoint, args := d.getEntrypointAndArgs(strslice.StrSlice{}, cmd) keys := []byte{} if config.DetachKeys != "" { @@ -137,18 +138,19 @@ func (d *Daemon) ContainerExecStart(name string, stdin io.ReadCloser, stdout io. ec, err := d.getExecConfig(name) if err != nil { - return derr.ErrorCodeNoExecID.WithArgs(name) + return errExecNotFound(name) } ec.Lock() if ec.ExitCode != nil { ec.Unlock() - return derr.ErrorCodeExecExited.WithArgs(ec.ID) + err := fmt.Errorf("Error: Exec command %s has already run", ec.ID) + return errors.NewRequestConflictError(err) } if ec.Running { ec.Unlock() - return derr.ErrorCodeExecRunning.WithArgs(ec.ID) + return fmt.Errorf("Error: Exec command %s is already running", ec.ID) } ec.Running = true ec.Unlock() @@ -157,7 +159,7 @@ func (d *Daemon) ContainerExecStart(name string, stdin io.ReadCloser, stdout io. logrus.Debugf("starting exec command %s in container %s", ec.ID, c.ID) d.LogContainerEvent(c, "exec_start: "+ec.ProcessConfig.Entrypoint+" "+strings.Join(ec.ProcessConfig.Arguments, " ")) - if ec.OpenStdin { + if ec.OpenStdin && stdin != nil { r, w := io.Pipe() go func() { defer w.Close() @@ -194,12 +196,12 @@ func (d *Daemon) ContainerExecStart(name string, stdin io.ReadCloser, stdout io. select { case err := <-attachErr: if err != nil { - return derr.ErrorCodeExecAttach.WithArgs(err) + return fmt.Errorf("attach failed with error: %v", err) } return nil case err := <-execErr: if aErr := <-attachErr; aErr != nil && err == nil { - return derr.ErrorCodeExecAttach.WithArgs(aErr) + return fmt.Errorf("attach failed with error: %v", aErr) } if err == nil { return nil @@ -207,9 +209,9 @@ func (d *Daemon) ContainerExecStart(name string, stdin io.ReadCloser, stdout io. // Maybe the container stopped while we were trying to exec if !c.IsRunning() { - return derr.ErrorCodeExecContainerStopped + return fmt.Errorf("container stopped while running exec: %s", c.ID) } - return derr.ErrorCodeExecCantRun.WithArgs(ec.ID, c.ID, err) + return fmt.Errorf("Cannot run exec command %s in container %s: %s", ec.ID, c.ID, err) } } diff --git a/components/engine/daemon/exec/exec.go b/components/engine/daemon/exec/exec.go index 6941cde689..2efb20ee9a 100644 --- a/components/engine/daemon/exec/exec.go +++ b/components/engine/daemon/exec/exec.go @@ -1,11 +1,11 @@ package exec import ( + "fmt" "sync" "time" "github.com/docker/docker/daemon/execdriver" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/runconfig" ) @@ -116,7 +116,7 @@ func (c *Config) Resize(h, w int) error { select { case <-c.waitStart: case <-time.After(time.Second): - return derr.ErrorCodeExecResize.WithArgs(c.ID) + return fmt.Errorf("Exec %s is not running, so it can not be resized.", c.ID) } return c.ProcessConfig.Terminal.Resize(h, w) } diff --git a/components/engine/daemon/execdriver/driver_unix.go b/components/engine/daemon/execdriver/driver_unix.go index 19550b3419..988737df19 100644 --- a/components/engine/daemon/execdriver/driver_unix.go +++ b/components/engine/daemon/execdriver/driver_unix.go @@ -50,6 +50,7 @@ type Resources struct { CPUPeriod int64 `json:"cpu_period"` Rlimits []*units.Rlimit `json:"rlimits"` OomKillDisable bool `json:"oom_kill_disable"` + PidsLimit int64 `json:"pids_limit"` MemorySwappiness int64 `json:"memory_swappiness"` } @@ -124,6 +125,7 @@ type Command struct { SeccompProfile string `json:"seccomp_profile"` UIDMapping []idtools.IDMap `json:"uidmapping"` UTS *UTS `json:"uts"` + NoNewPrivileges bool `json:"no_new_privileges"` } // SetRootPropagation sets the root mount propagation mode. @@ -140,7 +142,7 @@ func InitContainer(c *Command) *configs.Config { container.Hostname = getEnv("HOSTNAME", c.ProcessConfig.Env) container.Cgroups.Name = c.ID container.Cgroups.Resources.AllowedDevices = c.AllowedDevices - container.Devices = c.AutoCreatedDevices + container.Devices = filterDevices(c.AutoCreatedDevices, (c.RemappedRoot.UID != 0)) container.Rootfs = c.Rootfs container.Readonlyfs = c.ReadonlyRootfs // This can be overridden later by driver during mount setup based @@ -154,6 +156,24 @@ func InitContainer(c *Command) *configs.Config { return container } +func filterDevices(devices []*configs.Device, userNamespacesEnabled bool) []*configs.Device { + if !userNamespacesEnabled { + return devices + } + + filtered := []*configs.Device{} + // if we have user namespaces enabled, these devices will not be created + // because of the mknod limitation in the kernel for an unprivileged process. + // Rather, they will be bind-mounted, which will only work if they exist; + // check for existence and remove non-existent entries from the list + for _, device := range devices { + if _, err := os.Stat(device.Path); err == nil { + filtered = append(filtered, device) + } + } + return filtered +} + func getEnv(key string, env []string) string { for _, pair := range env { parts := strings.SplitN(pair, "=", 2) @@ -183,6 +203,7 @@ func SetupCgroups(container *configs.Config, c *Command) error { container.Cgroups.Resources.BlkioThrottleReadIOPSDevice = c.Resources.BlkioThrottleReadIOpsDevice container.Cgroups.Resources.BlkioThrottleWriteIOPSDevice = c.Resources.BlkioThrottleWriteIOpsDevice container.Cgroups.Resources.OomKillDisable = c.Resources.OomKillDisable + container.Cgroups.Resources.PidsLimit = c.Resources.PidsLimit container.Cgroups.Resources.MemorySwappiness = c.Resources.MemorySwappiness } diff --git a/components/engine/daemon/execdriver/driver_windows.go b/components/engine/daemon/execdriver/driver_windows.go index ec482cd30f..2fdb533729 100644 --- a/components/engine/daemon/execdriver/driver_windows.go +++ b/components/engine/daemon/execdriver/driver_windows.go @@ -49,11 +49,11 @@ type Command struct { // Fields below here are platform specific - FirstStart bool `json:"first_start"` // Optimisation for first boot of Windows + FirstStart bool `json:"first_start"` // Optimization for first boot of Windows Hostname string `json:"hostname"` // Windows sets the hostname in the execdriver LayerFolder string `json:"layer_folder"` // Layer folder for a command LayerPaths []string `json:"layer_paths"` // Layer paths for a command - Isolation string `json:"isolation"` // Isolation level for the container + Isolation string `json:"isolation"` // Isolation technology for the container ArgsEscaped bool `json:"args_escaped"` // True if args are already escaped HvPartition bool `json:"hv_partition"` // True if it's an hypervisor partition } diff --git a/components/engine/daemon/execdriver/native/create.go b/components/engine/daemon/execdriver/native/create.go index 14df913638..d85898f673 100644 --- a/components/engine/daemon/execdriver/native/create.go +++ b/components/engine/daemon/execdriver/native/create.go @@ -9,7 +9,6 @@ import ( "syscall" "github.com/docker/docker/daemon/execdriver" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/mount" "github.com/docker/docker/profiles/seccomp" @@ -72,7 +71,10 @@ func (d *Driver) createContainer(c *execdriver.Command, hooks execdriver.Hooks) } if c.SeccompProfile == "" { - container.Seccomp = seccomp.GetDefaultProfile() + container.Seccomp, err = seccomp.GetDefaultProfile() + if err != nil { + return nil, err + } } } // add CAP_ prefix to all caps for new libcontainer update to match @@ -104,7 +106,7 @@ func (d *Driver) createContainer(c *execdriver.Command, hooks execdriver.Hooks) if container.Readonlyfs { for i := range container.Mounts { switch container.Mounts[i].Destination { - case "/proc", "/dev", "/dev/pts": + case "/proc", "/dev", "/dev/pts", "/dev/mqueue": continue } container.Mounts[i].Flags |= syscall.MS_RDONLY @@ -120,6 +122,8 @@ func (d *Driver) createContainer(c *execdriver.Command, hooks execdriver.Hooks) d.setupLabels(container, c) d.setupRlimits(container, c) + + container.NoNewPrivileges = c.NoNewPrivileges return container, nil } @@ -427,7 +431,7 @@ func (d *Driver) setupMounts(container *configs.Config, c *execdriver.Command) e for _, m := range c.Mounts { for _, cm := range container.Mounts { if cm.Destination == m.Destination { - return derr.ErrorCodeMountDup.WithArgs(m.Destination) + return fmt.Errorf("Duplicate mount point '%s'", m.Destination) } } diff --git a/components/engine/daemon/execdriver/native/driver.go b/components/engine/daemon/execdriver/native/driver.go index 93ad481d17..98d64a32cf 100644 --- a/components/engine/daemon/execdriver/native/driver.go +++ b/components/engine/daemon/execdriver/native/driver.go @@ -123,11 +123,6 @@ func NewDriver(root string, options []string) (*Driver, error) { }, nil } -type execOutput struct { - exitCode int - err error -} - // Run implements the exec driver Driver interface, // it calls libcontainer APIs to run a container. func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execdriver.Hooks) (execdriver.ExitStatus, error) { @@ -152,7 +147,9 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd User: c.ProcessConfig.User, } - if err := setupPipes(container, &c.ProcessConfig, p, pipes); err != nil { + wg := sync.WaitGroup{} + writers, err := setupPipes(container, &c.ProcessConfig, p, pipes, &wg) + if err != nil { return execdriver.ExitStatus{ExitCode: -1}, err } @@ -160,6 +157,10 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd if err != nil { return execdriver.ExitStatus{ExitCode: -1}, err } + + if err := cont.Start(p); err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err + } d.Lock() d.activeContainers[c.ID] = cont d.Unlock() @@ -170,10 +171,10 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd d.cleanContainer(c.ID) }() - if err := cont.Start(p); err != nil { - return execdriver.ExitStatus{ExitCode: -1}, err + //close the write end of any opened pipes now that they are dup'ed into the container + for _, writer := range writers { + writer.Close() } - // 'oom' is used to emit 'oom' events to the eventstream, 'oomKilled' is used // to set the 'OOMKilled' flag in state oom := notifyOnOOM(cont) @@ -202,6 +203,9 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd } ps = execErr.ProcessState } + // wait for all IO goroutine copiers to finish + wg.Wait() + cont.Destroy() destroyed = true // oomKilled will have an oom event if any process within the container was @@ -298,6 +302,9 @@ func (d *Driver) Kill(c *execdriver.Command, sig int) error { if err != nil { return err } + if state.InitProcessPid == -1 { + return fmt.Errorf("avoid sending signal %d to container %s with pid -1", sig, c.ID) + } return syscall.Kill(state.InitProcessPid, syscall.Signal(sig)) } @@ -480,24 +487,26 @@ func (t *TtyConsole) Close() error { return t.console.Close() } -func setupPipes(container *configs.Config, processConfig *execdriver.ProcessConfig, p *libcontainer.Process, pipes *execdriver.Pipes) error { +func setupPipes(container *configs.Config, processConfig *execdriver.ProcessConfig, p *libcontainer.Process, pipes *execdriver.Pipes, wg *sync.WaitGroup) ([]io.WriteCloser, error) { + + writers := []io.WriteCloser{} rootuid, err := container.HostUID() if err != nil { - return err + return writers, err } if processConfig.Tty { cons, err := p.NewConsole(rootuid) if err != nil { - return err + return writers, err } term, err := NewTtyConsole(cons, pipes) if err != nil { - return err + return writers, err } processConfig.Terminal = term - return nil + return writers, nil } // not a tty--set up stdio pipes term := &execdriver.StdConsole{} @@ -512,7 +521,7 @@ func setupPipes(container *configs.Config, processConfig *execdriver.ProcessConf r, w, err := os.Pipe() if err != nil { - return err + return writers, err } if pipes.Stdin != nil { go func() { @@ -521,23 +530,32 @@ func setupPipes(container *configs.Config, processConfig *execdriver.ProcessConf }() p.Stdin = r } - return nil + return writers, nil } // if we have user namespaces enabled (rootuid != 0), we will set // up os pipes for stderr, stdout, stdin so we can chown them to // the proper ownership to allow for proper access to the underlying // fds - var fds []int + var fds []uintptr + + copyPipes := func(out io.Writer, in io.ReadCloser) { + defer wg.Done() + io.Copy(out, in) + in.Close() + } //setup stdout r, w, err := os.Pipe() if err != nil { - return err + w.Close() + return writers, err } - fds = append(fds, int(r.Fd()), int(w.Fd())) + writers = append(writers, w) + fds = append(fds, r.Fd(), w.Fd()) if pipes.Stdout != nil { - go io.Copy(pipes.Stdout, r) + wg.Add(1) + go copyPipes(pipes.Stdout, r) } term.Closers = append(term.Closers, r) p.Stdout = w @@ -545,11 +563,14 @@ func setupPipes(container *configs.Config, processConfig *execdriver.ProcessConf //setup stderr r, w, err = os.Pipe() if err != nil { - return err + w.Close() + return writers, err } - fds = append(fds, int(r.Fd()), int(w.Fd())) + writers = append(writers, w) + fds = append(fds, r.Fd(), w.Fd()) if pipes.Stderr != nil { - go io.Copy(pipes.Stderr, r) + wg.Add(1) + go copyPipes(pipes.Stderr, r) } term.Closers = append(term.Closers, r) p.Stderr = w @@ -557,9 +578,10 @@ func setupPipes(container *configs.Config, processConfig *execdriver.ProcessConf //setup stdin r, w, err = os.Pipe() if err != nil { - return err + r.Close() + return writers, err } - fds = append(fds, int(r.Fd()), int(w.Fd())) + fds = append(fds, r.Fd(), w.Fd()) if pipes.Stdin != nil { go func() { io.Copy(w, pipes.Stdin) @@ -568,11 +590,11 @@ func setupPipes(container *configs.Config, processConfig *execdriver.ProcessConf p.Stdin = r } for _, fd := range fds { - if err := syscall.Fchown(fd, rootuid, rootuid); err != nil { - return fmt.Errorf("Failed to chown pipes fd: %v", err) + if err := syscall.Fchown(int(fd), rootuid, rootuid); err != nil { + return writers, fmt.Errorf("Failed to chown pipes fd: %v", err) } } - return nil + return writers, nil } // SupportsHooks implements the execdriver Driver interface. diff --git a/components/engine/daemon/execdriver/native/exec.go b/components/engine/daemon/execdriver/native/exec.go index 0af5670239..d62fe5f405 100644 --- a/components/engine/daemon/execdriver/native/exec.go +++ b/components/engine/daemon/execdriver/native/exec.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "strings" + "sync" "syscall" "github.com/docker/docker/daemon/execdriver" @@ -52,13 +53,19 @@ func (d *Driver) Exec(c *execdriver.Command, processConfig *execdriver.ProcessCo } config := active.Config() - if err := setupPipes(&config, processConfig, p, pipes); err != nil { + wg := sync.WaitGroup{} + writers, err := setupPipes(&config, processConfig, p, pipes, &wg) + if err != nil { return -1, err } if err := active.Start(p); err != nil { return -1, err } + //close the write end of any opened pipes now that they are dup'ed into the container + for _, writer := range writers { + writer.Close() + } if hooks.Start != nil { pid, err := p.Pid() @@ -83,5 +90,7 @@ func (d *Driver) Exec(c *execdriver.Command, processConfig *execdriver.ProcessCo } ps = exitErr.ProcessState } + // wait for all IO goroutine copiers to finish + wg.Wait() return utils.ExitStatus(ps.Sys().(syscall.WaitStatus)), nil } diff --git a/components/engine/daemon/execdriver/windows/exec.go b/components/engine/daemon/execdriver/windows/exec.go index 1b6c06bd14..c9129a8bbd 100644 --- a/components/engine/daemon/execdriver/windows/exec.go +++ b/components/engine/daemon/execdriver/windows/exec.go @@ -4,6 +4,7 @@ package windows import ( "fmt" + "syscall" "github.com/Microsoft/hcsshim" "github.com/Sirupsen/logrus" @@ -17,7 +18,6 @@ func (d *Driver) Exec(c *execdriver.Command, processConfig *execdriver.ProcessCo term execdriver.Terminal err error exitCode int32 - errno uint32 ) active := d.activeContainers[c.ID] @@ -41,13 +41,13 @@ func (d *Driver) Exec(c *execdriver.Command, processConfig *execdriver.ProcessCo } // Start the command running in the container. - pid, stdin, stdout, stderr, rc, err := hcsshim.CreateProcessInComputeSystem(c.ID, pipes.Stdin != nil, true, !processConfig.Tty, createProcessParms) + pid, stdin, stdout, stderr, err := hcsshim.CreateProcessInComputeSystem(c.ID, pipes.Stdin != nil, true, !processConfig.Tty, createProcessParms) if err != nil { // TODO Windows: TP4 Workaround. In Hyper-V containers, there is a limitation // of one exec per container. This should be fixed post TP4. CreateProcessInComputeSystem // will return a specific error which we handle here to give a good error message // back to the user instead of an inactionable "An invalid argument was supplied" - if rc == hcsshim.Win32InvalidArgument { + if herr, ok := err.(*hcsshim.HcsError); ok && herr.Err == hcsshim.WSAEINVAL { return -1, fmt.Errorf("The limit of docker execs per Hyper-V container has been exceeded") } logrus.Errorf("CreateProcessInComputeSystem() failed %s", err) @@ -75,12 +75,12 @@ func (d *Driver) Exec(c *execdriver.Command, processConfig *execdriver.ProcessCo hooks.Start(&c.ProcessConfig, int(pid), chOOM) } - if exitCode, errno, err = hcsshim.WaitForProcessInComputeSystem(c.ID, pid, hcsshim.TimeoutInfinite); err != nil { - if errno == hcsshim.Win32PipeHasBeenEnded { - logrus.Debugf("Exiting Run() after WaitForProcessInComputeSystem failed with recognised error 0x%X", errno) + if exitCode, err = hcsshim.WaitForProcessInComputeSystem(c.ID, pid, hcsshim.TimeoutInfinite); err != nil { + if herr, ok := err.(*hcsshim.HcsError); ok && herr.Err == syscall.ERROR_BROKEN_PIPE { + logrus.Debugf("Exiting Run() after WaitForProcessInComputeSystem failed with recognised error %s", err) return hcsshim.WaitErrExecFailed, nil } - logrus.Warnf("WaitForProcessInComputeSystem failed (container may have been killed): 0x%X %s", errno, err) + logrus.Warnf("WaitForProcessInComputeSystem failed (container may have been killed): %s", err) return -1, err } diff --git a/components/engine/daemon/execdriver/windows/run.go b/components/engine/daemon/execdriver/windows/run.go index 0837ba85ee..b02fa747d9 100644 --- a/components/engine/daemon/execdriver/windows/run.go +++ b/components/engine/daemon/execdriver/windows/run.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strconv" "strings" + "syscall" "time" "github.com/Microsoft/hcsshim" @@ -20,6 +21,16 @@ import ( // preconfigured on the server. const defaultContainerNAT = "ContainerNAT" +// Win32 error codes that are used for various workarounds +// These really should be ALL_CAPS to match golangs syscall library and standard +// Win32 error conventions, but golint insists on CamelCase. +const ( + CoEClassstring = syscall.Errno(0x800401F3) // Invalid class string + ErrorNoNetwork = syscall.Errno(1222) // The network is not present or not started + ErrorBadPathname = syscall.Errno(161) // The specified path is invalid + ErrorInvalidObject = syscall.Errno(0x800710D8) // The object identifier does not represent a valid object +) + type layer struct { ID string Path string @@ -70,10 +81,10 @@ type containerInit struct { IsDummy bool // Used for development purposes. VolumePath string // Windows volume path for scratch space Devices []device // Devices used by the container - IgnoreFlushesDuringBoot bool // Optimisation hint for container startup in Windows + IgnoreFlushesDuringBoot bool // Optimization hint for container startup in Windows LayerFolderPath string // Where the layer folders are located Layers []layer // List of storage layers - ProcessorWeight int64 `json:",omitempty"` // CPU Shares 0..10000 on Windows; where 0 will be ommited and HCS will default. + ProcessorWeight int64 `json:",omitempty"` // CPU Shares 0..10000 on Windows; where 0 will be omitted and HCS will default. HostName string // Hostname MappedDirectories []mappedDir // List of mapped directories (volumes/mounts) SandboxPath string // Location of unmounted sandbox (used for Hyper-V containers, not Windows Server containers) @@ -227,30 +238,30 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd // TODO Windows TP5 timeframe. Remove when TP4 is no longer supported. // The following a workaround for Windows TP4 which has a networking // bug which fairly frequently returns an error. Back off and retry. - maxAttempts := 1 - if TP4RetryHack { - maxAttempts = 5 - } - i := 0 - for i < maxAttempts { - i++ + maxAttempts := 5 + for i := 0; i < maxAttempts; i++ { err = hcsshim.CreateComputeSystem(c.ID, configuration) - if err != nil { - if TP4RetryHack { - if !strings.Contains(err.Error(), `Win32 API call returned error r1=0x800401f3`) && // Invalid class string - !strings.Contains(err.Error(), `Win32 API call returned error r1=0x80070490`) && // Element not found - !strings.Contains(err.Error(), `Win32 API call returned error r1=0x80070002`) && // The system cannot find the file specified - !strings.Contains(err.Error(), `Win32 API call returned error r1=0x800704c6`) && // The network is not present or not started - !strings.Contains(err.Error(), `Win32 API call returned error r1=0x800700a1`) { // The specified path is invalid - logrus.Debugln("Failed to create temporary container ", err) - return execdriver.ExitStatus{ExitCode: -1}, err - } - logrus.Warnf("Invoking Windows TP4 retry hack (%d of %d)", i, maxAttempts-1) - time.Sleep(50 * time.Millisecond) - } - } else { + if err == nil { break } + + if !TP4RetryHack { + return execdriver.ExitStatus{ExitCode: -1}, err + } + + if herr, ok := err.(*hcsshim.HcsError); ok { + if herr.Err != syscall.ERROR_NOT_FOUND && // Element not found + herr.Err != syscall.ERROR_FILE_NOT_FOUND && // The system cannot find the file specified + herr.Err != ErrorNoNetwork && // The network is not present or not started + herr.Err != ErrorBadPathname && // The specified path is invalid + herr.Err != CoEClassstring && // Invalid class string + herr.Err != ErrorInvalidObject { // The object identifier does not represent a valid object + logrus.Debugln("Failed to create temporary container ", err) + return execdriver.ExitStatus{ExitCode: -1}, err + } + logrus.Warnf("Invoking Windows TP4 retry hack (%d of %d)", i, maxAttempts-1) + time.Sleep(50 * time.Millisecond) + } } // Start the container @@ -264,16 +275,17 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd // Stop the container if forceKill { logrus.Debugf("Forcibly terminating container %s", c.ID) - if errno, err := hcsshim.TerminateComputeSystem(c.ID, hcsshim.TimeoutInfinite, "exec-run-defer"); err != nil { - logrus.Warnf("Ignoring error from TerminateComputeSystem 0x%X %s", errno, err) + if err := hcsshim.TerminateComputeSystem(c.ID, hcsshim.TimeoutInfinite, "exec-run-defer"); err != nil { + logrus.Warnf("Ignoring error from TerminateComputeSystem %s", err) } } else { logrus.Debugf("Shutting down container %s", c.ID) - if errno, err := hcsshim.ShutdownComputeSystem(c.ID, hcsshim.TimeoutInfinite, "exec-run-defer"); err != nil { - if errno != hcsshim.Win32SystemShutdownIsInProgress && - errno != hcsshim.Win32SpecifiedPathInvalid && - errno != hcsshim.Win32SystemCannotFindThePathSpecified { - logrus.Warnf("Ignoring error from ShutdownComputeSystem 0x%X %s", errno, err) + if err := hcsshim.ShutdownComputeSystem(c.ID, hcsshim.TimeoutInfinite, "exec-run-defer"); err != nil { + if herr, ok := err.(*hcsshim.HcsError); !ok || + (herr.Err != hcsshim.ERROR_SHUTDOWN_IN_PROGRESS && + herr.Err != ErrorBadPathname && + herr.Err != syscall.ERROR_PATH_NOT_FOUND) { + logrus.Warnf("Ignoring error from ShutdownComputeSystem %s", err) } } } @@ -295,7 +307,7 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd } // Start the command running in the container. - pid, stdin, stdout, stderr, _, err := hcsshim.CreateProcessInComputeSystem(c.ID, pipes.Stdin != nil, true, !c.ProcessConfig.Tty, createProcessParms) + pid, stdin, stdout, stderr, err := hcsshim.CreateProcessInComputeSystem(c.ID, pipes.Stdin != nil, true, !c.ProcessConfig.Tty, createProcessParms) if err != nil { logrus.Errorf("CreateProcessInComputeSystem() failed %s", err) return execdriver.ExitStatus{ExitCode: -1}, err @@ -332,13 +344,9 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd hooks.Start(&c.ProcessConfig, int(pid), chOOM) } - var ( - exitCode int32 - errno uint32 - ) - exitCode, errno, err = hcsshim.WaitForProcessInComputeSystem(c.ID, pid, hcsshim.TimeoutInfinite) + exitCode, err := hcsshim.WaitForProcessInComputeSystem(c.ID, pid, hcsshim.TimeoutInfinite) if err != nil { - if errno != hcsshim.Win32PipeHasBeenEnded { + if herr, ok := err.(*hcsshim.HcsError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE { logrus.Warnf("WaitForProcessInComputeSystem failed (container may have been killed): %s", err) } // Do NOT return err here as the container would have diff --git a/components/engine/daemon/execdriver/windows/terminatekill.go b/components/engine/daemon/execdriver/windows/terminatekill.go index 7068a68fa9..d20b0e2b9d 100644 --- a/components/engine/daemon/execdriver/windows/terminatekill.go +++ b/components/engine/daemon/execdriver/windows/terminatekill.go @@ -28,8 +28,8 @@ func kill(id string, pid int, sig syscall.Signal) error { if sig == syscall.SIGKILL || forceKill { // Terminate the compute system - if errno, err := hcsshim.TerminateComputeSystem(id, hcsshim.TimeoutInfinite, context); err != nil { - logrus.Errorf("Failed to terminate %s - 0x%X %q", id, errno, err) + if err := hcsshim.TerminateComputeSystem(id, hcsshim.TimeoutInfinite, context); err != nil { + logrus.Errorf("Failed to terminate %s - %q", id, err) } } else { @@ -41,8 +41,8 @@ func kill(id string, pid int, sig syscall.Signal) error { } // Shutdown the compute system - if errno, err := hcsshim.ShutdownComputeSystem(id, hcsshim.TimeoutInfinite, context); err != nil { - logrus.Errorf("Failed to shutdown %s - 0x%X %q", id, errno, err) + if err := hcsshim.ShutdownComputeSystem(id, hcsshim.TimeoutInfinite, context); err != nil { + logrus.Errorf("Failed to shutdown %s - %q", id, err) } } return err diff --git a/components/engine/daemon/execdriver/windows/update.go b/components/engine/daemon/execdriver/windows/update.go index 33c0b9ef1a..a4c42a6b08 100644 --- a/components/engine/daemon/execdriver/windows/update.go +++ b/components/engine/daemon/execdriver/windows/update.go @@ -3,12 +3,12 @@ package windows import ( - "fmt" - "github.com/docker/docker/daemon/execdriver" ) // Update updates resource configs for a container. func (d *Driver) Update(c *execdriver.Command) error { - return fmt.Errorf("Windows: Update not implemented") + // Updating resource isn't supported on Windows + // but we should return nil for enabling updating container + return nil } diff --git a/components/engine/daemon/execdriver/windows/windows.go b/components/engine/daemon/execdriver/windows/windows.go index 0be7c0b02d..e6c8f10b91 100644 --- a/components/engine/daemon/execdriver/windows/windows.go +++ b/components/engine/daemon/execdriver/windows/windows.go @@ -4,16 +4,15 @@ package windows import ( "fmt" - "strconv" "strings" "sync" + "github.com/Microsoft/hcsshim" "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/execdriver" "github.com/docker/docker/dockerversion" "github.com/docker/docker/pkg/parsers" "github.com/docker/engine-api/types/container" - "golang.org/x/sys/windows/registry" ) // TP4RetryHack is a hack to retry CreateComputeSystem if it fails with @@ -28,11 +27,11 @@ var dummyMode bool // This allows the daemon to force kill (HCS terminate) rather than shutdown var forceKill bool -// DefaultIsolation allows users to specify a default isolation mode for +// DefaultIsolation allows users to specify a default isolation technology for // when running a container on Windows. For example docker daemon -D // --exec-opt isolation=hyperv will cause Windows to always run containers // as Hyper-V containers unless otherwise specified. -var DefaultIsolation container.IsolationLevel = "process" +var DefaultIsolation container.Isolation = "process" // Define name and version for windows var ( @@ -83,13 +82,13 @@ func NewDriver(root string, options []string) (*Driver, error) { } case "isolation": - if !container.IsolationLevel(val).IsValid() { + if !container.Isolation(val).IsValid() { return nil, fmt.Errorf("Unrecognised exec driver option 'isolation':'%s'", val) } - if container.IsolationLevel(val).IsHyperV() { + if container.Isolation(val).IsHyperV() { DefaultIsolation = "hyperv" } - logrus.Infof("Windows default isolation level: '%s'", val) + logrus.Infof("Windows default isolation: '%s'", val) default: return nil, fmt.Errorf("Unrecognised exec driver option %s\n", key) } @@ -98,33 +97,11 @@ func NewDriver(root string, options []string) (*Driver, error) { // TODO Windows TP5 timeframe. Remove this next block of code once TP4 // is no longer supported. Also remove the workaround in run.go. // - // Hack for TP4 - determine the version of Windows from the registry. + // Hack for TP4. // This overcomes an issue on TP4 which causes CreateComputeSystem to // intermittently fail. It's predominantly here to make Windows to Windows // CI more reliable. - TP4RetryHack = false - k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE) - if err != nil { - return &Driver{}, err - } - defer k.Close() - - s, _, err := k.GetStringValue("BuildLab") - if err != nil { - return &Driver{}, err - } - parts := strings.Split(s, ".") - if len(parts) < 1 { - return &Driver{}, err - } - var val int - if val, err = strconv.Atoi(parts[0]); err != nil { - return &Driver{}, err - } - if val < 14250 { - TP4RetryHack = true - } - // End of Windows TP4 hack + TP4RetryHack = hcsshim.IsTP4() return &Driver{ root: root, diff --git a/components/engine/daemon/export.go b/components/engine/daemon/export.go index 9e3d8da340..80d7dbb2e1 100644 --- a/components/engine/daemon/export.go +++ b/components/engine/daemon/export.go @@ -1,10 +1,10 @@ package daemon import ( + "fmt" "io" "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/ioutils" ) @@ -19,13 +19,13 @@ func (daemon *Daemon) ContainerExport(name string, out io.Writer) error { data, err := daemon.containerExport(container) if err != nil { - return derr.ErrorCodeExportFailed.WithArgs(name, err) + return fmt.Errorf("Error exporting container %s: %v", name, err) } defer data.Close() // Stream the entire contents of the container (basically a volatile snapshot) if _, err := io.Copy(out, data); err != nil { - return derr.ErrorCodeExportFailed.WithArgs(name, err) + return fmt.Errorf("Error exporting container %s: %v", name, err) } return nil } diff --git a/components/engine/daemon/graphdriver/aufs/aufs.go b/components/engine/daemon/graphdriver/aufs/aufs.go index 51054fa6ef..ac0bc5f483 100644 --- a/components/engine/daemon/graphdriver/aufs/aufs.go +++ b/components/engine/daemon/graphdriver/aufs/aufs.go @@ -34,6 +34,7 @@ import ( "syscall" "github.com/Sirupsen/logrus" + "github.com/vbatts/tar-split/tar/storage" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" @@ -227,10 +228,14 @@ func (a *Driver) Create(id, parent, mountLabel string) error { } } } + a.Lock() a.active[id] = &data{} + a.Unlock() return nil } +// createDirsFor creates two directories for the given id. +// mnt and diff func (a *Driver) createDirsFor(id string) error { paths := []string{ "mnt", @@ -241,6 +246,9 @@ func (a *Driver) createDirsFor(id string) error { if err != nil { return err } + // Directory permission is 0755. + // The path of directories are /mnt/ + // and /diff/ for _, p := range paths { if err := idtools.MkdirAllAs(path.Join(a.rootPath(), p, id), 0755, rootUID, rootGID); err != nil { return err @@ -285,20 +293,15 @@ func (a *Driver) Remove(id string) error { if err := os.Remove(path.Join(a.rootPath(), "layers", id)); err != nil && !os.IsNotExist(err) { return err } + if m != nil { + delete(a.active, id) + } return nil } // Get returns the rootfs path for the id. // This will mount the dir at it's given path func (a *Driver) Get(id, mountLabel string) (string, error) { - ids, err := getParentIds(a.rootPath(), id) - if err != nil { - if !os.IsNotExist(err) { - return "", err - } - ids = []string{} - } - // Protect the a.active from concurrent access a.Lock() defer a.Unlock() @@ -309,13 +312,18 @@ func (a *Driver) Get(id, mountLabel string) (string, error) { a.active[id] = m } + parents, err := a.getParentLayerPaths(id) + if err != nil && !os.IsNotExist(err) { + return "", err + } + // If a dir does not have a parent ( no layers )do not try to mount // just return the diff path to the data m.path = path.Join(a.rootPath(), "diff", id) - if len(ids) > 0 { + if len(parents) > 0 { m.path = path.Join(a.rootPath(), "mnt", id) if m.referenceCount == 0 { - if err := a.mount(id, m, mountLabel); err != nil { + if err := a.mount(id, m, mountLabel, parents); err != nil { return "", err } } @@ -367,10 +375,19 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) { }) } -// DiffPath returns path to the directory that contains files for the layer -// differences. Used for direct access for tar-split. -func (a *Driver) DiffPath(id string) (string, func() error, error) { - return path.Join(a.rootPath(), "diff", id), func() error { return nil }, nil +type fileGetNilCloser struct { + storage.FileGetter +} + +func (f fileGetNilCloser) Close() error { + return nil +} + +// DiffGetter returns a FileGetCloser that can read files from the directory that +// contains files for the layer differences. Used for direct access for tar-split. +func (a *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { + p := path.Join(a.rootPath(), "diff", id) + return fileGetNilCloser{storage.NewPathFileGetter(p)}, nil } func (a *Driver) applyDiff(id string, diff archive.Reader) error { @@ -426,7 +443,7 @@ func (a *Driver) getParentLayerPaths(id string) ([]string, error) { return layers, nil } -func (a *Driver) mount(id string, m *data, mountLabel string) error { +func (a *Driver) mount(id string, m *data, mountLabel string, layers []string) error { // If the id is mounted or we get an error return if mounted, err := a.mounted(m); err != nil || mounted { return err @@ -437,11 +454,6 @@ func (a *Driver) mount(id string, m *data, mountLabel string) error { rw = path.Join(a.rootPath(), "diff", id) ) - layers, err := a.getParentLayerPaths(id) - if err != nil { - return err - } - if err := a.aufsMount(layers, rw, target, mountLabel); err != nil { return fmt.Errorf("error creating aufs mount to %s: %v", target, err) } @@ -456,7 +468,11 @@ func (a *Driver) unmount(m *data) error { } func (a *Driver) mounted(m *data) (bool, error) { - return mountpk.Mounted(m.path) + var buf syscall.Statfs_t + if err := syscall.Statfs(m.path, &buf); err != nil { + return false, nil + } + return graphdriver.FsMagic(buf.Type) == graphdriver.FsMagicAufs, nil } // Cleanup aufs and unmount all mountpoints diff --git a/components/engine/daemon/graphdriver/aufs/aufs_test.go b/components/engine/daemon/graphdriver/aufs/aufs_test.go index 761b5b6872..0f6d59d054 100644 --- a/components/engine/daemon/graphdriver/aufs/aufs_test.go +++ b/components/engine/daemon/graphdriver/aufs/aufs_test.go @@ -9,11 +9,13 @@ import ( "io/ioutil" "os" "path" + "sync" "testing" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/reexec" + "github.com/docker/docker/pkg/stringid" ) var ( @@ -25,7 +27,7 @@ func init() { reexec.Init() } -func testInit(dir string, t *testing.T) graphdriver.Driver { +func testInit(dir string, t testing.TB) graphdriver.Driver { d, err := Init(dir, nil, nil, nil) if err != nil { if err == graphdriver.ErrNotSupported { @@ -37,7 +39,7 @@ func testInit(dir string, t *testing.T) graphdriver.Driver { return d } -func newDriver(t *testing.T) *Driver { +func newDriver(t testing.TB) *Driver { if err := os.MkdirAll(tmp, 0755); err != nil { t.Fatal(err) } @@ -732,3 +734,68 @@ func TestMountMoreThan42LayersMatchingPathLength(t *testing.T) { zeroes += "0" } } + +func BenchmarkConcurrentAccess(b *testing.B) { + b.StopTimer() + b.ResetTimer() + + d := newDriver(b) + defer os.RemoveAll(tmp) + defer d.Cleanup() + + numConcurent := 256 + // create a bunch of ids + var ids []string + for i := 0; i < numConcurent; i++ { + ids = append(ids, stringid.GenerateNonCryptoID()) + } + + if err := d.Create(ids[0], "", ""); err != nil { + b.Fatal(err) + } + + if err := d.Create(ids[1], ids[0], ""); err != nil { + b.Fatal(err) + } + + parent := ids[1] + ids = append(ids[2:]) + + chErr := make(chan error, numConcurent) + var outerGroup sync.WaitGroup + outerGroup.Add(len(ids)) + b.StartTimer() + + // here's the actual bench + for _, id := range ids { + go func(id string) { + defer outerGroup.Done() + if err := d.Create(id, parent, ""); err != nil { + b.Logf("Create %s failed", id) + chErr <- err + return + } + var innerGroup sync.WaitGroup + for i := 0; i < b.N; i++ { + innerGroup.Add(1) + go func() { + d.Get(id, "") + d.Put(id) + innerGroup.Done() + }() + } + innerGroup.Wait() + d.Remove(id) + }(id) + } + + outerGroup.Wait() + b.StopTimer() + close(chErr) + for err := range chErr { + if err != nil { + b.Log(err) + b.Fail() + } + } +} diff --git a/components/engine/daemon/graphdriver/aufs/mount.go b/components/engine/daemon/graphdriver/aufs/mount.go index d7e9bf9fd7..36fa62e41b 100644 --- a/components/engine/daemon/graphdriver/aufs/mount.go +++ b/components/engine/daemon/graphdriver/aufs/mount.go @@ -12,7 +12,7 @@ import ( // Unmount the target specified. func Unmount(target string) error { if err := exec.Command("auplink", target, "flush").Run(); err != nil { - logrus.Errorf("Couldn't run auplink before unmount: %s", err) + logrus.Errorf("Couldn't run auplink before unmount %s: %s", target, err) } if err := syscall.Unmount(target, 0); err != nil { return err diff --git a/components/engine/daemon/graphdriver/btrfs/btrfs.go b/components/engine/daemon/graphdriver/btrfs/btrfs.go index 48388f72fe..5ca86a5b6f 100644 --- a/components/engine/daemon/graphdriver/btrfs/btrfs.go +++ b/components/engine/daemon/graphdriver/btrfs/btrfs.go @@ -262,7 +262,7 @@ func (d *Driver) Create(id, parent, mountLabel string) error { return err } if !st.IsDir() { - return fmt.Errorf("%s: not a direcotory", parentDir) + return fmt.Errorf("%s: not a directory", parentDir) } if err := subvolSnapshot(parentDir, subvolumes, id); err != nil { return err diff --git a/components/engine/daemon/graphdriver/devmapper/deviceset.go b/components/engine/daemon/graphdriver/devmapper/deviceset.go index d8522349b8..7748413c13 100644 --- a/components/engine/daemon/graphdriver/devmapper/deviceset.go +++ b/components/engine/daemon/graphdriver/devmapper/deviceset.go @@ -573,7 +573,7 @@ func determineDefaultFS() string { return "xfs" } - logrus.Warn("devmapper: XFS is not supported in your system. Either the kernel doesnt support it or mkfs.xfs is not in your PATH. Defaulting to ext4 filesystem") + logrus.Warn("devmapper: XFS is not supported in your system. Either the kernel doesn't support it or mkfs.xfs is not in your PATH. Defaulting to ext4 filesystem") return "ext4" } diff --git a/components/engine/daemon/graphdriver/driver.go b/components/engine/daemon/graphdriver/driver.go index d9ab839c2e..abc400083d 100644 --- a/components/engine/daemon/graphdriver/driver.go +++ b/components/engine/daemon/graphdriver/driver.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/Sirupsen/logrus" + "github.com/vbatts/tar-split/tar/storage" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/idtools" @@ -92,6 +93,23 @@ type Driver interface { DiffSize(id, parent string) (size int64, err error) } +// DiffGetterDriver is the interface for layered file system drivers that +// provide a specialized function for getting file contents for tar-split. +type DiffGetterDriver interface { + Driver + // DiffGetter returns an interface to efficiently retrieve the contents + // of files in a layer. + DiffGetter(id string) (FileGetCloser, error) +} + +// FileGetCloser extends the storage.FileGetter interface with a Close method +// for cleaning up. +type FileGetCloser interface { + storage.FileGetter + // Close cleans up any resources associated with the FileGetCloser. + Close() error +} + func init() { drivers = make(map[string]InitFunc) } diff --git a/components/engine/daemon/graphdriver/windows/windows.go b/components/engine/daemon/graphdriver/windows/windows.go index 77f4f1b774..2b5b549e20 100644 --- a/components/engine/daemon/graphdriver/windows/windows.go +++ b/components/engine/daemon/graphdriver/windows/windows.go @@ -3,17 +3,23 @@ package windows import ( + "bufio" "crypto/sha512" "encoding/json" "fmt" + "io" "io/ioutil" "os" + "path" "path/filepath" - "strconv" "strings" "sync" + "syscall" "time" + "github.com/Microsoft/go-winio" + "github.com/Microsoft/go-winio/archive/tar" + "github.com/Microsoft/go-winio/backuptar" "github.com/Microsoft/hcsshim" "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" @@ -21,7 +27,7 @@ import ( "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/random" + "github.com/vbatts/tar-split/tar/storage" ) // init registers the windows graph drivers to the register. @@ -47,6 +53,8 @@ type Driver struct { active map[string]int } +var _ graphdriver.DiffGetterDriver = &Driver{} + // InitFilter returns a new Windows storage filter driver. func InitFilter(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { logrus.Debugf("WindowsGraphDriver InitFilter at %s", home) @@ -262,7 +270,7 @@ func (d *Driver) Cleanup() error { // Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". -func (d *Driver) Diff(id, parent string) (arch archive.Archive, err error) { +func (d *Driver) Diff(id, parent string) (_ archive.Archive, err error) { rID, err := d.resolveID(id) if err != nil { return @@ -274,6 +282,8 @@ func (d *Driver) Diff(id, parent string) (arch archive.Archive, err error) { return } + var undo func() + d.Lock() // To support export, a layer must be activated but not prepared. @@ -283,6 +293,56 @@ func (d *Driver) Diff(id, parent string) (arch archive.Archive, err error) { d.Unlock() return } + undo = func() { + if err := hcsshim.DeactivateLayer(d.info, rID); err != nil { + logrus.Warnf("Failed to Deactivate %s: %s", rID, err) + } + } + } else { + if err = hcsshim.UnprepareLayer(d.info, rID); err != nil { + d.Unlock() + return + } + undo = func() { + if err := hcsshim.PrepareLayer(d.info, rID, layerChain); err != nil { + logrus.Warnf("Failed to re-PrepareLayer %s: %s", rID, err) + } + } + } + } + + d.Unlock() + + arch, err := d.exportLayer(rID, layerChain) + if err != nil { + undo() + return + } + return ioutils.NewReadCloserWrapper(arch, func() error { + defer undo() + return arch.Close() + }), nil +} + +// Changes produces a list of changes between the specified layer +// and its parent layer. If parent is "", then all changes will be ADD changes. +func (d *Driver) Changes(id, parent string) ([]archive.Change, error) { + rID, err := d.resolveID(id) + if err != nil { + return nil, err + } + parentChain, err := d.getLayerChain(rID) + if err != nil { + return nil, err + } + + d.Lock() + if d.info.Flavour == filterDriver { + if d.active[rID] == 0 { + if err = hcsshim.ActivateLayer(d.info, rID); err != nil { + d.Unlock() + return nil, err + } defer func() { if err := hcsshim.DeactivateLayer(d.info, rID); err != nil { logrus.Warnf("Failed to Deactivate %s: %s", rID, err) @@ -291,25 +351,41 @@ func (d *Driver) Diff(id, parent string) (arch archive.Archive, err error) { } else { if err = hcsshim.UnprepareLayer(d.info, rID); err != nil { d.Unlock() - return + return nil, err } defer func() { - if err := hcsshim.PrepareLayer(d.info, rID, layerChain); err != nil { + if err := hcsshim.PrepareLayer(d.info, rID, parentChain); err != nil { logrus.Warnf("Failed to re-PrepareLayer %s: %s", rID, err) } }() } } - d.Unlock() - return d.exportLayer(rID, layerChain) -} + r, err := hcsshim.NewLayerReader(d.info, id, parentChain) + if err != nil { + return nil, err + } + defer r.Close() -// Changes produces a list of changes between the specified layer -// and its parent layer. If parent is "", then all changes will be ADD changes. -func (d *Driver) Changes(id, parent string) ([]archive.Change, error) { - return nil, fmt.Errorf("The Windows graphdriver does not support Changes()") + var changes []archive.Change + for { + name, _, fileInfo, err := r.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + name = filepath.ToSlash(name) + if fileInfo == nil { + changes = append(changes, archive.Change{name, archive.ChangeDelete}) + } else { + // Currently there is no way to tell between an add and a modify. + changes = append(changes, archive.Change{name, archive.ChangeModify}) + } + } + return changes, nil } // ApplyDiff extracts the changeset from the given diff into the @@ -441,71 +517,162 @@ func (d *Driver) GetMetadata(id string) (map[string]string, error) { return m, nil } -// exportLayer generates an archive from a layer based on the given ID. -func (d *Driver) exportLayer(id string, parentLayerPaths []string) (arch archive.Archive, err error) { - layerFolder := d.dir(id) - - tempFolder := layerFolder + "-" + strconv.FormatUint(uint64(random.Rand.Uint32()), 10) - if err = os.MkdirAll(tempFolder, 0755); err != nil { - logrus.Errorf("Could not create %s %s", tempFolder, err) - return - } - defer func() { +func writeTarFromLayer(r hcsshim.LayerReader, w io.Writer) error { + t := tar.NewWriter(w) + for { + name, size, fileInfo, err := r.Next() + if err == io.EOF { + break + } if err != nil { - _, folderName := filepath.Split(tempFolder) - if err2 := hcsshim.DestroyLayer(d.info, folderName); err2 != nil { - logrus.Warnf("Couldn't clean-up tempFolder: %s %s", tempFolder, err2) + return err + } + if fileInfo == nil { + // Write a whiteout file. + hdr := &tar.Header{ + Name: filepath.ToSlash(filepath.Join(filepath.Dir(name), archive.WhiteoutPrefix+filepath.Base(name))), + } + err := t.WriteHeader(hdr) + if err != nil { + return err + } + } else { + err = backuptar.WriteTarFileFromBackupStream(t, r, name, size, fileInfo) + if err != nil { + return err } } + } + return t.Close() +} + +// exportLayer generates an archive from a layer based on the given ID. +func (d *Driver) exportLayer(id string, parentLayerPaths []string) (archive.Archive, error) { + if hcsshim.IsTP4() { + // Export in TP4 format to maintain compatibility with existing images and + // because ExportLayer is somewhat broken on TP4 and can't work with the new + // scheme. + tempFolder, err := ioutil.TempDir("", "hcs") + if err != nil { + return nil, err + } + defer func() { + if err != nil { + os.RemoveAll(tempFolder) + } + }() + + if err = hcsshim.ExportLayer(d.info, id, tempFolder, parentLayerPaths); err != nil { + return nil, err + } + archive, err := archive.Tar(tempFolder, archive.Uncompressed) + if err != nil { + return nil, err + } + return ioutils.NewReadCloserWrapper(archive, func() error { + err := archive.Close() + os.RemoveAll(tempFolder) + return err + }), nil + } + + var r hcsshim.LayerReader + r, err := hcsshim.NewLayerReader(d.info, id, parentLayerPaths) + if err != nil { + return nil, err + } + + archive, w := io.Pipe() + go func() { + err := writeTarFromLayer(r, w) + cerr := r.Close() + if err == nil { + err = cerr + } + w.CloseWithError(err) }() - if err = hcsshim.ExportLayer(d.info, id, tempFolder, parentLayerPaths); err != nil { - return - } + return archive, nil +} - archive, err := archive.Tar(tempFolder, archive.Uncompressed) - if err != nil { - return - } - return ioutils.NewReadCloserWrapper(archive, func() error { - err := archive.Close() - d.Put(id) - _, folderName := filepath.Split(tempFolder) - if err2 := hcsshim.DestroyLayer(d.info, folderName); err2 != nil { - logrus.Warnf("Couldn't clean-up tempFolder: %s %s", tempFolder, err2) +func writeLayerFromTar(r archive.Reader, w hcsshim.LayerWriter) (int64, error) { + t := tar.NewReader(r) + hdr, err := t.Next() + totalSize := int64(0) + buf := bufio.NewWriter(nil) + for err == nil { + base := path.Base(hdr.Name) + if strings.HasPrefix(base, archive.WhiteoutPrefix) { + name := path.Join(path.Dir(hdr.Name), base[len(archive.WhiteoutPrefix):]) + err = w.Remove(filepath.FromSlash(name)) + if err != nil { + return 0, err + } + hdr, err = t.Next() + } else { + var ( + name string + size int64 + fileInfo *winio.FileBasicInfo + ) + name, size, fileInfo, err = backuptar.FileInfoFromHeader(hdr) + if err != nil { + return 0, err + } + err = w.Add(filepath.FromSlash(name), fileInfo) + if err != nil { + return 0, err + } + buf.Reset(w) + hdr, err = backuptar.WriteBackupStreamFromTarFile(buf, t, hdr) + ferr := buf.Flush() + if ferr != nil { + err = ferr + } + totalSize += size } - return err - }), nil - + } + if err != io.EOF { + return 0, err + } + return totalSize, nil } // importLayer adds a new layer to the tag and graph store based on the given data. func (d *Driver) importLayer(id string, layerData archive.Reader, parentLayerPaths []string) (size int64, err error) { - layerFolder := d.dir(id) - - tempFolder := layerFolder + "-" + strconv.FormatUint(uint64(random.Rand.Uint32()), 10) - if err = os.MkdirAll(tempFolder, 0755); err != nil { - logrus.Errorf("Could not create %s %s", tempFolder, err) - return - } - defer func() { - _, folderName := filepath.Split(tempFolder) - if err2 := hcsshim.DestroyLayer(d.info, folderName); err2 != nil { - logrus.Warnf("Couldn't clean-up tempFolder: %s %s", tempFolder, err2) + if hcsshim.IsTP4() { + // Import from TP4 format to maintain compatibility with existing images. + var tempFolder string + tempFolder, err = ioutil.TempDir("", "hcs") + if err != nil { + return } - }() + defer os.RemoveAll(tempFolder) - start := time.Now().UTC() - logrus.Debugf("Start untar layer") - if size, err = chrootarchive.ApplyLayer(tempFolder, layerData); err != nil { - return - } - logrus.Debugf("Untar time: %vs", time.Now().UTC().Sub(start).Seconds()) - - if err = hcsshim.ImportLayer(d.info, id, tempFolder, parentLayerPaths); err != nil { + if size, err = chrootarchive.ApplyLayer(tempFolder, layerData); err != nil { + return + } + if err = hcsshim.ImportLayer(d.info, id, tempFolder, parentLayerPaths); err != nil { + return + } return } + var w hcsshim.LayerWriter + w, err = hcsshim.NewLayerWriter(d.info, id, parentLayerPaths) + if err != nil { + return + } + + size, err = writeLayerFromTar(layerData, w) + if err != nil { + w.Close() + return + } + err = w.Close() + if err != nil { + return + } return } @@ -564,42 +731,78 @@ func (d *Driver) setLayerChain(id string, chain []string) error { return nil } -// DiffPath returns a directory that contains files needed to construct layer diff. -func (d *Driver) DiffPath(id string) (path string, release func() error, err error) { - id, err = d.resolveID(id) - if err != nil { - return - } +type fileGetCloserWithBackupPrivileges struct { + path string +} + +func (fg *fileGetCloserWithBackupPrivileges) Get(filename string) (io.ReadCloser, error) { + var f *os.File + // Open the file while holding the Windows backup privilege. This ensures that the + // file can be opened even if the caller does not actually have access to it according + // to the security descriptor. + err := winio.RunWithPrivilege(winio.SeBackupPrivilege, func() error { + path := filepath.Join(fg.path, filename) + p, err := syscall.UTF16FromString(path) + if err != nil { + return err + } + h, err := syscall.CreateFile(&p[0], syscall.GENERIC_READ, syscall.FILE_SHARE_READ, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + if err != nil { + return &os.PathError{Op: "open", Path: path, Err: err} + } + f = os.NewFile(uintptr(h), path) + return nil + }) + return f, err +} + +func (fg *fileGetCloserWithBackupPrivileges) Close() error { + return nil +} + +type fileGetDestroyCloser struct { + storage.FileGetter + path string +} + +func (f *fileGetDestroyCloser) Close() error { + // TODO: activate layers and release here? + return os.RemoveAll(f.path) +} - // Getting the layer paths must be done outside of the lock. - layerChain, err := d.getLayerChain(id) +// DiffGetter returns a FileGetCloser that can read files from the directory that +// contains files for the layer differences. Used for direct access for tar-split. +func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { + id, err := d.resolveID(id) if err != nil { - return + return nil, err } - layerFolder := d.dir(id) - tempFolder := layerFolder + "-" + strconv.FormatUint(uint64(random.Rand.Uint32()), 10) - if err = os.MkdirAll(tempFolder, 0755); err != nil { - logrus.Errorf("Could not create %s %s", tempFolder, err) - return - } + if hcsshim.IsTP4() { + // The export format for TP4 is different from the contents of the layer, so + // fall back to exporting the layer and getting file contents from there. + layerChain, err := d.getLayerChain(id) + if err != nil { + return nil, err + } - defer func() { + var tempFolder string + tempFolder, err = ioutil.TempDir("", "hcs") if err != nil { - _, folderName := filepath.Split(tempFolder) - if err2 := hcsshim.DestroyLayer(d.info, folderName); err2 != nil { - logrus.Warnf("Couldn't clean-up tempFolder: %s %s", tempFolder, err2) + return nil, err + } + defer func() { + if err != nil { + os.RemoveAll(tempFolder) } + }() + + if err = hcsshim.ExportLayer(d.info, id, tempFolder, layerChain); err != nil { + return nil, err } - }() - if err = hcsshim.ExportLayer(d.info, id, tempFolder, layerChain); err != nil { - return + return &fileGetDestroyCloser{storage.NewPathFileGetter(tempFolder), tempFolder}, nil } - return tempFolder, func() error { - // TODO: activate layers and release here? - _, folderName := filepath.Split(tempFolder) - return hcsshim.DestroyLayer(d.info, folderName) - }, nil + return &fileGetCloserWithBackupPrivileges{d.dir(id)}, nil } diff --git a/components/engine/daemon/graphdriver/zfs/zfs.go b/components/engine/daemon/graphdriver/zfs/zfs.go index 5cc10d2e26..28a94dd0b5 100644 --- a/components/engine/daemon/graphdriver/zfs/zfs.go +++ b/components/engine/daemon/graphdriver/zfs/zfs.go @@ -22,6 +22,12 @@ import ( "github.com/opencontainers/runc/libcontainer/label" ) +type activeMount struct { + count int + path string + mounted bool +} + type zfsOptions struct { fsName string mountPath string @@ -103,6 +109,7 @@ func Init(base string, opt []string, uidMaps, gidMaps []idtools.IDMap) (graphdri dataset: rootDataset, options: options, filesystemsCache: filesystemsCache, + active: make(map[string]*activeMount), uidMaps: uidMaps, gidMaps: gidMaps, } @@ -159,6 +166,7 @@ type Driver struct { options zfsOptions sync.Mutex // protects filesystem cache against concurrent access filesystemsCache map[string]bool + active map[string]*activeMount uidMaps []idtools.IDMap gidMaps []idtools.IDMap } @@ -294,6 +302,17 @@ func (d *Driver) Remove(id string) error { // Get returns the mountpoint for the given id after creating the target directories if necessary. func (d *Driver) Get(id, mountLabel string) (string, error) { + d.Lock() + defer d.Unlock() + + mnt := d.active[id] + if mnt != nil { + mnt.count++ + return mnt.path, nil + } + + mnt = &activeMount{count: 1} + mountpoint := d.mountPath(id) filesystem := d.zfsPath(id) options := label.FormatMountLabel("", mountLabel) @@ -316,17 +335,43 @@ func (d *Driver) Get(id, mountLabel string) (string, error) { if err := os.Chown(mountpoint, rootUID, rootGID); err != nil { return "", fmt.Errorf("error modifying zfs mountpoint (%s) directory ownership: %v", mountpoint, err) } + mnt.path = mountpoint + mnt.mounted = true + d.active[id] = mnt return mountpoint, nil } // Put removes the existing mountpoint for the given id if it exists. func (d *Driver) Put(id string) error { - mountpoint := d.mountPath(id) - logrus.Debugf(`[zfs] unmount("%s")`, mountpoint) + d.Lock() + defer d.Unlock() - if err := mount.Unmount(mountpoint); err != nil { - return fmt.Errorf("error unmounting to %s: %v", mountpoint, err) + mnt := d.active[id] + if mnt == nil { + logrus.Debugf("[zfs] Put on a non-mounted device %s", id) + // but it might be still here + if d.Exists(id) { + err := mount.Unmount(d.mountPath(id)) + if err != nil { + logrus.Debugf("[zfs] Failed to unmount %s zfs fs: %v", id, err) + } + } + return nil + } + + mnt.count-- + if mnt.count > 0 { + return nil + } + + defer delete(d.active, id) + if mnt.mounted { + logrus.Debugf(`[zfs] unmount("%s")`, mnt.path) + + if err := mount.Unmount(mnt.path); err != nil { + return fmt.Errorf("error unmounting to %s: %v", mnt.path, err) + } } return nil } diff --git a/components/engine/daemon/image_delete.go b/components/engine/daemon/image_delete.go index 0f32a903e6..7c6329a669 100644 --- a/components/engine/daemon/image_delete.go +++ b/components/engine/daemon/image_delete.go @@ -5,7 +5,7 @@ import ( "strings" "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" + "github.com/docker/docker/errors" "github.com/docker/docker/image" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/reference" @@ -43,7 +43,7 @@ const ( // // Hard Conflict: // - a pull or build using the image. -// - any descendent image. +// - any descendant image. // - any running container using the image. // // Soft Conflict: @@ -82,7 +82,8 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I // this image would remain "dangling" and since // we really want to avoid that the client must // explicitly force its removal. - return nil, derr.ErrorCodeImgDelUsed.WithArgs(imageRef, stringid.TruncateID(container.ID), stringid.TruncateID(imgID.String())) + err := fmt.Errorf("conflict: unable to remove repository reference %q (must force) - container %s is using its referenced image %s", imageRef, stringid.TruncateID(container.ID), stringid.TruncateID(imgID.String())) + return nil, errors.NewRequestConflictError(err) } } @@ -313,7 +314,7 @@ func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDe // image or any stopped container using the image. If ignoreSoftConflicts is // true, this function will not check for soft conflict conditions. func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType) *imageDeleteConflict { - // Check if the image has any descendent images. + // Check if the image has any descendant images. if mask&conflictDependentChild != 0 && len(daemon.imageStore.Children(imgID)) > 0 { return &imageDeleteConflict{ hard: true, diff --git a/components/engine/daemon/import.go b/components/engine/daemon/import.go index c04e8a38f2..4961a30fd9 100644 --- a/components/engine/daemon/import.go +++ b/components/engine/daemon/import.go @@ -11,6 +11,7 @@ import ( "github.com/docker/docker/dockerversion" "github.com/docker/docker/image" "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/streamformatter" @@ -24,13 +25,13 @@ import ( // the repo and tag arguments, respectively. func (daemon *Daemon) ImportImage(src string, newRef reference.Named, msg string, inConfig io.ReadCloser, outStream io.Writer, config *container.Config) error { var ( - sf = streamformatter.NewJSONStreamFormatter() - archive io.ReadCloser - resp *http.Response + sf = streamformatter.NewJSONStreamFormatter() + rc io.ReadCloser + resp *http.Response ) if src == "-" { - archive = inConfig + rc = inConfig } else { inConfig.Close() u, err := url.Parse(src) @@ -48,15 +49,20 @@ func (daemon *Daemon) ImportImage(src string, newRef reference.Named, msg string return err } progressOutput := sf.NewProgressOutput(outStream, true) - archive = progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Importing") + rc = progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Importing") } - defer archive.Close() + defer rc.Close() if len(msg) == 0 { msg = "Imported from " + src } + + inflatedLayerData, err := archive.DecompressStream(rc) + if err != nil { + return err + } // TODO: support windows baselayer? - l, err := daemon.layerStore.Register(archive, "") + l, err := daemon.layerStore.Register(inflatedLayerData, "") if err != nil { return err } diff --git a/components/engine/daemon/info.go b/components/engine/daemon/info.go index 20d8356d44..1ee367add1 100644 --- a/components/engine/daemon/info.go +++ b/components/engine/daemon/info.go @@ -20,6 +20,7 @@ import ( "github.com/docker/docker/utils" "github.com/docker/docker/volume/drivers" "github.com/docker/engine-api/types" + "github.com/docker/go-connections/sockets" ) // SystemInfo returns information about the host server the daemon is running on. @@ -74,14 +75,15 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { DriverStatus: daemon.layerStore.DriverStatus(), Plugins: daemon.showPluginsInfo(), IPv4Forwarding: !sysInfo.IPv4ForwardingDisabled, - BridgeNfIptables: !sysInfo.BridgeNfCallIptablesDisabled, - BridgeNfIP6tables: !sysInfo.BridgeNfCallIP6tablesDisabled, + BridgeNfIptables: !sysInfo.BridgeNFCallIPTablesDisabled, + BridgeNfIP6tables: !sysInfo.BridgeNFCallIP6TablesDisabled, Debug: utils.IsDebugEnabled(), NFd: fileutils.GetTotalUsedFds(), NGoroutines: runtime.NumGoroutine(), SystemTime: time.Now().Format(time.RFC3339Nano), ExecutionDriver: daemon.ExecutionDriver().Name(), LoggingDriver: daemon.defaultLogConfig.Type, + CgroupDriver: daemon.getCgroupDriver(), NEventsListener: daemon.EventsService.SubscribersCount(), KernelVersion: kernelVersion, OperatingSystem: operatingSystem, @@ -97,9 +99,9 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { ServerVersion: dockerversion.Version, ClusterStore: daemon.configStore.ClusterStore, ClusterAdvertise: daemon.configStore.ClusterAdvertise, - HTTPProxy: getProxyEnv("http_proxy"), - HTTPSProxy: getProxyEnv("https_proxy"), - NoProxy: getProxyEnv("no_proxy"), + HTTPProxy: sockets.GetProxyEnv("http_proxy"), + HTTPSProxy: sockets.GetProxyEnv("https_proxy"), + NoProxy: sockets.GetProxyEnv("no_proxy"), } // TODO Windows. Refactor this more once sysinfo is refactored into @@ -109,6 +111,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { if runtime.GOOS != "windows" { v.MemoryLimit = sysInfo.MemoryLimit v.SwapLimit = sysInfo.SwapLimit + v.KernelMemory = sysInfo.KernelMemory v.OomKillDisable = sysInfo.OomKillDisable v.CPUCfsPeriod = sysInfo.CPUCfsPeriod v.CPUCfsQuota = sysInfo.CPUCfsQuota diff --git a/components/engine/daemon/inspect.go b/components/engine/daemon/inspect.go index feb7de28f1..511454f9c7 100644 --- a/components/engine/daemon/inspect.go +++ b/components/engine/daemon/inspect.go @@ -4,8 +4,8 @@ import ( "fmt" "time" + "github.com/docker/docker/api/types/backend" "github.com/docker/docker/container" - "github.com/docker/docker/daemon/exec" "github.com/docker/docker/daemon/network" "github.com/docker/docker/pkg/version" "github.com/docker/engine-api/types" @@ -175,12 +175,26 @@ func (daemon *Daemon) getInspectData(container *container.Container, size bool) // ContainerExecInspect returns low-level information about the exec // command. An error is returned if the exec cannot be found. -func (daemon *Daemon) ContainerExecInspect(id string) (*exec.Config, error) { - eConfig, err := daemon.getExecConfig(id) +func (daemon *Daemon) ContainerExecInspect(id string) (*backend.ExecInspect, error) { + e, err := daemon.getExecConfig(id) if err != nil { return nil, err } - return eConfig, nil + + pc := inspectExecProcessConfig(e) + + return &backend.ExecInspect{ + ID: e.ID, + Running: e.Running, + ExitCode: e.ExitCode, + ProcessConfig: pc, + OpenStdin: e.OpenStdin, + OpenStdout: e.OpenStdout, + OpenStderr: e.OpenStderr, + CanRemove: e.CanRemove, + ContainerID: e.ContainerID, + DetachKeys: e.DetachKeys, + }, nil } // VolumeInspect looks up a volume by name. An error is returned if diff --git a/components/engine/daemon/inspect_unix.go b/components/engine/daemon/inspect_unix.go index b9321f34c3..bb224c8796 100644 --- a/components/engine/daemon/inspect_unix.go +++ b/components/engine/daemon/inspect_unix.go @@ -3,7 +3,9 @@ package daemon import ( + "github.com/docker/docker/api/types/backend" "github.com/docker/docker/container" + "github.com/docker/docker/daemon/exec" "github.com/docker/engine-api/types" "github.com/docker/engine-api/types/versions/v1p19" ) @@ -77,3 +79,13 @@ func addMountPoints(container *container.Container) []types.MountPoint { } return mountPoints } + +func inspectExecProcessConfig(e *exec.Config) *backend.ExecProcessConfig { + return &backend.ExecProcessConfig{ + Tty: e.ProcessConfig.Tty, + Entrypoint: e.ProcessConfig.Entrypoint, + Arguments: e.ProcessConfig.Arguments, + Privileged: &e.ProcessConfig.Privileged, + User: e.ProcessConfig.User, + } +} diff --git a/components/engine/daemon/inspect_windows.go b/components/engine/daemon/inspect_windows.go index e42a61dadc..f20571d052 100644 --- a/components/engine/daemon/inspect_windows.go +++ b/components/engine/daemon/inspect_windows.go @@ -1,7 +1,9 @@ package daemon import ( + "github.com/docker/docker/api/types/backend" "github.com/docker/docker/container" + "github.com/docker/docker/daemon/exec" "github.com/docker/engine-api/types" ) @@ -28,3 +30,11 @@ func addMountPoints(container *container.Container) []types.MountPoint { func (daemon *Daemon) containerInspectPre120(name string) (*types.ContainerJSON, error) { return daemon.containerInspectCurrent(name, false) } + +func inspectExecProcessConfig(e *exec.Config) *backend.ExecProcessConfig { + return &backend.ExecProcessConfig{ + Tty: e.ProcessConfig.Tty, + Entrypoint: e.ProcessConfig.Entrypoint, + Arguments: e.ProcessConfig.Arguments, + } +} diff --git a/components/engine/daemon/kill.go b/components/engine/daemon/kill.go index 2bb695699c..4d29346d2f 100644 --- a/components/engine/daemon/kill.go +++ b/components/engine/daemon/kill.go @@ -8,10 +8,25 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/signal" ) +type errNoSuchProcess struct { + pid int + signal int +} + +func (e errNoSuchProcess) Error() string { + return fmt.Sprintf("Cannot kill process (pid=%d) with signal %d: no such process.", e.pid, e.signal) +} + +// isErrNoSuchProcess returns true if the error +// is an instance of errNoSuchProcess. +func isErrNoSuchProcess(err error) bool { + _, ok := err.(errNoSuchProcess) + return ok +} + // ContainerKill send signal to the container // If no signal is given (sig 0), then Kill with SIGKILL and wait // for the container to exit. @@ -45,11 +60,11 @@ func (daemon *Daemon) killWithSignal(container *container.Container, sig int) er // We could unpause the container for them rather than returning this error if container.Paused { - return derr.ErrorCodeUnpauseContainer.WithArgs(container.ID) + return fmt.Errorf("Container %s is paused. Unpause the container before stopping", container.ID) } if !container.Running { - return derr.ErrorCodeNotRunning.WithArgs(container.ID) + return errNotRunning{container.ID} } container.ExitOnNext() @@ -62,7 +77,7 @@ func (daemon *Daemon) killWithSignal(container *container.Container, sig int) er } if err := daemon.kill(container, sig); err != nil { - return derr.ErrorCodeCantKill.WithArgs(container.ID, err) + return fmt.Errorf("Cannot kill container %s: %s", container.ID, err) } attributes := map[string]string{ @@ -75,7 +90,7 @@ func (daemon *Daemon) killWithSignal(container *container.Container, sig int) er // Kill forcefully terminates a container. func (daemon *Daemon) Kill(container *container.Container) error { if !container.IsRunning() { - return derr.ErrorCodeNotRunning.WithArgs(container.ID) + return errNotRunning{container.ID} } // 1. Send SIGKILL @@ -90,6 +105,9 @@ func (daemon *Daemon) Kill(container *container.Container) error { // So, instead we'll give it up to 2 more seconds to complete and if // by that time the container is still running, then the error // we got is probably valid and so we return it to the caller. + if isErrNoSuchProcess(err) { + return nil + } if container.IsRunning() { container.WaitStop(2 * time.Second) @@ -101,6 +119,9 @@ func (daemon *Daemon) Kill(container *container.Container) error { // 2. Wait for the process to die, in last resort, try to kill the process directly if err := killProcessDirectly(container); err != nil { + if isErrNoSuchProcess(err) { + return nil + } return err } @@ -112,8 +133,9 @@ func (daemon *Daemon) Kill(container *container.Container) error { func (daemon *Daemon) killPossiblyDeadProcess(container *container.Container, sig int) error { err := daemon.killWithSignal(container, sig) if err == syscall.ESRCH { - logrus.Debugf("Cannot kill process (pid=%d) with signal %d: no such process.", container.GetPID(), sig) - return nil + e := errNoSuchProcess{container.GetPID(), sig} + logrus.Debug(e) + return e } return err } diff --git a/components/engine/daemon/list.go b/components/engine/daemon/list.go index 472aa8894f..e84353419f 100644 --- a/components/engine/daemon/list.go +++ b/components/engine/daemon/list.go @@ -9,6 +9,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/container" "github.com/docker/docker/image" + "github.com/docker/docker/volume" "github.com/docker/engine-api/types" "github.com/docker/engine-api/types/filters" networktypes "github.com/docker/engine-api/types/network" @@ -225,8 +226,24 @@ func (daemon *Daemon) foldFilter(config *types.ContainerListOptions) (*listConte // includeContainerInList decides whether a containers should be include in the output or not based in the filter. // It also decides if the iteration should be stopped or not. func includeContainerInList(container *container.Container, ctx *listContext) iterationAction { + // Do not include container if it's in the list before the filter container. + // Set the filter container to nil to include the rest of containers after this one. + if ctx.beforeFilter != nil { + if container.ID == ctx.beforeFilter.ID { + ctx.beforeFilter = nil + } + return excludeContainer + } + + // Stop iteration when the container arrives to the filter container + if ctx.sinceFilter != nil { + if container.ID == ctx.sinceFilter.ID { + return stopIteration + } + } + // Do not include container if it's stopped and we're not filters - // FIXME remove the ctx.beforContainer part of the condition for 1.12 as --since and --before are deprecated + // FIXME remove the ctx.beforContainer and ctx.sinceContainer part of the condition for 1.12 as --since and --before are deprecated if !container.Running && !ctx.All && ctx.Limit <= 0 && ctx.beforeContainer == nil && ctx.sinceContainer == nil { return excludeContainer } @@ -246,7 +263,7 @@ func includeContainerInList(container *container.Container, ctx *listContext) it return excludeContainer } - // Do not include container if the isolation mode doesn't match + // Do not include container if isolation doesn't match if excludeContainer == excludeByIsolation(container, ctx) { return excludeContainer } @@ -266,22 +283,6 @@ func includeContainerInList(container *container.Container, ctx *listContext) it } } - // Do not include container if it's in the list before the filter container. - // Set the filter container to nil to include the rest of containers after this one. - if ctx.beforeFilter != nil { - if container.ID == ctx.beforeFilter.ID { - ctx.beforeFilter = nil - } - return excludeContainer - } - - // Stop iteration when the container arrives to the filter container - if ctx.sinceFilter != nil { - if container.ID == ctx.sinceFilter.ID { - return stopIteration - } - } - // Stop iteration when the index is over the limit if ctx.Limit > 0 && ctx.idx == ctx.Limit { return stopIteration @@ -306,6 +307,27 @@ func includeContainerInList(container *container.Container, ctx *listContext) it return excludeContainer } + if ctx.filters.Include("volume") { + volumesByName := make(map[string]*volume.MountPoint) + for _, m := range container.MountPoints { + volumesByName[m.Name] = m + } + + volumeExist := fmt.Errorf("volume mounted in container") + err := ctx.filters.WalkValues("volume", func(value string) error { + if _, exist := container.MountPoints[value]; exist { + return volumeExist + } + if _, exist := volumesByName[value]; exist { + return volumeExist + } + return nil + }) + if err != volumeExist { + return excludeContainer + } + } + if ctx.ancestorFilter { if len(ctx.images) == 0 { return excludeContainer @@ -419,6 +441,7 @@ func (daemon *Daemon) transformContainer(container *container.Container, ctx *li newC.SizeRootFs = sizeRootFs } newC.Labels = container.Config.Labels + newC.Mounts = addMountPoints(container) return newC, nil } diff --git a/components/engine/daemon/logdrivers_linux.go b/components/engine/daemon/logdrivers_linux.go index 0abc6269de..89fe49a858 100644 --- a/components/engine/daemon/logdrivers_linux.go +++ b/components/engine/daemon/logdrivers_linux.go @@ -5,6 +5,7 @@ import ( // therefore they register themselves to the logdriver factory. _ "github.com/docker/docker/daemon/logger/awslogs" _ "github.com/docker/docker/daemon/logger/fluentd" + _ "github.com/docker/docker/daemon/logger/gcplogs" _ "github.com/docker/docker/daemon/logger/gelf" _ "github.com/docker/docker/daemon/logger/journald" _ "github.com/docker/docker/daemon/logger/jsonfilelog" diff --git a/components/engine/daemon/logdrivers_windows.go b/components/engine/daemon/logdrivers_windows.go index d3710ec174..129b06650b 100644 --- a/components/engine/daemon/logdrivers_windows.go +++ b/components/engine/daemon/logdrivers_windows.go @@ -4,6 +4,7 @@ import ( // Importing packages here only to make sure their init gets called and // therefore they register themselves to the logdriver factory. _ "github.com/docker/docker/daemon/logger/awslogs" + _ "github.com/docker/docker/daemon/logger/etwlogs" _ "github.com/docker/docker/daemon/logger/jsonfilelog" _ "github.com/docker/docker/daemon/logger/splunk" ) diff --git a/components/engine/daemon/logger/etwlogs/etwlogs_windows.go b/components/engine/daemon/logger/etwlogs/etwlogs_windows.go new file mode 100644 index 0000000000..de128a2e93 --- /dev/null +++ b/components/engine/daemon/logger/etwlogs/etwlogs_windows.go @@ -0,0 +1,183 @@ +// Package etwlogs provides a log driver for forwarding container logs +// as ETW events.(ETW stands for Event Tracing for Windows) +// A client can then create an ETW listener to listen for events that are sent +// by the ETW provider that we register, using the provider's GUID "a3693192-9ed6-46d2-a981-f8226c8363bd". +// Here is an example of how to do this using the logman utility: +// 1. logman start -ets DockerContainerLogs -p {a3693192-9ed6-46d2-a981-f8226c8363bd} 0 0 -o trace.etl +// 2. Run container(s) and generate log messages +// 3. logman stop -ets DockerContainerLogs +// 4. You can then convert the etl log file to XML using: tracerpt -y trace.etl +// +// Each container log message generates a ETW event that also contains: +// the container name and ID, the timestamp, and the stream type. +package etwlogs + +import ( + "errors" + "fmt" + "sync" + "syscall" + "unsafe" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/daemon/logger" +) + +type etwLogs struct { + containerName string + imageName string + containerID string + imageID string +} + +const ( + name = "etwlogs" + win32CallSuccess = 0 +) + +var win32Lib *syscall.DLL +var providerHandle syscall.Handle +var refCount int +var mu sync.Mutex + +func init() { + providerHandle = syscall.InvalidHandle + if err := logger.RegisterLogDriver(name, New); err != nil { + logrus.Fatal(err) + } +} + +// New creates a new etwLogs logger for the given container and registers the EWT provider. +func New(ctx logger.Context) (logger.Logger, error) { + if err := registerETWProvider(); err != nil { + return nil, err + } + logrus.Debugf("logging driver etwLogs configured for container: %s.", ctx.ContainerID) + + return &etwLogs{ + containerName: fixContainerName(ctx.ContainerName), + imageName: ctx.ContainerImageName, + containerID: ctx.ContainerID, + imageID: ctx.ContainerImageID, + }, nil +} + +// Log logs the message to the ETW stream. +func (etwLogger *etwLogs) Log(msg *logger.Message) error { + if providerHandle == syscall.InvalidHandle { + // This should never be hit, if it is, it indicates a programming error. + errorMessage := "ETWLogs cannot log the message, because the event provider has not been registered." + logrus.Error(errorMessage) + return errors.New(errorMessage) + } + return callEventWriteString(createLogMessage(etwLogger, msg)) +} + +// Close closes the logger by unregistering the ETW provider. +func (etwLogger *etwLogs) Close() error { + unregisterETWProvider() + return nil +} + +func (etwLogger *etwLogs) Name() string { + return name +} + +func createLogMessage(etwLogger *etwLogs, msg *logger.Message) string { + return fmt.Sprintf("container_name: %s, image_name: %s, container_id: %s, image_id: %s, source: %s, log: %s", + etwLogger.containerName, + etwLogger.imageName, + etwLogger.containerID, + etwLogger.imageID, + msg.Source, + msg.Line) +} + +// fixContainerName removes the initial '/' from the container name. +func fixContainerName(cntName string) string { + if len(cntName) > 0 && cntName[0] == '/' { + cntName = cntName[1:] + } + return cntName +} + +func registerETWProvider() error { + mu.Lock() + defer mu.Unlock() + if refCount == 0 { + var err error + if win32Lib, err = syscall.LoadDLL("Advapi32.dll"); err != nil { + return err + } + if err = callEventRegister(); err != nil { + win32Lib.Release() + win32Lib = nil + return err + } + } + + refCount++ + return nil +} + +func unregisterETWProvider() { + mu.Lock() + defer mu.Unlock() + if refCount == 1 { + if callEventUnregister() { + refCount-- + providerHandle = syscall.InvalidHandle + win32Lib.Release() + win32Lib = nil + } + // Not returning an error if EventUnregister fails, because etwLogs will continue to work + } else { + refCount-- + } +} + +func callEventRegister() error { + proc, err := win32Lib.FindProc("EventRegister") + if err != nil { + return err + } + // The provider's GUID is {a3693192-9ed6-46d2-a981-f8226c8363bd} + guid := syscall.GUID{ + 0xa3693192, 0x9ed6, 0x46d2, + [8]byte{0xa9, 0x81, 0xf8, 0x22, 0x6c, 0x83, 0x63, 0xbd}, + } + + ret, _, _ := proc.Call(uintptr(unsafe.Pointer(&guid)), 0, 0, uintptr(unsafe.Pointer(&providerHandle))) + if ret != win32CallSuccess { + errorMessage := fmt.Sprintf("Failed to register ETW provider. Error: %d", ret) + logrus.Error(errorMessage) + return errors.New(errorMessage) + } + return nil +} + +func callEventWriteString(message string) error { + proc, err := win32Lib.FindProc("EventWriteString") + if err != nil { + return err + } + ret, _, _ := proc.Call(uintptr(providerHandle), 0, 0, uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(message)))) + if ret != win32CallSuccess { + errorMessage := fmt.Sprintf("ETWLogs provider failed to log message. Error: %d", ret) + logrus.Error(errorMessage) + return errors.New(errorMessage) + } + return nil +} + +func callEventUnregister() bool { + proc, err := win32Lib.FindProc("EventUnregister") + if err != nil { + return false + } + ret, _, _ := proc.Call(uintptr(providerHandle)) + if ret != win32CallSuccess { + return false + } + return true +} diff --git a/components/engine/daemon/logger/gcplogs/gcplogging.go b/components/engine/daemon/logger/gcplogs/gcplogging.go new file mode 100644 index 0000000000..b9b8af5871 --- /dev/null +++ b/components/engine/daemon/logger/gcplogs/gcplogging.go @@ -0,0 +1,181 @@ +package gcplogs + +import ( + "fmt" + "sync/atomic" + "time" + + "github.com/docker/docker/daemon/logger" + + "github.com/Sirupsen/logrus" + "golang.org/x/net/context" + "google.golang.org/cloud/compute/metadata" + "google.golang.org/cloud/logging" +) + +const ( + name = "gcplogs" + + projectOptKey = "gcp-project" + logLabelsKey = "labels" + logEnvKey = "env" + logCmdKey = "gcp-log-cmd" +) + +var ( + // The number of logs the gcplogs driver has dropped. + droppedLogs uint64 + + onGCE = metadata.OnGCE() + + // instance metadata populated from the metadata server if available + projectID string + zone string + instanceName string + instanceID string +) + +func init() { + if onGCE { + // These will fail on instances if the metadata service is + // down or the client is compiled with an API version that + // has been removed. Since these are not vital, let's ignore + // them and make their fields in the dockeLogEntry ,omitempty + projectID, _ = metadata.ProjectID() + zone, _ = metadata.Zone() + instanceName, _ = metadata.InstanceName() + instanceID, _ = metadata.InstanceID() + } + + if err := logger.RegisterLogDriver(name, New); err != nil { + logrus.Fatal(err) + } + + if err := logger.RegisterLogOptValidator(name, ValidateLogOpts); err != nil { + logrus.Fatal(err) + } +} + +type gcplogs struct { + client *logging.Client + instance *instanceInfo + container *containerInfo +} + +type dockerLogEntry struct { + Instance *instanceInfo `json:"instance,omitempty"` + Container *containerInfo `json:"container,omitempty"` + Data string `json:"data,omitempty"` +} + +type instanceInfo struct { + Zone string `json:"zone,omitempty"` + Name string `json:"name,omitempty"` + ID string `json:"id,omitempty"` +} + +type containerInfo struct { + Name string `json:"name,omitempty"` + ID string `json:"id,omitempty"` + ImageName string `json:"imageName,omitempty"` + ImageID string `json:"imageId,omitempty"` + Created time.Time `json:"created,omitempty"` + Command string `json:"command,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// New creates a new logger that logs to Google Cloud Logging using the application +// default credentials. +// +// See https://developers.google.com/identity/protocols/application-default-credentials +func New(ctx logger.Context) (logger.Logger, error) { + + var project string + if projectID != "" { + project = projectID + } + if projectID, found := ctx.Config[projectOptKey]; found { + project = projectID + } + if project == "" { + return nil, fmt.Errorf("No project was specified and couldn't read project from the meatadata server. Please specify a project") + } + + c, err := logging.NewClient(context.Background(), project, "gcplogs-docker-driver") + if err != nil { + return nil, err + } + + if err := c.Ping(); err != nil { + return nil, fmt.Errorf("unable to connect or authenticate with Google Cloud Logging: %v", err) + } + + l := &gcplogs{ + client: c, + container: &containerInfo{ + Name: ctx.ContainerName, + ID: ctx.ContainerID, + ImageName: ctx.ContainerImageName, + ImageID: ctx.ContainerImageID, + Created: ctx.ContainerCreated, + Metadata: ctx.ExtraAttributes(nil), + }, + } + + if ctx.Config[logCmdKey] == "true" { + l.container.Command = ctx.Command() + } + + if onGCE { + l.instance = &instanceInfo{ + Zone: zone, + Name: instanceName, + ID: instanceID, + } + } + + // The logger "overflows" at a rate of 10,000 logs per second and this + // overflow func is called. We want to surface the error to the user + // without overly spamming /var/log/docker.log so we log the first time + // we overflow and every 1000th time after. + c.Overflow = func(_ *logging.Client, _ logging.Entry) error { + if i := atomic.AddUint64(&droppedLogs, 1); i%1000 == 1 { + logrus.Errorf("gcplogs driver has dropped %v logs", i) + } + return nil + } + + return l, nil +} + +// ValidateLogOpts validates the opts passed to the gcplogs driver. Currently, the gcplogs +// driver doesn't take any arguments. +func ValidateLogOpts(cfg map[string]string) error { + for k := range cfg { + switch k { + case projectOptKey, logLabelsKey, logEnvKey, logCmdKey: + default: + return fmt.Errorf("%q is not a valid option for the gcplogs driver", k) + } + } + return nil +} + +func (l *gcplogs) Log(m *logger.Message) error { + return l.client.Log(logging.Entry{ + Time: m.Timestamp, + Payload: &dockerLogEntry{ + Instance: l.instance, + Container: l.container, + Data: string(m.Line), + }, + }) +} + +func (l *gcplogs) Close() error { + return l.client.Flush() +} + +func (l *gcplogs) Name() string { + return name +} diff --git a/components/engine/daemon/logger/journald/read.go b/components/engine/daemon/logger/journald/read.go index 80c1fbda7a..8d94c302fb 100644 --- a/components/engine/daemon/logger/journald/read.go +++ b/components/engine/daemon/logger/journald/read.go @@ -2,7 +2,6 @@ package journald -// #cgo pkg-config: libsystemd-journal // #include // #include // #include diff --git a/components/engine/daemon/logger/journald/read_native.go b/components/engine/daemon/logger/journald/read_native.go new file mode 100644 index 0000000000..bba6de55be --- /dev/null +++ b/components/engine/daemon/logger/journald/read_native.go @@ -0,0 +1,6 @@ +// +build linux,cgo,!static_build,journald,!journald_compat + +package journald + +// #cgo pkg-config: libsystemd +import "C" diff --git a/components/engine/daemon/logger/journald/read_native_compat.go b/components/engine/daemon/logger/journald/read_native_compat.go new file mode 100644 index 0000000000..3f7a43c59e --- /dev/null +++ b/components/engine/daemon/logger/journald/read_native_compat.go @@ -0,0 +1,6 @@ +// +build linux,cgo,!static_build,journald,journald_compat + +package journald + +// #cgo pkg-config: libsystemd-journal +import "C" diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go index 86baa316b9..9faa4e02db 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go @@ -25,7 +25,6 @@ type JSONFileLogger struct { buf *bytes.Buffer writer *loggerutils.RotateFileWriter mu sync.Mutex - ctx logger.Context readers map[*logger.LogWatcher]struct{} // stores the active log followers extra []byte // json-encoded extra attributes } @@ -91,7 +90,6 @@ func (l *JSONFileLogger) Log(msg *logger.Message) error { return err } l.mu.Lock() - defer l.mu.Unlock() err = (&jsonlog.JSONLogs{ Log: append(msg.Line, '\n'), Stream: msg.Source, @@ -99,12 +97,14 @@ func (l *JSONFileLogger) Log(msg *logger.Message) error { RawAttrs: l.extra, }).MarshalJSONBuf(l.buf) if err != nil { + l.mu.Unlock() return err } l.buf.WriteByte('\n') _, err = l.writer.Write(l.buf.Bytes()) l.buf.Reset() + l.mu.Unlock() return err } diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go index 162c685b24..ef840531a1 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go @@ -199,3 +199,50 @@ func TestJSONFileLoggerWithLabelsEnv(t *testing.T) { t.Fatalf("Wrong log attrs: %q, expected %q", extra, expected) } } + +func BenchmarkJSONFileLoggerWithReader(b *testing.B) { + b.StopTimer() + b.ResetTimer() + cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657" + dir, err := ioutil.TempDir("", "json-logger-bench") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(dir) + + l, err := New(logger.Context{ + ContainerID: cid, + LogPath: filepath.Join(dir, "container.log"), + }) + if err != nil { + b.Fatal(err) + } + defer l.Close() + msg := &logger.Message{ContainerID: cid, Line: []byte("line"), Source: "src1"} + jsonlog, err := (&jsonlog.JSONLog{Log: string(msg.Line) + "\n", Stream: msg.Source, Created: msg.Timestamp}).MarshalJSON() + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(jsonlog)+1) * 30) + + b.StartTimer() + + go func() { + for i := 0; i < b.N; i++ { + for j := 0; j < 30; j++ { + l.Log(msg) + } + } + l.Close() + }() + + lw := l.(logger.LogReader).ReadLogs(logger.ReadConfig{Follow: true}) + watchClose := lw.WatchClose() + for { + select { + case <-lw.Msg: + case <-watchClose: + return + } + } +} diff --git a/components/engine/daemon/logger/loggerutils/rotatefilewriter.go b/components/engine/daemon/logger/loggerutils/rotatefilewriter.go index 0d2553a13f..de7112896c 100644 --- a/components/engine/daemon/logger/loggerutils/rotatefilewriter.go +++ b/components/engine/daemon/logger/loggerutils/rotatefilewriter.go @@ -13,6 +13,7 @@ type RotateFileWriter struct { f *os.File // store for closing mu sync.Mutex capacity int64 //maximum size of each file + currentSize int64 // current size of the latest file maxFiles int //maximum number of files notifyRotate *pubsub.Publisher } @@ -21,12 +22,18 @@ type RotateFileWriter struct { func NewRotateFileWriter(logPath string, capacity int64, maxFiles int) (*RotateFileWriter, error) { log, err := os.OpenFile(logPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640) if err != nil { - return &RotateFileWriter{}, err + return nil, err + } + + size, err := log.Seek(0, os.SEEK_END) + if err != nil { + return nil, err } return &RotateFileWriter{ f: log, capacity: capacity, + currentSize: size, maxFiles: maxFiles, notifyRotate: pubsub.NewPublisher(0, 1), }, nil @@ -35,12 +42,17 @@ func NewRotateFileWriter(logPath string, capacity int64, maxFiles int) (*RotateF //WriteLog write log message to File func (w *RotateFileWriter) Write(message []byte) (int, error) { w.mu.Lock() - defer w.mu.Unlock() if err := w.checkCapacityAndRotate(); err != nil { + w.mu.Unlock() return -1, err } - return w.f.Write(message) + n, err := w.f.Write(message) + if err == nil { + w.currentSize += int64(n) + } + w.mu.Unlock() + return n, err } func (w *RotateFileWriter) checkCapacityAndRotate() error { @@ -48,12 +60,7 @@ func (w *RotateFileWriter) checkCapacityAndRotate() error { return nil } - meta, err := w.f.Stat() - if err != nil { - return err - } - - if meta.Size() >= w.capacity { + if w.currentSize >= w.capacity { name := w.f.Name() if err := w.f.Close(); err != nil { return err @@ -66,6 +73,7 @@ func (w *RotateFileWriter) checkCapacityAndRotate() error { return err } w.f = file + w.currentSize = 0 w.notifyRotate.Publish(struct{}{}) } diff --git a/components/engine/daemon/logs.go b/components/engine/daemon/logs.go index eb6fa54ae6..8172df175c 100644 --- a/components/engine/daemon/logs.go +++ b/components/engine/daemon/logs.go @@ -1,6 +1,7 @@ package daemon import ( + "fmt" "io" "strconv" "time" @@ -10,9 +11,9 @@ import ( "github.com/docker/docker/container" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/jsonfilelog" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stdcopy" + containertypes "github.com/docker/engine-api/types/container" timetypes "github.com/docker/engine-api/types/time" ) @@ -21,11 +22,11 @@ import ( func (daemon *Daemon) ContainerLogs(containerName string, config *backend.ContainerLogsConfig, started chan struct{}) error { container, err := daemon.GetContainer(containerName) if err != nil { - return derr.ErrorCodeNoSuchContainer.WithArgs(containerName) + return err } if !(config.ShowStdout || config.ShowStderr) { - return derr.ErrorCodeNeedStream + return fmt.Errorf("You must choose at least one stream") } cLog, err := daemon.getLogger(container) @@ -103,7 +104,7 @@ func (daemon *Daemon) getLogger(container *container.Container) (logger.Logger, if container.LogDriver != nil && container.IsRunning() { return container.LogDriver, nil } - cfg := container.GetLogConfig(daemon.defaultLogConfig) + cfg := daemon.getLogConfig(container.HostConfig.LogConfig) if err := logger.ValidateLogOpts(cfg.Type, cfg.Config); err != nil { return nil, err } @@ -112,7 +113,7 @@ func (daemon *Daemon) getLogger(container *container.Container) (logger.Logger, // StartLogging initializes and starts the container logging stream. func (daemon *Daemon) StartLogging(container *container.Container) error { - cfg := container.GetLogConfig(daemon.defaultLogConfig) + cfg := daemon.getLogConfig(container.HostConfig.LogConfig) if cfg.Type == "none" { return nil // do not start logging routines } @@ -122,7 +123,7 @@ func (daemon *Daemon) StartLogging(container *container.Container) error { } l, err := container.StartLogger(cfg) if err != nil { - return derr.ErrorCodeInitLogger.WithArgs(err) + return fmt.Errorf("Failed to initialize logging driver: %v", err) } copier := logger.NewCopier(container.ID, map[string]io.Reader{"stdout": container.StdoutPipe(), "stderr": container.StderrPipe()}, l) @@ -137,3 +138,16 @@ func (daemon *Daemon) StartLogging(container *container.Container) error { return nil } + +// getLogConfig returns the log configuration for the container. +func (daemon *Daemon) getLogConfig(cfg containertypes.LogConfig) containertypes.LogConfig { + if cfg.Type != "" || len(cfg.Config) > 0 { // container has log driver configured + if cfg.Type == "" { + cfg.Type = jsonfilelog.Name + } + return cfg + } + + // Use daemon's default log config for containers + return daemon.defaultLogConfig +} diff --git a/components/engine/daemon/mounts.go b/components/engine/daemon/mounts.go index 276301d130..d4f24b2812 100644 --- a/components/engine/daemon/mounts.go +++ b/components/engine/daemon/mounts.go @@ -1,10 +1,10 @@ package daemon import ( + "fmt" "strings" "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" volumestore "github.com/docker/docker/volume/store" ) @@ -42,7 +42,7 @@ func (daemon *Daemon) removeMountPoints(container *container.Container, rm bool) } } if len(rmErrors) > 0 { - return derr.ErrorCodeRemovingVolume.WithArgs(strings.Join(rmErrors, "\n")) + return fmt.Errorf("Error removing volumes:\n%v", strings.Join(rmErrors, "\n")) } return nil } diff --git a/components/engine/daemon/network.go b/components/engine/daemon/network.go index 07d0771f7c..e937391b83 100644 --- a/components/engine/daemon/network.go +++ b/components/engine/daemon/network.go @@ -3,9 +3,10 @@ package daemon import ( "fmt" "net" + "net/http" "strings" - derr "github.com/docker/docker/errors" + "github.com/docker/docker/errors" "github.com/docker/docker/runconfig" "github.com/docker/engine-api/types/network" "github.com/docker/libnetwork" @@ -90,7 +91,7 @@ func (daemon *Daemon) GetAllNetworks() []libnetwork.Network { } // CreateNetwork creates a network with the given name, driver and other optional parameters -func (daemon *Daemon) CreateNetwork(name, driver string, ipam network.IPAM, options map[string]string, internal bool) (libnetwork.Network, error) { +func (daemon *Daemon) CreateNetwork(name, driver string, ipam network.IPAM, netOption map[string]string, internal bool, enableIPv6 bool) (libnetwork.Network, error) { c := daemon.netController if driver == "" { driver = c.Config().Daemon.DefaultDriver @@ -104,7 +105,8 @@ func (daemon *Daemon) CreateNetwork(name, driver string, ipam network.IPAM, opti } nwOptions = append(nwOptions, libnetwork.NetworkOptionIpam(ipam.Driver, "", v4Conf, v6Conf, ipam.Options)) - nwOptions = append(nwOptions, libnetwork.NetworkOptionDriverOpts(options)) + nwOptions = append(nwOptions, libnetwork.NetworkOptionEnableIPv6(enableIPv6)) + nwOptions = append(nwOptions, libnetwork.NetworkOptionDriverOpts(netOption)) if internal { nwOptions = append(nwOptions, libnetwork.NetworkOptionInternalNetwork()) } @@ -190,7 +192,8 @@ func (daemon *Daemon) DeleteNetwork(networkID string) error { } if runconfig.IsPreDefinedNetwork(nw.Name()) { - return derr.ErrorCodeCantDeletePredefinedNetwork.WithArgs(nw.Name()) + err := fmt.Errorf("%s is a pre-defined network and cannot be removed", nw.Name()) + return errors.NewErrorWithStatusCode(err, http.StatusForbidden) } if err := nw.Delete(); err != nil { diff --git a/components/engine/daemon/pause.go b/components/engine/daemon/pause.go index a8ce013d4f..2ec0df7030 100644 --- a/components/engine/daemon/pause.go +++ b/components/engine/daemon/pause.go @@ -1,8 +1,9 @@ package daemon import ( + "fmt" + "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" ) // ContainerPause pauses a container @@ -27,21 +28,21 @@ func (daemon *Daemon) containerPause(container *container.Container) error { // We cannot Pause the container which is not running if !container.Running { - return derr.ErrorCodeNotRunning.WithArgs(container.ID) + return errNotRunning{container.ID} } // We cannot Pause the container which is already paused if container.Paused { - return derr.ErrorCodeAlreadyPaused.WithArgs(container.ID) + return fmt.Errorf("Container %s is already paused", container.ID) } // We cannot Pause the container which is restarting if container.Restarting { - return derr.ErrorCodeContainerRestarting.WithArgs(container.ID) + return errContainerIsRestarting(container.ID) } if err := daemon.execDriver.Pause(container.Command); err != nil { - return derr.ErrorCodeCantPause.WithArgs(container.ID, err) + return fmt.Errorf("Cannot pause container %s: %s", container.ID, err) } container.Paused = true daemon.LogContainerEvent(container, "pause") diff --git a/components/engine/daemon/rename.go b/components/engine/daemon/rename.go index 2f903d9324..363a7f8bf4 100644 --- a/components/engine/daemon/rename.go +++ b/components/engine/daemon/rename.go @@ -1,10 +1,10 @@ package daemon import ( + "fmt" "strings" "github.com/Sirupsen/logrus" - derr "github.com/docker/docker/errors" "github.com/docker/libnetwork" ) @@ -18,7 +18,7 @@ func (daemon *Daemon) ContainerRename(oldName, newName string) error { ) if oldName == "" || newName == "" { - return derr.ErrorCodeEmptyRename + return fmt.Errorf("Neither old nor new names may be empty") } container, err := daemon.GetContainer(oldName) @@ -31,7 +31,7 @@ func (daemon *Daemon) ContainerRename(oldName, newName string) error { container.Lock() defer container.Unlock() if newName, err = daemon.reserveName(container.ID, newName); err != nil { - return derr.ErrorCodeRenameTaken.WithArgs(err) + return fmt.Errorf("Error when allocating new name: %v", err) } container.Name = newName diff --git a/components/engine/daemon/resize.go b/components/engine/daemon/resize.go index c326248c92..d7bb105b36 100644 --- a/components/engine/daemon/resize.go +++ b/components/engine/daemon/resize.go @@ -1,10 +1,6 @@ package daemon -import ( - "fmt" - - derr "github.com/docker/docker/errors" -) +import "fmt" // ContainerResize changes the size of the TTY of the process running // in the container with the given name to the given height and width. @@ -15,7 +11,7 @@ func (daemon *Daemon) ContainerResize(name string, height, width int) error { } if !container.IsRunning() { - return derr.ErrorCodeNotRunning.WithArgs(container.ID) + return errNotRunning{container.ID} } if err = container.Resize(height, width); err == nil { diff --git a/components/engine/daemon/restart.go b/components/engine/daemon/restart.go index 8ee16918cb..3779116cfa 100644 --- a/components/engine/daemon/restart.go +++ b/components/engine/daemon/restart.go @@ -1,8 +1,9 @@ package daemon import ( + "fmt" + "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" ) // ContainerRestart stops and starts a container. It attempts to @@ -17,7 +18,7 @@ func (daemon *Daemon) ContainerRestart(name string, seconds int) error { return err } if err := daemon.containerRestart(container, seconds); err != nil { - return derr.ErrorCodeCantRestart.WithArgs(name, err) + return fmt.Errorf("Cannot restart container %s: %v", name, err) } return nil } diff --git a/components/engine/daemon/start.go b/components/engine/daemon/start.go index 883e9d06d9..09842e3ae3 100644 --- a/components/engine/daemon/start.go +++ b/components/engine/daemon/start.go @@ -2,11 +2,12 @@ package daemon import ( "fmt" + "net/http" "runtime" "github.com/Sirupsen/logrus" "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" + "github.com/docker/docker/errors" "github.com/docker/docker/runconfig" containertypes "github.com/docker/engine-api/types/container" ) @@ -19,11 +20,12 @@ func (daemon *Daemon) ContainerStart(name string, hostConfig *containertypes.Hos } if container.IsPaused() { - return derr.ErrorCodeStartPaused + return fmt.Errorf("Cannot start a paused container, try unpause instead.") } if container.IsRunning() { - return derr.ErrorCodeAlreadyStarted + err := fmt.Errorf("Container already started") + return errors.NewErrorWithStatusCode(err, http.StatusNotModified) } // Windows does not have the backwards compatibility issue here. @@ -52,13 +54,13 @@ func (daemon *Daemon) ContainerStart(name string, hostConfig *containertypes.Hos } } else { if hostConfig != nil { - return derr.ErrorCodeHostConfigStart + return fmt.Errorf("Supplying a hostconfig on start is not supported. It should be supplied on create") } } // check if hostConfig is in line with the current system settings. // It may happen cgroups are umounted or the like. - if _, err = daemon.verifyContainerSettings(container.HostConfig, nil); err != nil { + if _, err = daemon.verifyContainerSettings(container.HostConfig, nil, false); err != nil { return err } // Adapt for old containers in case we have updates in this function and @@ -88,7 +90,7 @@ func (daemon *Daemon) containerStart(container *container.Container) (err error) } if container.RemovalInProgress || container.Dead { - return derr.ErrorCodeContainerBeingRemoved + return fmt.Errorf("Container is marked for removal and cannot be started.") } // if we encounter an error during start we need to ensure that any other @@ -124,7 +126,8 @@ func (daemon *Daemon) containerStart(container *container.Container) (err error) if err != nil { return err } - if err := container.SetupWorkingDirectory(); err != nil { + rootUID, rootGID := daemon.GetRemappedUIDGID() + if err := container.SetupWorkingDirectory(rootUID, rootGID); err != nil { return err } env := container.CreateDaemonEnvironment(linkedEnv) @@ -154,7 +157,7 @@ func (daemon *Daemon) containerStart(container *container.Container) (err error) } func (daemon *Daemon) waitForStart(container *container.Container) error { - return container.StartMonitor(daemon, container.HostConfig.RestartPolicy) + return container.StartMonitor(daemon) } // Cleanup releases any network resources allocated to the container along with any rules diff --git a/components/engine/daemon/stats_collector_unix.go b/components/engine/daemon/stats_collector_unix.go index 2fd368cd34..a8de5a2062 100644 --- a/components/engine/daemon/stats_collector_unix.go +++ b/components/engine/daemon/stats_collector_unix.go @@ -4,6 +4,7 @@ package daemon import ( "bufio" + "fmt" "os" "strconv" "strings" @@ -13,7 +14,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/container" "github.com/docker/docker/daemon/execdriver" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/pubsub" "github.com/opencontainers/runc/libcontainer/system" ) @@ -163,13 +163,13 @@ func (s *statsCollector) getSystemCPUUsage() (uint64, error) { switch parts[0] { case "cpu": if len(parts) < 8 { - return 0, derr.ErrorCodeBadCPUFields + return 0, fmt.Errorf("invalid number of cpu fields") } var totalClockTicks uint64 for _, i := range parts[1:8] { v, err := strconv.ParseUint(i, 10, 64) if err != nil { - return 0, derr.ErrorCodeBadCPUInt.WithArgs(i, err) + return 0, fmt.Errorf("Unable to convert value %s to int: %s", i, err) } totalClockTicks += v } @@ -177,5 +177,5 @@ func (s *statsCollector) getSystemCPUUsage() (uint64, error) { s.clockTicksPerSecond, nil } } - return 0, derr.ErrorCodeBadStatFormat + return 0, fmt.Errorf("invalid stat format. Error trying to parse the '/proc/stat' file") } diff --git a/components/engine/daemon/stats_linux.go b/components/engine/daemon/stats_linux.go index 201552a4e1..1a907e015a 100644 --- a/components/engine/daemon/stats_linux.go +++ b/components/engine/daemon/stats_linux.go @@ -61,6 +61,10 @@ func convertStatsToAPITypes(ls *libcontainer.Stats) *types.StatsJSON { Stats: mem.Stats, Failcnt: mem.Usage.Failcnt, } + pids := cs.PidsStats + s.PidsStats = types.PidsStats{ + Current: pids.Current, + } } return s diff --git a/components/engine/daemon/stop.go b/components/engine/daemon/stop.go index 55e3787751..701743008a 100644 --- a/components/engine/daemon/stop.go +++ b/components/engine/daemon/stop.go @@ -1,11 +1,13 @@ package daemon import ( + "fmt" + "net/http" "time" "github.com/Sirupsen/logrus" "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" + "github.com/docker/docker/errors" ) // ContainerStop looks for the given container and terminates it, @@ -20,10 +22,11 @@ func (daemon *Daemon) ContainerStop(name string, seconds int) error { return err } if !container.IsRunning() { - return derr.ErrorCodeStopped.WithArgs(name) + err := fmt.Errorf("Container %s is already stopped", name) + return errors.NewErrorWithStatusCode(err, http.StatusNotModified) } if err := daemon.containerStop(container, seconds); err != nil { - return derr.ErrorCodeCantStop.WithArgs(name, err) + return fmt.Errorf("Cannot stop container %s: %v", name, err) } return nil } diff --git a/components/engine/daemon/top_unix.go b/components/engine/daemon/top_unix.go index 6d92592eb6..1f8ab07f04 100644 --- a/components/engine/daemon/top_unix.go +++ b/components/engine/daemon/top_unix.go @@ -3,11 +3,11 @@ package daemon import ( + "fmt" "os/exec" "strconv" "strings" - derr "github.com/docker/docker/errors" "github.com/docker/engine-api/types" ) @@ -27,11 +27,11 @@ func (daemon *Daemon) ContainerTop(name string, psArgs string) (*types.Container } if !container.IsRunning() { - return nil, derr.ErrorCodeNotRunning.WithArgs(name) + return nil, errNotRunning{container.ID} } if container.IsRestarting() { - return nil, derr.ErrorCodeContainerRestarting.WithArgs(name) + return nil, errContainerIsRestarting(container.ID) } pids, err := daemon.ExecutionDriver().GetPidsForContainer(container.ID) if err != nil { @@ -40,7 +40,7 @@ func (daemon *Daemon) ContainerTop(name string, psArgs string) (*types.Container output, err := exec.Command("ps", strings.Split(psArgs, " ")...).Output() if err != nil { - return nil, derr.ErrorCodePSError.WithArgs(err) + return nil, fmt.Errorf("Error running ps: %v", err) } procList := &types.ContainerProcessList{} @@ -55,7 +55,7 @@ func (daemon *Daemon) ContainerTop(name string, psArgs string) (*types.Container } } if pidIndex == -1 { - return nil, derr.ErrorCodeNoPID + return nil, fmt.Errorf("Couldn't find PID field in ps output") } // loop through the output and extract the PID from each line @@ -66,7 +66,7 @@ func (daemon *Daemon) ContainerTop(name string, psArgs string) (*types.Container fields := strings.Fields(line) p, err := strconv.Atoi(fields[pidIndex]) if err != nil { - return nil, derr.ErrorCodeBadPID.WithArgs(fields[pidIndex], err) + return nil, fmt.Errorf("Unexpected pid '%s': %s", fields[pidIndex], err) } for _, pid := range pids { diff --git a/components/engine/daemon/top_windows.go b/components/engine/daemon/top_windows.go index dc4cace65d..8b4fb2c6f0 100644 --- a/components/engine/daemon/top_windows.go +++ b/components/engine/daemon/top_windows.go @@ -1,11 +1,12 @@ package daemon import ( - derr "github.com/docker/docker/errors" + "fmt" + "github.com/docker/engine-api/types" ) // ContainerTop is not supported on Windows and returns an error. func (daemon *Daemon) ContainerTop(name string, psArgs string) (*types.ContainerProcessList, error) { - return nil, derr.ErrorCodeNoTop + return nil, fmt.Errorf("Top is not supported on Windows") } diff --git a/components/engine/daemon/unpause.go b/components/engine/daemon/unpause.go index ace52593d7..4af6f11222 100644 --- a/components/engine/daemon/unpause.go +++ b/components/engine/daemon/unpause.go @@ -1,8 +1,9 @@ package daemon import ( + "fmt" + "github.com/docker/docker/container" - derr "github.com/docker/docker/errors" ) // ContainerUnpause unpauses a container @@ -26,16 +27,16 @@ func (daemon *Daemon) containerUnpause(container *container.Container) error { // We cannot unpause the container which is not running if !container.Running { - return derr.ErrorCodeNotRunning.WithArgs(container.ID) + return errNotRunning{container.ID} } // We cannot unpause the container which is not paused if !container.Paused { - return derr.ErrorCodeNotPaused.WithArgs(container.ID) + return fmt.Errorf("Container %s is not paused", container.ID) } if err := daemon.execDriver.Unpause(container.Command); err != nil { - return derr.ErrorCodeCantUnpause.WithArgs(container.ID, err) + return fmt.Errorf("Cannot unpause container %s: %s", container.ID, err) } container.Paused = false diff --git a/components/engine/daemon/update.go b/components/engine/daemon/update.go index 181b399c96..ffdcc852bd 100644 --- a/components/engine/daemon/update.go +++ b/components/engine/daemon/update.go @@ -2,16 +2,16 @@ package daemon import ( "fmt" + "time" - derr "github.com/docker/docker/errors" "github.com/docker/engine-api/types/container" ) -// ContainerUpdate updates resources of the container +// ContainerUpdate updates configuration of the container func (daemon *Daemon) ContainerUpdate(name string, hostConfig *container.HostConfig) ([]string, error) { var warnings []string - warnings, err := daemon.verifyContainerSettings(hostConfig, nil) + warnings, err := daemon.verifyContainerSettings(hostConfig, nil, true) if err != nil { return warnings, err } @@ -44,27 +44,46 @@ func (daemon *Daemon) update(name string, hostConfig *container.HostConfig) erro return err } + restoreConfig := false + backupHostConfig := *container.HostConfig + defer func() { + if restoreConfig { + container.Lock() + container.HostConfig = &backupHostConfig + container.ToDisk() + container.Unlock() + } + }() + if container.RemovalInProgress || container.Dead { - errMsg := fmt.Errorf("Container is marked for removal and cannot be \"update\".") - return derr.ErrorCodeCantUpdate.WithArgs(container.ID, errMsg) + return errCannotUpdate(container.ID, fmt.Errorf("Container is marked for removal and cannot be \"update\".")) } if container.IsRunning() && hostConfig.KernelMemory != 0 { - errMsg := fmt.Errorf("Can not update kernel memory to a running container, please stop it first.") - return derr.ErrorCodeCantUpdate.WithArgs(container.ID, errMsg) + return errCannotUpdate(container.ID, fmt.Errorf("Can not update kernel memory to a running container, please stop it first.")) } if err := container.UpdateContainer(hostConfig); err != nil { - return derr.ErrorCodeCantUpdate.WithArgs(container.ID, err.Error()) + restoreConfig = true + return errCannotUpdate(container.ID, err) + } + + // if Restart Policy changed, we need to update container monitor + container.UpdateMonitor(hostConfig.RestartPolicy) + + // if container is restarting, wait 5 seconds until it's running + if container.IsRestarting() { + container.WaitRunning(5 * time.Second) } // If container is not running, update hostConfig struct is enough, // resources will be updated when the container is started again. // If container is running (including paused), we need to update configs // to the real world. - if container.IsRunning() { + if container.IsRunning() && !container.IsRestarting() { if err := daemon.execDriver.Update(container.Command); err != nil { - return derr.ErrorCodeCantUpdate.WithArgs(container.ID, err.Error()) + restoreConfig = true + return errCannotUpdate(container.ID, err) } } @@ -72,3 +91,7 @@ func (daemon *Daemon) update(name string, hostConfig *container.HostConfig) erro return nil } + +func errCannotUpdate(containerID string, err error) error { + return fmt.Errorf("Cannot update container %s: %v", containerID, err) +} diff --git a/components/engine/daemon/volumes.go b/components/engine/daemon/volumes.go index 7e2c417b2b..d32715997d 100644 --- a/components/engine/daemon/volumes.go +++ b/components/engine/daemon/volumes.go @@ -2,13 +2,13 @@ package daemon import ( "errors" + "fmt" "os" "path/filepath" "strings" "github.com/docker/docker/container" "github.com/docker/docker/daemon/execdriver" - derr "github.com/docker/docker/errors" "github.com/docker/docker/volume" "github.com/docker/engine-api/types" containertypes "github.com/docker/engine-api/types/container" @@ -114,10 +114,10 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo } if binds[bind.Destination] { - return derr.ErrorCodeMountDup.WithArgs(bind.Destination) + return fmt.Errorf("Duplicate mount point '%s'", bind.Destination) } - if len(bind.Name) > 0 && len(bind.Driver) > 0 { + if len(bind.Name) > 0 { // create the volume v, err := daemon.volumes.CreateWithRef(bind.Name, bind.Driver, container.ID, nil) if err != nil { @@ -128,7 +128,9 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo // bind.Name is an already existing volume, we need to use that here bind.Driver = v.DriverName() bind.Named = true - bind = setBindModeIfNull(bind) + if bind.Driver == "local" { + bind = setBindModeIfNull(bind) + } } if label.RelabelNeeded(bind.Mode) { if err := label.Relabel(bind.Source, container.MountLabel, label.IsShared(bind.Mode)); err != nil { diff --git a/components/engine/daemon/volumes_windows.go b/components/engine/daemon/volumes_windows.go index 05a45c385d..23c6a3b5e3 100644 --- a/components/engine/daemon/volumes_windows.go +++ b/components/engine/daemon/volumes_windows.go @@ -3,11 +3,11 @@ package daemon import ( + "fmt" "sort" "github.com/docker/docker/container" "github.com/docker/docker/daemon/execdriver" - derr "github.com/docker/docker/errors" "github.com/docker/docker/volume" ) @@ -27,7 +27,7 @@ func (daemon *Daemon) setupMounts(container *container.Container) ([]execdriver. s = mount.Volume.Path() } if s == "" { - return nil, derr.ErrorCodeVolumeNoSourceForMount.WithArgs(mount.Name, mount.Driver, mount.Destination) + return nil, fmt.Errorf("No source for mount name '%s' driver %q destination '%s'", mount.Name, mount.Driver, mount.Destination) } mnts = append(mnts, execdriver.Mount{ Source: s, diff --git a/components/engine/distribution/errors.go b/components/engine/distribution/errors.go new file mode 100644 index 0000000000..1cb34fdd51 --- /dev/null +++ b/components/engine/distribution/errors.go @@ -0,0 +1,106 @@ +package distribution + +import ( + "net/url" + "strings" + "syscall" + + "github.com/docker/distribution/registry/api/errcode" + "github.com/docker/distribution/registry/api/v2" + "github.com/docker/distribution/registry/client" + "github.com/docker/docker/distribution/xfer" +) + +// ErrNoSupport is an error type used for errors indicating that an operation +// is not supported. It encapsulates a more specific error. +type ErrNoSupport struct{ Err error } + +func (e ErrNoSupport) Error() string { + if e.Err == nil { + return "not supported" + } + return e.Err.Error() +} + +// fallbackError wraps an error that can possibly allow fallback to a different +// endpoint. +type fallbackError struct { + // err is the error being wrapped. + err error + // confirmedV2 is set to true if it was confirmed that the registry + // supports the v2 protocol. This is used to limit fallbacks to the v1 + // protocol. + confirmedV2 bool + // transportOK is set to true if we managed to speak HTTP with the + // registry. This confirms that we're using appropriate TLS settings + // (or lack of TLS). + transportOK bool +} + +// Error renders the FallbackError as a string. +func (f fallbackError) Error() string { + return f.err.Error() +} + +// shouldV2Fallback returns true if this error is a reason to fall back to v1. +func shouldV2Fallback(err errcode.Error) bool { + switch err.Code { + case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown, v2.ErrorCodeNameUnknown: + return true + } + return false +} + +// continueOnError returns true if we should fallback to the next endpoint +// as a result of this error. +func continueOnError(err error) bool { + switch v := err.(type) { + case errcode.Errors: + if len(v) == 0 { + return true + } + return continueOnError(v[0]) + case ErrNoSupport: + return continueOnError(v.Err) + case errcode.Error: + return shouldV2Fallback(v) + case *client.UnexpectedHTTPResponseError: + return true + case ImageConfigPullError: + return false + case error: + return !strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())) + } + // let's be nice and fallback if the error is a completely + // unexpected one. + // If new errors have to be handled in some way, please + // add them to the switch above. + return true +} + +// retryOnError wraps the error in xfer.DoNotRetry if we should not retry the +// operation after this error. +func retryOnError(err error) error { + switch v := err.(type) { + case errcode.Errors: + return retryOnError(v[0]) + case errcode.Error: + switch v.Code { + case errcode.ErrorCodeUnauthorized, errcode.ErrorCodeUnsupported, errcode.ErrorCodeDenied: + return xfer.DoNotRetry{Err: err} + } + case *url.Error: + return retryOnError(v.Err) + case *client.UnexpectedHTTPResponseError: + return xfer.DoNotRetry{Err: err} + case error: + if strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())) { + return xfer.DoNotRetry{Err: err} + } + } + // let's be nice and fallback if the error is a completely + // unexpected one. + // If new errors have to be handled in some way, please + // add them to the switch above. + return err +} diff --git a/components/engine/distribution/pull.go b/components/engine/distribution/pull.go index 603d5de5be..4b42371b90 100644 --- a/components/engine/distribution/pull.go +++ b/components/engine/distribution/pull.go @@ -2,7 +2,6 @@ package distribution import ( "fmt" - "os" "github.com/Sirupsen/logrus" "github.com/docker/docker/api" @@ -89,7 +88,7 @@ func Pull(ctx context.Context, ref reference.Named, imagePullConfig *ImagePullCo return err } - endpoints, err := imagePullConfig.RegistryService.LookupPullEndpoints(repoInfo) + endpoints, err := imagePullConfig.RegistryService.LookupPullEndpoints(repoInfo.Hostname()) if err != nil { return err } @@ -110,12 +109,25 @@ func Pull(ctx context.Context, ref reference.Named, imagePullConfig *ImagePullCo // confirm that it was talking to a v2 registry. This will // prevent fallback to the v1 protocol. confirmedV2 bool + + // confirmedTLSRegistries is a map indicating which registries + // are known to be using TLS. There should never be a plaintext + // retry for any of these. + confirmedTLSRegistries = make(map[string]struct{}) ) for _, endpoint := range endpoints { if confirmedV2 && endpoint.Version == registry.APIVersion1 { logrus.Debugf("Skipping v1 endpoint %s because v2 registry was detected", endpoint.URL) continue } + + if endpoint.URL.Scheme != "https" { + if _, confirmedTLS := confirmedTLSRegistries[endpoint.URL.Host]; confirmedTLS { + logrus.Debugf("Skipping non-TLS endpoint %s for host/port that appears to use TLS", endpoint.URL) + continue + } + } + logrus.Debugf("Trying to pull %s from %s %s", repoInfo.Name(), endpoint.URL, endpoint.Version) puller, err := newPuller(endpoint, repoInfo, imagePullConfig) @@ -133,11 +145,14 @@ func Pull(ctx context.Context, ref reference.Named, imagePullConfig *ImagePullCo if fallbackErr, ok := err.(fallbackError); ok { fallback = true confirmedV2 = confirmedV2 || fallbackErr.confirmedV2 + if fallbackErr.transportOK && endpoint.URL.Scheme == "https" { + confirmedTLSRegistries[endpoint.URL.Host] = struct{}{} + } err = fallbackErr.err } } if fallback { - if _, ok := err.(registry.ErrNoSupport); !ok { + if _, ok := err.(ErrNoSupport); !ok { // Because we found an error that's not ErrNoSupport, discard all subsequent ErrNoSupport errors. discardNoSupportErrors = true // append subsequent errors @@ -148,9 +163,10 @@ func Pull(ctx context.Context, ref reference.Named, imagePullConfig *ImagePullCo // append subsequent errors lastErr = err } + logrus.Errorf("Attempting next endpoint for pull after error: %v", err) continue } - logrus.Debugf("Not continuing with error: %v", err) + logrus.Errorf("Not continuing with pull after error: %v", err) return err } @@ -187,16 +203,3 @@ func validateRepoName(name string) error { } return nil } - -// tmpFileClose creates a closer function for a temporary file that closes the file -// and also deletes it. -func tmpFileCloser(tmpFile *os.File) func() error { - return func() error { - tmpFile.Close() - if err := os.RemoveAll(tmpFile.Name()); err != nil { - logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name()) - } - - return nil - } -} diff --git a/components/engine/distribution/pull_v1.go b/components/engine/distribution/pull_v1.go index 312f7e30e2..3e0cbdb46c 100644 --- a/components/engine/distribution/pull_v1.go +++ b/components/engine/distribution/pull_v1.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net" "net/url" + "os" "strings" "time" @@ -37,7 +38,7 @@ type v1Puller struct { func (p *v1Puller) Pull(ctx context.Context, ref reference.Named) error { if _, isCanonical := ref.(reference.Canonical); isCanonical { // Allowing fallback, because HTTPS v1 is before HTTP v2 - return fallbackError{err: registry.ErrNoSupport{Err: errors.New("Cannot pull by digest with v1 registry")}} + return fallbackError{err: ErrNoSupport{Err: errors.New("Cannot pull by digest with v1 registry")}} } tlsConfig, err := p.config.RegistryService.TLSConfig(p.repoInfo.Index.Name) @@ -279,6 +280,7 @@ type v1LayerDescriptor struct { layersDownloaded *bool layerSize int64 session *registry.Session + tmpFile *os.File } func (ld *v1LayerDescriptor) Key() string { @@ -308,7 +310,7 @@ func (ld *v1LayerDescriptor) Download(ctx context.Context, progressOutput progre } *ld.layersDownloaded = true - tmpFile, err := ioutil.TempFile("", "GetImageBlob") + ld.tmpFile, err = ioutil.TempFile("", "GetImageBlob") if err != nil { layerReader.Close() return nil, 0, err @@ -317,17 +319,28 @@ func (ld *v1LayerDescriptor) Download(ctx context.Context, progressOutput progre reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerReader), progressOutput, ld.layerSize, ld.ID(), "Downloading") defer reader.Close() - _, err = io.Copy(tmpFile, reader) + _, err = io.Copy(ld.tmpFile, reader) if err != nil { + ld.Close() return nil, 0, err } progress.Update(progressOutput, ld.ID(), "Download complete") - logrus.Debugf("Downloaded %s to tempfile %s", ld.ID(), tmpFile.Name()) + logrus.Debugf("Downloaded %s to tempfile %s", ld.ID(), ld.tmpFile.Name()) - tmpFile.Seek(0, 0) - return ioutils.NewReadCloserWrapper(tmpFile, tmpFileCloser(tmpFile)), ld.layerSize, nil + ld.tmpFile.Seek(0, 0) + return ld.tmpFile, ld.layerSize, nil +} + +func (ld *v1LayerDescriptor) Close() { + if ld.tmpFile != nil { + ld.tmpFile.Close() + if err := os.RemoveAll(ld.tmpFile.Name()); err != nil { + logrus.Errorf("Failed to remove temp file: %s", ld.tmpFile.Name()) + } + ld.tmpFile = nil + } } func (ld *v1LayerDescriptor) Registered(diffID layer.DiffID) { diff --git a/components/engine/distribution/pull_v2.go b/components/engine/distribution/pull_v2.go index 04d05e02f4..596d1c1321 100644 --- a/components/engine/distribution/pull_v2.go +++ b/components/engine/distribution/pull_v2.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "io/ioutil" + "net/url" "os" "runtime" @@ -17,6 +18,8 @@ import ( "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/registry/api/errcode" "github.com/docker/distribution/registry/client" + "github.com/docker/distribution/registry/client/auth" + "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/distribution/metadata" "github.com/docker/docker/distribution/xfer" "github.com/docker/docker/image" @@ -32,6 +35,17 @@ import ( var errRootFSMismatch = errors.New("layers from manifest don't match image configuration") +// ImageConfigPullError is an error pulling the image config blob +// (only applies to schema2). +type ImageConfigPullError struct { + Err error +} + +// Error returns the error string for ImageConfigPullError. +func (e ImageConfigPullError) Error() string { + return "error pulling image configuration: " + e.Err.Error() +} + type v2Puller struct { V2MetadataService *metadata.V2MetadataService endpoint registry.APIEndpoint @@ -48,16 +62,20 @@ func (p *v2Puller) Pull(ctx context.Context, ref reference.Named) (err error) { p.repo, p.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull") if err != nil { logrus.Warnf("Error getting v2 registry: %v", err) - return fallbackError{err: err, confirmedV2: p.confirmedV2} + return err } if err = p.pullV2Repository(ctx, ref); err != nil { if _, ok := err.(fallbackError); ok { return err } - if registry.ContinueOnError(err) { - logrus.Debugf("Error trying v2 registry: %v", err) - return fallbackError{err: err, confirmedV2: p.confirmedV2} + if continueOnError(err) { + logrus.Errorf("Error trying v2 registry: %v", err) + return fallbackError{ + err: err, + confirmedV2: p.confirmedV2, + transportOK: true, + } } } return err @@ -114,6 +132,8 @@ type v2LayerDescriptor struct { repoInfo *registry.RepositoryInfo repo distribution.Repository V2MetadataService *metadata.V2MetadataService + tmpFile *os.File + verifier digest.Verifier } func (ld *v2LayerDescriptor) Key() string { @@ -131,17 +151,56 @@ func (ld *v2LayerDescriptor) DiffID() (layer.DiffID, error) { func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) { logrus.Debugf("pulling blob %q", ld.digest) + var ( + err error + offset int64 + ) + + if ld.tmpFile == nil { + ld.tmpFile, err = createDownloadFile() + if err != nil { + return nil, 0, xfer.DoNotRetry{Err: err} + } + } else { + offset, err = ld.tmpFile.Seek(0, os.SEEK_END) + if err != nil { + logrus.Debugf("error seeking to end of download file: %v", err) + offset = 0 + + ld.tmpFile.Close() + if err := os.Remove(ld.tmpFile.Name()); err != nil { + logrus.Errorf("Failed to remove temp file: %s", ld.tmpFile.Name()) + } + ld.tmpFile, err = createDownloadFile() + if err != nil { + return nil, 0, xfer.DoNotRetry{Err: err} + } + } else if offset != 0 { + logrus.Debugf("attempting to resume download of %q from %d bytes", ld.digest, offset) + } + } + + tmpFile := ld.tmpFile blobs := ld.repo.Blobs(ctx) layerDownload, err := blobs.Open(ctx, ld.digest) if err != nil { - logrus.Debugf("Error statting layer: %v", err) + logrus.Errorf("Error initiating layer download: %v", err) if err == distribution.ErrBlobUnknown { return nil, 0, xfer.DoNotRetry{Err: err} } return nil, 0, retryOnError(err) } + if offset != 0 { + _, err := layerDownload.Seek(offset, os.SEEK_SET) + if err != nil { + if err := ld.truncateDownloadFile(); err != nil { + return nil, 0, xfer.DoNotRetry{Err: err} + } + return nil, 0, err + } + } size, err := layerDownload.Seek(0, os.SEEK_END) if err != nil { // Seek failed, perhaps because there was no Content-Length @@ -149,46 +208,59 @@ func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progre // still continue without a progress bar. size = 0 } else { - // Restore the seek offset at the beginning of the stream. - _, err = layerDownload.Seek(0, os.SEEK_SET) + if size != 0 && offset > size { + logrus.Debugf("Partial download is larger than full blob. Starting over") + offset = 0 + if err := ld.truncateDownloadFile(); err != nil { + return nil, 0, xfer.DoNotRetry{Err: err} + } + } + + // Restore the seek offset either at the beginning of the + // stream, or just after the last byte we have from previous + // attempts. + _, err = layerDownload.Seek(offset, os.SEEK_SET) if err != nil { return nil, 0, err } } - reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerDownload), progressOutput, size, ld.ID(), "Downloading") + reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerDownload), progressOutput, size-offset, ld.ID(), "Downloading") defer reader.Close() - verifier, err := digest.NewDigestVerifier(ld.digest) - if err != nil { - return nil, 0, xfer.DoNotRetry{Err: err} + if ld.verifier == nil { + ld.verifier, err = digest.NewDigestVerifier(ld.digest) + if err != nil { + return nil, 0, xfer.DoNotRetry{Err: err} + } } - tmpFile, err := ioutil.TempFile("", "GetImageBlob") + _, err = io.Copy(tmpFile, io.TeeReader(reader, ld.verifier)) if err != nil { - return nil, 0, xfer.DoNotRetry{Err: err} - } - - _, err = io.Copy(tmpFile, io.TeeReader(reader, verifier)) - if err != nil { - tmpFile.Close() - if err := os.Remove(tmpFile.Name()); err != nil { - logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name()) + if err == transport.ErrWrongCodeForByteRange { + if err := ld.truncateDownloadFile(); err != nil { + return nil, 0, xfer.DoNotRetry{Err: err} + } + return nil, 0, err } return nil, 0, retryOnError(err) } progress.Update(progressOutput, ld.ID(), "Verifying Checksum") - if !verifier.Verified() { + if !ld.verifier.Verified() { err = fmt.Errorf("filesystem layer verification failed for digest %s", ld.digest) logrus.Error(err) - tmpFile.Close() - if err := os.Remove(tmpFile.Name()); err != nil { - logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name()) - } + // Allow a retry if this digest verification error happened + // after a resumed download. + if offset != 0 { + if err := ld.truncateDownloadFile(); err != nil { + return nil, 0, xfer.DoNotRetry{Err: err} + } + return nil, 0, err + } return nil, 0, xfer.DoNotRetry{Err: err} } @@ -202,9 +274,37 @@ func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progre if err := os.Remove(tmpFile.Name()); err != nil { logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name()) } + ld.tmpFile = nil + ld.verifier = nil return nil, 0, xfer.DoNotRetry{Err: err} } - return ioutils.NewReadCloserWrapper(tmpFile, tmpFileCloser(tmpFile)), size, nil + return tmpFile, size, nil +} + +func (ld *v2LayerDescriptor) Close() { + if ld.tmpFile != nil { + ld.tmpFile.Close() + if err := os.RemoveAll(ld.tmpFile.Name()); err != nil { + logrus.Errorf("Failed to remove temp file: %s", ld.tmpFile.Name()) + } + } +} + +func (ld *v2LayerDescriptor) truncateDownloadFile() error { + // Need a new hash context since we will be redoing the download + ld.verifier = nil + + if _, err := ld.tmpFile.Seek(0, os.SEEK_SET); err != nil { + logrus.Errorf("error seeking to beginning of download file: %v", err) + return err + } + + if err := ld.tmpFile.Truncate(0); err != nil { + logrus.Errorf("error truncating download file: %v", err) + return err + } + + return nil } func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) { @@ -399,7 +499,7 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s go func() { configJSON, err := p.pullSchema2ImageConfig(ctx, target.Digest) if err != nil { - errChan <- err + errChan <- ImageConfigPullError{Err: err} cancel() return } @@ -619,12 +719,24 @@ func allowV1Fallback(err error) error { switch v := err.(type) { case errcode.Errors: if len(v) != 0 { - if v0, ok := v[0].(errcode.Error); ok && registry.ShouldV2Fallback(v0) { - return fallbackError{err: err, confirmedV2: false} + if v0, ok := v[0].(errcode.Error); ok && shouldV2Fallback(v0) { + return fallbackError{ + err: err, + confirmedV2: false, + transportOK: true, + } } } case errcode.Error: - if registry.ShouldV2Fallback(v) { + if shouldV2Fallback(v) { + return fallbackError{ + err: err, + confirmedV2: false, + transportOK: true, + } + } + case *url.Error: + if v.Err == auth.ErrNoBasicAuthCredentials { return fallbackError{err: err, confirmedV2: false} } } @@ -711,3 +823,7 @@ func fixManifestLayers(m *schema1.Manifest) error { return nil } + +func createDownloadFile() (*os.File, error) { + return ioutil.TempFile("", "GetImageBlob") +} diff --git a/components/engine/distribution/pull_v2_test.go b/components/engine/distribution/pull_v2_test.go index 53995bf663..8555c81e6b 100644 --- a/components/engine/distribution/pull_v2_test.go +++ b/components/engine/distribution/pull_v2_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "io/ioutil" "reflect" + "runtime" "strings" "testing" @@ -62,6 +63,10 @@ func TestFixManifestLayers(t *testing.T) { // TestFixManifestLayersBaseLayerParent makes sure that fixManifestLayers fails // if the base layer configuration specifies a parent. func TestFixManifestLayersBaseLayerParent(t *testing.T) { + // TODO Windows: Fix this unit text + if runtime.GOOS == "windows" { + t.Skip("Needs fixing on Windows") + } duplicateLayerManifest := schema1.Manifest{ FSLayers: []schema1.FSLayer{ {BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")}, @@ -104,6 +109,10 @@ func TestFixManifestLayersBadParent(t *testing.T) { // TestValidateManifest verifies the validateManifest function func TestValidateManifest(t *testing.T) { + // TODO Windows: Fix this unit text + if runtime.GOOS == "windows" { + t.Skip("Needs fixing on Windows") + } expectedDigest, err := reference.ParseNamed("repo@sha256:02fee8c3220ba806531f606525eceb83f4feb654f62b207191b1c9209188dedd") if err != nil { t.Fatal("could not parse reference") diff --git a/components/engine/distribution/push.go b/components/engine/distribution/push.go index d0622f82c9..52ee8e77e2 100644 --- a/components/engine/distribution/push.go +++ b/components/engine/distribution/push.go @@ -100,7 +100,7 @@ func Push(ctx context.Context, ref reference.Named, imagePushConfig *ImagePushCo return err } - endpoints, err := imagePushConfig.RegistryService.LookupPushEndpoints(repoInfo) + endpoints, err := imagePushConfig.RegistryService.LookupPushEndpoints(repoInfo.Hostname()) if err != nil { return err } @@ -119,6 +119,11 @@ func Push(ctx context.Context, ref reference.Named, imagePushConfig *ImagePushCo // confirm that it was talking to a v2 registry. This will // prevent fallback to the v1 protocol. confirmedV2 bool + + // confirmedTLSRegistries is a map indicating which registries + // are known to be using TLS. There should never be a plaintext + // retry for any of these. + confirmedTLSRegistries = make(map[string]struct{}) ) for _, endpoint := range endpoints { @@ -127,6 +132,13 @@ func Push(ctx context.Context, ref reference.Named, imagePushConfig *ImagePushCo continue } + if endpoint.URL.Scheme != "https" { + if _, confirmedTLS := confirmedTLSRegistries[endpoint.URL.Host]; confirmedTLS { + logrus.Debugf("Skipping non-TLS endpoint %s for host/port that appears to use TLS", endpoint.URL) + continue + } + } + logrus.Debugf("Trying to push %s to %s %s", repoInfo.FullName(), endpoint.URL, endpoint.Version) pusher, err := NewPusher(ref, endpoint, repoInfo, imagePushConfig) @@ -142,13 +154,17 @@ func Push(ctx context.Context, ref reference.Named, imagePushConfig *ImagePushCo default: if fallbackErr, ok := err.(fallbackError); ok { confirmedV2 = confirmedV2 || fallbackErr.confirmedV2 + if fallbackErr.transportOK && endpoint.URL.Scheme == "https" { + confirmedTLSRegistries[endpoint.URL.Host] = struct{}{} + } err = fallbackErr.err lastErr = err + logrus.Errorf("Attempting next endpoint for push after error: %v", err) continue } } - logrus.Debugf("Not continuing with error: %v", err) + logrus.Errorf("Not continuing with push after error: %v", err) return err } diff --git a/components/engine/distribution/push_v1.go b/components/engine/distribution/push_v1.go index 8be1df97f1..e9b1065f7a 100644 --- a/components/engine/distribution/push_v1.go +++ b/components/engine/distribution/push_v1.go @@ -21,7 +21,6 @@ import ( ) type v1Pusher struct { - ctx context.Context v1IDService *metadata.V1IDService endpoint registry.APIEndpoint ref reference.Named diff --git a/components/engine/distribution/push_v2.go b/components/engine/distribution/push_v2.go index ac2017a8d0..e812c1da8c 100644 --- a/components/engine/distribution/push_v2.go +++ b/components/engine/distribution/push_v2.go @@ -42,7 +42,7 @@ type v2Pusher struct { config *ImagePushConfig repo distribution.Repository - // pushState is state built by the Download functions. + // pushState is state built by the Upload functions. pushState pushState } @@ -64,12 +64,16 @@ func (p *v2Pusher) Push(ctx context.Context) (err error) { p.repo, p.pushState.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "push", "pull") if err != nil { logrus.Debugf("Error getting v2 registry: %v", err) - return fallbackError{err: err, confirmedV2: p.pushState.confirmedV2} + return err } if err = p.pushV2Repository(ctx); err != nil { - if registry.ContinueOnError(err) { - return fallbackError{err: err, confirmedV2: p.pushState.confirmedV2} + if continueOnError(err) { + return fallbackError{ + err: err, + confirmedV2: p.pushState.confirmedV2, + transportOK: true, + } } } return err @@ -220,6 +224,7 @@ type v2PushDescriptor struct { repoInfo reference.Named repo distribution.Repository pushState *pushState + remoteDescriptor distribution.Descriptor } func (pd *v2PushDescriptor) Key() string { @@ -234,16 +239,16 @@ func (pd *v2PushDescriptor) DiffID() layer.DiffID { return pd.layer.DiffID() } -func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) error { +func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) { diffID := pd.DiffID() pd.pushState.Lock() - if _, ok := pd.pushState.remoteLayers[diffID]; ok { + if descriptor, ok := pd.pushState.remoteLayers[diffID]; ok { // it is already known that the push is not needed and // therefore doing a stat is unnecessary pd.pushState.Unlock() progress.Update(progressOutput, pd.ID(), "Layer already exists") - return nil + return descriptor, nil } pd.pushState.Unlock() @@ -253,14 +258,14 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. descriptor, exists, err := layerAlreadyExists(ctx, v2Metadata, pd.repoInfo, pd.repo, pd.pushState) if err != nil { progress.Update(progressOutput, pd.ID(), "Image push failed") - return retryOnError(err) + return distribution.Descriptor{}, retryOnError(err) } if exists { progress.Update(progressOutput, pd.ID(), "Layer already exists") pd.pushState.Lock() pd.pushState.remoteLayers[diffID] = descriptor pd.pushState.Unlock() - return nil + return descriptor, nil } } @@ -270,27 +275,29 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. // then push the blob. bs := pd.repo.Blobs(ctx) - var mountFrom metadata.V2Metadata + var layerUpload distribution.BlobWriter + mountAttemptsRemaining := 3 - // Attempt to find another repository in the same registry to mount the layer from to avoid an unnecessary upload - for _, metadata := range v2Metadata { - sourceRepo, err := reference.ParseNamed(metadata.SourceRepository) + // Attempt to find another repository in the same registry to mount the layer + // from to avoid an unnecessary upload. + // Note: metadata is stored from oldest to newest, so we iterate through this + // slice in reverse to maximize our chances of the blob still existing in the + // remote repository. + for i := len(v2Metadata) - 1; i >= 0 && mountAttemptsRemaining > 0; i-- { + mountFrom := v2Metadata[i] + + sourceRepo, err := reference.ParseNamed(mountFrom.SourceRepository) if err != nil { continue } - if pd.repoInfo.Hostname() == sourceRepo.Hostname() { - logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, metadata.Digest, sourceRepo.FullName()) - mountFrom = metadata - break + if pd.repoInfo.Hostname() != sourceRepo.Hostname() { + // don't mount blobs from another registry + continue } - } - var createOpts []distribution.BlobCreateOption - - if mountFrom.SourceRepository != "" { namedRef, err := reference.WithName(mountFrom.SourceRepository) if err != nil { - return err + continue } // TODO (brianbland): We need to construct a reference where the Name is @@ -298,51 +305,55 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. // richer reference package remoteRef, err := distreference.WithName(namedRef.RemoteName()) if err != nil { - return err + continue } canonicalRef, err := distreference.WithDigest(remoteRef, mountFrom.Digest) if err != nil { - return err + continue } - createOpts = append(createOpts, client.WithMountFrom(canonicalRef)) - } + logrus.Debugf("attempting to mount layer %s (%s) from %s", diffID, mountFrom.Digest, sourceRepo.FullName()) - // Send the layer - layerUpload, err := bs.Create(ctx, createOpts...) - switch err := err.(type) { - case distribution.ErrBlobMounted: - progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name()) + layerUpload, err = bs.Create(ctx, client.WithMountFrom(canonicalRef)) + switch err := err.(type) { + case distribution.ErrBlobMounted: + progress.Updatef(progressOutput, pd.ID(), "Mounted from %s", err.From.Name()) - err.Descriptor.MediaType = schema2.MediaTypeLayer + err.Descriptor.MediaType = schema2.MediaTypeLayer - pd.pushState.Lock() - pd.pushState.confirmedV2 = true - pd.pushState.remoteLayers[diffID] = err.Descriptor - pd.pushState.Unlock() + pd.pushState.Lock() + pd.pushState.confirmedV2 = true + pd.pushState.remoteLayers[diffID] = err.Descriptor + pd.pushState.Unlock() - // Cache mapping from this layer's DiffID to the blobsum - if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: mountFrom.Digest, SourceRepository: pd.repoInfo.FullName()}); err != nil { - return xfer.DoNotRetry{Err: err} + // Cache mapping from this layer's DiffID to the blobsum + if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: mountFrom.Digest, SourceRepository: pd.repoInfo.FullName()}); err != nil { + return distribution.Descriptor{}, xfer.DoNotRetry{Err: err} + } + return err.Descriptor, nil + case nil: + // blob upload session created successfully, so begin the upload + mountAttemptsRemaining = 0 + default: + // unable to mount layer from this repository, so this source mapping is no longer valid + logrus.Debugf("unassociating layer %s (%s) with %s", diffID, mountFrom.Digest, mountFrom.SourceRepository) + pd.v2MetadataService.Remove(mountFrom) + mountAttemptsRemaining-- } - - return nil - } - if mountFrom.SourceRepository != "" { - // unable to mount layer from this repository, so this source mapping is no longer valid - logrus.Debugf("unassociating layer %s (%s) with %s", diffID, mountFrom.Digest, mountFrom.SourceRepository) - pd.v2MetadataService.Remove(mountFrom) } - if err != nil { - return retryOnError(err) + if layerUpload == nil { + layerUpload, err = bs.Create(ctx) + if err != nil { + return distribution.Descriptor{}, retryOnError(err) + } } defer layerUpload.Close() arch, err := pd.layer.TarStream() if err != nil { - return xfer.DoNotRetry{Err: err} + return distribution.Descriptor{}, xfer.DoNotRetry{Err: err} } // don't care if this fails; best effort @@ -361,12 +372,12 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. nn, err := layerUpload.ReadFrom(tee) compressedReader.Close() if err != nil { - return retryOnError(err) + return distribution.Descriptor{}, retryOnError(err) } pushDigest := digester.Digest() if _, err := layerUpload.Commit(ctx, distribution.Descriptor{Digest: pushDigest}); err != nil { - return retryOnError(err) + return distribution.Descriptor{}, retryOnError(err) } logrus.Debugf("uploaded layer %s (%s), %d bytes", diffID, pushDigest, nn) @@ -374,32 +385,33 @@ func (pd *v2PushDescriptor) Upload(ctx context.Context, progressOutput progress. // Cache mapping from this layer's DiffID to the blobsum if err := pd.v2MetadataService.Add(diffID, metadata.V2Metadata{Digest: pushDigest, SourceRepository: pd.repoInfo.FullName()}); err != nil { - return xfer.DoNotRetry{Err: err} + return distribution.Descriptor{}, xfer.DoNotRetry{Err: err} } pd.pushState.Lock() - // If Commit succeded, that's an indication that the remote registry + // If Commit succeeded, that's an indication that the remote registry // speaks the v2 protocol. pd.pushState.confirmedV2 = true - pd.pushState.remoteLayers[diffID] = distribution.Descriptor{ + descriptor := distribution.Descriptor{ Digest: pushDigest, MediaType: schema2.MediaTypeLayer, Size: nn, } + pd.pushState.remoteLayers[diffID] = descriptor pd.pushState.Unlock() - return nil + return descriptor, nil +} + +func (pd *v2PushDescriptor) SetRemoteDescriptor(descriptor distribution.Descriptor) { + pd.remoteDescriptor = descriptor } func (pd *v2PushDescriptor) Descriptor() distribution.Descriptor { - // Not necessary to lock pushStatus because this is always - // called after all the mutation in pushStatus. - // By the time this function is called, every layer will have - // an entry in remoteLayers. - return pd.pushState.remoteLayers[pd.DiffID()] + return pd.remoteDescriptor } // layerAlreadyExists checks if the registry already know about any of the diff --git a/components/engine/distribution/registry.go b/components/engine/distribution/registry.go index 4a8988f13f..4946c0b573 100644 --- a/components/engine/distribution/registry.go +++ b/components/engine/distribution/registry.go @@ -5,39 +5,19 @@ import ( "net" "net/http" "net/url" - "strings" - "syscall" "time" "github.com/docker/distribution" distreference "github.com/docker/distribution/reference" - "github.com/docker/distribution/registry/api/errcode" "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client/auth" "github.com/docker/distribution/registry/client/transport" - "github.com/docker/docker/distribution/xfer" "github.com/docker/docker/dockerversion" "github.com/docker/docker/registry" "github.com/docker/engine-api/types" "golang.org/x/net/context" ) -// fallbackError wraps an error that can possibly allow fallback to a different -// endpoint. -type fallbackError struct { - // err is the error being wrapped. - err error - // confirmedV2 is set to true if it was confirmed that the registry - // supports the v2 protocol. This is used to limit fallbacks to the v1 - // protocol. - confirmedV2 bool -} - -// Error renders the FallbackError as a string. -func (f fallbackError) Error() string { - return f.err.Error() -} - type dumbCredentialStore struct { auth *types.AuthConfig } @@ -72,41 +52,19 @@ func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, end modifiers := registry.DockerHeaders(dockerversion.DockerUserAgent(), metaHeaders) authTransport := transport.NewTransport(base, modifiers...) - pingClient := &http.Client{ - Transport: authTransport, - Timeout: 15 * time.Second, - } - endpointStr := strings.TrimRight(endpoint.URL, "/") + "/v2/" - req, err := http.NewRequest("GET", endpointStr, nil) + + challengeManager, foundVersion, err := registry.PingV2Registry(endpoint, authTransport) if err != nil { - return nil, false, err - } - resp, err := pingClient.Do(req) - if err != nil { - return nil, false, err - } - defer resp.Body.Close() - - v2Version := auth.APIVersion{ - Type: "registry", - Version: "2.0", - } - - versions := auth.APIVersions(resp, registry.DefaultRegistryVersionHeader) - for _, pingVersion := range versions { - if pingVersion == v2Version { - // The version header indicates we're definitely - // talking to a v2 registry. So don't allow future - // fallbacks to the v1 protocol. - - foundVersion = true - break + transportOK := false + if responseErr, ok := err.(registry.PingResponseError); ok { + transportOK = true + err = responseErr.Err + } + return nil, foundVersion, fallbackError{ + err: err, + confirmedV2: foundVersion, + transportOK: transportOK, } - } - - challengeManager := auth.NewSimpleChallengeManager() - if err := challengeManager.AddResponse(resp); err != nil { - return nil, foundVersion, err } if authConfig.RegistryToken != "" { @@ -122,11 +80,22 @@ func NewV2Repository(ctx context.Context, repoInfo *registry.RepositoryInfo, end repoNameRef, err := distreference.ParseNamed(repoName) if err != nil { - return nil, foundVersion, err + return nil, foundVersion, fallbackError{ + err: err, + confirmedV2: foundVersion, + transportOK: true, + } } - repo, err = client.NewRepository(ctx, repoNameRef, endpoint.URL, tr) - return repo, foundVersion, err + repo, err = client.NewRepository(ctx, repoNameRef, endpoint.URL.String(), tr) + if err != nil { + err = fallbackError{ + err: err, + confirmedV2: foundVersion, + transportOK: true, + } + } + return } type existingTokenHandler struct { @@ -141,30 +110,3 @@ func (th *existingTokenHandler) AuthorizeRequest(req *http.Request, params map[s req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", th.token)) return nil } - -// retryOnError wraps the error in xfer.DoNotRetry if we should not retry the -// operation after this error. -func retryOnError(err error) error { - switch v := err.(type) { - case errcode.Errors: - return retryOnError(v[0]) - case errcode.Error: - switch v.Code { - case errcode.ErrorCodeUnauthorized, errcode.ErrorCodeUnsupported, errcode.ErrorCodeDenied: - return xfer.DoNotRetry{Err: err} - } - case *url.Error: - return retryOnError(v.Err) - case *client.UnexpectedHTTPResponseError: - return xfer.DoNotRetry{Err: err} - case error: - if strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())) { - return xfer.DoNotRetry{Err: err} - } - } - // let's be nice and fallback if the error is a completely - // unexpected one. - // If new errors have to be handled in some way, please - // add them to the switch above. - return err -} diff --git a/components/engine/distribution/registry_unit_test.go b/components/engine/distribution/registry_unit_test.go index 0702232943..b60a465d78 100644 --- a/components/engine/distribution/registry_unit_test.go +++ b/components/engine/distribution/registry_unit_test.go @@ -3,6 +3,7 @@ package distribution import ( "net/http" "net/http/httptest" + "net/url" "os" "strings" "testing" @@ -43,9 +44,14 @@ func testTokenPassThru(t *testing.T, ts *httptest.Server) { } defer os.RemoveAll(tmp) + uri, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("could not parse url from test server: %v", err) + } + endpoint := registry.APIEndpoint{ Mirror: false, - URL: ts.URL, + URL: uri, Version: 2, Official: false, TrimHostname: false, diff --git a/components/engine/distribution/xfer/download.go b/components/engine/distribution/xfer/download.go index 69c8bad031..2536f1dd23 100644 --- a/components/engine/distribution/xfer/download.go +++ b/components/engine/distribution/xfer/download.go @@ -59,6 +59,10 @@ type DownloadDescriptor interface { DiffID() (layer.DiffID, error) // Download is called to perform the download. Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) + // Close is called when the download manager is finished with this + // descriptor and will not call Download again or read from the reader + // that Download returned. + Close() } // DownloadDescriptorWithRegistered is a DownloadDescriptor that has an @@ -229,6 +233,8 @@ func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor, retries int ) + defer descriptor.Close() + for { downloadReader, size, err = descriptor.Download(d.Transfer.Context(), progressOutput) if err == nil { diff --git a/components/engine/distribution/xfer/download_test.go b/components/engine/distribution/xfer/download_test.go index 6dc6708531..9be9a24a3b 100644 --- a/components/engine/distribution/xfer/download_test.go +++ b/components/engine/distribution/xfer/download_test.go @@ -5,6 +5,7 @@ import ( "errors" "io" "io/ioutil" + "runtime" "sync/atomic" "testing" "time" @@ -199,6 +200,9 @@ func (d *mockDownloadDescriptor) Download(ctx context.Context, progressOutput pr return d.mockTarStream(), 0, nil } +func (d *mockDownloadDescriptor) Close() { +} + func downloadDescriptors(currentDownloads *int32) []DownloadDescriptor { return []DownloadDescriptor{ &mockDownloadDescriptor{ @@ -236,6 +240,10 @@ func downloadDescriptors(currentDownloads *int32) []DownloadDescriptor { } func TestSuccessfulDownload(t *testing.T) { + // TODO Windows: Fix this unit text + if runtime.GOOS == "windows" { + t.Skip("Needs fixing on Windows") + } layerStore := &mockLayerStore{make(map[layer.ChainID]*mockLayer)} ldm := NewLayerDownloadManager(layerStore, maxDownloadConcurrency) diff --git a/components/engine/distribution/xfer/upload.go b/components/engine/distribution/xfer/upload.go index 8da6a89e39..20fe045ac6 100644 --- a/components/engine/distribution/xfer/upload.go +++ b/components/engine/distribution/xfer/upload.go @@ -5,6 +5,7 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/docker/distribution" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/progress" "golang.org/x/net/context" @@ -28,8 +29,8 @@ func NewLayerUploadManager(concurrencyLimit int) *LayerUploadManager { type uploadTransfer struct { Transfer - diffID layer.DiffID - err error + remoteDescriptor distribution.Descriptor + err error } // An UploadDescriptor references a layer that may need to be uploaded. @@ -41,7 +42,12 @@ type UploadDescriptor interface { // DiffID should return the DiffID for this layer. DiffID() layer.DiffID // Upload is called to perform the Upload. - Upload(ctx context.Context, progressOutput progress.Output) error + Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) + // SetRemoteDescriptor provides the distribution.Descriptor that was + // returned by Upload. This descriptor is not to be confused with + // the UploadDescriptor interface, which is used for internally + // identifying layers that are being uploaded. + SetRemoteDescriptor(descriptor distribution.Descriptor) } // Upload is a blocking function which ensures the listed layers are present on @@ -50,7 +56,7 @@ type UploadDescriptor interface { func (lum *LayerUploadManager) Upload(ctx context.Context, layers []UploadDescriptor, progressOutput progress.Output) error { var ( uploads []*uploadTransfer - dedupDescriptors = make(map[string]struct{}) + dedupDescriptors = make(map[string]*uploadTransfer) ) for _, descriptor := range layers { @@ -60,12 +66,12 @@ func (lum *LayerUploadManager) Upload(ctx context.Context, layers []UploadDescri if _, present := dedupDescriptors[key]; present { continue } - dedupDescriptors[key] = struct{}{} xferFunc := lum.makeUploadFunc(descriptor) upload, watcher := lum.tm.Transfer(descriptor.Key(), xferFunc, progressOutput) defer upload.Release(watcher) uploads = append(uploads, upload.(*uploadTransfer)) + dedupDescriptors[key] = upload.(*uploadTransfer) } for _, upload := range uploads { @@ -78,6 +84,9 @@ func (lum *LayerUploadManager) Upload(ctx context.Context, layers []UploadDescri } } } + for _, l := range layers { + l.SetRemoteDescriptor(dedupDescriptors[l.Key()].remoteDescriptor) + } return nil } @@ -86,7 +95,6 @@ func (lum *LayerUploadManager) makeUploadFunc(descriptor UploadDescriptor) DoFun return func(progressChan chan<- progress.Progress, start <-chan struct{}, inactive chan<- struct{}) Transfer { u := &uploadTransfer{ Transfer: NewTransfer(), - diffID: descriptor.DiffID(), } go func() { @@ -105,8 +113,9 @@ func (lum *LayerUploadManager) makeUploadFunc(descriptor UploadDescriptor) DoFun retries := 0 for { - err := descriptor.Upload(u.Transfer.Context(), progressOutput) + remoteDescriptor, err := descriptor.Upload(u.Transfer.Context(), progressOutput) if err == nil { + u.remoteDescriptor = remoteDescriptor break } diff --git a/components/engine/distribution/xfer/upload_test.go b/components/engine/distribution/xfer/upload_test.go index d87dfcaa25..275d24268d 100644 --- a/components/engine/distribution/xfer/upload_test.go +++ b/components/engine/distribution/xfer/upload_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/progress" @@ -35,13 +36,17 @@ func (u *mockUploadDescriptor) DiffID() layer.DiffID { return u.diffID } +// SetRemoteDescriptor is not used in the mock. +func (u *mockUploadDescriptor) SetRemoteDescriptor(remoteDescriptor distribution.Descriptor) { +} + // Upload is called to perform the upload. -func (u *mockUploadDescriptor) Upload(ctx context.Context, progressOutput progress.Output) error { +func (u *mockUploadDescriptor) Upload(ctx context.Context, progressOutput progress.Output) (distribution.Descriptor, error) { if u.currentUploads != nil { defer atomic.AddInt32(u.currentUploads, -1) if atomic.AddInt32(u.currentUploads, 1) > maxUploadConcurrency { - return errors.New("concurrency limit exceeded") + return distribution.Descriptor{}, errors.New("concurrency limit exceeded") } } @@ -49,7 +54,7 @@ func (u *mockUploadDescriptor) Upload(ctx context.Context, progressOutput progre for i := int64(0); i <= 10; i++ { select { case <-ctx.Done(): - return ctx.Err() + return distribution.Descriptor{}, ctx.Err() case <-time.After(10 * time.Millisecond): progressOutput.WriteProgress(progress.Progress{ID: u.ID(), Current: i, Total: 10}) } @@ -57,10 +62,10 @@ func (u *mockUploadDescriptor) Upload(ctx context.Context, progressOutput progre if u.simulateRetries != 0 { u.simulateRetries-- - return errors.New("simulating retry") + return distribution.Descriptor{}, errors.New("simulating retry") } - return nil + return distribution.Descriptor{}, nil } func uploadDescriptors(currentUploads *int32) []UploadDescriptor { diff --git a/components/engine/docker/daemon.go b/components/engine/docker/daemon.go index 821b3f993b..efb459f6f9 100644 --- a/components/engine/docker/daemon.go +++ b/components/engine/docker/daemon.go @@ -14,10 +14,19 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/uuid" apiserver "github.com/docker/docker/api/server" + "github.com/docker/docker/api/server/router" + "github.com/docker/docker/api/server/router/build" + "github.com/docker/docker/api/server/router/container" + "github.com/docker/docker/api/server/router/image" + "github.com/docker/docker/api/server/router/network" + systemrouter "github.com/docker/docker/api/server/router/system" + "github.com/docker/docker/api/server/router/volume" + "github.com/docker/docker/builder/dockerfile" "github.com/docker/docker/cli" "github.com/docker/docker/cliconfig" "github.com/docker/docker/daemon" "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/docker/listeners" "github.com/docker/docker/dockerversion" "github.com/docker/docker/opts" "github.com/docker/docker/pkg/jsonlog" @@ -226,6 +235,9 @@ func (cli *DaemonCli) CmdDaemon(args ...string) error { if len(cli.Config.Hosts) == 0 { cli.Config.Hosts = make([]string, 1) } + + api := apiserver.New(serverConfig) + for i := 0; i < len(cli.Config.Hosts); i++ { var err error if cli.Config.Hosts[i], err = opts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil { @@ -237,12 +249,13 @@ func (cli *DaemonCli) CmdDaemon(args ...string) error { if len(protoAddrParts) != 2 { logrus.Fatalf("bad format %s, expected PROTO://ADDR", protoAddr) } - serverConfig.Addrs = append(serverConfig.Addrs, apiserver.Addr{Proto: protoAddrParts[0], Addr: protoAddrParts[1]}) - } + l, err := listeners.Init(protoAddrParts[0], protoAddrParts[1], serverConfig.SocketGroup, serverConfig.TLSConfig) + if err != nil { + logrus.Fatal(err) + } - api, err := apiserver.New(serverConfig) - if err != nil { - logrus.Fatal(err) + logrus.Debugf("Listener created for HTTP on %s (%s)", protoAddrParts[0], protoAddrParts[1]) + api.Accept(protoAddrParts[1], l...) } if err := migrateKey(); err != nil { @@ -270,14 +283,25 @@ func (cli *DaemonCli) CmdDaemon(args ...string) error { "graphdriver": d.GraphDriverName(), }).Info("Docker daemon") - api.InitRouters(d) + initRouter(api, d) reload := func(config *daemon.Config) { if err := d.Reload(config); err != nil { logrus.Errorf("Error reconfiguring the daemon: %v", err) return } - api.Reload(config) + if config.IsValueSet("debug") { + debugEnabled := utils.IsDebugEnabled() + switch { + case debugEnabled && !config.Debug: // disable debug + utils.DisableDebug() + api.DisableProfiler() + case config.Debug && !debugEnabled: // enable debug + utils.EnableDebug() + api.EnableProfiler() + } + + } } setupConfigReloadTrap(*configFile, cli.flags, reload) @@ -373,3 +397,18 @@ func loadDaemonCliConfig(config *daemon.Config, daemonFlags *flag.FlagSet, commo return config, nil } + +func initRouter(s *apiserver.Server, d *daemon.Daemon) { + routers := []router.Router{ + container.NewRouter(d), + image.NewRouter(d), + systemrouter.NewRouter(d), + volume.NewRouter(d), + build.NewRouter(dockerfile.NewBuildManager(d)), + } + if d.NetworkControllerEnabled() { + routers = append(routers, network.NewRouter(d)) + } + + s.InitRouter(utils.IsDebugEnabled(), routers...) +} diff --git a/components/engine/docker/daemon_test.go b/components/engine/docker/daemon_test.go index 5afdfb3bde..322e0b7604 100644 --- a/components/engine/docker/daemon_test.go +++ b/components/engine/docker/daemon_test.go @@ -247,47 +247,3 @@ func TestLoadDaemonConfigWithEmbeddedOptions(t *testing.T) { t.Fatalf("expected LogConfig type syslog, got %v", loadedConfig.LogConfig.Type) } } - -func TestLoadDaemonConfigWithMapOptions(t *testing.T) { - c := &daemon.Config{} - common := &cli.CommonFlags{} - flags := mflag.NewFlagSet("test", mflag.ContinueOnError) - - flags.Var(opts.NewNamedMapOpts("cluster-store-opts", c.ClusterOpts, nil), []string{"-cluster-store-opt"}, "") - flags.Var(opts.NewNamedMapOpts("log-opts", c.LogConfig.Config, nil), []string{"-log-opt"}, "") - - f, err := ioutil.TempFile("", "docker-config-") - if err != nil { - t.Fatal(err) - } - - configFile := f.Name() - f.Write([]byte(`{ - "cluster-store-opts": {"kv.cacertfile": "/var/lib/docker/discovery_certs/ca.pem"}, - "log-opts": {"tag": "test"} -}`)) - f.Close() - - loadedConfig, err := loadDaemonCliConfig(c, flags, common, configFile) - if err != nil { - t.Fatal(err) - } - if loadedConfig == nil { - t.Fatal("expected configuration, got nil") - } - if loadedConfig.ClusterOpts == nil { - t.Fatal("expected cluster options, got nil") - } - - expectedPath := "/var/lib/docker/discovery_certs/ca.pem" - if caPath := loadedConfig.ClusterOpts["kv.cacertfile"]; caPath != expectedPath { - t.Fatalf("expected %s, got %s", expectedPath, caPath) - } - - if loadedConfig.LogConfig.Config == nil { - t.Fatal("expected log config options, got nil") - } - if tag := loadedConfig.LogConfig.Config["tag"]; tag != "test" { - t.Fatalf("expected log tag `test`, got %s", tag) - } -} diff --git a/components/engine/docker/daemon_unix.go b/components/engine/docker/daemon_unix.go index a89bdc73bb..c76700f014 100644 --- a/components/engine/docker/daemon_unix.go +++ b/components/engine/docker/daemon_unix.go @@ -8,6 +8,7 @@ import ( "os/signal" "syscall" + "github.com/Sirupsen/logrus" apiserver "github.com/docker/docker/api/server" "github.com/docker/docker/daemon" "github.com/docker/docker/pkg/mflag" @@ -58,7 +59,9 @@ func setupConfigReloadTrap(configFile string, flags *mflag.FlagSet, reload func( signal.Notify(c, syscall.SIGHUP) go func() { for range c { - daemon.ReloadConfiguration(configFile, flags, reload) + if err := daemon.ReloadConfiguration(configFile, flags, reload); err != nil { + logrus.Error(err) + } } }() } diff --git a/components/engine/docker/daemon_unix_test.go b/components/engine/docker/daemon_unix_test.go index 889482b007..58b692b532 100644 --- a/components/engine/docker/daemon_unix_test.go +++ b/components/engine/docker/daemon_unix_test.go @@ -8,6 +8,7 @@ import ( "github.com/docker/docker/cli" "github.com/docker/docker/daemon" + "github.com/docker/docker/opts" "github.com/docker/docker/pkg/mflag" ) @@ -41,3 +42,124 @@ func TestLoadDaemonConfigWithNetwork(t *testing.T) { t.Fatalf("expected DefaultIP 127.0.0.1, got %s", loadedConfig.DefaultIP) } } + +func TestLoadDaemonConfigWithMapOptions(t *testing.T) { + c := &daemon.Config{} + common := &cli.CommonFlags{} + flags := mflag.NewFlagSet("test", mflag.ContinueOnError) + + flags.Var(opts.NewNamedMapOpts("cluster-store-opts", c.ClusterOpts, nil), []string{"-cluster-store-opt"}, "") + flags.Var(opts.NewNamedMapOpts("log-opts", c.LogConfig.Config, nil), []string{"-log-opt"}, "") + + f, err := ioutil.TempFile("", "docker-config-") + if err != nil { + t.Fatal(err) + } + + configFile := f.Name() + f.Write([]byte(`{ + "cluster-store-opts": {"kv.cacertfile": "/var/lib/docker/discovery_certs/ca.pem"}, + "log-opts": {"tag": "test"} +}`)) + f.Close() + + loadedConfig, err := loadDaemonCliConfig(c, flags, common, configFile) + if err != nil { + t.Fatal(err) + } + if loadedConfig == nil { + t.Fatal("expected configuration, got nil") + } + if loadedConfig.ClusterOpts == nil { + t.Fatal("expected cluster options, got nil") + } + + expectedPath := "/var/lib/docker/discovery_certs/ca.pem" + if caPath := loadedConfig.ClusterOpts["kv.cacertfile"]; caPath != expectedPath { + t.Fatalf("expected %s, got %s", expectedPath, caPath) + } + + if loadedConfig.LogConfig.Config == nil { + t.Fatal("expected log config options, got nil") + } + if tag := loadedConfig.LogConfig.Config["tag"]; tag != "test" { + t.Fatalf("expected log tag `test`, got %s", tag) + } +} + +func TestLoadDaemonConfigWithTrueDefaultValues(t *testing.T) { + c := &daemon.Config{} + common := &cli.CommonFlags{} + flags := mflag.NewFlagSet("test", mflag.ContinueOnError) + flags.BoolVar(&c.EnableUserlandProxy, []string{"-userland-proxy"}, true, "") + + f, err := ioutil.TempFile("", "docker-config-") + if err != nil { + t.Fatal(err) + } + + if err := flags.ParseFlags([]string{}, false); err != nil { + t.Fatal(err) + } + + configFile := f.Name() + f.Write([]byte(`{ + "userland-proxy": false +}`)) + f.Close() + + loadedConfig, err := loadDaemonCliConfig(c, flags, common, configFile) + if err != nil { + t.Fatal(err) + } + if loadedConfig == nil { + t.Fatal("expected configuration, got nil") + } + + if loadedConfig.EnableUserlandProxy { + t.Fatal("expected userland proxy to be disabled, got enabled") + } + + // make sure reloading doesn't generate configuration + // conflicts after normalizing boolean values. + err = daemon.ReloadConfiguration(configFile, flags, func(reloadedConfig *daemon.Config) { + if reloadedConfig.EnableUserlandProxy { + t.Fatal("expected userland proxy to be disabled, got enabled") + } + }) + if err != nil { + t.Fatal(err) + } +} + +func TestLoadDaemonConfigWithTrueDefaultValuesLeaveDefaults(t *testing.T) { + c := &daemon.Config{} + common := &cli.CommonFlags{} + flags := mflag.NewFlagSet("test", mflag.ContinueOnError) + flags.BoolVar(&c.EnableUserlandProxy, []string{"-userland-proxy"}, true, "") + + f, err := ioutil.TempFile("", "docker-config-") + if err != nil { + t.Fatal(err) + } + + if err := flags.ParseFlags([]string{}, false); err != nil { + t.Fatal(err) + } + + configFile := f.Name() + f.Write([]byte(`{}`)) + f.Close() + + loadedConfig, err := loadDaemonCliConfig(c, flags, common, configFile) + if err != nil { + t.Fatal(err) + } + if loadedConfig == nil { + t.Fatal("expected configuration, got nil") + } + + if !loadedConfig.EnableUserlandProxy { + t.Fatal("expected userland proxy to be enabled, got disabled") + } +} diff --git a/components/engine/docker/daemon_windows.go b/components/engine/docker/daemon_windows.go index 307bbcc39b..52649daf0b 100644 --- a/components/engine/docker/daemon_windows.go +++ b/components/engine/docker/daemon_windows.go @@ -50,7 +50,9 @@ func setupConfigReloadTrap(configFile string, flags *mflag.FlagSet, reload func( logrus.Debugf("Config reload - waiting signal at %s", ev) for { syscall.WaitForSingleObject(h, syscall.INFINITE) - daemon.ReloadConfiguration(configFile, flags, reload) + if err := daemon.ReloadConfiguration(configFile, flags, reload); err != nil { + logrus.Error(err) + } } } }() diff --git a/components/engine/docker/listeners/listeners.go b/components/engine/docker/listeners/listeners.go new file mode 100644 index 0000000000..8150ba0c23 --- /dev/null +++ b/components/engine/docker/listeners/listeners.go @@ -0,0 +1,22 @@ +package listeners + +import ( + "crypto/tls" + "net" + + "github.com/Sirupsen/logrus" + "github.com/docker/go-connections/sockets" +) + +func initTCPSocket(addr string, tlsConfig *tls.Config) (l net.Listener, err error) { + if tlsConfig == nil || tlsConfig.ClientAuth != tls.RequireAndVerifyClientCert { + logrus.Warn("/!\\ DON'T BIND ON ANY IP ADDRESS WITHOUT setting -tlsverify IF YOU DON'T KNOW WHAT YOU'RE DOING /!\\") + } + if l, err = sockets.NewTCPSocket(addr, tlsConfig); err != nil { + return nil, err + } + if err := allocateDaemonPort(addr); err != nil { + return nil, err + } + return +} diff --git a/components/engine/api/server/server_unix.go b/components/engine/docker/listeners/listeners_unix.go similarity index 73% rename from components/engine/api/server/server_unix.go rename to components/engine/docker/listeners/listeners_unix.go index a4fc639575..732565a386 100644 --- a/components/engine/api/server/server_unix.go +++ b/components/engine/docker/listeners/listeners_unix.go @@ -1,42 +1,35 @@ -// +build freebsd linux +// +build !windows -package server +package listeners import ( "crypto/tls" "fmt" "net" - "net/http" "strconv" "github.com/Sirupsen/logrus" + "github.com/coreos/go-systemd/activation" "github.com/docker/go-connections/sockets" "github.com/docker/libnetwork/portallocator" - - systemdActivation "github.com/coreos/go-systemd/activation" ) -// newServer sets up the required HTTPServers and does protocol specific checking. -// newServer does not set any muxers, you should set it later to Handler field -func (s *Server) newServer(proto, addr string) ([]*HTTPServer, error) { - var ( - err error - ls []net.Listener - ) +// Init creates new listeners for the server. +func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) (ls []net.Listener, err error) { switch proto { case "fd": - ls, err = listenFD(addr, s.cfg.TLSConfig) + ls, err = listenFD(addr, tlsConfig) if err != nil { return nil, err } case "tcp": - l, err := s.initTCPSocket(addr) + l, err := initTCPSocket(addr, tlsConfig) if err != nil { return nil, err } ls = append(ls, l) case "unix": - l, err := sockets.NewUnixSocket(addr, s.cfg.SocketGroup) + l, err := sockets.NewUnixSocket(addr, socketGroup) if err != nil { return nil, fmt.Errorf("can't create unix socket %s: %v", addr, err) } @@ -44,43 +37,8 @@ func (s *Server) newServer(proto, addr string) ([]*HTTPServer, error) { default: return nil, fmt.Errorf("Invalid protocol format: %q", proto) } - var res []*HTTPServer - for _, l := range ls { - res = append(res, &HTTPServer{ - &http.Server{ - Addr: addr, - }, - l, - }) - } - return res, nil -} -func allocateDaemonPort(addr string) error { - host, port, err := net.SplitHostPort(addr) - if err != nil { - return err - } - - intPort, err := strconv.Atoi(port) - if err != nil { - return err - } - - var hostIPs []net.IP - if parsedIP := net.ParseIP(host); parsedIP != nil { - hostIPs = append(hostIPs, parsedIP) - } else if hostIPs, err = net.LookupIP(host); err != nil { - return fmt.Errorf("failed to lookup %s address in host specification", host) - } - - pa := portallocator.Get() - for _, hostIP := range hostIPs { - if _, err := pa.RequestPort(hostIP, "tcp", intPort); err != nil { - return fmt.Errorf("failed to allocate daemon listening port %d (err: %v)", intPort, err) - } - } - return nil + return } // listenFD returns the specified socket activated files as a slice of @@ -92,16 +50,16 @@ func listenFD(addr string, tlsConfig *tls.Config) ([]net.Listener, error) { ) // socket activation if tlsConfig != nil { - listeners, err = systemdActivation.TLSListeners(false, tlsConfig) + listeners, err = activation.TLSListeners(false, tlsConfig) } else { - listeners, err = systemdActivation.Listeners(false) + listeners, err = activation.Listeners(false) } if err != nil { return nil, err } if len(listeners) == 0 { - return nil, fmt.Errorf("No sockets found") + return nil, fmt.Errorf("No sockets found. Make sure the docker daemon was started by systemd.") } // default to all fds just like unix:// and tcp:// @@ -130,3 +88,32 @@ func listenFD(addr string, tlsConfig *tls.Config) ([]net.Listener, error) { } return []net.Listener{listeners[fdOffset]}, nil } + +// allocateDaemonPort ensures that there are no containers +// that try to use any port allocated for the docker server. +func allocateDaemonPort(addr string) error { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return err + } + + intPort, err := strconv.Atoi(port) + if err != nil { + return err + } + + var hostIPs []net.IP + if parsedIP := net.ParseIP(host); parsedIP != nil { + hostIPs = append(hostIPs, parsedIP) + } else if hostIPs, err = net.LookupIP(host); err != nil { + return fmt.Errorf("failed to lookup %s address in host specification", host) + } + + pa := portallocator.Get() + for _, hostIP := range hostIPs { + if _, err := pa.RequestPort(hostIP, "tcp", intPort); err != nil { + return fmt.Errorf("failed to allocate daemon listening port %d (err: %v)", intPort, err) + } + } + return nil +} diff --git a/components/engine/api/server/server_windows.go b/components/engine/docker/listeners/listeners_windows.go similarity index 59% rename from components/engine/api/server/server_windows.go rename to components/engine/docker/listeners/listeners_windows.go index 613d185522..282b285256 100644 --- a/components/engine/api/server/server_windows.go +++ b/components/engine/docker/listeners/listeners_windows.go @@ -1,24 +1,20 @@ -// +build windows - -package server +package listeners import ( + "crypto/tls" "errors" "fmt" - "github.com/Microsoft/go-winio" "net" - "net/http" "strings" + + "github.com/Microsoft/go-winio" ) -// NewServer sets up the required Server and does protocol specific checking. -func (s *Server) newServer(proto, addr string) ([]*HTTPServer, error) { - var ( - ls []net.Listener - ) +// Init creates new listeners for the server. +func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) (ls []net.Listener, err error) { switch proto { case "tcp": - l, err := s.initTCPSocket(addr) + l, err := initTCPSocket(addr, tlsConfig) if err != nil { return nil, err } @@ -27,8 +23,8 @@ func (s *Server) newServer(proto, addr string) ([]*HTTPServer, error) { case "npipe": // allow Administrators and SYSTEM, plus whatever additional users or groups were specified sddl := "D:P(A;;GA;;;BA)(A;;GA;;;SY)" - if s.cfg.SocketGroup != "" { - for _, g := range strings.Split(s.cfg.SocketGroup, ",") { + if socketGroup != "" { + for _, g := range strings.Split(socketGroup, ",") { sid, err := winio.LookupSidByName(g) if err != nil { return nil, err @@ -46,19 +42,11 @@ func (s *Server) newServer(proto, addr string) ([]*HTTPServer, error) { return nil, errors.New("Invalid protocol format. Windows only supports tcp and npipe.") } - var res []*HTTPServer - for _, l := range ls { - res = append(res, &HTTPServer{ - &http.Server{ - Addr: addr, - }, - l, - }) - } - return res, nil - + return } +// allocateDaemonPort ensures that there are no containers +// that try to use any port allocated for the docker server. func allocateDaemonPort(addr string) error { return nil } diff --git a/components/engine/docs/Dockerfile b/components/engine/docs/Dockerfile index 2f29b214eb..3690d1572a 100644 --- a/components/engine/docs/Dockerfile +++ b/components/engine/docs/Dockerfile @@ -5,8 +5,9 @@ RUN svn checkout https://github.com/docker/compose/trunk/docs /docs/content/comp RUN svn checkout https://github.com/docker/swarm/trunk/docs /docs/content/swarm RUN svn checkout https://github.com/docker/machine/trunk/docs /docs/content/machine RUN svn checkout https://github.com/docker/distribution/trunk/docs /docs/content/registry -RUN svn checkout https://github.com/kitematic/kitematic/trunk/docs /docs/content/kitematic -RUN svn checkout https://github.com/docker/tutorials/trunk/docs /docs/content/ +RUN svn checkout https://github.com/docker/notary/trunk/docs /docs/content/notary +RUN svn checkout https://github.com/docker/kitematic/trunk/docs /docs/content/kitematic +RUN svn checkout https://github.com/docker/toolbox/trunk/docs /docs/content/toolbox RUN svn checkout https://github.com/docker/opensource/trunk/docs /docs/content/opensource ENV PROJECT=engine diff --git a/components/engine/docs/admin/logging/etwlogs.md b/components/engine/docs/admin/logging/etwlogs.md new file mode 100644 index 0000000000..5b98fd5495 --- /dev/null +++ b/components/engine/docs/admin/logging/etwlogs.md @@ -0,0 +1,69 @@ + + + +# ETW logging driver + +The ETW logging driver forwards container logs as ETW events. +ETW stands for Event Tracing in Windows, and is the common framework +for tracing applications in Windows. Each ETW event contains a message +with both the log and its context information. A client can then create +an ETW listener to listen to these events. + +The ETW provider that this logging driver registers with Windows, has the +GUID identifier of: `{a3693192-9ed6-46d2-a981-f8226c8363bd}`. A client creates an +ETW listener and registers to listen to events from the logging driver's provider. +It does not matter the order in which the provider and listener are created. +A client can create their ETW listener and start listening for events from the provider, +before the provider has been registered with the system. + +## Usage + +Here is an example of how to listen to these events using the logman utility program +included in most installations of Windows: + + 1. `logman start -ets DockerContainerLogs -p {a3693192-9ed6-46d2-a981-f8226c8363bd} 0 0 -o trace.etl` + 2. Run your container(s) with the etwlogs driver, by adding `--log-driver=etwlogs` + to the Docker run command, and generate log messages. + 3. `logman stop -ets DockerContainerLogs` + 4. This will generate an etl file that contains the events. One way to convert this file into + human-readable form is to run: `tracerpt -y trace.etl`. + +Each ETW event will contain a structured message string in this format: + + container_name: %s, image_name: %s, container_id: %s, image_id: %s, source: [stdout | stderr], log: %s + +Details on each item in the message can be found below: + +| Field | Description | +-----------------------|-------------------------------------------------| +| `container_name` | The container name at the time it was started. | +| `image_name` | The name of the container's image. | +| `container_id` | The full 64-character container ID. | +| `image_id` | The full ID of the container's image. | +| `source` | `stdout` or `stderr`. | +| `log` | The container log message. | + +Here is an example event message: + + container_name: backstabbing_spence, + image_name: windowsservercore, + container_id: f14bb55aa862d7596b03a33251c1be7dbbec8056bbdead1da8ec5ecebbe29731, + image_id: sha256:2f9e19bd998d3565b4f345ac9aaf6e3fc555406239a4fb1b1ba879673713824b, + source: stdout, + log: Hello world! + +A client can parse this message string to get both the log message, as well as its +context information. Note that the time stamp is also available within the ETW event. + +**Note** This ETW provider emits only a message string, and not a specially +structured ETW event. Therefore, it is not required to register a manifest file +with the system to read and interpret its ETW events. diff --git a/components/engine/docs/admin/logging/fluentd.md b/components/engine/docs/admin/logging/fluentd.md index a87b1dca69..86bddecb72 100644 --- a/components/engine/docs/admin/logging/fluentd.md +++ b/components/engine/docs/admin/logging/fluentd.md @@ -101,7 +101,7 @@ and [its documents](http://docs.fluentd.org/). To use this logging driver, start the `fluentd` daemon on a host. We recommend that you use [the Fluentd docker -image](https://registry.hub.docker.com/u/fluent/fluentd/). This image is +image](https://hub.docker.com/r/fluent/fluentd/). This image is especially useful if you want to aggregate multiple container logs on a each host then, later, transfer the logs to another Fluentd node to create an aggregate store. diff --git a/components/engine/docs/admin/logging/gcplogs.md b/components/engine/docs/admin/logging/gcplogs.md new file mode 100644 index 0000000000..08fd858da0 --- /dev/null +++ b/components/engine/docs/admin/logging/gcplogs.md @@ -0,0 +1,70 @@ + + +# Google Cloud Logging driver + +The Google Cloud Logging driver sends container logs to Google Cloud +Logging. + +## Usage + +You can configure the default logging driver by passing the `--log-driver` +option to the Docker daemon: + + docker daemon --log-driver=gcplogs + +You can set the logging driver for a specific container by using the +`--log-driver` option to `docker run`: + + docker run --log-driver=gcplogs ... + +This log driver does not implement a reader so it is incompatible with +`docker logs`. + +If Docker detects that it is running in a Google Cloud Project, it will discover configuration +from the instance metadata service. +Otherwise, the user must specify which project to log to using the `--gcp-project` +log option and Docker will attempt to obtain credentials from the +Google Application Default Credential. +The `--gcp-project` takes precedence over information discovered from the metadata server +so a Docker daemon running in a Google Cloud Project can be overriden to log to a different +Google Cloud Project using `--gcp-project`. + +## gcplogs options + +You can use the `--log-opt NAME=VALUE` flag to specify these additional Google +Cloud Logging driver options: + +| Option | Required | Description | +|-----------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------| +| `gcp-project` | optional | Which GCP project to log to. Defaults to discovering this value from the GCE metadata service. | +| `gcp-log-cmd` | optional | Whether to log the command that the container was started with. Defaults to false. | +| `labels` | optional | Comma-separated list of keys of labels, which should be included in message, if these labels are specified for container. | +| `env` | optional | Comma-separated list of keys of environment variables, which should be included in message, if these variables are specified for container. | + +If there is collision between `label` and `env` keys, the value of the `env` +takes precedence. Both options add additional fields to the attributes of a +logging message. + +Below is an example of the logging options required to log to the default +logging destination which is discovered by querying the GCE metadata server. + + docker run --log-driver=gcplogs \ + --log-opt labels=location + --log-opt env=TEST + --log-opt gcp-log-cmd=true + --env "TEST=false" + --label location=west + your/application + +This configuration also directs the driver to include in the payload the label +`location`, the environment variable `ENV`, and the command used to start the +container. diff --git a/components/engine/docs/admin/logging/index.md b/components/engine/docs/admin/logging/index.md index 64f9c526aa..4300565e8f 100644 --- a/components/engine/docs/admin/logging/index.md +++ b/components/engine/docs/admin/logging/index.md @@ -20,3 +20,4 @@ weight=8 * [Journald logging driver](journald.md) * [Amazon CloudWatch Logs logging driver](awslogs.md) * [Splunk logging driver](splunk.md) +* [ETW logging driver](etwlogs.md) diff --git a/components/engine/docs/admin/logging/overview.md b/components/engine/docs/admin/logging/overview.md index 531b338ed0..e3d3d11256 100644 --- a/components/engine/docs/admin/logging/overview.md +++ b/components/engine/docs/admin/logging/overview.md @@ -26,6 +26,8 @@ container's logging driver. The following options are supported: | `fluentd` | Fluentd logging driver for Docker. Writes log messages to `fluentd` (forward input). | | `awslogs` | Amazon CloudWatch Logs logging driver for Docker. Writes log messages to Amazon CloudWatch Logs. | | `splunk` | Splunk logging driver for Docker. Writes log messages to `splunk` using HTTP Event Collector. | +| `etwlogs` | ETW logging driver for Docker on Windows. Writes log messages as ETW events. | +| `gcplogs` | Google Cloud Logging driver for Docker. Writes log messages to Google Cloud Logging. | The `docker logs`command is available only for the `json-file` and `journald` logging drivers. @@ -204,3 +206,22 @@ The Splunk logging driver requires the following options: For detailed information about working with this logging driver, see the [Splunk logging driver](splunk.md) reference documentation. + +## ETW logging driver options + +The etwlogs logging driver does not require any options to be specified. This logging driver will forward each log message +as an ETW event. An ETW listener can then be created to listen for these events. + +For detailed information on working with this logging driver, see [the ETW logging driver](etwlogs.md) reference documentation. + +## Google Cloud Logging + +The Google Cloud Logging driver supports the following options: + + --log-opt gcp-project= + --log-opt labels=, + --log-opt env=, + --log-opt log-cmd=true + +For detailed information about working with this logging driver, see the [Google Cloud Logging driver](gcplogs.md). +reference documentation. diff --git a/components/engine/docs/deprecated.md b/components/engine/docs/deprecated.md index 4ed127f953..7a351ac176 100644 --- a/components/engine/docs/deprecated.md +++ b/components/engine/docs/deprecated.md @@ -14,6 +14,13 @@ weight=80 The following list of features are deprecated in Engine. +### `-e` and `--email` flags on `docker login` +**Deprecated In Release: v1.11** + +**Target For Removal In Release: v1.13** + +The docker login command is removing the ability to automatically register for an account with the target registry if the given username doesn't exist. Due to this change, the email flag is no longer required, and will be deprecated. + ### Ambiguous event fields in API **Deprecated In Release: v1.10** diff --git a/components/engine/docs/examples/mongodb.md b/components/engine/docs/examples/mongodb.md index f6498e0cc5..3173aa1b7e 100644 --- a/components/engine/docs/examples/mongodb.md +++ b/components/engine/docs/examples/mongodb.md @@ -17,7 +17,7 @@ MongoDB pre-installed. We'll also see how to `push` that image to the [Docker Hub registry](https://hub.docker.com) and share it with others! > **Note:** This guide will show the mechanics of building a MongoDB container, but -> you will probably want to use the official image on [Docker Hub]( https://registry.hub.docker.com/_/mongo/) +> you will probably want to use the official image on [Docker Hub]( https://hub.docker.com/_/mongo/) Using Docker and containers for deploying [MongoDB](https://www.mongodb.org/) instances will bring several benefits, such as: @@ -49,7 +49,7 @@ Although optional, it is handy to have comments at the beginning of a > the *parent* of your *Dockerized MongoDB* image. We will build our image using the latest version of Ubuntu from the -[Docker Hub Ubuntu](https://registry.hub.docker.com/_/ubuntu/) repository. +[Docker Hub Ubuntu](https://hub.docker.com/_/ubuntu/) repository. # Format: FROM repository[:version] FROM ubuntu:latest diff --git a/components/engine/docs/examples/nodejs_web_app.md b/components/engine/docs/examples/nodejs_web_app.md index 55425c0672..149f5b47e4 100644 --- a/components/engine/docs/examples/nodejs_web_app.md +++ b/components/engine/docs/examples/nodejs_web_app.md @@ -67,7 +67,7 @@ Open the `Dockerfile` in your favorite text editor Define the parent image you want to use to build your own image on top of. Here, we'll use -[CentOS](https://registry.hub.docker.com/_/centos/) (tag: `centos6`) +[CentOS](https://hub.docker.com/_/centos/) (tag: `centos6`) available on the [Docker Hub](https://hub.docker.com/): FROM centos:centos6 @@ -89,7 +89,7 @@ Install your app dependencies using the `npm` binary: # Install app dependencies COPY package.json /src/package.json - RUN cd /src; npm install + RUN cd /src; npm install --production To bundle your app's source code inside the Docker image, use the `COPY` instruction: @@ -119,7 +119,7 @@ Your `Dockerfile` should now look like this: # Install app dependencies COPY package.json /src/package.json - RUN cd /src; npm install + RUN cd /src; npm install --production # Bundle app source COPY . /src diff --git a/components/engine/docs/examples/running_riak_service.md b/components/engine/docs/examples/running_riak_service.md index a6c3d3f4d4..f17969fe48 100644 --- a/components/engine/docs/examples/running_riak_service.md +++ b/components/engine/docs/examples/running_riak_service.md @@ -20,7 +20,7 @@ Create an empty file called `Dockerfile`: $ touch Dockerfile Next, define the parent image you want to use to build your image on top -of. We'll use [Ubuntu](https://registry.hub.docker.com/_/ubuntu/) (tag: +of. We'll use [Ubuntu](https://hub.docker.com/_/ubuntu/) (tag: `trusty`), which is available on [Docker Hub](https://hub.docker.com): # Riak diff --git a/components/engine/docs/extend/index.md b/components/engine/docs/extend/index.md index f491926e9a..8a061e4e29 100644 --- a/components/engine/docs/extend/index.md +++ b/components/engine/docs/extend/index.md @@ -18,5 +18,5 @@ Currently, you can extend Docker Engine by adding a plugin. This section contain * [Understand Docker plugins](plugins.md) * [Write a volume plugin](plugins_volume.md) * [Write a network plugin](plugins_network.md) -* [Write an authorization plugin](authorization.md) +* [Write an authorization plugin](plugins_authorization.md) * [Docker plugin API](plugin_api.md) diff --git a/components/engine/docs/extend/plugin_api.md b/components/engine/docs/extend/plugin_api.md index c6793a3dda..1a4237e2c7 100644 --- a/components/engine/docs/extend/plugin_api.md +++ b/components/engine/docs/extend/plugin_api.md @@ -96,6 +96,43 @@ directory and activates it with a handshake. See Handshake API below. Plugins are *not* activated automatically at Docker daemon startup. Rather, they are activated only lazily, or on-demand, when they are needed. +## Systemd socket activation + +Plugins may also be socket activated by `systemd`. The official [Plugins helpers](https://github.com/docker/go-plugins-helpers) +natively supports socket activation. In order for a plugin to be socket activated it needs +a `service` file and a `socket` file. + +The `service` file (for example `/lib/systemd/system/your-plugin.service`): + +``` +[Unit] +Description=Your plugin +Before=docker.service +After=network.target your-plugin.socket +Requires=your-plugin.socket docker.service + +[Service] +ExecStart=/usr/lib/docker/your-plugin + +[Install] +WantedBy=multi-user.target +``` +The `socket` file (for example `/lib/systemd/system/your-plugin.socket`): +``` +[Unit] +Description=Your plugin + +[Socket] +ListenStream=/run/docker/plugins/your-plugin.sock + +[Install] +WantedBy=sockets.target +``` + +This will allow plugins to be actually started when the Docker daemon connects to +the sockets they're listening on (for instance the first time the daemon uses them +or if one of the plugin goes down accidentally). + ## API design The Plugin API is RPC-style JSON over HTTP, much like webhooks. @@ -127,9 +164,20 @@ Plugins are activated via the following "handshake" API call. Responds with a list of Docker subsystems which this plugin implements. After activation, the plugin will then be sent events from this subsystem. +Possible values are: + - [`authz`](plugins_authorization.md) + - [`NetworkDriver`](plugins_network.md) + - [`VolumeDriver`](plugins_volume.md) + + ## Plugin retries Attempts to call a method on a plugin are retried with an exponential backoff for up to 30 seconds. This may help when packaging plugins as containers, since it gives plugin containers a chance to start up before failing any user containers which depend on them. + +## Plugins helpers + +To ease plugins development, we're providing an `sdk` for each kind of plugins +currently supported by Docker at [docker/go-plugins-helpers](https://github.com/docker/go-plugins-helpers). diff --git a/components/engine/docs/extend/plugins.md b/components/engine/docs/extend/plugins.md index 997c9dfbc5..da46a16ed7 100644 --- a/components/engine/docs/extend/plugins.md +++ b/components/engine/docs/extend/plugins.md @@ -99,6 +99,11 @@ The following plugins exist: Neutron, the OpenStack networking service. It includes an IPAM driver as well. +* The [Local Persist Plugin](https://github.com/CWSpear/local-persist) + extends the default `local` driver's functionality by allowing you specify + a mountpoint anywhere on the host, which enables the files to *always persist*, + even if the volume is removed via `docker volume rm`. + ## Troubleshooting a plugin If you are having problems with Docker after loading a plugin, ask the authors diff --git a/components/engine/docs/extend/authorization.md b/components/engine/docs/extend/plugins_authorization.md similarity index 99% rename from components/engine/docs/extend/authorization.md rename to components/engine/docs/extend/plugins_authorization.md index 3512c56ccd..7db31b85c4 100644 --- a/components/engine/docs/extend/authorization.md +++ b/components/engine/docs/extend/plugins_authorization.md @@ -3,6 +3,7 @@ title = "Access authorization plugin" description = "How to create authorization plugins to manage access control to your Docker daemon." keywords = ["security, authorization, authentication, docker, documentation, plugin, extend"] +aliases = ["/engine/extend/authorization/"] [menu.main] parent = "engine_extend" weight = -1 diff --git a/components/engine/docs/installation/cloud/cloud-ex-aws.md b/components/engine/docs/installation/cloud/cloud-ex-aws.md index 3163865ad6..0484f2481d 100644 --- a/components/engine/docs/installation/cloud/cloud-ex-aws.md +++ b/components/engine/docs/installation/cloud/cloud-ex-aws.md @@ -1,6 +1,6 @@ -# Example: Manual install on a cloud provider +# Example: Manual install on cloud provider You can install Docker Engine directly to servers you have on cloud providers. This example shows how to create an Amazon Web Services (AWS) EC2 instance, and install Docker Engine on it. @@ -197,8 +197,12 @@ For Ubuntu Trusty (and some other versions), it’s recommended to install the ` ## Where to go next -* Would you like a quicker way to do Docker cloud installs? See [Digital Ocean Example: Use Docker Machine to provision Docker on cloud hosts](cloud-ex-aws.md). +_Looking for a quicker way to do Docker cloud installs and provision multiple hosts?_ You can use [Docker Machine](https://docs.docker.com/machine/overview/) to provision hosts. -* To learn more about options for installing Docker Engine on cloud providers, see [Understand cloud install options and choose one](cloud.md). + * [Use Docker Machine to provision hosts on cloud providers](https://docs.docker.com/machine/get-started-cloud/) -* To get started with Docker, see Docker User Guide . + * [Docker Machine driver reference](https://docs.docker.com/machine/drivers/) + +* [Install Docker Engine](../index.md) + +* [Docker User Guide](../../userguide/intro.md) diff --git a/components/engine/docs/installation/cloud/cloud-ex-machine-ocean.md b/components/engine/docs/installation/cloud/cloud-ex-machine-ocean.md index 2164f4bf22..ac00a84c2a 100644 --- a/components/engine/docs/installation/cloud/cloud-ex-machine-ocean.md +++ b/components/engine/docs/installation/cloud/cloud-ex-machine-ocean.md @@ -14,7 +14,7 @@ Docker Machine driver plugins are available for many cloud platforms, so you can You'll need to install and run Docker Machine, and create an account with the cloud provider. -Then you provide account verification, security credentials, and configuration options for the providers as flags to `docker-machine create`. The flags are unique for each cloud-specific driver. For instance, to pass a Digital Ocean access token you use the `--digitalocean-access-token` flag. +Then you provide account verification, security credentials, and configuration options for the providers as flags to `docker-machine create`. The flags are unique for each cloud-specific driver. For instance, to pass a Digital Ocean access token, you use the `--digitalocean-access-token` flag. As an example, let's take a look at how to create a Dockerized Digital Ocean _Droplet_ (cloud server). @@ -44,7 +44,7 @@ To generate your access token: This is the personal access token you'll use in the next step to create your cloud server. -### Step 3. Start Docker Machine +### Step 3. Install Docker Machine 1. If you have not done so already, install Docker Machine on your local host. @@ -56,30 +56,6 @@ To generate your access token: 2. At a command terminal, use `docker-machine ls` to get a list of Docker Machines and their status. - $ docker-machine ls - NAME ACTIVE DRIVER STATE URL SWARM - default - virtualbox Stopped - -3. If Machine is stopped, start it. - - $ docker-machine start default - (default) OUT | Starting VM... - Started machines may have new IP addresses. You may need to re-run the `docker-machine env` command. - -4. Set environment variables to connect your shell to the local VM. - - $ docker-machine env default - export DOCKER_TLS_VERIFY="1" - export DOCKER_HOST="tcp://xxx.xxx.xx.xxx:xxxx" - export DOCKER_CERT_PATH="/Users/londoncalling/.docker/machine/machines/default" - export DOCKER_MACHINE_NAME="default" - # Run this command to configure your shell: - # eval "$(docker-machine env default)" - - eval "$(docker-machine env default)" - -5. Re-run `docker-machine ls` to check that it's now running. - $ docker-machine ls NAME ACTIVE DRIVER STATE URL SWARM default * virtualbox Running tcp:////xxx.xxx.xx.xxx:xxxx @@ -92,28 +68,15 @@ To generate your access token: Hello from Docker. This message shows that your installation appears to be working correctly. + ... - To generate this message, Docker took the following steps: - 1. The Docker client contacted the Docker daemon. - 2. The Docker daemon pulled the "hello-world" image from the Docker Hub. - 3. The Docker daemon created a new container from that image which runs the executable that produces the output you are currently reading. - 4. The Docker daemon streamed that output to the Docker client, which sent it to your terminal. - - To try something more ambitious, you can run an Ubuntu container with: - $ docker run -it ubuntu bash - - Share images, automate workflows, and more with a free Docker Hub account: https://hub.docker.com - - For more examples and ideas, visit: - https://docs.docker.com/userguide/ - -### Step 4. Use Docker Machine to Create the Droplet +### Step 4. Use Machine to Create the Droplet 1. Run `docker-machine create` with the `digitalocean` driver and pass your key to the `--digitalocean-access-token` flag, along with a name for the new cloud server. For this example, we'll call our new Droplet "docker-sandbox". - $ docker-machine create --driver digitalocean --digitalocean-access-token 455275108641c7716462d6f35d08b76b246b6b6151a816cf75de63c5ef918872 docker-sandbox + $ docker-machine create --driver digitalocean --digitalocean-access-token xxxxx docker-sandbox Running pre-create checks... Creating machine... (docker-sandbox) OUT | Creating SSH key... @@ -163,45 +126,52 @@ To generate your access token: default - virtualbox Running tcp://192.168.99.100:2376 docker-sandbox * digitalocean Running tcp://45.55.222.72:2376 -6. Log in to the Droplet with the `docker-machine ssh` command. +6. Run some `docker-machine` commands to inspect the remote host. For example, `docker-machine ip ` gets the host IP adddress and `docker-machine inspect ` lists all the details. - $ docker-machine ssh docker-sandbox - Welcome to Ubuntu 14.04.3 LTS (GNU/Linux 3.13.0-71-generic x86_64) + $ docker-machine ip docker-sandbox + 104.131.43.236 - * Documentation: https://help.ubuntu.com/ + $ docker-machine inspect docker-sandbox + { + "ConfigVersion": 3, + "Driver": { + "IPAddress": "104.131.43.236", + "MachineName": "docker-sandbox", + "SSHUser": "root", + "SSHPort": 22, + "SSHKeyPath": "/Users/samanthastevens/.docker/machine/machines/docker-sandbox/id_rsa", + "StorePath": "/Users/samanthastevens/.docker/machine", + "SwarmMaster": false, + "SwarmHost": "tcp://0.0.0.0:3376", + "SwarmDiscovery": "", + ... - System information as of Mon Dec 21 21:38:53 EST 2015 +7. Verify Docker Engine is installed correctly by running `docker` commands. - System load: 0.77 Processes: 70 - Usage of /: 11.4% of 19.56GB Users logged in: 0 - Memory usage: 15% IP address for eth0: 45.55.139.48 - Swap usage: 0% IP address for docker0: 172.17.0.1 + Start with something basic like `docker run hello-world`, or for a more interesting test, run a Dockerized webserver on your new remote machine. - Graph this data and manage this system at: - https://landscape.canonical.com/ + In this example, the `-p` option is used to expose port 80 from the `nginx` container and make it accessible on port `8000` of the `docker-sandbox` host. -7. Verify Docker Engine is installed correctly by running `docker run hello-world`. + $ docker run -d -p 8000:80 --name webserver kitematic/hello-world-nginx + Unable to find image 'kitematic/hello-world-nginx:latest' locally + latest: Pulling from kitematic/hello-world-nginx + a285d7f063ea: Pull complete + 2d7baf27389b: Pull complete + ... + Digest: sha256:ec0ca6dcb034916784c988b4f2432716e2e92b995ac606e080c7a54b52b87066 + Status: Downloaded newer image for kitematic/hello-world-nginx:latest + 942dfb4a0eaae75bf26c9785ade4ff47ceb2ec2a152be82b9d7960e8b5777e65 - ubuntu@ip-172-31-0-151:~$ sudo docker run hello-world - Unable to find image 'hello-world:latest' locally - latest: Pulling from library/hello-world - b901d36b6f2f: Pull complete - 0a6ba66e537a: Pull complete - Digest: sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7 - Status: Downloaded newer image for hello-world:latest + In a web browser, go to `http://:8000` to bring up the webserver home page. You got the `` from the output of the `docker-machine ip ` command you ran in a previous step. Use the port you exposed in the `docker run` command. - Hello from Docker. - This message shows that your installation appears to be working correctly. - . . . - - You can type keyboard command Control-D or `exit` to log out of the remote server. + ![nginx webserver](../images/nginx-webserver.png) #### Understand the defaults and options on the create command For convenience, `docker-machine` will use sensible defaults for choosing settings such as the image that the server is based on, but you override the defaults using the respective flags (e.g. `--digitalocean-image`). This is useful if, for example, you want to create a cloud server with a lot of memory and CPUs (by default `docker-machine` creates a small server). For a full list of the flags/settings available and their defaults, see the output of `docker-machine create -h` at the command line. See also Driver options and operating system defaults and information about the create command in the Docker Machine documentation. -### Step 5. Use Docker Machine to remove the Droplet +### Step 5. Use Machine to remove the Droplet To remove a host and all of its containers and images, first stop the machine, then use `docker-machine rm`: @@ -220,8 +190,12 @@ If you create a host with Docker Machine, but remove it through the cloud provid ## Where to go next -* To learn more about options for installing Docker Engine on cloud providers, see [Understand cloud install options and choose one](cloud.md). +* [Docker Machine driver reference](https://docs.docker.com/machine/drivers/) -* To learn more about using Docker Machine to provision cloud hosts, see Using Docker Machine with a cloud provider. +* [Docker Machine Overview](https://docs.docker.com/machine/overview/) -* To get started with Docker, see Docker User Guide. +* [Use Docker Machine to provision hosts on cloud providers](https://docs.docker.com/machine/get-started-cloud/) + +* [Install Docker Engine](../../installation/index.md) + +* [Docker User Guide](../../userguide/intro.md) diff --git a/components/engine/docs/installation/cloud/index.md b/components/engine/docs/installation/cloud/index.md index 96589c4608..c7a83d31d1 100644 --- a/components/engine/docs/installation/cloud/index.md +++ b/components/engine/docs/installation/cloud/index.md @@ -8,7 +8,7 @@ aliases = [ "/engine/installation/rackspace/", "/engine/installation/joyent/" ] -title = "In the cloud" +title = "On cloud providers" description = "Cloud Installations" keywords = ["Docker install "] [menu.main] @@ -20,6 +20,6 @@ weight="-60" # Install Engine in the cloud -* [Understand cloud install options and choose one](cloud.md) -* [Example: Use Docker Machine to provision cloud hosts](cloud-ex-machine-ocean.md) +* [Understand cloud install options and choose one](overview.md) +* [Example: Use Machine to provision cloud hosts](cloud-ex-machine-ocean.md) * [Example: Manual install on a cloud provider](cloud-ex-aws.md) diff --git a/components/engine/docs/installation/cloud/cloud.md b/components/engine/docs/installation/cloud/overview.md similarity index 72% rename from components/engine/docs/installation/cloud/cloud.md rename to components/engine/docs/installation/cloud/overview.md index d5ba411b9c..e8b3bb7e94 100644 --- a/components/engine/docs/installation/cloud/cloud.md +++ b/components/engine/docs/installation/cloud/overview.md @@ -1,5 +1,8 @@ -# Understand cloud install options and choose one +# Choose how to install You can install Docker Engine on any cloud platform that runs an operating system (OS) that Docker supports. This includes many flavors and versions of Linux, along with Mac and Windows. @@ -26,11 +29,11 @@ To install on a cloud provider: 2. Decide which OS you want to run on the cloud host. -3. Understand the Docker prerequisites and install process for the chosen OS. See [Install Docker Engine](index.md) for a list of supported systems and links to the install guides. +3. Understand the Docker prerequisites and install process for the chosen OS. See [Install Docker Engine](../index.md) for a list of supported systems and links to the install guides. 4. Create a host with a Docker supported OS, and install Docker per the instructions for that OS. -[Example: Manual install on a cloud provider](cloud-ex-aws.md) shows how to create an Amazon Web Services (AWS) EC2 instance, and install Docker Engine on it. +[Example (AWS): Manual install on a cloud provider](cloud-ex-aws.md) shows how to create an Amazon Web Services (AWS) EC2 instance, and install Docker Engine on it. ## Use Docker Machine to provision cloud hosts @@ -41,13 +44,13 @@ With Docker Machine, you can use the same interface to create cloud hosts with D To do this, you use the `docker-machine create` command with the driver for the cloud provider, and provider-specific flags for account verification, security credentials, and other configuration details. -[Example: Use Docker Machine to provision cloud hosts](cloud-ex-machine-ocean.md) walks you through the steps to set up Docker Machine and provision a Dockerized host on [Digital Ocean](https://www.digitalocean.com/). +[Example: Use Docker Machine to provision cloud hosts](cloud-ex-machine-ocean.md) walks you through the steps to set up Docker Machine and provision a Dockerized host on Digital Ocean). ## Where to go next * [Example: Manual install on a cloud provider](cloud-ex-aws.md) (AWS EC2) * [Example: Use Docker Machine to provision cloud hosts](cloud-ex-machine-ocean.md) (Digital Ocean) -* [Using Docker Machine with a cloud provider](https://docs.docker.com/machine/get-started-cloud/) +* For supported platforms, see [Install Docker Engine](../index.md). -* Docker User Guide (after your install is complete, get started using Docker) +* To get started with Docker post-install, see [Docker User Guide](../../userguide/intro.md). diff --git a/components/engine/docs/installation/images/nginx-webserver.png b/components/engine/docs/installation/images/nginx-webserver.png new file mode 100644 index 0000000000..941fdaaf63 Binary files /dev/null and b/components/engine/docs/installation/images/nginx-webserver.png differ diff --git a/components/engine/docs/installation/index.md b/components/engine/docs/installation/index.md index 54d85c107c..f4c530a10e 100644 --- a/components/engine/docs/installation/index.md +++ b/components/engine/docs/installation/index.md @@ -30,7 +30,7 @@ Docker Engine is supported on Linux, Cloud, Windows, and OS X. Installation inst If your linux distribution is not listed above, don't give up yet. To try out Docker on a distribution that is not listed above, go here: [Installation from binaries](binaries.md). ## On Cloud -* [Choose how to Install](cloud/cloud.md) +* [Choose how to Install](cloud/overview.md) * [Example: Manual install on a cloud provider](cloud/cloud-ex-aws.md) * [Example: Use Docker Machine to provision cloud hosts](cloud/cloud-ex-machine-ocean.md) diff --git a/components/engine/docs/installation/linux/centos.md b/components/engine/docs/installation/linux/centos.md index d0569f61bb..1647f76611 100644 --- a/components/engine/docs/installation/linux/centos.md +++ b/components/engine/docs/installation/linux/centos.md @@ -32,7 +32,7 @@ display your kernel version: $ uname -r 3.10.0-229.el7.x86_64 -Finally, is it recommended that you fully update your system. Please keep in +Finally, it is recommended that you fully update your system. Please keep in mind that your system should be fully patched to fix any potential kernel bugs. Any reported kernel bugs may have already been fixed on the latest kernel packages. @@ -141,15 +141,19 @@ To create the `docker` group and add your user: 1. Log into Centos as a user with `sudo` privileges. -2. Create the `docker` group and add your user. +2. Create the `docker` group. + + `sudo groupadd docker` + +3. Add your user to `docker` group. `sudo usermod -aG docker your_username` -3. Log out and log back in. +4. Log out and log back in. This ensures your user is running with the correct permissions. -4. Verify your work by running `docker` without `sudo`. +5. Verify your work by running `docker` without `sudo`. $ docker run hello-world diff --git a/components/engine/docs/installation/linux/cruxlinux.md b/components/engine/docs/installation/linux/cruxlinux.md index 583f3f45ba..6c95110b40 100644 --- a/components/engine/docs/installation/linux/cruxlinux.md +++ b/components/engine/docs/installation/linux/cruxlinux.md @@ -64,7 +64,7 @@ or use it as part of your `FROM` line in your `Dockerfile(s)`. $ docker pull crux $ docker run -i -t crux -There are also user contributed [CRUX based image(s)](https://registry.hub.docker.com/repos/crux/) on the Docker Hub. +There are also user contributed [CRUX based image(s)](https://hub.docker.com/_/crux/) on the Docker Hub. ## Uninstallation diff --git a/components/engine/docs/installation/linux/fedora.md b/components/engine/docs/installation/linux/fedora.md index 3fd46e9a5a..782adc6735 100644 --- a/components/engine/docs/installation/linux/fedora.md +++ b/components/engine/docs/installation/linux/fedora.md @@ -135,15 +135,19 @@ To create the `docker` group and add your user: 1. Log into your system as a user with `sudo` privileges. -2. Create the `docker` group and add your user. +2. Create the `docker` group. + + `sudo groupadd docker` + +3. Add your user to `docker` group. `sudo usermod -aG docker your_username` -3. Log out and log back in. +4. Log out and log back in. This ensures your user is running with the correct permissions. -4. Verify your work by running `docker` without `sudo`. +5. Verify your work by running `docker` without `sudo`. $ docker run hello-world diff --git a/components/engine/docs/installation/linux/gentoolinux.md b/components/engine/docs/installation/linux/gentoolinux.md index dac0497465..3b33ee453b 100644 --- a/components/engine/docs/installation/linux/gentoolinux.md +++ b/components/engine/docs/installation/linux/gentoolinux.md @@ -76,6 +76,7 @@ To use Docker, the `docker` daemon must be running as **root**. To use Docker as a **non-root** user, add yourself to the **docker** group by running the following command: + $ sudo groupadd docker $ sudo usermod -a -G docker user ### OpenRC diff --git a/components/engine/docs/installation/linux/oracle.md b/components/engine/docs/installation/linux/oracle.md index a154346494..9513f8b85d 100644 --- a/components/engine/docs/installation/linux/oracle.md +++ b/components/engine/docs/installation/linux/oracle.md @@ -113,15 +113,19 @@ To create the `docker` group and add your user: 1. Log into Oracle Linux as a user with `sudo` privileges. -2. Create the `docker` group and add your user. +2. Create the `docker` group. + + sudo groupadd docker + +3. Add your user to `docker` group. sudo usermod -aG docker username -3. Log out and log back in. +4. Log out and log back in. This ensures your user is running with the correct permissions. -4. Verify your work by running `docker` without `sudo`. +5. Verify your work by running `docker` without `sudo`. $ docker run hello-world diff --git a/components/engine/docs/installation/linux/rhel.md b/components/engine/docs/installation/linux/rhel.md index d35d09a1b3..abf7b30ba8 100644 --- a/components/engine/docs/installation/linux/rhel.md +++ b/components/engine/docs/installation/linux/rhel.md @@ -133,15 +133,19 @@ To create the `docker` group and add your user: 1. Log into your machine as a user with `sudo` or `root` privileges. -2. Create the `docker` group and add your user. +2. Create the `docker` group. + + `sudo groupadd docker` + +3. Add your user to `docker` group. `sudo usermod -aG docker your_username` -3. Log out and log back in. +4. Log out and log back in. This ensures your user is running with the correct permissions. -4. Verify your work by running `docker` without `sudo`. +5. Verify your work by running `docker` without `sudo`. $ docker run hello-world diff --git a/components/engine/docs/installation/linux/ubuntulinux.md b/components/engine/docs/installation/linux/ubuntulinux.md index 8db5e3f931..4a9fa71dee 100644 --- a/components/engine/docs/installation/linux/ubuntulinux.md +++ b/components/engine/docs/installation/linux/ubuntulinux.md @@ -85,7 +85,8 @@ packages from the new repository: deb https://apt.dockerproject.org/repo ubuntu-wily main - > **Note**: Docker does not provide packages for all architectures. To install docker on + > **Note**: Docker does not provide packages for all architectures. You can find + > nightly built binaries in https://master.dockerproject.org. To install docker on > a multi-architecture system, add an `[arch=...]` clause to the entry. Refer to the > [Debian Multiarch wiki](https://wiki.debian.org/Multiarch/HOWTO#Setting_up_apt_sources) > for details. @@ -109,10 +110,9 @@ packages from the new repository: ### Prerequisites by Ubuntu Version - Ubuntu Wily 15.10 -- Ubuntu Vivid 15.04 - Ubuntu Trusty 14.04 (LTS) -For Ubuntu Trusty, Vivid, and Wily, it's recommended to install the +For Ubuntu Trusty and Wily, it's recommended to install the `linux-image-extra` kernel package. The `linux-image-extra` package allows you use the `aufs` storage driver. @@ -239,15 +239,19 @@ To create the `docker` group and add your user: This procedure assumes you log in as the `ubuntu` user. -3. Create the `docker` group and add your user. +2. Create the `docker` group. + + $ sudo groupadd docker + +3. Add your user to `docker` group. $ sudo usermod -aG docker ubuntu -3. Log out and log back in. +4. Log out and log back in. This ensures your user is running with the correct permissions. -4. Verify your work by running `docker` without `sudo`. +5. Verify your work by running `docker` without `sudo`. $ docker run hello-world diff --git a/components/engine/docs/installation/mac.md b/components/engine/docs/installation/mac.md index 67b56b15ce..3d520b3781 100644 --- a/components/engine/docs/installation/mac.md +++ b/components/engine/docs/installation/mac.md @@ -11,10 +11,6 @@ weight="-90" # Mac OS X -> **Note**: This release of Docker deprecates the Boot2Docker command line in -> favor of Docker Machine. Use the Docker Toolbox to install Docker Machine as -> well as the other Docker tools. - You install Docker using Docker Toolbox. Docker Toolbox includes the following Docker tools: * Docker Machine for running the `docker-machine` binary @@ -67,7 +63,7 @@ installer. 1. Go to the [Docker Toolbox](https://www.docker.com/toolbox) page. -2. Click the installer link to download. +2. Click the Download link. 3. Install Docker Toolbox by double-clicking the package or by right-clicking and choosing "Open" from the pop-up menu. @@ -89,7 +85,7 @@ and choosing "Open" from the pop-up menu. * makes these binaries available to all users * installs VirtualBox; or updates any existing installation - Change these defaults by pressing "Customize" or "Change + To change these defaults, press "Customize" or "Change Install Location." 5. Press "Install" to perform the standard installation. @@ -107,16 +103,15 @@ and choosing "Open" from the pop-up menu. 7. Press "Close" to exit. - ## Running a Docker Container To run a Docker container, you: -* create a new (or start an existing) virtual machine that runs Docker. -* switch your environment to your new VM -* use the `docker` client to create, load, and manage containers +* Create a new (or start an existing) virtual machine +* Switch your environment to your new VM +* Use the `docker` client to create, load, and manage containers -Once you create a machine, you can reuse it as often as you like. Like any +You can reuse this virtual machine as often as you like. Like any VirtualBox VM, it maintains its configuration between uses. There are two ways to use the installed tools, from the Docker Quickstart Terminal or @@ -130,9 +125,9 @@ There are two ways to use the installed tools, from the Docker Quickstart Termin The application: - * opens a terminal window - * creates a `default` VM if it doesn't exists, and starts the VM after - * points the terminal environment to this VM + * Opens a terminal window + * Creates a `default` VM if it doesn't exists, and starts the VM after + * Points the terminal environment to this VM Once the launch completes, the Docker Quickstart Terminal reports: @@ -222,7 +217,6 @@ different shell such as C Shell but the commands are the same. $ docker run hello-world - ## Learn about your Toolbox installation Toolbox installs the Docker Engine binary, the Docker binary on your system. When you @@ -271,7 +265,7 @@ and what it does: | upgrade | upgrade | Upgrades a machine's Docker client to the latest stable release. | -## Example of Docker on Mac OS X +## Examples on Mac OS X Work through this section to try some practical container tasks on a VM. At this point, you should have a VM running and be connected to it through your shell. @@ -361,7 +355,7 @@ The next exercise demonstrates how to do this. $ docker run -d -P -v $HOME/site:/usr/share/nginx/html \ --name mysite nginx -6. Get the `mysite` container's port. +6. View the `mysite` container's port. $ docker port mysite 80/tcp -> 0.0.0.0:49166 @@ -371,7 +365,7 @@ The next exercise demonstrates how to do this. ![My site page](images/newsite_view.png) -8. Try adding a page to your `$HOME/site` in real time. +8. Add a page to your `$HOME/site` in real time. $ echo "This is cool" > cool.html @@ -391,7 +385,7 @@ The next exercise demonstrates how to do this. ## Upgrade Docker Toolbox -To upgrade Docker Toolbox, download and re-run [the Docker Toolbox +To upgrade Docker Toolbox, download and re-run the [Docker Toolbox installer](https://docker.com/toolbox/). @@ -427,9 +421,9 @@ To uninstall, do the following: ## Learning more Use `docker-machine help` to list the full command line reference for Docker Machine. For more -information about using SSH or SCP to access a VM, see [the Docker Machine +information about using SSH or SCP to access a VM, see the [Docker Machine documentation](https://docs.docker.com/machine/). -You can continue with the [Docker User Guide](../userguide/index.md). If you are +You can continue with the [Docker Engine User Guide](../userguide/index.md). If you are interested in using the Kitematic GUI, see the [Kitematic user guide](https://docs.docker.com/kitematic/userguide/). diff --git a/components/engine/docs/installation/windows.md b/components/engine/docs/installation/windows.md index dfb650f8e9..696ee3394a 100644 --- a/components/engine/docs/installation/windows.md +++ b/components/engine/docs/installation/windows.md @@ -144,9 +144,9 @@ installer. To run a Docker container, you: -* create a new (or start an existing) Docker virtual machine -* switch your environment to your new VM -* use the `docker` client to create, load, and manage containers +* Create a new (or start an existing) Docker virtual machine +* Switch your environment to your new VM +* Use the `docker` client to create, load, and manage containers Once you create a machine, you can reuse it as often as you like. Like any VirtualBox VM, it maintains its configuration between uses. @@ -160,9 +160,9 @@ There are several ways to use the installed tools, from the Docker Quickstart Te The application: - * opens a terminal window - * creates a `default` VM if it doesn't exist, and starts the VM after - * points the terminal environment to this VM + * Opens a terminal window + * Creates a `default` VM if it doesn't exist, and starts the VM after + * Points the terminal environment to this VM Once the launch completes, you can run `docker` commands. @@ -353,9 +353,9 @@ DHCP implementation. ## Login with PUTTY instead of using the CMD Docker Machine generates and uses the public/private key pair in your -`%USERPROFILE%\.ssh` directory so to log in you need to use the private key from -this same directory. The private key needs to be converted into the format PuTTY -uses. You can do this with +`%USERPROFILE%\.docker\machine\machines\` directory. To +log in you need to use the private key from this same directory. The private key +needs to be converted into the format PuTTY uses. You can do this with [puttygen](http://www.chiark.greenend.org.uk/~sgtatham/putty/download.html): 1. Open `puttygen.exe` and load ("File"->"Load" menu) the private key from (you may need to change to the `All Files (*.*)` filter) @@ -374,6 +374,6 @@ delete that file yourself. ## Learn more -You can continue with the [Docker User Guide](../userguide/index.md). If you are +You can continue with the [Docker Engine User Guide](../userguide/index.md). If you are interested in using the Kitematic GUI, see the [Kitematic user guide](https://docs.docker.com/kitematic/userguide/). diff --git a/components/engine/docs/quickstart.md b/components/engine/docs/quickstart.md index d8a93227f2..4f282b75dc 100644 --- a/components/engine/docs/quickstart.md +++ b/components/engine/docs/quickstart.md @@ -1,7 +1,7 @@ -# Quickstart Docker Engine +# Docker Engine Quickstart -This quickstart assumes you have a working installation of Docker Engine. To verify Engine is installed, use the following command: +This quickstart assumes you have a working installation of Docker Engine. To verify Engine is installed and configured, use the following command: # Check that you have a working install $ docker info -If you get `docker: command not found` or something like +If you have a successful install, the system information appears. If you get `docker: command not found` or something like `/var/lib/docker/repositories: permission denied` you may have an incomplete Docker installation or insufficient privileges to access Engine on your machine. With the default installation of Engine `docker` @@ -25,9 +25,9 @@ commands need to be run by a user that is in the `docker` group or by the `root` user. Depending on your Engine system configuration, you may be required -to preface each `docker` command with `sudo`. One way to avoid having to use -`sudo` with the `docker` commands is to create a Unix group called `docker` and -add users that will be entering `docker` commands to the 'docker' group. +to preface each `docker` command with `sudo`. If you want to run without using +`sudo` with the `docker` commands, then create a Unix group called `docker` and +add the user to the 'docker' group. For more information about installing Docker Engine or `sudo` configuration, refer to the [installation](installation/index.md) instructions for your operating system. @@ -35,34 +35,40 @@ the [installation](installation/index.md) instructions for your operating system ## Download a pre-built image +To pull an `ubuntu` image, run: + # Download an ubuntu image $ docker pull ubuntu -This will find the `ubuntu` image by name on -[*Docker Hub*](userguide/containers/dockerrepos.md#searching-for-images) -and download it from [Docker Hub](https://hub.docker.com) to a local -image cache. +This downloads the `ubuntu` image by name from [Docker Hub](https://hub.docker.com) to a local +image cache. To search for an image, run `docker search`. For more information, go to: +[Searching images](userguide/containers/dockerrepos.md#searching-for-images) + > **Note**: > When the image is successfully downloaded, you see a 12 character > hash `539c0211cd76: Download complete` which is the -> short form of the image ID. These short image IDs are the first 12 -> characters of the full image ID - which can be found using +> short form of the Image ID. These short Image IDs are the first 12 +> characters of the full Image ID. To view this information, run > `docker inspect` or `docker images --no-trunc=true`. +To display a list of downloaded images, run `docker images`. + ## Running an interactive shell To run an interactive shell in the Ubuntu image: $ docker run -i -t ubuntu /bin/bash -The `-i` flag starts an interactive container. The `-t` flag creates a -pseudo-TTY that attaches `stdin` and `stdout`. +The `-i` flag starts an interactive container. +The `-t` flag creates a pseudo-TTY that attaches `stdin` and `stdout`. +The image is `ubuntu`. +The command `/bin/bash` starts a shell you can log in. To detach the `tty` without exiting the shell, use the escape sequence -`Ctrl-p` + `Ctrl-q`. The container will continue to exist in a stopped state -once exited. To list all containers, stopped and running, use the `docker ps -a` -command. +`Ctrl-p` + `Ctrl-q`. The container continues to exist in a stopped state +once exited. To list all running containers, run `docker ps`. To view stopped and running containers, +run `docker ps -a`. ## Bind Docker to another host/port or a Unix socket @@ -179,16 +185,14 @@ TCP and a Unix socket ## Committing (saving) a container state -Save your containers state to an image, so the state can be -re-used. +To save the current state of a container as an image: -When you commit your container, Docker only stores the diff (difference) between -the source image and the current state of the container's image. To list images -you already have, use the `docker images` command. - - # Commit your container to a new named image $ docker commit +When you commit your container, Docker Engine only stores the diff (difference) between +the source image and the current state of the container's image. To list images +you already have, run: + # List your images $ docker images @@ -196,6 +200,6 @@ You now have an image state from which you can create new instances. ## Where to go next -* Work your way through the [Docker User Guide](userguide/index.md) -* Read more about [*Share Images via Repositories*](userguide/containers/dockerrepos.md) -* Review [*Command Line*](reference/commandline/cli.md) +* Work your way through the [Docker Engine User Guide](userguide/index.md) +* Read more about [Store Images on Docker Hub](userguide/containers/dockerrepos.md) +* Review [Command Line](reference/commandline/cli.md) diff --git a/components/engine/docs/reference/api/docker_remote_api.md b/components/engine/docs/reference/api/docker_remote_api.md index debf9f0454..75edd928bf 100644 --- a/components/engine/docs/reference/api/docker_remote_api.md +++ b/components/engine/docs/reference/api/docker_remote_api.md @@ -24,6 +24,17 @@ client must have `root` access to interact with the daemon. If a group named `docker` exists on your system, `docker` applies ownership of the socket to the group. +To connect to the Docker daemon with cURL you need to use cURL 7.40 or +later, as these versions have the `--unix-socket` flag available. To +run `curl` against the daemon on the default socket, use the +following: + + curl --unix-socket /var/run/docker.sock http://containers/json + +If you have bound the Docker daemon to a different socket path or TCP +port, you would reference that in your cURL rather than the +default. + The current version of the API is v1.23 which means calling `/info` is the same as calling `/v1.23/info`. To call an older version of the API use `/v1.22/info`. @@ -94,7 +105,7 @@ Some container-related events are not affected by container state, so they are n Running `docker rmi` emits an **untag** event when removing an image name. The `rmi` command may also emit **delete** events when images are deleted by ID directly or by deleting the last tag referring to the image. -> **Acknowledgement**: This diagram and the accompanying text were used with the permission of Matt Good and Gilder Labs. See Matt's original blog post [Docker Events Explained](http://gliderlabs.com/blog/2015/04/14/docker-events-explained/). +> **Acknowledgment**: This diagram and the accompanying text were used with the permission of Matt Good and Gilder Labs. See Matt's original blog post [Docker Events Explained](https://gliderlabs.com/blog/2015/04/14/docker-events-explained/). ## Version history @@ -105,8 +116,15 @@ This section lists each version from latest to oldest. Each listing includes a [Docker Remote API v1.23](docker_remote_api_v1.23.md) documentation * `GET /containers/json` returns the state of the container, one of `created`, `restarting`, `running`, `paused`, `exited` or `dead`. +* `GET /containers/json` returns the mount points for the container. * `GET /networks/(name)` now returns an `Internal` field showing whether the network is internal or not. - +* `GET /networks/(name)` now returns an `EnableIPv6` field showing whether the network has ipv6 enabled or not. +* `POST /containers/(name)/update` now supports updating container's restart policy. +* `POST /networks/create` now supports enabling ipv6 on the network by setting the `EnableIPv6` field (doing this with a label will no longer work). +* `GET /info` now returns `CgroupDriver` field showing what cgroup driver the daemon is using; `cgroupfs` or `systemd`. +* `GET /info` now returns `KernelMemory` field, showing if "kernel memory limit" is supported. +* `POST /containers/create` now takes `PidsLimit` field, if the kernel is >= 4.3 and the pids cgroup is supported. +* `GET /containers/(id or name)/stats` now returns `pids_stats`, if the kernel is >= 4.3 and the pids cgroup is supported. ### v1.22 API changes @@ -131,7 +149,7 @@ This section lists each version from latest to oldest. Each listing includes a * `POST /containers/create` now allows you to set the static IPv4 and/or IPv6 address for the container. * `POST /networks/(id)/connect` now allows you to set the static IPv4 and/or IPv6 address for the container. * `GET /info` now includes the number of containers running, stopped, and paused. -* `POST /networks/create` now supports restricting external access to the network by setting the `internal` field. +* `POST /networks/create` now supports restricting external access to the network by setting the `Internal` field. * `POST /networks/(id)/disconnect` now includes a `Force` option to forcefully disconnect a container from network * `GET /containers/(id)/json` now returns the `NetworkID` of containers. * `POST /networks/create` Now supports an options field in the IPAM config that provides options diff --git a/components/engine/docs/reference/api/docker_remote_api_v1.14.md b/components/engine/docs/reference/api/docker_remote_api_v1.14.md index 3b8c9030b0..8aa7871c30 100644 --- a/components/engine/docs/reference/api/docker_remote_api_v1.14.md +++ b/components/engine/docs/reference/api/docker_remote_api_v1.14.md @@ -180,7 +180,7 @@ Status Codes: ### Inspect a container -`GET /containers/(id)/json` +`GET /containers/(id or name)/json` Return low-level information on the container `id` @@ -267,7 +267,7 @@ Status Codes: ### List processes running inside a container -`GET /containers/(id)/top` +`GET /containers/(id or name)/top` List processes running inside the container `id`. On Unix systems this is done by running the `ps` command. This endpoint is not @@ -331,7 +331,7 @@ Status Codes: ### Get container logs -`GET /containers/(id)/logs` +`GET /containers/(id or name)/logs` Get stdout and stderr logs from the container ``id`` @@ -364,7 +364,7 @@ Status Codes: ### Inspect changes on a container's filesystem -`GET /containers/(id)/changes` +`GET /containers/(id or name)/changes` Inspect changes on container `id`'s filesystem @@ -400,7 +400,7 @@ Status Codes: ### Export a container -`GET /containers/(id)/export` +`GET /containers/(id or name)/export` Export the contents of container `id` @@ -423,13 +423,13 @@ Status Codes: ### Start a container -`POST /containers/(id)/start` +`POST /containers/(id or name)/start` Start the container `id` **Example request**: - POST /containers/(id)/start HTTP/1.1 + POST /containers/e90e34656806/start HTTP/1.1 Content-Type: application/json { @@ -462,7 +462,7 @@ Status Codes: ### Stop a container -`POST /containers/(id)/stop` +`POST /containers/(id or name)/stop` Stop the container `id` @@ -487,7 +487,7 @@ Status Codes: ### Restart a container -`POST /containers/(id)/restart` +`POST /containers/(id or name)/restart` Restart the container `id` @@ -511,7 +511,7 @@ Status Codes: ### Kill a container -`POST /containers/(id)/kill` +`POST /containers/(id or name)/kill` Kill the container `id` @@ -536,7 +536,7 @@ Status Codes: ### Pause a container -`POST /containers/(id)/pause` +`POST /containers/(id or name)/pause` Pause the container `id` @@ -556,7 +556,7 @@ Status Codes: ### Unpause a container -`POST /containers/(id)/unpause` +`POST /containers/(id or name)/unpause` Unpause the container `id` @@ -576,7 +576,7 @@ Status Codes: ### Attach to a container -`POST /containers/(id)/attach` +`POST /containers/(id or name)/attach` Attach to the container `id` @@ -654,7 +654,7 @@ Status Codes: ### Attach to a container (websocket) -`GET /containers/(id)/attach/ws` +`GET /containers/(id or name)/attach/ws` Attach to the container `id` via websocket @@ -689,7 +689,7 @@ Status Codes: ### Wait a container -`POST /containers/(id)/wait` +`POST /containers/(id or name)/wait` Block until container `id` stops, then returns the exit code @@ -712,7 +712,7 @@ Status Codes: ### Remove a container -`DELETE /containers/(id)` +`DELETE /containers/(id or name)` Remove the container `id` from the filesystem @@ -740,7 +740,7 @@ Status Codes: ### Copy files or folders from a container -`POST /containers/(id)/copy` +`POST /containers/(id or name)/copy` Copy files or folders of container `id` diff --git a/components/engine/docs/reference/api/docker_remote_api_v1.15.md b/components/engine/docs/reference/api/docker_remote_api_v1.15.md index d7c7abd076..9988aafef1 100644 --- a/components/engine/docs/reference/api/docker_remote_api_v1.15.md +++ b/components/engine/docs/reference/api/docker_remote_api_v1.15.md @@ -249,7 +249,7 @@ Json Parameters: An ever increasing delay (double the previous delay, starting at 100mS) is added before each restart to prevent flooding the server. - **NetworkMode** - Sets the networking mode for the container. Supported - values are: `bridge`, `host`, and `container:` + values are: `bridge`, `host`, `none`, and `container:` - **Devices** - A list of devices to add to the container specified in the form `{ "PathOnHost": "/dev/deviceName", "PathInContainer": "/dev/deviceName", "CgroupPermissions": "mrw"}` @@ -268,7 +268,7 @@ Status Codes: ### Inspect a container -`GET /containers/(id)/json` +`GET /containers/(id or name)/json` Return low-level information on the container `id` @@ -355,7 +355,7 @@ Status Codes: ### List processes running inside a container -`GET /containers/(id)/top` +`GET /containers/(id or name)/top` List processes running inside the container `id`. On Unix systems this is done by running the `ps` command. This endpoint is not @@ -419,7 +419,7 @@ Status Codes: ### Get container logs -`GET /containers/(id)/logs` +`GET /containers/(id or name)/logs` Get stdout and stderr logs from the container ``id`` @@ -451,7 +451,7 @@ Status Codes: ### Inspect changes on a container's filesystem -`GET /containers/(id)/changes` +`GET /containers/(id or name)/changes` Inspect changes on container `id`'s filesystem @@ -487,7 +487,7 @@ Status Codes: ### Export a container -`GET /containers/(id)/export` +`GET /containers/(id or name)/export` Export the contents of container `id` @@ -510,7 +510,7 @@ Status Codes: ### Resize a container TTY -`GET /containers/(id)/resize?h=&w=` +`GET /containers/(id or name)/resize?h=&w=` Resize the TTY of container `id` @@ -532,13 +532,13 @@ Status Codes: ### Start a container -`POST /containers/(id)/start` +`POST /containers/(id or name)/start` Start the container `id` **Example request**: - POST /containers/(id)/start HTTP/1.1 + POST /containers/e90e34656806/start HTTP/1.1 Content-Type: application/json { @@ -596,7 +596,7 @@ Json Parameters: An ever increasing delay (double the previous delay, starting at 100mS) is added before each restart to prevent flooding the server. - **NetworkMode** - Sets the networking mode for the container. Supported - values are: `bridge`, `host`, and `container:` + values are: `bridge`, `host`, `none`, and `container:` - **Devices** - A list of devices to add to the container specified in the form `{ "PathOnHost": "/dev/deviceName", "PathInContainer": "/dev/deviceName", "CgroupPermissions": "mrw"}` @@ -610,7 +610,7 @@ Status Codes: ### Stop a container -`POST /containers/(id)/stop` +`POST /containers/(id or name)/stop` Stop the container `id` @@ -635,7 +635,7 @@ Status Codes: ### Restart a container -`POST /containers/(id)/restart` +`POST /containers/(id or name)/restart` Restart the container `id` @@ -659,7 +659,7 @@ Status Codes: ### Kill a container -`POST /containers/(id)/kill` +`POST /containers/(id or name)/kill` Kill the container `id` @@ -684,7 +684,7 @@ Status Codes: ### Pause a container -`POST /containers/(id)/pause` +`POST /containers/(id or name)/pause` Pause the container `id` @@ -704,7 +704,7 @@ Status Codes: ### Unpause a container -`POST /containers/(id)/unpause` +`POST /containers/(id or name)/unpause` Unpause the container `id` @@ -724,7 +724,7 @@ Status Codes: ### Attach to a container -`POST /containers/(id)/attach` +`POST /containers/(id or name)/attach` Attach to the container `id` @@ -803,7 +803,7 @@ Status Codes: ### Attach to a container (websocket) -`GET /containers/(id)/attach/ws` +`GET /containers/(id or name)/attach/ws` Attach to the container `id` via websocket @@ -838,7 +838,7 @@ Status Codes: ### Wait a container -`POST /containers/(id)/wait` +`POST /containers/(id or name)/wait` Block until container `id` stops, then returns the exit code @@ -861,7 +861,7 @@ Status Codes: ### Remove a container -`DELETE /containers/(id)` +`DELETE /containers/(id or name)` Remove the container `id` from the filesystem @@ -889,7 +889,7 @@ Status Codes: ### Copy files or folders from a container -`POST /containers/(id)/copy` +`POST /containers/(id or name)/copy` Copy files or folders of container `id` @@ -1623,7 +1623,7 @@ the root that contains a list of repository and tag names mapped to layer IDs. ### Exec Create -`POST /containers/(id)/exec` +`POST /containers/(id or name)/exec` Sets up an exec instance in a running container `id` @@ -1701,7 +1701,7 @@ Status Codes: - **404** – no such exec instance **Stream details**: - Similar to the stream behavior of `POST /container/(id)/attach` API + Similar to the stream behavior of `POST /containers/(id or name)/attach` API ### Exec Resize diff --git a/components/engine/docs/reference/api/docker_remote_api_v1.16.md b/components/engine/docs/reference/api/docker_remote_api_v1.16.md index df94a7320a..ad88c5eac5 100644 --- a/components/engine/docs/reference/api/docker_remote_api_v1.16.md +++ b/components/engine/docs/reference/api/docker_remote_api_v1.16.md @@ -249,7 +249,7 @@ Json Parameters: An ever increasing delay (double the previous delay, starting at 100mS) is added before each restart to prevent flooding the server. - **NetworkMode** - Sets the networking mode for the container. Supported - values are: `bridge`, `host`, and `container:` + values are: `bridge`, `host`, `none`, and `container:` - **Devices** - A list of devices to add to the container specified in the form `{ "PathOnHost": "/dev/deviceName", "PathInContainer": "/dev/deviceName", "CgroupPermissions": "mrw"}` @@ -268,7 +268,7 @@ Status Codes: ### Inspect a container -`GET /containers/(id)/json` +`GET /containers/(id or name)/json` Return low-level information on the container `id` @@ -355,7 +355,7 @@ Status Codes: ### List processes running inside a container -`GET /containers/(id)/top` +`GET /containers/(id or name)/top` List processes running inside the container `id`. On Unix systems this is done by running the `ps` command. This endpoint is not @@ -419,7 +419,7 @@ Status Codes: ### Get container logs -`GET /containers/(id)/logs` +`GET /containers/(id or name)/logs` Get stdout and stderr logs from the container ``id`` @@ -451,7 +451,7 @@ Status Codes: ### Inspect changes on a container's filesystem -`GET /containers/(id)/changes` +`GET /containers/(id or name)/changes` Inspect changes on container `id`'s filesystem @@ -487,7 +487,7 @@ Status Codes: ### Export a container -`GET /containers/(id)/export` +`GET /containers/(id or name)/export` Export the contents of container `id` @@ -510,7 +510,7 @@ Status Codes: ### Resize a container TTY -`POST /containers/(id)/resize?h=&w=` +`POST /containers/(id or name)/resize?h=&w=` Resize the TTY for container with `id`. The container must be restarted for the resize to take effect. @@ -532,7 +532,7 @@ Status Codes: ### Start a container -`POST /containers/(id)/start` +`POST /containers/(id or name)/start` Start the container `id` @@ -542,7 +542,7 @@ Start the container `id` **Example request**: - POST /containers/(id)/start HTTP/1.1 + POST /containers/e90e34656806/start HTTP/1.1 **Example response**: @@ -557,7 +557,7 @@ Status Codes: ### Stop a container -`POST /containers/(id)/stop` +`POST /containers/(id or name)/stop` Stop the container `id` @@ -582,7 +582,7 @@ Status Codes: ### Restart a container -`POST /containers/(id)/restart` +`POST /containers/(id or name)/restart` Restart the container `id` @@ -606,7 +606,7 @@ Status Codes: ### Kill a container -`POST /containers/(id)/kill` +`POST /containers/(id or name)/kill` Kill the container `id` @@ -631,7 +631,7 @@ Status Codes: ### Pause a container -`POST /containers/(id)/pause` +`POST /containers/(id or name)/pause` Pause the container `id` @@ -651,7 +651,7 @@ Status Codes: ### Unpause a container -`POST /containers/(id)/unpause` +`POST /containers/(id or name)/unpause` Unpause the container `id` @@ -671,7 +671,7 @@ Status Codes: ### Attach to a container -`POST /containers/(id)/attach` +`POST /containers/(id or name)/attach` Attach to the container `id` @@ -751,7 +751,7 @@ Status Codes: ### Attach to a container (websocket) -`GET /containers/(id)/attach/ws` +`GET /containers/(id or name)/attach/ws` Attach to the container `id` via websocket @@ -786,7 +786,7 @@ Status Codes: ### Wait a container -`POST /containers/(id)/wait` +`POST /containers/(id or name)/wait` Block until container `id` stops, then returns the exit code @@ -809,7 +809,7 @@ Status Codes: ### Remove a container -`DELETE /containers/(id)` +`DELETE /containers/(id or name)` Remove the container `id` from the filesystem @@ -837,7 +837,7 @@ Status Codes: ### Copy files or folders from a container -`POST /containers/(id)/copy` +`POST /containers/(id or name)/copy` Copy files or folders of container `id` @@ -1585,7 +1585,7 @@ the root that contains a list of repository and tag names mapped to layer IDs. ### Exec Create -`POST /containers/(id)/exec` +`POST /containers/(id or name)/exec` Sets up an exec instance in a running container `id` @@ -1663,7 +1663,7 @@ Status Codes: - **404** – no such exec instance **Stream details**: - Similar to the stream behavior of `POST /container/(id)/attach` API + Similar to the stream behavior of `POST /containers/(id or name)/attach` API ### Exec Resize diff --git a/components/engine/docs/reference/api/docker_remote_api_v1.17.md b/components/engine/docs/reference/api/docker_remote_api_v1.17.md index 6f2cfe5f03..5a85ba09b8 100644 --- a/components/engine/docs/reference/api/docker_remote_api_v1.17.md +++ b/components/engine/docs/reference/api/docker_remote_api_v1.17.md @@ -251,7 +251,7 @@ Json Parameters: An ever increasing delay (double the previous delay, starting at 100mS) is added before each restart to prevent flooding the server. - **NetworkMode** - Sets the networking mode for the container. Supported - values are: `bridge`, `host`, and `container:` + values are: `bridge`, `host`, `none`, and `container:` - **Devices** - A list of devices to add to the container specified in the form `{ "PathOnHost": "/dev/deviceName", "PathInContainer": "/dev/deviceName", "CgroupPermissions": "mrw"}` @@ -272,7 +272,7 @@ Status Codes: ### Inspect a container -`GET /containers/(id)/json` +`GET /containers/(id or name)/json` Return low-level information on the container `id` @@ -394,7 +394,7 @@ Status Codes: ### List processes running inside a container -`GET /containers/(id)/top` +`GET /containers/(id or name)/top` List processes running inside the container `id`. On Unix systems this is done by running the `ps` command. This endpoint is not @@ -458,7 +458,7 @@ Status Codes: ### Get container logs -`GET /containers/(id)/logs` +`GET /containers/(id or name)/logs` Get stdout and stderr logs from the container ``id`` @@ -493,7 +493,7 @@ Status Codes: ### Inspect changes on a container's filesystem -`GET /containers/(id)/changes` +`GET /containers/(id or name)/changes` Inspect changes on container `id`'s filesystem @@ -529,7 +529,7 @@ Status Codes: ### Export a container -`GET /containers/(id)/export` +`GET /containers/(id or name)/export` Export the contents of container `id` @@ -552,7 +552,7 @@ Status Codes: ### Get container stats based on resource usage -`GET /containers/(id)/stats` +`GET /containers/(id or name)/stats` This endpoint returns a live stream of a container's resource usage statistics. @@ -640,7 +640,7 @@ Status Codes: ### Resize a container TTY -`POST /containers/(id)/resize?h=&w=` +`POST /containers/(id or name)/resize?h=&w=` Resize the TTY for container with `id`. The container must be restarted for the resize to take effect. @@ -662,7 +662,7 @@ Status Codes: ### Start a container -`POST /containers/(id)/start` +`POST /containers/(id or name)/start` Start the container `id` @@ -672,7 +672,7 @@ Start the container `id` **Example request**: - POST /containers/(id)/start HTTP/1.1 + POST /containers/e90e34656806/start HTTP/1.1 **Example response**: @@ -687,7 +687,7 @@ Status Codes: ### Stop a container -`POST /containers/(id)/stop` +`POST /containers/(id or name)/stop` Stop the container `id` @@ -712,7 +712,7 @@ Status Codes: ### Restart a container -`POST /containers/(id)/restart` +`POST /containers/(id or name)/restart` Restart the container `id` @@ -736,7 +736,7 @@ Status Codes: ### Kill a container -`POST /containers/(id)/kill` +`POST /containers/(id or name)/kill` Kill the container `id` @@ -761,7 +761,7 @@ Status Codes: ### Rename a container -`POST /containers/(id)/rename` +`POST /containers/(id or name)/rename` Rename the container `id` to a `new_name` @@ -786,7 +786,7 @@ Status Codes: ### Pause a container -`POST /containers/(id)/pause` +`POST /containers/(id or name)/pause` Pause the container `id` @@ -806,7 +806,7 @@ Status Codes: ### Unpause a container -`POST /containers/(id)/unpause` +`POST /containers/(id or name)/unpause` Unpause the container `id` @@ -826,7 +826,7 @@ Status Codes: ### Attach to a container -`POST /containers/(id)/attach` +`POST /containers/(id or name)/attach` Attach to the container `id` @@ -909,7 +909,7 @@ Status Codes: ### Attach to a container (websocket) -`GET /containers/(id)/attach/ws` +`GET /containers/(id or name)/attach/ws` Attach to the container `id` via websocket @@ -944,7 +944,7 @@ Status Codes: ### Wait a container -`POST /containers/(id)/wait` +`POST /containers/(id or name)/wait` Block until container `id` stops, then returns the exit code @@ -967,7 +967,7 @@ Status Codes: ### Remove a container -`DELETE /containers/(id)` +`DELETE /containers/(id or name)` Remove the container `id` from the filesystem @@ -995,7 +995,7 @@ Status Codes: ### Copy files or folders from a container -`POST /containers/(id)/copy` +`POST /containers/(id or name)/copy` Copy files or folders of container `id` @@ -1748,7 +1748,7 @@ the root that contains a list of repository and tag names mapped to layer IDs. ### Exec Create -`POST /containers/(id)/exec` +`POST /containers/(id or name)/exec` Sets up an exec instance in a running container `id` @@ -1826,7 +1826,7 @@ Status Codes: - **404** – no such exec instance **Stream details**: - Similar to the stream behavior of `POST /container/(id)/attach` API + Similar to the stream behavior of `POST /containers/(id or name)/attach` API ### Exec Resize @@ -2005,4 +2005,4 @@ This might change in the future. To set cross origin requests to the remote api, please add flag "--api-enable-cors" when running docker in daemon mode. - $ docker -d -H="192.168.1.9:2375" --api-enable-cors + $ docker -d -H="192.168.1.9:2375" --api-enable-cors diff --git a/components/engine/docs/reference/api/docker_remote_api_v1.18.md b/components/engine/docs/reference/api/docker_remote_api_v1.18.md index f117a471a6..09d8ae27f8 100644 --- a/components/engine/docs/reference/api/docker_remote_api_v1.18.md +++ b/components/engine/docs/reference/api/docker_remote_api_v1.18.md @@ -270,7 +270,7 @@ Json Parameters: An ever increasing delay (double the previous delay, starting at 100mS) is added before each restart to prevent flooding the server. - **NetworkMode** - Sets the networking mode for the container. Supported - values are: `bridge`, `host`, and `container:` + values are: `bridge`, `host`, `none`, and `container:` - **Devices** - A list of devices to add to the container specified in the form `{ "PathOnHost": "/dev/deviceName", "PathInContainer": "/dev/deviceName", "CgroupPermissions": "mrw"}` @@ -299,7 +299,7 @@ Status Codes: ### Inspect a container -`GET /containers/(id)/json` +`GET /containers/(id or name)/json` Return low-level information on the container `id` @@ -432,7 +432,7 @@ Status Codes: ### List processes running inside a container -`GET /containers/(id)/top` +`GET /containers/(id or name)/top` List processes running inside the container `id`. On Unix systems this is done by running the `ps` command. This endpoint is not @@ -496,7 +496,7 @@ Status Codes: ### Get container logs -`GET /containers/(id)/logs` +`GET /containers/(id or name)/logs` Get stdout and stderr logs from the container ``id`` @@ -534,7 +534,7 @@ Status Codes: ### Inspect changes on a container's filesystem -`GET /containers/(id)/changes` +`GET /containers/(id or name)/changes` Inspect changes on container `id`'s filesystem @@ -576,7 +576,7 @@ Status Codes: ### Export a container -`GET /containers/(id)/export` +`GET /containers/(id or name)/export` Export the contents of container `id` @@ -599,7 +599,7 @@ Status Codes: ### Get container stats based on resource usage -`GET /containers/(id)/stats` +`GET /containers/(id or name)/stats` This endpoint returns a live stream of a container's resource usage statistics. @@ -687,7 +687,7 @@ Status Codes: ### Resize a container TTY -`POST /containers/(id)/resize?h=&w=` +`POST /containers/(id or name)/resize?h=&w=` Resize the TTY for container with `id`. The container must be restarted for the resize to take effect. @@ -709,7 +709,7 @@ Status Codes: ### Start a container -`POST /containers/(id)/start` +`POST /containers/(id or name)/start` Start the container `id` @@ -719,7 +719,7 @@ Start the container `id` **Example request**: - POST /containers/(id)/start HTTP/1.1 + POST /containers/e90e34656806/start HTTP/1.1 **Example response**: @@ -734,7 +734,7 @@ Status Codes: ### Stop a container -`POST /containers/(id)/stop` +`POST /containers/(id or name)/stop` Stop the container `id` @@ -759,7 +759,7 @@ Status Codes: ### Restart a container -`POST /containers/(id)/restart` +`POST /containers/(id or name)/restart` Restart the container `id` @@ -783,7 +783,7 @@ Status Codes: ### Kill a container -`POST /containers/(id)/kill` +`POST /containers/(id or name)/kill` Kill the container `id` @@ -808,7 +808,7 @@ Status Codes: ### Rename a container -`POST /containers/(id)/rename` +`POST /containers/(id or name)/rename` Rename the container `id` to a `new_name` @@ -833,7 +833,7 @@ Status Codes: ### Pause a container -`POST /containers/(id)/pause` +`POST /containers/(id or name)/pause` Pause the container `id` @@ -853,7 +853,7 @@ Status Codes: ### Unpause a container -`POST /containers/(id)/unpause` +`POST /containers/(id or name)/unpause` Unpause the container `id` @@ -873,7 +873,7 @@ Status Codes: ### Attach to a container -`POST /containers/(id)/attach` +`POST /containers/(id or name)/attach` Attach to the container `id` @@ -956,7 +956,7 @@ Status Codes: ### Attach to a container (websocket) -`GET /containers/(id)/attach/ws` +`GET /containers/(id or name)/attach/ws` Attach to the container `id` via websocket @@ -991,7 +991,7 @@ Status Codes: ### Wait a container -`POST /containers/(id)/wait` +`POST /containers/(id or name)/wait` Block until container `id` stops, then returns the exit code @@ -1014,7 +1014,7 @@ Status Codes: ### Remove a container -`DELETE /containers/(id)` +`DELETE /containers/(id or name)` Remove the container `id` from the filesystem @@ -1042,7 +1042,7 @@ Status Codes: ### Copy files or folders from a container -`POST /containers/(id)/copy` +`POST /containers/(id or name)/copy` Copy files or folders of container `id` @@ -1194,12 +1194,12 @@ or being killed. Query Parameters: -- **dockerfile** - path within the build context to the Dockerfile. This is +- **dockerfile** - path within the build context to the Dockerfile. This is ignored if `remote` is specified and points to an individual filename. - **t** – repository name (and optionally a tag) to be applied to the resulting image in case of success -- **remote** – A Git repository URI or HTTP/HTTPS URI build source. If the - URI specifies a filename, the file's contents are placed into a file +- **remote** – A Git repository URI or HTTP/HTTPS URI build source. If the + URI specifies a filename, the file's contents are placed into a file called `Dockerfile`. - **q** – suppress verbose build output - **nocache** – do not use the cache when building the image @@ -1599,7 +1599,7 @@ Display system-wide information "SwapLimit": 0, "SystemTime": "2015-03-10T11:11:23.730591467-07:00" } - + Status Codes: - **200** – no error @@ -1866,7 +1866,7 @@ the root that contains a list of repository and tag names mapped to layer IDs. ### Exec Create -`POST /containers/(id)/exec` +`POST /containers/(id or name)/exec` Sets up an exec instance in a running container `id` @@ -1945,7 +1945,7 @@ Status Codes: - **404** – no such exec instance **Stream details**: - Similar to the stream behavior of `POST /container/(id)/attach` API + Similar to the stream behavior of `POST /containers/(id or name)/attach` API ### Exec Resize @@ -2118,7 +2118,7 @@ This might change in the future. ## 3.3 CORS Requests -To set cross origin requests to the remote api please give values to +To set cross origin requests to the remote api please give values to "--api-cors-header" when running docker in daemon mode. Set * will allow all, default or blank means CORS disabled diff --git a/components/engine/docs/reference/api/docker_remote_api_v1.19.md b/components/engine/docs/reference/api/docker_remote_api_v1.19.md index 196d9f75c6..0175148ab8 100644 --- a/components/engine/docs/reference/api/docker_remote_api_v1.19.md +++ b/components/engine/docs/reference/api/docker_remote_api_v1.19.md @@ -220,7 +220,7 @@ Json Parameters: (ie. the relative weight vs other containers). - **CpuPeriod** - The length of a CPU period in microseconds. - **CpuQuota** - Microseconds of CPU time that the container can get in a CPU period. -- **Cpuset** - Deprecated please don't use. Use `CpusetCpus` instead. +- **Cpuset** - Deprecated please don't use. Use `CpusetCpus` instead. - **CpusetCpus** - String value containing the `cgroups CpusetCpus` to use. - **CpusetMems** - Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems. - **BlkioWeight** - Block IO weight (relative weight) accepts a weight value between 10 and 1000. @@ -281,7 +281,7 @@ Json Parameters: An ever increasing delay (double the previous delay, starting at 100mS) is added before each restart to prevent flooding the server. - **NetworkMode** - Sets the networking mode for the container. Supported - values are: `bridge`, `host`, and `container:` + values are: `bridge`, `host`, `none`, and `container:` - **Devices** - A list of devices to add to the container specified as a JSON object in the form `{ "PathOnHost": "/dev/deviceName", "PathInContainer": "/dev/deviceName", "CgroupPermissions": "mrw"}` @@ -310,7 +310,7 @@ Status Codes: ### Inspect a container -`GET /containers/(id)/json` +`GET /containers/(id or name)/json` Return low-level information on the container `id` @@ -447,7 +447,7 @@ Status Codes: ### List processes running inside a container -`GET /containers/(id)/top` +`GET /containers/(id or name)/top` List processes running inside the container `id`. On Unix systems this is done by running the `ps` command. This endpoint is not @@ -511,7 +511,7 @@ Status Codes: ### Get container logs -`GET /containers/(id)/logs` +`GET /containers/(id or name)/logs` Get `stdout` and `stderr` logs from the container ``id`` @@ -551,7 +551,7 @@ Status Codes: ### Inspect changes on a container's filesystem -`GET /containers/(id)/changes` +`GET /containers/(id or name)/changes` Inspect changes on container `id`'s filesystem @@ -593,7 +593,7 @@ Status Codes: ### Export a container -`GET /containers/(id)/export` +`GET /containers/(id or name)/export` Export the contents of container `id` @@ -616,7 +616,7 @@ Status Codes: ### Get container stats based on resource usage -`GET /containers/(id)/stats` +`GET /containers/(id or name)/stats` This endpoint returns a live stream of a container's resource usage statistics. @@ -682,20 +682,37 @@ This endpoint returns a live stream of a container's resource usage statistics. "cpu_stats" : { "cpu_usage" : { "percpu_usage" : [ - 16970827, - 1839451, - 7107380, - 10571290 + 8646879, + 24472255, + 36438778, + 30657443 ], - "usage_in_usermode" : 10000000, - "total_usage" : 36488948, - "usage_in_kernelmode" : 20000000 + "usage_in_usermode" : 50000000, + "total_usage" : 100215355, + "usage_in_kernelmode" : 30000000 }, - "system_cpu_usage" : 20091722000000000, - "throttling_data" : {} + "system_cpu_usage" : 739306590000000, + "throttling_data" : {"periods":0,"throttled_periods":0,"throttled_time":0} + }, + "precpu_stats" : { + "cpu_usage" : { + "percpu_usage" : [ + 8646879, + 24350896, + 36438778, + 30657443 + ], + "usage_in_usermode" : 50000000, + "total_usage" : 100093996, + "usage_in_kernelmode" : 30000000 + }, + "system_cpu_usage" : 9492140000000, + "throttling_data" : {"periods":0,"throttled_periods":0,"throttled_time":0} } } +The precpu_stats is the cpu statistic of last read, which is used for calculating the cpu usage percent. It is not the exact copy of the “cpu_stats” field. + Query Parameters: - **stream** – 1/True/true or 0/False/false, pull stats once then disconnect. Default `true`. @@ -708,7 +725,7 @@ Status Codes: ### Resize a container TTY -`POST /containers/(id)/resize?h=&w=` +`POST /containers/(id or name)/resize?h=&w=` Resize the TTY for container with `id`. You must restart the container for the resize to take effect. @@ -730,7 +747,7 @@ Status Codes: ### Start a container -`POST /containers/(id)/start` +`POST /containers/(id or name)/start` Start the container `id` @@ -740,7 +757,7 @@ Start the container `id` **Example request**: - POST /containers/(id)/start HTTP/1.1 + POST /containers/e90e34656806/start HTTP/1.1 **Example response**: @@ -755,7 +772,7 @@ Status Codes: ### Stop a container -`POST /containers/(id)/stop` +`POST /containers/(id or name)/stop` Stop the container `id` @@ -780,7 +797,7 @@ Status Codes: ### Restart a container -`POST /containers/(id)/restart` +`POST /containers/(id or name)/restart` Restart the container `id` @@ -804,7 +821,7 @@ Status Codes: ### Kill a container -`POST /containers/(id)/kill` +`POST /containers/(id or name)/kill` Kill the container `id` @@ -829,7 +846,7 @@ Status Codes: ### Rename a container -`POST /containers/(id)/rename` +`POST /containers/(id or name)/rename` Rename the container `id` to a `new_name` @@ -854,7 +871,7 @@ Status Codes: ### Pause a container -`POST /containers/(id)/pause` +`POST /containers/(id or name)/pause` Pause the container `id` @@ -874,7 +891,7 @@ Status Codes: ### Unpause a container -`POST /containers/(id)/unpause` +`POST /containers/(id or name)/unpause` Unpause the container `id` @@ -894,7 +911,7 @@ Status Codes: ### Attach to a container -`POST /containers/(id)/attach` +`POST /containers/(id or name)/attach` Attach to the container `id` @@ -977,7 +994,7 @@ Status Codes: ### Attach to a container (websocket) -`GET /containers/(id)/attach/ws` +`GET /containers/(id or name)/attach/ws` Attach to the container `id` via websocket @@ -1012,7 +1029,7 @@ Status Codes: ### Wait a container -`POST /containers/(id)/wait` +`POST /containers/(id or name)/wait` Block until container `id` stops, then returns the exit code @@ -1035,7 +1052,7 @@ Status Codes: ### Remove a container -`DELETE /containers/(id)` +`DELETE /containers/(id or name)` Remove the container `id` from the filesystem @@ -1063,7 +1080,7 @@ Status Codes: ### Copy files or folders from a container -`POST /containers/(id)/copy` +`POST /containers/(id or name)/copy` Copy files or folders of container `id` @@ -1365,37 +1382,37 @@ Return the history of the image `name` HTTP/1.1 200 OK Content-Type: application/json - [ - { + [ + { "Id": "3db9c44f45209632d6050b35958829c3a2aa256d81b9a7be45b362ff85c54710", "Created": 1398108230, "CreatedBy": "/bin/sh -c #(nop) ADD file:eb15dbd63394e063b805a3c32ca7bf0266ef64676d5a6fab4801f2e81e2a5148 in /", "Tags": [ "ubuntu:lucid", "ubuntu:10.04" - ], + ], "Size": 182964289, "Comment": "" - }, - { + }, + { "Id": "6cfa4d1f33fb861d4d114f43b25abd0ac737509268065cdfd69d544a59c85ab8", "Created": 1398108222, "CreatedBy": "/bin/sh -c #(nop) MAINTAINER Tianon Gravi - mkimage-debootstrap.sh -i iproute,iputils-ping,ubuntu-minimal -t lucid.tar.xz lucid http://archive.ubuntu.com/ubuntu/", "Tags": null, "Size": 0, "Comment": "" - }, - { + }, + { "Id": "511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158", "Created": 1371157430, "CreatedBy": "", "Tags": [ "scratch12:latest", "scratch:latest" - ], + ], "Size": 0, "Comment": "Imported from -" - } + } ] Status Codes: @@ -1932,7 +1949,7 @@ the root that contains a list of repository and tag names mapped to layer IDs. ### Exec Create -`POST /containers/(id)/exec` +`POST /containers/(id or name)/exec` Sets up an exec instance in a running container `id` @@ -2011,7 +2028,7 @@ Status Codes: - **404** – no such exec instance **Stream details**: - Similar to the stream behavior of `POST /container/(id)/attach` API + Similar to the stream behavior of `POST /containers/(id or name)/attach` API ### Exec Resize @@ -2182,7 +2199,7 @@ from **200 OK** to **101 UPGRADED** and resends the same headers. ## 3.3 CORS Requests -To set cross origin requests to the remote api please give values to +To set cross origin requests to the remote api please give values to `--api-cors-header` when running Docker in daemon mode. Set * (asterisk) allows all, default or blank means CORS disabled diff --git a/components/engine/docs/reference/api/docker_remote_api_v1.20.md b/components/engine/docs/reference/api/docker_remote_api_v1.20.md index f8ab5823d1..4ada5c26ac 100644 --- a/components/engine/docs/reference/api/docker_remote_api_v1.20.md +++ b/components/engine/docs/reference/api/docker_remote_api_v1.20.md @@ -289,7 +289,7 @@ Json Parameters: An ever increasing delay (double the previous delay, starting at 100mS) is added before each restart to prevent flooding the server. - **NetworkMode** - Sets the networking mode for the container. Supported - values are: `bridge`, `host`, and `container:` + values are: `bridge`, `host`, `none`, and `container:` - **Devices** - A list of devices to add to the container specified as a JSON object in the form `{ "PathOnHost": "/dev/deviceName", "PathInContainer": "/dev/deviceName", "CgroupPermissions": "mrw"}` @@ -318,7 +318,7 @@ Status Codes: ### Inspect a container -`GET /containers/(id)/json` +`GET /containers/(id or name)/json` Return low-level information on the container `id` @@ -460,7 +460,7 @@ Status Codes: ### List processes running inside a container -`GET /containers/(id)/top` +`GET /containers/(id or name)/top` List processes running inside the container `id`. On Unix systems this is done by running the `ps` command. This endpoint is not @@ -524,7 +524,7 @@ Status Codes: ### Get container logs -`GET /containers/(id)/logs` +`GET /containers/(id or name)/logs` Get `stdout` and `stderr` logs from the container ``id`` @@ -564,7 +564,7 @@ Status Codes: ### Inspect changes on a container's filesystem -`GET /containers/(id)/changes` +`GET /containers/(id or name)/changes` Inspect changes on container `id`'s filesystem @@ -606,7 +606,7 @@ Status Codes: ### Export a container -`GET /containers/(id)/export` +`GET /containers/(id or name)/export` Export the contents of container `id` @@ -629,7 +629,7 @@ Status Codes: ### Get container stats based on resource usage -`GET /containers/(id)/stats` +`GET /containers/(id or name)/stats` This endpoint returns a live stream of a container's resource usage statistics. @@ -695,20 +695,37 @@ This endpoint returns a live stream of a container's resource usage statistics. "cpu_stats" : { "cpu_usage" : { "percpu_usage" : [ - 16970827, - 1839451, - 7107380, - 10571290 + 8646879, + 24472255, + 36438778, + 30657443 ], - "usage_in_usermode" : 10000000, - "total_usage" : 36488948, - "usage_in_kernelmode" : 20000000 + "usage_in_usermode" : 50000000, + "total_usage" : 100215355, + "usage_in_kernelmode" : 30000000 }, - "system_cpu_usage" : 20091722000000000, - "throttling_data" : {} + "system_cpu_usage" : 739306590000000, + "throttling_data" : {"periods":0,"throttled_periods":0,"throttled_time":0} + }, + "precpu_stats" : { + "cpu_usage" : { + "percpu_usage" : [ + 8646879, + 24350896, + 36438778, + 30657443 + ], + "usage_in_usermode" : 50000000, + "total_usage" : 100093996, + "usage_in_kernelmode" : 30000000 + }, + "system_cpu_usage" : 9492140000000, + "throttling_data" : {"periods":0,"throttled_periods":0,"throttled_time":0} } } +The precpu_stats is the cpu statistic of last read, which is used for calculating the cpu usage percent. It is not the exact copy of the “cpu_stats” field. + Query Parameters: - **stream** – 1/True/true or 0/False/false, pull stats once then disconnect. Default `true`. @@ -721,7 +738,7 @@ Status Codes: ### Resize a container TTY -`POST /containers/(id)/resize?h=&w=` +`POST /containers/(id or name)/resize?h=&w=` Resize the TTY for container with `id`. You must restart the container for the resize to take effect. @@ -743,7 +760,7 @@ Status Codes: ### Start a container -`POST /containers/(id)/start` +`POST /containers/(id or name)/start` Start the container `id` @@ -753,7 +770,7 @@ Start the container `id` **Example request**: - POST /containers/(id)/start HTTP/1.1 + POST /containers/e90e34656806/start HTTP/1.1 **Example response**: @@ -768,7 +785,7 @@ Status Codes: ### Stop a container -`POST /containers/(id)/stop` +`POST /containers/(id or name)/stop` Stop the container `id` @@ -793,7 +810,7 @@ Status Codes: ### Restart a container -`POST /containers/(id)/restart` +`POST /containers/(id or name)/restart` Restart the container `id` @@ -817,7 +834,7 @@ Status Codes: ### Kill a container -`POST /containers/(id)/kill` +`POST /containers/(id or name)/kill` Kill the container `id` @@ -842,7 +859,7 @@ Status Codes: ### Rename a container -`POST /containers/(id)/rename` +`POST /containers/(id or name)/rename` Rename the container `id` to a `new_name` @@ -867,7 +884,7 @@ Status Codes: ### Pause a container -`POST /containers/(id)/pause` +`POST /containers/(id or name)/pause` Pause the container `id` @@ -887,7 +904,7 @@ Status Codes: ### Unpause a container -`POST /containers/(id)/unpause` +`POST /containers/(id or name)/unpause` Unpause the container `id` @@ -907,7 +924,7 @@ Status Codes: ### Attach to a container -`POST /containers/(id)/attach` +`POST /containers/(id or name)/attach` Attach to the container `id` @@ -990,7 +1007,7 @@ Status Codes: ### Attach to a container (websocket) -`GET /containers/(id)/attach/ws` +`GET /containers/(id or name)/attach/ws` Attach to the container `id` via websocket @@ -1025,7 +1042,7 @@ Status Codes: ### Wait a container -`POST /containers/(id)/wait` +`POST /containers/(id or name)/wait` Block until container `id` stops, then returns the exit code @@ -1048,7 +1065,7 @@ Status Codes: ### Remove a container -`DELETE /containers/(id)` +`DELETE /containers/(id or name)` Remove the container `id` from the filesystem @@ -1076,7 +1093,7 @@ Status Codes: ### Copy files or folders from a container -`POST /containers/(id)/copy` +`POST /containers/(id or name)/copy` Copy files or folders of container `id` @@ -1106,14 +1123,14 @@ Status Codes: ### Retrieving information about files and folders in a container -`HEAD /containers/(id)/archive` +`HEAD /containers/(id or name)/archive` See the description of the `X-Docker-Container-Path-Stat` header in the following section. ### Get an archive of a filesystem resource in a container -`GET /containers/(id)/archive` +`GET /containers/(id or name)/archive` Get an tar archive of a resource in the filesystem of container `id`. @@ -1174,7 +1191,7 @@ Status Codes: ### Extract an archive of files or folders to a directory in a container -`PUT /containers/(id)/archive` +`PUT /containers/(id or name)/archive` Upload a tar archive to be extracted to a path in the filesystem of container `id`. @@ -2078,7 +2095,7 @@ the root that contains a list of repository and tag names mapped to layer IDs. ### Exec Create -`POST /containers/(id)/exec` +`POST /containers/(id or name)/exec` Sets up an exec instance in a running container `id` @@ -2157,7 +2174,7 @@ Status Codes: - **404** – no such exec instance **Stream details**: - Similar to the stream behavior of `POST /container/(id)/attach` API + Similar to the stream behavior of `POST /containers/(id or name)/attach` API ### Exec Resize diff --git a/components/engine/docs/reference/api/docker_remote_api_v1.21.md b/components/engine/docs/reference/api/docker_remote_api_v1.21.md index a1bcd62585..467ec32fdd 100644 --- a/components/engine/docs/reference/api/docker_remote_api_v1.21.md +++ b/components/engine/docs/reference/api/docker_remote_api_v1.21.md @@ -305,7 +305,8 @@ Json Parameters: An ever increasing delay (double the previous delay, starting at 100mS) is added before each restart to prevent flooding the server. - **NetworkMode** - Sets the networking mode for the container. Supported - values are: `bridge`, `host`, and `container:` + standard values are: `bridge`, `host`, `none`, and `container:`. Any other value is taken + as a custom network's name to which this container should connect to. - **Devices** - A list of devices to add to the container specified as a JSON object in the form `{ "PathOnHost": "/dev/deviceName", "PathInContainer": "/dev/deviceName", "CgroupPermissions": "mrw"}` @@ -335,7 +336,7 @@ Status Codes: ### Inspect a container -`GET /containers/(id)/json` +`GET /containers/(id or name)/json` Return low-level information on the container `id` @@ -525,7 +526,7 @@ Status Codes: ### List processes running inside a container -`GET /containers/(id)/top` +`GET /containers/(id or name)/top` List processes running inside the container `id`. On Unix systems this is done by running the `ps` command. This endpoint is not @@ -589,7 +590,7 @@ Status Codes: ### Get container logs -`GET /containers/(id)/logs` +`GET /containers/(id or name)/logs` Get `stdout` and `stderr` logs from the container ``id`` @@ -629,7 +630,7 @@ Status Codes: ### Inspect changes on a container's filesystem -`GET /containers/(id)/changes` +`GET /containers/(id or name)/changes` Inspect changes on container `id`'s filesystem @@ -671,7 +672,7 @@ Status Codes: ### Export a container -`GET /containers/(id)/export` +`GET /containers/(id or name)/export` Export the contents of container `id` @@ -694,7 +695,7 @@ Status Codes: ### Get container stats based on resource usage -`GET /containers/(id)/stats` +`GET /containers/(id or name)/stats` This endpoint returns a live stream of a container's resource usage statistics. @@ -772,20 +773,37 @@ This endpoint returns a live stream of a container's resource usage statistics. "cpu_stats" : { "cpu_usage" : { "percpu_usage" : [ - 16970827, - 1839451, - 7107380, - 10571290 + 8646879, + 24472255, + 36438778, + 30657443 ], - "usage_in_usermode" : 10000000, - "total_usage" : 36488948, - "usage_in_kernelmode" : 20000000 + "usage_in_usermode" : 50000000, + "total_usage" : 100215355, + "usage_in_kernelmode" : 30000000 }, - "system_cpu_usage" : 20091722000000000, - "throttling_data" : {} + "system_cpu_usage" : 739306590000000, + "throttling_data" : {"periods":0,"throttled_periods":0,"throttled_time":0} + }, + "precpu_stats" : { + "cpu_usage" : { + "percpu_usage" : [ + 8646879, + 24350896, + 36438778, + 30657443 + ], + "usage_in_usermode" : 50000000, + "total_usage" : 100093996, + "usage_in_kernelmode" : 30000000 + }, + "system_cpu_usage" : 9492140000000, + "throttling_data" : {"periods":0,"throttled_periods":0,"throttled_time":0} } } +The precpu_stats is the cpu statistic of last read, which is used for calculating the cpu usage percent. It is not the exact copy of the “cpu_stats” field. + Query Parameters: - **stream** – 1/True/true or 0/False/false, pull stats once then disconnect. Default `true`. @@ -798,7 +816,7 @@ Status Codes: ### Resize a container TTY -`POST /containers/(id)/resize` +`POST /containers/(id or name)/resize` Resize the TTY for container with `id`. The unit is number of characters. You must restart the container for the resize to take effect. @@ -825,7 +843,7 @@ Status Codes: ### Start a container -`POST /containers/(id)/start` +`POST /containers/(id or name)/start` Start the container `id` @@ -835,7 +853,7 @@ Start the container `id` **Example request**: - POST /containers/(id)/start HTTP/1.1 + POST /containers/e90e34656806/start HTTP/1.1 **Example response**: @@ -850,7 +868,7 @@ Status Codes: ### Stop a container -`POST /containers/(id)/stop` +`POST /containers/(id or name)/stop` Stop the container `id` @@ -875,7 +893,7 @@ Status Codes: ### Restart a container -`POST /containers/(id)/restart` +`POST /containers/(id or name)/restart` Restart the container `id` @@ -899,7 +917,7 @@ Status Codes: ### Kill a container -`POST /containers/(id)/kill` +`POST /containers/(id or name)/kill` Kill the container `id` @@ -924,7 +942,7 @@ Status Codes: ### Rename a container -`POST /containers/(id)/rename` +`POST /containers/(id or name)/rename` Rename the container `id` to a `new_name` @@ -949,7 +967,7 @@ Status Codes: ### Pause a container -`POST /containers/(id)/pause` +`POST /containers/(id or name)/pause` Pause the container `id` @@ -969,7 +987,7 @@ Status Codes: ### Unpause a container -`POST /containers/(id)/unpause` +`POST /containers/(id or name)/unpause` Unpause the container `id` @@ -989,7 +1007,7 @@ Status Codes: ### Attach to a container -`POST /containers/(id)/attach` +`POST /containers/(id or name)/attach` Attach to the container `id` @@ -1072,7 +1090,7 @@ Status Codes: ### Attach to a container (websocket) -`GET /containers/(id)/attach/ws` +`GET /containers/(id or name)/attach/ws` Attach to the container `id` via websocket @@ -1107,7 +1125,7 @@ Status Codes: ### Wait a container -`POST /containers/(id)/wait` +`POST /containers/(id or name)/wait` Block until container `id` stops, then returns the exit code @@ -1130,7 +1148,7 @@ Status Codes: ### Remove a container -`DELETE /containers/(id)` +`DELETE /containers/(id or name)` Remove the container `id` from the filesystem @@ -1158,7 +1176,7 @@ Status Codes: ### Copy files or folders from a container -`POST /containers/(id)/copy` +`POST /containers/(id or name)/copy` Copy files or folders of container `id` @@ -1188,14 +1206,14 @@ Status Codes: ### Retrieving information about files and folders in a container -`HEAD /containers/(id)/archive` +`HEAD /containers/(id or name)/archive` See the description of the `X-Docker-Container-Path-Stat` header in the following section. ### Get an archive of a filesystem resource in a container -`GET /containers/(id)/archive` +`GET /containers/(id or name)/archive` Get an tar archive of a resource in the filesystem of container `id`. @@ -1256,7 +1274,7 @@ Status Codes: ### Extract an archive of files or folders to a directory in a container -`PUT /containers/(id)/archive` +`PUT /containers/(id or name)/archive` Upload a tar archive to be extracted to a path in the filesystem of container `id`. @@ -2231,7 +2249,7 @@ the root that contains a list of repository and tag names mapped to layer IDs. ### Exec Create -`POST /containers/(id)/exec` +`POST /containers/(id or name)/exec` Sets up an exec instance in a running container `id` @@ -2313,7 +2331,7 @@ Status Codes: - **409** - container is paused **Stream details**: - Similar to the stream behavior of `POST /container/(id)/attach` API + Similar to the stream behavior of `POST /containers/(id or name)/attach` API ### Exec Resize diff --git a/components/engine/docs/reference/api/docker_remote_api_v1.22.md b/components/engine/docs/reference/api/docker_remote_api_v1.22.md index 22edf39c20..3e4da1058b 100644 --- a/components/engine/docs/reference/api/docker_remote_api_v1.22.md +++ b/components/engine/docs/reference/api/docker_remote_api_v1.22.md @@ -58,9 +58,15 @@ List containers }, "SizeRw": 12288, "SizeRootFs": 0, + "HostConfig": { + "NetworkMode": "default" + }, "NetworkSettings": { "Networks": { "bridge": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, "NetworkID": "7ea29fc1412292a2d7bba362f9253545fecdfa8ce9a6e37dd10ba8bee7129812", "EndpointID": "2cdc4edb1ded3631c81f57966563e5c8525b81121bb3706a9a9a3ae102711f3f", "Gateway": "172.17.0.1", @@ -86,9 +92,15 @@ List containers "Labels": {}, "SizeRw": 12288, "SizeRootFs": 0, + "HostConfig": { + "NetworkMode": "default" + }, "NetworkSettings": { "Networks": { "bridge": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, "NetworkID": "7ea29fc1412292a2d7bba362f9253545fecdfa8ce9a6e37dd10ba8bee7129812", "EndpointID": "88eaed7b37b38c2a3f0c4bc796494fdf51b270c2d22656412a2ca5d559a64d7a", "Gateway": "172.17.0.1", @@ -115,9 +127,15 @@ List containers "Labels": {}, "SizeRw":12288, "SizeRootFs":0, + "HostConfig": { + "NetworkMode": "default" + }, "NetworkSettings": { "Networks": { "bridge": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, "NetworkID": "7ea29fc1412292a2d7bba362f9253545fecdfa8ce9a6e37dd10ba8bee7129812", "EndpointID": "8b27c041c30326d59cd6e6f510d4f8d1d570a228466f956edf7815508f78e30d", "Gateway": "172.17.0.1", @@ -144,9 +162,15 @@ List containers "Labels": {}, "SizeRw": 12288, "SizeRootFs": 0, + "HostConfig": { + "NetworkMode": "default" + }, "NetworkSettings": { "Networks": { "bridge": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, "NetworkID": "7ea29fc1412292a2d7bba362f9253545fecdfa8ce9a6e37dd10ba8bee7129812", "EndpointID": "d91c7b2f0644403d7ef3095985ea0e2370325cd2332ff3a3225c4247328e66e9", "Gateway": "172.17.0.1", @@ -386,7 +410,8 @@ Json Parameters: An ever increasing delay (double the previous delay, starting at 100mS) is added before each restart to prevent flooding the server. - **NetworkMode** - Sets the networking mode for the container. Supported - values are: `bridge`, `host`, and `container:` + standard values are: `bridge`, `host`, `none`, and `container:`. Any other value is taken + as a custom network's name to which this container should connect to. - **Devices** - A list of devices to add to the container specified as a JSON object in the form `{ "PathOnHost": "/dev/deviceName", "PathInContainer": "/dev/deviceName", "CgroupPermissions": "mrw"}` @@ -417,7 +442,7 @@ Status Codes: ### Inspect a container -`GET /containers/(id)/json` +`GET /containers/(id or name)/json` Return low-level information on the container `id` @@ -619,7 +644,7 @@ Status Codes: ### List processes running inside a container -`GET /containers/(id)/top` +`GET /containers/(id or name)/top` List processes running inside the container `id`. On Unix systems this is done by running the `ps` command. This endpoint is not @@ -683,7 +708,7 @@ Status Codes: ### Get container logs -`GET /containers/(id)/logs` +`GET /containers/(id or name)/logs` Get `stdout` and `stderr` logs from the container ``id`` @@ -723,7 +748,7 @@ Status Codes: ### Inspect changes on a container's filesystem -`GET /containers/(id)/changes` +`GET /containers/(id or name)/changes` Inspect changes on container `id`'s filesystem @@ -765,7 +790,7 @@ Status Codes: ### Export a container -`GET /containers/(id)/export` +`GET /containers/(id or name)/export` Export the contents of container `id` @@ -788,7 +813,7 @@ Status Codes: ### Get container stats based on resource usage -`GET /containers/(id)/stats` +`GET /containers/(id or name)/stats` This endpoint returns a live stream of a container's resource usage statistics. @@ -866,20 +891,37 @@ This endpoint returns a live stream of a container's resource usage statistics. "cpu_stats" : { "cpu_usage" : { "percpu_usage" : [ - 16970827, - 1839451, - 7107380, - 10571290 + 8646879, + 24472255, + 36438778, + 30657443 ], - "usage_in_usermode" : 10000000, - "total_usage" : 36488948, - "usage_in_kernelmode" : 20000000 + "usage_in_usermode" : 50000000, + "total_usage" : 100215355, + "usage_in_kernelmode" : 30000000 }, - "system_cpu_usage" : 20091722000000000, - "throttling_data" : {} + "system_cpu_usage" : 739306590000000, + "throttling_data" : {"periods":0,"throttled_periods":0,"throttled_time":0} + }, + "precpu_stats" : { + "cpu_usage" : { + "percpu_usage" : [ + 8646879, + 24350896, + 36438778, + 30657443 + ], + "usage_in_usermode" : 50000000, + "total_usage" : 100093996, + "usage_in_kernelmode" : 30000000 + }, + "system_cpu_usage" : 9492140000000, + "throttling_data" : {"periods":0,"throttled_periods":0,"throttled_time":0} } } +The precpu_stats is the cpu statistic of last read, which is used for calculating the cpu usage percent. It is not the exact copy of the “cpu_stats” field. + Query Parameters: - **stream** – 1/True/true or 0/False/false, pull stats once then disconnect. Default `true`. @@ -892,7 +934,7 @@ Status Codes: ### Resize a container TTY -`POST /containers/(id)/resize` +`POST /containers/(id or name)/resize` Resize the TTY for container with `id`. The unit is number of characters. You must restart the container for the resize to take effect. @@ -919,7 +961,7 @@ Status Codes: ### Start a container -`POST /containers/(id)/start` +`POST /containers/(id or name)/start` Start the container `id` @@ -929,7 +971,7 @@ Start the container `id` **Example request**: - POST /containers/(id)/start HTTP/1.1 + POST /containers/e90e34656806/start HTTP/1.1 **Example response**: @@ -950,7 +992,7 @@ Status Codes: ### Stop a container -`POST /containers/(id)/stop` +`POST /containers/(id or name)/stop` Stop the container `id` @@ -975,7 +1017,7 @@ Status Codes: ### Restart a container -`POST /containers/(id)/restart` +`POST /containers/(id or name)/restart` Restart the container `id` @@ -999,7 +1041,7 @@ Status Codes: ### Kill a container -`POST /containers/(id)/kill` +`POST /containers/(id or name)/kill` Kill the container `id` @@ -1024,13 +1066,13 @@ Status Codes: ### Update a container -`POST /containers/(id)/update` +`POST /containers/(id or name)/update` Update resource configs of one or more containers. **Example request**: - POST /containers/(id)/update HTTP/1.1 + POST /containers/e90e34656806/update HTTP/1.1 Content-Type: application/json { @@ -1064,7 +1106,7 @@ Status Codes: ### Rename a container -`POST /containers/(id)/rename` +`POST /containers/(id or name)/rename` Rename the container `id` to a `new_name` @@ -1089,7 +1131,7 @@ Status Codes: ### Pause a container -`POST /containers/(id)/pause` +`POST /containers/(id or name)/pause` Pause the container `id` @@ -1109,7 +1151,7 @@ Status Codes: ### Unpause a container -`POST /containers/(id)/unpause` +`POST /containers/(id or name)/unpause` Unpause the container `id` @@ -1129,7 +1171,7 @@ Status Codes: ### Attach to a container -`POST /containers/(id)/attach` +`POST /containers/(id or name)/attach` Attach to the container `id` @@ -1215,7 +1257,7 @@ Status Codes: ### Attach to a container (websocket) -`GET /containers/(id)/attach/ws` +`GET /containers/(id or name)/attach/ws` Attach to the container `id` via websocket @@ -1253,7 +1295,7 @@ Status Codes: ### Wait a container -`POST /containers/(id)/wait` +`POST /containers/(id or name)/wait` Block until container `id` stops, then returns the exit code @@ -1276,7 +1318,7 @@ Status Codes: ### Remove a container -`DELETE /containers/(id)` +`DELETE /containers/(id or name)` Remove the container `id` from the filesystem @@ -1304,7 +1346,7 @@ Status Codes: ### Copy files or folders from a container -`POST /containers/(id)/copy` +`POST /containers/(id or name)/copy` Copy files or folders of container `id` @@ -1334,14 +1376,14 @@ Status Codes: ### Retrieving information about files and folders in a container -`HEAD /containers/(id)/archive` +`HEAD /containers/(id or name)/archive` See the description of the `X-Docker-Container-Path-Stat` header in the following section. ### Get an archive of a filesystem resource in a container -`GET /containers/(id)/archive` +`GET /containers/(id or name)/archive` Get an tar archive of a resource in the filesystem of container `id`. @@ -1402,7 +1444,7 @@ Status Codes: ### Extract an archive of files or folders to a directory in a container -`PUT /containers/(id)/archive` +`PUT /containers/(id or name)/archive` Upload a tar archive to be extracted to a path in the filesystem of container `id`. @@ -2480,7 +2522,7 @@ the root that contains a list of repository and tag names mapped to layer IDs. ### Exec Create -`POST /containers/(id)/exec` +`POST /containers/(id or name)/exec` Sets up an exec instance in a running container `id` @@ -2566,7 +2608,7 @@ Status Codes: - **409** - container is paused **Stream details**: - Similar to the stream behavior of `POST /container/(id)/attach` API + Similar to the stream behavior of `POST /containers/(id or name)/attach` API ### Exec Resize @@ -3054,7 +3096,7 @@ Content-Type: application/json { "Container":"3613f73ba0e4", "EndpointConfig": { - "test_nw": { + "IPAMConfig": { "IPv4Address":"172.24.56.89", "IPv6Address":"2001:db8::5689" } diff --git a/components/engine/docs/reference/api/docker_remote_api_v1.23.md b/components/engine/docs/reference/api/docker_remote_api_v1.23.md index 25ebb1fc5b..8ff2652368 100644 --- a/components/engine/docs/reference/api/docker_remote_api_v1.23.md +++ b/components/engine/docs/reference/api/docker_remote_api_v1.23.md @@ -59,9 +59,15 @@ List containers }, "SizeRw": 12288, "SizeRootFs": 0, + "HostConfig": { + "NetworkMode": "default" + }, "NetworkSettings": { "Networks": { "bridge": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, "NetworkID": "7ea29fc1412292a2d7bba362f9253545fecdfa8ce9a6e37dd10ba8bee7129812", "EndpointID": "2cdc4edb1ded3631c81f57966563e5c8525b81121bb3706a9a9a3ae102711f3f", "Gateway": "172.17.0.1", @@ -73,7 +79,18 @@ List containers "MacAddress": "02:42:ac:11:00:02" } } - } + }, + "Mounts": [ + { + "Name": "fac362...80535", + "Source": "/data", + "Destination": "/data", + "Driver": "local", + "Mode": "ro,Z", + "RW": false, + "Propagation": "" + } + ] }, { "Id": "9cd87474be90", @@ -88,9 +105,15 @@ List containers "Labels": {}, "SizeRw": 12288, "SizeRootFs": 0, + "HostConfig": { + "NetworkMode": "default" + }, "NetworkSettings": { "Networks": { "bridge": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, "NetworkID": "7ea29fc1412292a2d7bba362f9253545fecdfa8ce9a6e37dd10ba8bee7129812", "EndpointID": "88eaed7b37b38c2a3f0c4bc796494fdf51b270c2d22656412a2ca5d559a64d7a", "Gateway": "172.17.0.1", @@ -102,8 +125,8 @@ List containers "MacAddress": "02:42:ac:11:00:08" } } - } - + }, + "Mounts": [] }, { "Id": "3176a2479c92", @@ -118,9 +141,15 @@ List containers "Labels": {}, "SizeRw":12288, "SizeRootFs":0, + "HostConfig": { + "NetworkMode": "default" + }, "NetworkSettings": { "Networks": { "bridge": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, "NetworkID": "7ea29fc1412292a2d7bba362f9253545fecdfa8ce9a6e37dd10ba8bee7129812", "EndpointID": "8b27c041c30326d59cd6e6f510d4f8d1d570a228466f956edf7815508f78e30d", "Gateway": "172.17.0.1", @@ -132,8 +161,8 @@ List containers "MacAddress": "02:42:ac:11:00:06" } } - } - + }, + "Mounts": [] }, { "Id": "4cb07b47f9fb", @@ -148,9 +177,15 @@ List containers "Labels": {}, "SizeRw": 12288, "SizeRootFs": 0, + "HostConfig": { + "NetworkMode": "default" + }, "NetworkSettings": { "Networks": { "bridge": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, "NetworkID": "7ea29fc1412292a2d7bba362f9253545fecdfa8ce9a6e37dd10ba8bee7129812", "EndpointID": "d91c7b2f0644403d7ef3095985ea0e2370325cd2332ff3a3225c4247328e66e9", "Gateway": "172.17.0.1", @@ -162,8 +197,8 @@ List containers "MacAddress": "02:42:ac:11:00:05" } } - } - + }, + "Mounts": [] } ] @@ -184,6 +219,10 @@ Query Parameters: - `status=`(`created`|`restarting`|`running`|`paused`|`exited`|`dead`) - `label=key` or `label="key=value"` of a container label - `isolation=`(`default`|`process`|`hyperv`) (Windows daemon only) + - `ancestor`=(`[:]`, `` or ``) + - `before`=(`` or ``) + - `since`=(`` or ``) + - `volume`=(`` or ``) Status Codes: @@ -331,6 +370,7 @@ Json Parameters: - **MemorySwappiness** - Tune a container's memory swappiness behavior. Accepts an integer between 0 and 100. - **OomKillDisable** - Boolean value, whether to disable OOM Killer for the container or not. - **OomScoreAdj** - An integer value containing the score given to the container in order to tune OOM killer preferences. +- **PidsLimit** - Tune a container's pids limit. Set -1 for unlimited. - **AttachStdin** - Boolean value, attaches to `stdin`. - **AttachStdout** - Boolean value, attaches to `stdout`. - **AttachStderr** - Boolean value, attaches to `stderr`. @@ -390,7 +430,8 @@ Json Parameters: An ever increasing delay (double the previous delay, starting at 100mS) is added before each restart to prevent flooding the server. - **NetworkMode** - Sets the networking mode for the container. Supported - values are: `bridge`, `host`, and `container:` + standard values are: `bridge`, `host`, `none`, and `container:`. Any other value is taken + as a custom network's name to which this container should connect to. - **Devices** - A list of devices to add to the container specified as a JSON object in the form `{ "PathOnHost": "/dev/deviceName", "PathInContainer": "/dev/deviceName", "CgroupPermissions": "mrw"}` @@ -401,7 +442,7 @@ Json Parameters: systems, such as SELinux. - **LogConfig** - Log configuration for the container, specified as a JSON object in the form `{ "Type": "", "Config": {"key1": "val1"}}`. - Available types: `json-file`, `syslog`, `journald`, `gelf`, `awslogs`, `splunk`, `none`. + Available types: `json-file`, `syslog`, `journald`, `gelf`, `fluentd`, `awslogs`, `splunk`, `etwlogs`, `none`. `json-file` logging driver. - **CgroupParent** - Path to `cgroups` under which the container's `cgroup` is created. If the path is not absolute, the path is considered to be relative to the `cgroups` path of the init process. Cgroups are created if they do not already exist. - **VolumeDriver** - Driver that this container users to mount volumes. @@ -421,7 +462,7 @@ Status Codes: ### Inspect a container -`GET /containers/(id)/json` +`GET /containers/(id or name)/json` Return low-level information on the container `id` @@ -623,7 +664,7 @@ Status Codes: ### List processes running inside a container -`GET /containers/(id)/top` +`GET /containers/(id or name)/top` List processes running inside the container `id`. On Unix systems this is done by running the `ps` command. This endpoint is not @@ -687,7 +728,7 @@ Status Codes: ### Get container logs -`GET /containers/(id)/logs` +`GET /containers/(id or name)/logs` Get `stdout` and `stderr` logs from the container ``id`` @@ -727,7 +768,7 @@ Status Codes: ### Inspect changes on a container's filesystem -`GET /containers/(id)/changes` +`GET /containers/(id or name)/changes` Inspect changes on container `id`'s filesystem @@ -769,7 +810,7 @@ Status Codes: ### Export a container -`GET /containers/(id)/export` +`GET /containers/(id or name)/export` Export the contents of container `id` @@ -792,7 +833,7 @@ Status Codes: ### Get container stats based on resource usage -`GET /containers/(id)/stats` +`GET /containers/(id or name)/stats` This endpoint returns a live stream of a container's resource usage statistics. @@ -807,6 +848,9 @@ This endpoint returns a live stream of a container's resource usage statistics. { "read" : "2015-01-08T22:57:31.547920715Z", + "pids_stats": { + "current": 3 + }, "networks": { "eth0": { "rx_bytes": 5338, @@ -870,20 +914,37 @@ This endpoint returns a live stream of a container's resource usage statistics. "cpu_stats" : { "cpu_usage" : { "percpu_usage" : [ - 16970827, - 1839451, - 7107380, - 10571290 + 8646879, + 24472255, + 36438778, + 30657443 ], - "usage_in_usermode" : 10000000, - "total_usage" : 36488948, - "usage_in_kernelmode" : 20000000 + "usage_in_usermode" : 50000000, + "total_usage" : 100215355, + "usage_in_kernelmode" : 30000000 }, - "system_cpu_usage" : 20091722000000000, - "throttling_data" : {} + "system_cpu_usage" : 739306590000000, + "throttling_data" : {"periods":0,"throttled_periods":0,"throttled_time":0} + }, + "precpu_stats" : { + "cpu_usage" : { + "percpu_usage" : [ + 8646879, + 24350896, + 36438778, + 30657443 + ], + "usage_in_usermode" : 50000000, + "total_usage" : 100093996, + "usage_in_kernelmode" : 30000000 + }, + "system_cpu_usage" : 9492140000000, + "throttling_data" : {"periods":0,"throttled_periods":0,"throttled_time":0} } } +The precpu_stats is the cpu statistic of last read, which is used for calculating the cpu usage percent. It is not the exact copy of the “cpu_stats” field. + Query Parameters: - **stream** – 1/True/true or 0/False/false, pull stats once then disconnect. Default `true`. @@ -896,7 +957,7 @@ Status Codes: ### Resize a container TTY -`POST /containers/(id)/resize` +`POST /containers/(id or name)/resize` Resize the TTY for container with `id`. The unit is number of characters. You must restart the container for the resize to take effect. @@ -923,7 +984,7 @@ Status Codes: ### Start a container -`POST /containers/(id)/start` +`POST /containers/(id or name)/start` Start the container `id` @@ -933,7 +994,7 @@ Start the container `id` **Example request**: - POST /containers/(id)/start HTTP/1.1 + POST /containers/e90e34656806/start HTTP/1.1 **Example response**: @@ -954,7 +1015,7 @@ Status Codes: ### Stop a container -`POST /containers/(id)/stop` +`POST /containers/(id or name)/stop` Stop the container `id` @@ -979,7 +1040,7 @@ Status Codes: ### Restart a container -`POST /containers/(id)/restart` +`POST /containers/(id or name)/restart` Restart the container `id` @@ -1003,7 +1064,7 @@ Status Codes: ### Kill a container -`POST /containers/(id)/kill` +`POST /containers/(id or name)/kill` Kill the container `id` @@ -1028,13 +1089,13 @@ Status Codes: ### Update a container -`POST /containers/(id)/update` +`POST /containers/(id or name)/update` -Update resource configs of one or more containers. +Update configuration of one or more containers. **Example request**: - POST /containers/(id)/update HTTP/1.1 + POST /containers/e90e34656806/update HTTP/1.1 Content-Type: application/json { @@ -1048,6 +1109,10 @@ Update resource configs of one or more containers. "MemorySwap": 514288000, "MemoryReservation": 209715200, "KernelMemory": 52428800, + "RestartPolicy": { + "MaximumRetryCount": 4, + "Name": "on-failure" + }, } **Example response**: @@ -1068,7 +1133,7 @@ Status Codes: ### Rename a container -`POST /containers/(id)/rename` +`POST /containers/(id or name)/rename` Rename the container `id` to a `new_name` @@ -1093,7 +1158,7 @@ Status Codes: ### Pause a container -`POST /containers/(id)/pause` +`POST /containers/(id or name)/pause` Pause the container `id` @@ -1113,7 +1178,7 @@ Status Codes: ### Unpause a container -`POST /containers/(id)/unpause` +`POST /containers/(id or name)/unpause` Unpause the container `id` @@ -1133,7 +1198,7 @@ Status Codes: ### Attach to a container -`POST /containers/(id)/attach` +`POST /containers/(id or name)/attach` Attach to the container `id` @@ -1219,7 +1284,7 @@ Status Codes: ### Attach to a container (websocket) -`GET /containers/(id)/attach/ws` +`GET /containers/(id or name)/attach/ws` Attach to the container `id` via websocket @@ -1257,7 +1322,7 @@ Status Codes: ### Wait a container -`POST /containers/(id)/wait` +`POST /containers/(id or name)/wait` Block until container `id` stops, then returns the exit code @@ -1280,7 +1345,7 @@ Status Codes: ### Remove a container -`DELETE /containers/(id)` +`DELETE /containers/(id or name)` Remove the container `id` from the filesystem @@ -1308,7 +1373,7 @@ Status Codes: ### Copy files or folders from a container -`POST /containers/(id)/copy` +`POST /containers/(id or name)/copy` Copy files or folders of container `id` @@ -1338,14 +1403,14 @@ Status Codes: ### Retrieving information about files and folders in a container -`HEAD /containers/(id)/archive` +`HEAD /containers/(id or name)/archive` See the description of the `X-Docker-Container-Path-Stat` header in the following section. ### Get an archive of a filesystem resource in a container -`GET /containers/(id)/archive` +`GET /containers/(id or name)/archive` Get an tar archive of a resource in the filesystem of container `id`. @@ -1406,7 +1471,7 @@ Status Codes: ### Extract an archive of files or folders to a directory in a container -`PUT /containers/(id)/archive` +`PUT /containers/(id or name)/archive` Upload a tar archive to be extracted to a path in the filesystem of container `id`. @@ -2091,6 +2156,7 @@ Display system-wide information { "Architecture": "x86_64", + "CgroupDriver": "cgroupfs", "Containers": 11, "ContainersRunning": 7, "ContainersStopped": 3, @@ -2123,6 +2189,7 @@ Display system-wide information "IndexServerAddress": "https://index.docker.io/v1/", "InitPath": "/usr/bin/docker", "InitSha1": "", + "KernelMemory": true, "KernelVersion": "3.12.0-1-amd64", "Labels": [ "storage=ssd" @@ -2484,7 +2551,7 @@ the root that contains a list of repository and tag names mapped to layer IDs. ### Exec Create -`POST /containers/(id)/exec` +`POST /containers/(id or name)/exec` Sets up an exec instance in a running container `id` @@ -2570,7 +2637,7 @@ Status Codes: - **409** - container is paused **Stream details**: - Similar to the stream behavior of `POST /container/(id)/attach` API + Similar to the stream behavior of `POST /containers/(id or name)/attach` API ### Exec Resize @@ -2868,6 +2935,8 @@ Content-Type: application/json "Id": "f2de39df4171b0dc801e8002d1d999b77256983dfc63041c0f34030aa3977566", "Scope": "local", "Driver": "bridge", + "EnableIPv6": false, + "Internal": false, "IPAM": { "Driver": "default", "Config": [ @@ -2898,6 +2967,8 @@ Content-Type: application/json "Id": "e086a3893b05ab69242d3c44e49483a3bbbd3a26b46baa8f61ab797c1088d794", "Scope": "local", "Driver": "null", + "EnableIPv6": false, + "Internal": false, "IPAM": { "Driver": "default", "Config": [] @@ -2910,6 +2981,8 @@ Content-Type: application/json "Id": "13e871235c677f196c4e1ecebb9dc733b9b2d2ab589e30c539efeda84a24215e", "Scope": "local", "Driver": "host", + "EnableIPv6": false, + "Internal": false, "IPAM": { "Driver": "default", "Config": [] @@ -2951,6 +3024,7 @@ Content-Type: application/json "Id": "7d86d31b1478e7cca9ebed7e73aa0fdeec46c5ca29497431d3007d2d9e15ed99", "Scope": "local", "Driver": "bridge", + "EnableIPv6": false, "IPAM": { "Driver": "default", "Config": [ @@ -3004,6 +3078,7 @@ Content-Type: application/json { "Name":"isolated_nw", "Driver":"bridge", + "EnableIPv6": false, "IPAM":{ "Config":[{ "Subnet":"172.20.0.0/16", @@ -3040,7 +3115,9 @@ JSON Parameters: - **Name** - The new network's name. this is a mandatory field - **Driver** - Name of the network driver plugin to use. Defaults to `bridge` driver +- **Internal** - Restrict external access to the network - **IPAM** - Optional custom IP scheme for the network +- **EnableIPv6** - Enable IPv6 on the network - **Options** - Network specific options to be used by the drivers - **CheckDuplicate** - Requests daemon to check for networks with same name @@ -3059,7 +3136,7 @@ Content-Type: application/json { "Container":"3613f73ba0e4", "EndpointConfig": { - "test_nw": { + "IPAMConfig": { "IPv4Address":"172.24.56.89", "IPv6Address":"2001:db8::5689" } diff --git a/components/engine/docs/reference/builder.md b/components/engine/docs/reference/builder.md index 3b9a6d5ea4..8a01d41d52 100644 --- a/components/engine/docs/reference/builder.md +++ b/components/engine/docs/reference/builder.md @@ -516,7 +516,7 @@ feature](../userguide/networking/index.md)). ENV = ... The `ENV` instruction sets the environment variable `` to the value -``. This value will be in the environment of all "descendent" +``. This value will be in the environment of all "descendant" `Dockerfile` commands and can be [replaced inline](#environment-replacement) in many as well. @@ -579,7 +579,7 @@ The `` is an absolute path, or a path relative to `WORKDIR`, into which the source will be copied inside the destination container. ADD test relativeDir/ # adds "test" to `WORKDIR`/relativeDir/ - ADD test /absoluteDir # adds "test" to /absoluteDir + ADD test /absoluteDir/ # adds "test" to /absoluteDir/ All new files and directories are created with a UID and GID of 0. @@ -646,7 +646,7 @@ guide](../userguide/eng-image/dockerfile_best-practices.md#build-cache) for more > **Note**: > Whether a file is identified as a recognized compression format or not - > is done soley based on the contents of the file, not the name of the file. + > is done solely based on the contents of the file, not the name of the file. > For example, if an empty file happens to end with `.tar.gz` this will not > be recognized as a compressed file and **will not** generate any kind of > decompression error message, rather the file will simply be copied to the @@ -691,7 +691,7 @@ The `` is an absolute path, or a path relative to `WORKDIR`, into which the source will be copied inside the destination container. COPY test relativeDir/ # adds "test" to `WORKDIR`/relativeDir/ - COPY test /absoluteDir # adds "test" to /absoluteDir + COPY test /absoluteDir/ # adds "test" to /absoluteDir/ All new files and directories are created with a UID and GID of 0. @@ -950,6 +950,29 @@ If you then run `docker stop test`, the container will not exit cleanly - the user 0m 0.04s sys 0m 0.03s +### Understand how CMD and ENTRYPOINT interact + +Both `CMD` and `ENTRYPOINT` instructions define what command gets executed when running a container. +There are few rules that describe their co-operation. + +1. Dockerfile should specify at least one of `CMD` or `ENTRYPOINT` commands. + +2. `ENTRYPOINT` should be defined when using the container as an executable. + +3. `CMD` should be used as a way of defining default arguments for an `ENTRYPOINT` command +or for executing an ad-hoc command in a container. + +4. `CMD` will be overridden when running the container with alternative arguments. + +The table below shows what command is executed for different `ENTRYPOINT` / `CMD` combinations: + +| | No ENTRYPOINT | ENTRYPOINT exec_entry p1_entry | ENTRYPOINT ["exec_entry", "p1_entry"] | +|--------------------------------|----------------------------|-----------------------------------------------------------|------------------------------------------------| +| **No CMD** | *error, not allowed* | /bin/sh -c exec_entry p1_entry | exec_entry p1_entry | +| **CMD ["exec_cmd", "p1_cmd"]** | exec_cmd p1_cmd | /bin/sh -c exec_entry p1_entry exec_cmd p1_cmd | exec_entry p1_entry exec_cmd p1_cmd | +| **CMD ["p1_cmd", "p2_cmd"]** | p1_cmd p2_cmd | /bin/sh -c exec_entry p1_entry p1_cmd p2_cmd | exec_entry p1_entry p1_cmd p2_cmd | +| **CMD exec_cmd p1_cmd** | /bin/sh -c exec_cmd p1_cmd | /bin/sh -c exec_entry p1_entry /bin/sh -c exec_cmd p1_cmd | exec_entry p1_entry /bin/sh -c exec_cmd p1_cmd | + ## VOLUME VOLUME ["/data"] diff --git a/components/engine/docs/reference/commandline/commit.md b/components/engine/docs/reference/commandline/commit.md index 13dd3340b0..df64e957ac 100644 --- a/components/engine/docs/reference/commandline/commit.md +++ b/components/engine/docs/reference/commandline/commit.md @@ -31,7 +31,7 @@ volumes mounted inside the container. By default, the container being committed and its processes will be paused while the image is committed. This reduces the likelihood of encountering data corruption during the process of creating the commit. If this behavior is -undesired, set the 'p' option to false. +undesired, set the `--pause` option to false. The `--change` option will apply `Dockerfile` instructions to the image that is created. Supported `Dockerfile` instructions: diff --git a/components/engine/docs/reference/commandline/cp.md b/components/engine/docs/reference/commandline/cp.md index 50179fb4cf..841aeb36e0 100644 --- a/components/engine/docs/reference/commandline/cp.md +++ b/components/engine/docs/reference/commandline/cp.md @@ -23,7 +23,7 @@ You can copy from the container's file system to the local machine or the reverse, from the local filesystem to the container. If `-` is specified for either the `SRC_PATH` or `DEST_PATH`, you can also stream a tar archive from `STDIN` or to `STDOUT`. The `CONTAINER` can be a running or stopped container. -The `SRC_PATH` or `DEST_PATH` be a file or directory. +The `SRC_PATH` or `DEST_PATH` can be a file or directory. The `docker cp` command assumes container paths are relative to the container's `/` (root) directory. This means supplying the initial forward slash is optional; @@ -39,7 +39,8 @@ the user and primary group at the destination. For example, files copied to a container are created with `UID:GID` of the root user. Files copied to the local machine are created with the `UID:GID` of the user which invoked the `docker cp` command. If you specify the `-L` option, `docker cp` follows any symbolic link -in the `SRC_PATH`. +in the `SRC_PATH`. `docker cp` does *not* create parent directories for +`DEST_PATH` if they do not exist. Assuming a path separator of `/`, a first argument of `SRC_PATH` and second argument of `DEST_PATH`, the behavior is as follows: @@ -85,4 +86,4 @@ It is not possible to copy certain system files such as resources under Using `-` as the `SRC_PATH` streams the contents of `STDIN` as a tar archive. The command extracts the content of the tar to the `DEST_PATH` in container's filesystem. In this case, `DEST_PATH` must specify a directory. Using `-` as -`DEST_PATH` streams the contents of the resource as a tar archive to `STDOUT`. +the `DEST_PATH` streams the contents of the resource as a tar archive to `STDOUT`. diff --git a/components/engine/docs/reference/commandline/create.md b/components/engine/docs/reference/commandline/create.md index ad23995ac1..fa68b0feb1 100644 --- a/components/engine/docs/reference/commandline/create.md +++ b/components/engine/docs/reference/commandline/create.md @@ -74,6 +74,7 @@ Creates a new container. -P, --publish-all Publish all exposed ports to random ports -p, --publish=[] Publish a container's port(s) to the host --pid="" PID namespace to use + --pids-limit=-1 Tune container pids limit (set -1 for unlimited), kernel >= 4.3 --privileged Give extended privileges to this container --read-only Mount the container's root filesystem as read only --restart="no" Restart policy (no, on-failure[:max-retry], always, unless-stopped) diff --git a/components/engine/docs/reference/commandline/daemon.md b/components/engine/docs/reference/commandline/daemon.md index 1986295414..aef28d576b 100644 --- a/components/engine/docs/reference/commandline/daemon.md +++ b/components/engine/docs/reference/commandline/daemon.md @@ -644,7 +644,7 @@ multiple plugins installed, at least one must allow the request for it to complete. For information about how to create an authorization plugin, see [authorization -plugin](../../extend/authorization.md) section in the Docker extend section of this documentation. +plugin](../../extend/plugins_authorization.md) section in the Docker extend section of this documentation. ## Daemon user namespace options @@ -696,11 +696,17 @@ these resources are name-based, not id-based. If the numeric ID information provided does not exist as entries in `/etc/passwd` or `/etc/group`, daemon startup will fail with an error message. +> **Note:** On Fedora 22, you have to `touch` the `/etc/subuid` and `/etc/subgid` +> files to have ranges assigned when users are created. This must be done +> *before* the `--userns-remap` option is enabled. Once these files exist, the +> daemon can be (re)started and range assignment on user creation works properly. + *Example: starting with default Docker user management:* +```bash +$ docker daemon --userns-remap=default ``` - $ docker daemon --userns-remap=default -``` + When `default` is provided, Docker will create - or find the existing - user and group named `dockremap`. If the user is created, and the Linux distribution has appropriate support, the `/etc/subuid` and `/etc/subgid` files will be populated @@ -709,16 +715,11 @@ at an offset based on prior entries in those files. For example, Ubuntu will create the following range, based on an existing user named `user1` already owning the first 65536 range: +```bash +$ cat /etc/subuid +user1:100000:65536 +dockremap:165536:65536 ``` - $ cat /etc/subuid - user1:100000:65536 - dockremap:165536:65536 -``` - -> **Note:** On a fresh Fedora install, we had to `touch` the -> `/etc/subuid` and `/etc/subgid` files to have ranges assigned when users -> were created. Once these files existed, range assignment on user creation -> worked properly. If you have a preferred/self-managed user with subordinate ID mappings already configured, you can provide that username or uid to the `--userns-remap` flag. @@ -802,9 +803,17 @@ cgroup. Assuming the daemon is running in cgroup `daemoncgroup`, `--cgroup-parent=/foobar` creates a cgroup in -`/sys/fs/cgroup/memory/foobar`, wheras using `--cgroup-parent=foobar` +`/sys/fs/cgroup/memory/foobar`, whereas using `--cgroup-parent=foobar` creates the cgroup in `/sys/fs/cgroup/memory/daemoncgroup/foobar` +The systemd cgroup driver has different rules for `--cgroup-parent`. Systemd +represents hierarchy by slice and the name of the slice encodes the location in +the tree. So `--cgroup-parent` for systemd cgroups should be a slice name. A +name can consist of a dash-separated series of names, which describes the path +to the slice from the root slice. For example, `--cgroup-parent=user-a-b.slice` +means the memory cgroup for the container is created in +`/sys/fs/cgroup/memory/user.slice/user-a.slice/user-a-b.slice/docker-.scope`. + This setting can also be set per container, using the `--cgroup-parent` option on `docker create` and `docker run`, and takes precedence over the `--cgroup-parent` option on the daemon. @@ -890,4 +899,15 @@ if there are conflicts, but it won't stop execution. The list of currently supported options that can be reconfigured is this: - `debug`: it changes the daemon to debug mode when set to true. +- `cluster-store`: it reloads the discovery store with the new address. +- `cluster-store-opts`: it uses the new options to reload the discovery store. +- `cluster-advertise`: it modifies the address advertised after reloading. - `labels`: it replaces the daemon labels with a new set of labels. + +Updating and reloading the cluster configurations such as `--cluster-store`, +`--cluster-advertise` and `--cluster-store-opts` will take effect only if +these configurations were not previously configured. If `--cluster-store` +has been provided in flags and `cluster-advertise` not, `cluster-advertise` +can be added in the configuration file without accompanied by `--cluster-store` +Configuration reload will log a warning message if it detects a change in +previously configured cluster configurations. diff --git a/components/engine/docs/reference/commandline/events.md b/components/engine/docs/reference/commandline/events.md index 30eae105d9..22e94609d2 100644 --- a/components/engine/docs/reference/commandline/events.md +++ b/components/engine/docs/reference/commandline/events.md @@ -37,7 +37,7 @@ Docker networks report the following events: The `--since` and `--until` parameters can be Unix timestamps, date formatted timestamps, or Go duration strings (e.g. `10m`, `1h30m`) computed -relative to the client machine’s time. If you do not provide the --since option, +relative to the client machine’s time. If you do not provide the `--since` option, the command returns only new and/or live events. Supported formats for date formatted time stamps include RFC3339Nano, RFC3339, `2006-01-02T15:04:05`, `2006-01-02T15:04:05.999999999`, `2006-01-02Z07:00`, and `2006-01-02`. The local diff --git a/components/engine/docs/reference/commandline/import.md b/components/engine/docs/reference/commandline/import.md index 3b36a52bad..d4ca8d5775 100644 --- a/components/engine/docs/reference/commandline/import.md +++ b/components/engine/docs/reference/commandline/import.md @@ -47,7 +47,7 @@ Import to docker via pipe and `STDIN`. $ cat exampleimage.tgz | docker import - exampleimagelocal:new -Import with a commit message +Import with a commit message. $ cat exampleimage.tgz | docker import --message "New image imported from tarball" - exampleimagelocal:new diff --git a/components/engine/docs/reference/commandline/info.md b/components/engine/docs/reference/commandline/info.md index 5bb6fb299f..06f7848d5a 100644 --- a/components/engine/docs/reference/commandline/info.md +++ b/components/engine/docs/reference/commandline/info.md @@ -33,6 +33,7 @@ For example: Dirperm1 Supported: true Execution Driver: native-0.2 Logging Driver: json-file + Cgroup Driver: cgroupfs Plugins: Volume: local Network: bridge null host @@ -44,6 +45,7 @@ For example: Total Memory: 62.86 GiB Name: docker ID: I54V:OLXT:HVMM:TPKO:JPHQ:CQCD:JNLC:O3BZ:4ZVJ:43XJ:PFHZ:6N2S + Docker Root Dir: /var/lib/docker Debug mode (client): true Debug mode (server): true File Descriptors: 59 diff --git a/components/engine/docs/reference/commandline/login.md b/components/engine/docs/reference/commandline/login.md index faf3615a00..34a7228427 100644 --- a/components/engine/docs/reference/commandline/login.md +++ b/components/engine/docs/reference/commandline/login.md @@ -12,10 +12,9 @@ parent = "smn_cli" Usage: docker login [OPTIONS] [SERVER] - Register or log in to a Docker registry server, if no server is + Log in to a Docker registry server, if no server is specified "https://index.docker.io/v1/" is the default. - -e, --email="" Email --help Print usage -p, --password="" Password -u, --username="" Username @@ -27,10 +26,10 @@ adding the server name. $ docker login localhost:8080 -`docker login` requires user to use `sudo` or be `root`, except when: +`docker login` requires user to use `sudo` or be `root`, except when: 1. connecting to a remote daemon, such as a `docker-machine` provisioned `docker engine`. -2. user is added to the `docker` group. This will impact the security of your system; the `docker` group is `root` equivalent. See [Docker Daemon Attack Surface](https://docs.docker.com/security/security/#docker-daemon-attack-surface) for details. +2. user is added to the `docker` group. This will impact the security of your system; the `docker` group is `root` equivalent. See [Docker Daemon Attack Surface](https://docs.docker.com/security/security/#docker-daemon-attack-surface) for details. You can log into any public or private repository for which you have credentials. When you log in, the command stores encoded credentials in @@ -38,3 +37,78 @@ credentials. When you log in, the command stores encoded credentials in > **Note**: When running `sudo docker login` credentials are saved in `/root/.docker/config.json`. > + +## Credentials store + +The Docker Engine can keep user credentials in an external credentials store, +such as the native keychain of the operating system. Using an external store +is more secure than storing credentials in the Docker configuration file. + +To use a credentials store, you need an external helper program to interact +with a specific keychain or external store. Docker requires the helper +program to be in the client's host `$PATH`. + +This is the list of currently available credentials helpers and where +you can download them from: + +- D-Bus Secret Service: https://github.com/docker/docker-credential-helpers/releases +- Apple OS X keychain: https://github.com/docker/docker-credential-helpers/releases +- Microsoft Windows Credential Manager: https://github.com/docker/docker-credential-helpers/releases + +### Usage + +You need to speficy the credentials store in `HOME/.docker/config.json` +to tell the docker engine to use it: + +```json +{ + "credsStore": "osxkeychain" +} +``` + +If you are currently logged in, run `docker logout` to remove +the credentials from the file and run `docker login` again. + +### Protocol + +Credential helpers can be any program or script that follows a very simple protocol. +This protocol is heavily inspired by Git, but it differs in the information shared. + +The helpers always use the first argument in the command to identify the action. +There are only three possible values for that argument: `store`, `get`, and `erase`. + +The `store` command takes a JSON payload from the standard input. That payload carries +the server address, to identify the credential, the user name and the password. +This is an example of that payload: + +```json +{ + "ServerURL": "https://index.docker.io/v1", + "Username": "david", + "Password": "passw0rd1" +} +``` + +The `store` command can write error messages to `STDOUT` that the docker engine +will show if there was an issue. + +The `get` command takes a string payload from the standard input. That payload carries +the server address that the docker engine needs credentials for. This is +an example of that payload: `https://index.docker.io/v1`. + +The `get` command writes a JSON payload to `STDOUT`. Docker reads the user name +and password from this payload: + +```json +{ + "Username": "david", + "Password": "passw0rd1" +} +``` + +The `erase` command takes a string payload from `STDIN`. That payload carries +the server address that the docker engine wants to remove credentials for. This is +an example of that payload: `https://index.docker.io/v1`. + +The `erase` command can write error messages to `STDOUT` that the docker engine +will show if there was an issue. diff --git a/components/engine/docs/reference/commandline/network_connect.md b/components/engine/docs/reference/commandline/network_connect.md index 49f0b3d7f4..a815ca38ec 100644 --- a/components/engine/docs/reference/commandline/network_connect.md +++ b/components/engine/docs/reference/commandline/network_connect.md @@ -40,7 +40,7 @@ You can specify the IP address you want to be assigned to the container's interf $ docker network connect --ip 10.10.36.122 multi-host-network container2 ``` -You can use `--link` option to link another container with a prefered alias +You can use `--link` option to link another container with a preferred alias ```bash $ docker network connect --link container1:c1 multi-host-network container2 diff --git a/components/engine/docs/reference/commandline/network_create.md b/components/engine/docs/reference/commandline/network_create.md index bb85cc0462..6ae18e6e73 100644 --- a/components/engine/docs/reference/commandline/network_create.md +++ b/components/engine/docs/reference/commandline/network_create.md @@ -22,6 +22,7 @@ parent = "smn_cli" --ip-range=[] Allocate container ip from a sub-range --ipam-driver=default IP Address Management Driver --ipam-opt=map[] Set custom IPAM driver specific options + --ipv6 Enable IPv6 networking -o --opt=map[] Set custom driver specific options --subnet=[] Subnet in CIDR format that represents a network segment @@ -134,7 +135,17 @@ The following are those options and the equivalent docker daemon flags used for | `com.docker.network.bridge.enable_icc` | `--icc` | Enable or Disable Inter Container Connectivity | | `com.docker.network.bridge.host_binding_ipv4` | `--ip` | Default IP when binding container ports | | `com.docker.network.mtu` | `--mtu` | Set the containers network MTU | -| `com.docker.network.enable_ipv6` | `--ipv6` | Enable IPv6 networking | + +The following arguments can be passed to `docker network create` for any network driver, again with their approximate +equivalents to `docker daemon`. + +| Argument | Equivalent | Description | +|--------------|----------------|--------------------------------------------| +| `--gateway` | - | ipv4 or ipv6 Gateway for the master subnet | +| `--ip-range` | `--fixed-cidr` | Allocate IPs from a range | +| `--internal` | - | Restricts external access to the network | +| `--ipv6` | `--ipv6` | Enable IPv6 networking | +| `--subnet` | `--bip` | Subnet for network | For example, let's use `-o` or `--opt` options to specify an IP address binding when publishing ports: diff --git a/components/engine/docs/reference/commandline/network_ls.md b/components/engine/docs/reference/commandline/network_ls.md index 06733cd3ca..b12957a3a4 100644 --- a/components/engine/docs/reference/commandline/network_ls.md +++ b/components/engine/docs/reference/commandline/network_ls.md @@ -94,7 +94,7 @@ NETWORK ID NAME DRIVER You can also filter for a substring in a name as this shows: ```bash -$ docker ps --filter name=foo +$ docker network ls --filter name=foo NETWORK ID NAME DRIVER 95e74588f40d foo bridge 06e7eef0a170 foobar bridge @@ -104,8 +104,8 @@ NETWORK ID NAME DRIVER The `id` filter matches on all or part of a network's ID. -The following filter matches all networks with a name containing the -`06e7eef01700` string. +The following filter matches all networks with an ID containing the +`63d1ff1f77b0...` string. ```bash $ docker network ls --filter id=63d1ff1f77b07ca51070a8c227e962238358bd310bde1529cf62e6c307ade161 @@ -113,14 +113,14 @@ NETWORK ID NAME DRIVER 63d1ff1f77b0 dev bridge ``` -You can also filter for a substring in a ID as this shows: +You can also filter for a substring in an ID as this shows: ```bash -$ docker ps --filter id=95e74588f40d +$ docker network ls --filter id=95e74588f40d NETWORK ID NAME DRIVER 95e74588f40d foo bridge -$ docker ps --filter id=95e +$ docker network ls --filter id=95e NETWORK ID NAME DRIVER 95e74588f40d foo bridge ``` diff --git a/components/engine/docs/reference/commandline/network_rm.md b/components/engine/docs/reference/commandline/network_rm.md index 516eb4ecfc..0653458f9d 100644 --- a/components/engine/docs/reference/commandline/network_rm.md +++ b/components/engine/docs/reference/commandline/network_rm.md @@ -25,7 +25,7 @@ To remove the network named 'my-network': ``` To delete multiple networks in a single `docker network rm` command, provide -multiple network names or id's. The following example deletes a network with id +multiple network names or ids. The following example deletes a network with id `3695c422697f` and a network named `my-network`: ```bash diff --git a/components/engine/docs/reference/commandline/ps.md b/components/engine/docs/reference/commandline/ps.md index 328e674264..af82322e87 100644 --- a/components/engine/docs/reference/commandline/ps.md +++ b/components/engine/docs/reference/commandline/ps.md @@ -57,7 +57,10 @@ The currently supported filters are: * exited (int - the code of exited containers. Only useful with `--all`) * status (created|restarting|running|paused|exited|dead) * ancestor (`[:]`, `` or ``) - filters containers that were created from the given image or a descendant. +* before (container's id or name) - filters containers created before given id or name +* since (container's id or name) - filters containers created since given id or name * isolation (default|process|hyperv) (Windows daemon only) +* volume (volume name or mount point) - filters containers that mount volumes. #### Label @@ -163,6 +166,46 @@ in it's layer stack. CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 82a598284012 ubuntu:12.04.5 "top" 3 minutes ago Up 3 minutes sleepy_bose +#### Before + +The `before` filter shows only containers created before the container with given id or name. For example, +having these containers created: + + $ docker ps + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 9c3527ed70ce busybox "top" 14 seconds ago Up 15 seconds desperate_dubinsky + 4aace5031105 busybox "top" 48 seconds ago Up 49 seconds focused_hamilton + 6e63f6ff38b0 busybox "top" About a minute ago Up About a minute distracted_fermat + +Filtering with `before` would give: + + $ docker ps -f before=9c3527ed70ce + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 4aace5031105 busybox "top" About a minute ago Up About a minute focused_hamilton + 6e63f6ff38b0 busybox "top" About a minute ago Up About a minute distracted_fermat + +#### Since + +The `since` filter shows only containers created since the container with given id or name. For example, +with the same containers as in `before` filter: + + $ docker ps -f since=6e63f6ff38b0 + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 9c3527ed70ce busybox "top" 10 minutes ago Up 10 minutes desperate_dubinsky + 4aace5031105 busybox "top" 10 minutes ago Up 10 minutes focused_hamilton + +#### Volume + +The `volume` filter shows only containers that mount a specific volume or have a volume mounted in a specific path: + + $ docker ps --filter volume=remote-volume --format "table {{.ID}}\t{{.Mounts}}" + CONTAINER ID MOUNTS + 9c3527ed70ce remote-volume + + $ docker ps --filter volume=/data --format "table {{.ID}}\t{{.Mounts}}" + CONTAINER ID MOUNTS + 9c3527ed70ce remote-volume + ## Formatting @@ -183,6 +226,7 @@ Placeholder | Description `.Names` | Container names. `.Labels` | All labels assigned to the container. `.Label` | Value of a specific label for this container. For example `{{.Label "com.docker.swarm.cpu"}}` +`.Mounts` | Names of the volumes mounted in this container. When using the `--format` option, the `ps` command will either output the data exactly as the template declares or, when using the `table` directive, will include column headers as well. diff --git a/components/engine/docs/reference/commandline/rm.md b/components/engine/docs/reference/commandline/rm.md index 514b92c27e..bf615b55b8 100644 --- a/components/engine/docs/reference/commandline/rm.md +++ b/components/engine/docs/reference/commandline/rm.md @@ -46,15 +46,15 @@ This command will delete all stopped containers. The command the `rm` command which will delete them. Any running containers will not be deleted. - $ docker rm -v redis - redis + $ docker rm -v redis + redis This command will remove the container and any volumes associated with it. Note that if a volume was specified with a name, it will not be removed. - $ docker create -v awesome:/foo -v /bar --name hello redis - hello - $ docker rm -v hello + $ docker create -v awesome:/foo -v /bar --name hello redis + hello + $ docker rm -v hello In this example, the volume for `/foo` will remain intact, but the volume for `/bar` will be removed. The same behavior holds for volumes inherited with diff --git a/components/engine/docs/reference/commandline/run.md b/components/engine/docs/reference/commandline/run.md index e9b913dff2..496ff4865d 100644 --- a/components/engine/docs/reference/commandline/run.md +++ b/components/engine/docs/reference/commandline/run.md @@ -74,6 +74,7 @@ parent = "smn_cli" -P, --publish-all Publish all exposed ports to random ports -p, --publish=[] Publish a container's port(s) to the host --pid="" PID namespace to use + --pids-limit=-1 Tune container pids limit (set -1 for unlimited), kernel >= 4.3 --privileged Give extended privileges to this container --read-only Mount the container's root filesystem as read only --restart="no" Restart policy (no, on-failure[:max-retry], always, unless-stopped) @@ -110,7 +111,7 @@ For information on connecting a container to a network, see the ["*Docker networ ## Examples -### Assign name and allocate psuedo-TTY (--name, -it) +### Assign name and allocate pseudo-TTY (--name, -it) $ docker run --name test -it debian root@d6c0fe130dba:/# exit 13 @@ -326,17 +327,17 @@ Guide. ### Connect a container to a network (--net) When you start a container use the `--net` flag to connect it to a network. -This adds the `busybox` container to the `mynet` network. +This adds the `busybox` container to the `my-net` network. ```bash -$ docker run -itd --net=my-multihost-network busybox +$ docker run -itd --net=my-net busybox ``` You can also choose the IP addresses for the container with `--ip` and `--ip6` flags when you start the container on a user-defined network. ```bash -$ docker run -itd --net=my-multihost-network --ip=10.10.9.75 busybox +$ docker run -itd --net=my-net --ip=10.10.9.75 busybox ``` If you want to add a running container to a network use the `docker network connect` subcommand. diff --git a/components/engine/docs/reference/commandline/update.md b/components/engine/docs/reference/commandline/update.md index bcbfab6ab9..24fb1f290d 100644 --- a/components/engine/docs/reference/commandline/update.md +++ b/components/engine/docs/reference/commandline/update.md @@ -12,7 +12,7 @@ parent = "smn_cli" Usage: docker update [OPTIONS] CONTAINER [CONTAINER...] - Updates container resource limits + Update configuration of one or more containers --help=false Print usage --blkio-weight=0 Block IO (relative weight), between 10 and 1000 @@ -25,11 +25,12 @@ parent = "smn_cli" --memory-reservation="" Memory soft limit --memory-swap="" A positive integer equal to memory plus swap. Specify -1 to enable unlimited swap --kernel-memory="" Kernel memory limit: container must be stopped + --restart Restart policy to apply when a container exits -The `docker update` command dynamically updates container resources. Use this -command to prevent containers from consuming too many resources from their -Docker host. With a single command, you can place limits on a single -container or on many. To specify more than one container, provide +The `docker update` command dynamically updates container configuration. +You can use this command to prevent containers from consuming too many resources +from their Docker host. With a single command, you can place limits on +a single container or on many. To specify more than one container, provide space-separated list of container names or IDs. With the exception of the `--kernel-memory` value, you can specify these @@ -38,6 +39,10 @@ options on a running or a stopped container. You can only update stopped container, the next time you restart it, the container uses those values. +Another configuration you can change with this command is restart policy, +new restart policy will take effect instantly after you run `docker update` +on a container. + ## EXAMPLES The following sections illustrate ways to use this command. @@ -59,3 +64,10 @@ To update multiple resource configurations for multiple containers: ```bash $ docker update --cpu-shares 512 -m 300M abebf7571666 hopeful_morse ``` + +### Update a container's restart policy + +To update restart policy for one or more containers: +```bash +$ docker update --restart=on-failure:3 abebf7571666 hopeful_morse +``` diff --git a/components/engine/docs/reference/commandline/volume_create.md b/components/engine/docs/reference/commandline/volume_create.md index 744cfd1ffa..da2c66de80 100644 --- a/components/engine/docs/reference/commandline/volume_create.md +++ b/components/engine/docs/reference/commandline/volume_create.md @@ -21,10 +21,12 @@ parent = "smn_cli" Creates a new volume that containers can consume and store data in. If a name is not specified, Docker generates a random name. You create a volume and then configure the container to use it, for example: - $ docker volume create --name hello - hello +```bash +$ docker volume create --name hello +hello - $ docker run -d -v hello:/world busybox ls /world +$ docker run -d -v hello:/world busybox ls /world +``` The mount is created inside the container's `/world` directory. Docker does not support relative paths for mount points inside the container. @@ -42,9 +44,32 @@ If you specify a volume name already in use on the current driver, Docker assume Some volume drivers may take options to customize the volume creation. Use the `-o` or `--opt` flags to pass driver options: - $ docker volume create --driver fake --opt tardis=blue --opt timey=wimey +```bash +$ docker volume create --driver fake --opt tardis=blue --opt timey=wimey +``` These options are passed directly to the volume driver. Options for different volume drivers may do different things (or nothing at all). -*Note*: The built-in `local` volume driver does not currently accept any options. +The built-in `local` driver on Windows does not support any options. + +The built-in `local` driver on Linux accepts options similar to the linux `mount` +command: + +```bash +$ docker volume create --driver local --opt type=tmpfs --opt device=tmpfs --opt o=size=100m,uid=1000 +``` + +Another example: + +```bash +$ docker volume create --driver local --opt type=btrfs --opt device=/dev/sda2 +``` + + +## Related information + +* [volume inspect](volume_inspect.md) +* [volume ls](volume_ls.md) +* [volume rm](volume_rm.md) +* [Understand Data Volumes](../../userguide/containers/dockervolumes.md) diff --git a/components/engine/docs/reference/commandline/volume_inspect.md b/components/engine/docs/reference/commandline/volume_inspect.md index 519f80c23e..8fdd34d93b 100644 --- a/components/engine/docs/reference/commandline/volume_inspect.md +++ b/components/engine/docs/reference/commandline/volume_inspect.md @@ -12,7 +12,7 @@ parent = "smn_cli" Usage: docker volume inspect [OPTIONS] VOLUME [VOLUME...] - Inspect one or more volumes + Return low-level information on a volume -f, --format= Format the output using the given go template. --help Print usage @@ -38,3 +38,10 @@ Example output: $ docker volume inspect --format '{{ .Mountpoint }}' 85bffb0677236974f93955d8ecc4df55ef5070117b0e53333cc1b443777be24d /var/lib/docker/volumes/85bffb0677236974f93955d8ecc4df55ef5070117b0e53333cc1b443777be24d/_data + +## Related information + +* [volume create](volume_create.md) +* [volume ls](volume_ls.md) +* [volume rm](volume_rm.md) +* [Understand Data Volumes](../../userguide/containers/dockervolumes.md) \ No newline at end of file diff --git a/components/engine/docs/reference/commandline/volume_ls.md b/components/engine/docs/reference/commandline/volume_ls.md index 3361959439..0388e8ae2d 100644 --- a/components/engine/docs/reference/commandline/volume_ls.md +++ b/components/engine/docs/reference/commandline/volume_ls.md @@ -32,3 +32,10 @@ Example output: DRIVER VOLUME NAME local rose local tyler + +## Related information + +* [volume create](volume_create.md) +* [volume inspect](volume_inspect.md) +* [volume rm](volume_rm.md) +* [Understand Data Volumes](../../userguide/containers/dockervolumes.md) \ No newline at end of file diff --git a/components/engine/docs/reference/commandline/volume_rm.md b/components/engine/docs/reference/commandline/volume_rm.md index 495e746553..ff5ce24a4b 100644 --- a/components/engine/docs/reference/commandline/volume_rm.md +++ b/components/engine/docs/reference/commandline/volume_rm.md @@ -20,3 +20,10 @@ Removes one or more volumes. You cannot remove a volume that is in use by a cont $ docker volume rm hello hello + +## Related information + +* [volume create](volume_create.md) +* [volume inspect](volume_inspect.md) +* [volume ls](volume_ls.md) +* [Understand Data Volumes](../../userguide/containers/dockervolumes.md) \ No newline at end of file diff --git a/components/engine/docs/reference/glossary.md b/components/engine/docs/reference/glossary.md index ff4398c249..22c2d36d40 100644 --- a/components/engine/docs/reference/glossary.md +++ b/components/engine/docs/reference/glossary.md @@ -178,8 +178,8 @@ A repository is a set of Docker images. A repository can be shared by pushing it to a [registry](#registry) server. The different images in the repository can be labeled using [tags](#tag). -Here is an example of the shared [nginx repository](https://registry.hub.docker.com/_/nginx/) -and its [tags](https://registry.hub.docker.com/_/nginx/tags/manage/) +Here is an example of the shared [nginx repository](https://hub.docker.com/_/nginx/) +and its [tags](https://hub.docker.com/r/library/nginx/tags/) ## Swarm diff --git a/components/engine/docs/reference/run.md b/components/engine/docs/reference/run.md index e38fd1f18d..4be50a2d02 100644 --- a/components/engine/docs/reference/run.md +++ b/components/engine/docs/reference/run.md @@ -605,6 +605,8 @@ with the same logic -- if the original volume was specified with a name it will --security-opt="label:disable" : Turn off label confinement for the container --security-opt="apparmor:PROFILE" : Set the apparmor profile to be applied to the container + --security-opt="no-new-privileges" : Disable container processes from gaining + new privileges You can override the default labeling scheme for each container by specifying the `--security-opt` flag. For example, you can specify the MCS/MLS level, a @@ -631,6 +633,13 @@ command: > **Note**: You would have to write policy defining a `svirt_apache_t` type. +If you want to prevent your container processes from gaining additional +privileges, you can execute the following command: + + $ docker run --security-opt no-new-privileges -it centos bash + +For more details, see [kernel documentation](https://www.kernel.org/doc/Documentation/prctl/no_new_privs.txt). + ## Specifying custom cgroups Using the `--cgroup-parent` flag, you can pass a specific cgroup to run a @@ -1049,8 +1058,8 @@ By default, the docker container process runs with the supplementary groups look up for the specified user. If one wants to add more to that list of groups, then one can use this flag: - $ docker run -it --rm --group-add audio --group-add dbus --group-add 777 busybox id - uid=0(root) gid=0(root) groups=10(wheel),29(audio),81(dbus),777 + $ docker run --rm --group-add audio --group-add nogroup --group-add 777 busybox id + uid=0(root) gid=0(root) groups=10(wheel),29(audio),99(nogroup),777 ## Runtime privilege and Linux capabilities @@ -1059,6 +1068,14 @@ one can use this flag: --privileged=false: Give extended privileges to this container --device=[]: Allows you to run devices inside the container without the --privileged flag. +> **Note:** +> With Docker 1.10 and greater, the default seccomp profile will also block +> syscalls, regardless of `--cap-add` passed to the container. We recommend in +> these cases to create your own custom seccomp profile based off our +> [default](https://github.com/docker/docker/blob/master/profiles/seccomp/default.json). +> Or if you don't want to run with the default seccomp profile, you can pass +> `--security-opt=seccomp:unconfined` on run. + By default, Docker containers are "unprivileged" and cannot, for example, run a Docker daemon inside a Docker container. This is because by default a container is not allowed to access any devices, but a @@ -1429,7 +1446,10 @@ The developer can set a default user to run the first process with the Dockerfile `USER` instruction. When starting a container, the operator can override the `USER` instruction by passing the `-u` option. - -u="": Username or UID + -u="", --user="": Sets the username or UID used and optionally the groupname or GID for the specified command. + + The followings examples are all valid: + --user=[ user | user:group | uid | uid:gid | user:gid | uid:group ] > **Note:** if you pass a numeric uid, it must be in the range of 0-2147483647. diff --git a/components/engine/docs/security/https/README.md b/components/engine/docs/security/https/README.md index 1369913ed7..ff5538911c 100644 --- a/components/engine/docs/security/https/README.md +++ b/components/engine/docs/security/https/README.md @@ -7,26 +7,27 @@ draft = true This is an initial attempt to make it easier to test the examples in the https.md -doc +doc. -at this point, it has to be a manual thing, and I've been running it in boot2docker +At this point, it has to be a manual thing, and I've been running it in boot2docker. -so my process is +My process is as following: + + $ boot2docker ssh + $$ git clone https://github.com/docker/docker + $$ cd docker/docs/articles/https + $$ make cert -$ boot2docker ssh -$$ git clone https://github.com/docker/docker -$$ cd docker/docs/articles/https -$$ make cert lots of things to see and manually answer, as openssl wants to be interactive + **NOTE:** make sure you enter the hostname (`boot2docker` in my case) when prompted for `Computer Name`) -$$ sudo make run -start another terminal + $$ sudo make run -$ boot2docker ssh -$$ cd docker/docs/articles/https -$$ make client +Start another terminal: -the last will connect first with `--tls` and then with `--tlsverify` + $ boot2docker ssh + $$ cd docker/docs/articles/https + $$ make client -both should succeed +The last will connect first with `--tls` and then with `--tlsverify`, both should succeed. diff --git a/components/engine/docs/security/seccomp.md b/components/engine/docs/security/seccomp.md index 5bfed0f647..196d93810d 100644 --- a/components/engine/docs/security/seccomp.md +++ b/components/engine/docs/security/seccomp.md @@ -16,48 +16,48 @@ restrict the actions available within the container. The `seccomp()` system call operates on the seccomp state of the calling process. You can use this feature to restrict your application's access. -This feature is available only if the kernel is configured with `CONFIG_SECCOMP` -enabled. +This feature is available only if Docker has been built with seccomp and the +kernel is configured with `CONFIG_SECCOMP` enabled. To check if your kernel +supports seccomp: -> **Note**: On Ubuntu 14.04, Debian Wheezy, and Debian Jessie, you must download -> the [latest static Docker Linux binary](../installation/binaries.md) to use -> seccomp. +```bash +$ cat /boot/config-`uname -r` | grep CONFIG_SECCOMP= +CONFIG_SECCOMP=y +``` + +> **Note**: seccomp profiles require seccomp 2.2.1 and are only +> available starting with Debian 9 "Stretch", Ubuntu 15.10 "Wily", and +> Fedora 22. To use this feature on Ubuntu 14.04, Debian Wheezy, or +> Debian Jessie, you must download the [latest static Docker Linux binary](../installation/binaries.md). +> This feature is currently *not* available on other distributions. ## Passing a profile for a container The default seccomp profile provides a sane default for running containers with -seccomp. It is moderately protective while providing wide application -compatibility. The default Docker profile has layout in the following form: +seccomp and disables around 44 system calls out of 300+. It is moderately protective while providing wide application +compatibility. The default Docker profile (found [here](https://github.com/docker/docker/blob/master/profiles/seccomp/default.json) has a JSON layout in the following form: -``` +```json { - "defaultAction": "SCMP_ACT_ALLOW", - "syscalls": [ - { - "name": "getcwd", - "action": "SCMP_ACT_ERRNO" - }, - { - "name": "mount", - "action": "SCMP_ACT_ERRNO" - }, - { - "name": "setns", - "action": "SCMP_ACT_ERRNO" - }, - { - "name": "create_module", - "action": "SCMP_ACT_ERRNO" - }, - { - "name": "chown", - "action": "SCMP_ACT_ERRNO" - }, - { - "name": "chmod", - "action": "SCMP_ACT_ERRNO" - } - ] + "defaultAction": "SCMP_ACT_ERRNO", + "architectures": [ + "SCMP_ARCH_X86_64", + "SCMP_ARCH_X86", + "SCMP_ARCH_X32" + ], + "syscalls": [ + { + "name": "accept", + "action": "SCMP_ACT_ALLOW", + "args": [] + }, + { + "name": "accept4", + "action": "SCMP_ACT_ALLOW", + "args": [] + }, + ... + ] } ``` @@ -69,7 +69,7 @@ specifies the default policy: $ docker run --rm -it --security-opt seccomp:/path/to/seccomp/profile.json hello-world ``` -### Syscalls blocked by the default profile +### Significant syscalls blocked by the default profile Docker's default seccomp profile is a whitelist which specifies the calls that are allowed. The table below lists the significant (but not all) syscalls that diff --git a/components/engine/docs/security/security.md b/components/engine/docs/security/security.md index ec24d879d8..9eea663788 100644 --- a/components/engine/docs/security/security.md +++ b/components/engine/docs/security/security.md @@ -243,26 +243,16 @@ with e.g., special network topologies or shared filesystems, you can expect to see tools to harden existing Docker containers without affecting Docker's core. -Recent improvements in Linux namespaces will soon allow to run -full-featured containers without root privileges, thanks to the new user -namespace. This is covered in detail [here]( -http://s3hh.wordpress.com/2013/07/19/creating-and-using-containers-without-privilege/). -Moreover, this will solve the problem caused by sharing filesystems -between host and guest, since the user namespace allows users within -containers (including the root user) to be mapped to other users in the -host system. +As of Docker 1.10 User Namespaces are supported directly by the docker +daemon. This feature allows for the root user in a container to be mapped +to a non uid-0 user outside the container, which can help to mitigate the +risks of container breakout. This facility is available but not enabled +by default. -Today, Docker does not directly support user namespaces, but they -may still be utilized by Docker containers on supported kernels, -by directly using the clone syscall, or utilizing the 'unshare' -utility. Using this, some users may find it possible to drop -more capabilities from their process as user namespaces provide -an artificial capabilities set. Likewise, however, this artificial -capabilities set may require use of 'capsh' to restrict the -user-namespace capabilities set when using 'unshare'. - -Eventually, it is expected that Docker will have direct, native support -for user-namespaces, simplifying the process of hardening containers. +Refer to the [daemon command](../reference/commandline/daemon.md#daemon-user-namespace-options) +in the command line reference for more information on this feature. +Additional information on the implementation of User Namespaces in Docker +can be found in this blog post. ## Conclusions diff --git a/components/engine/docs/understanding-docker.md b/components/engine/docs/understanding-docker.md index 1278f3902c..1dba248e0b 100644 --- a/components/engine/docs/understanding-docker.md +++ b/components/engine/docs/understanding-docker.md @@ -11,7 +11,6 @@ weight = -82 # Understand the architecture -**What is Docker?** Docker is an open platform for developing, shipping, and running applications. Docker is designed to deliver your applications faster. With Docker you can @@ -32,11 +31,11 @@ your hardware. Surrounding the container is tooling and a platform which can help you in several ways: -* getting your applications (and supporting components) into Docker containers -* distributing and shipping those containers to your teams for further development +* Get your applications (and supporting components) into Docker containers +* Distribute and ship those containers to your teams for further development and testing -* deploying those applications to your production environment, - whether it is in a local data center or the Cloud. +* Deploy those applications to your production environment, + whether it is in a local data center or the Cloud ## What can I use Docker for? @@ -75,7 +74,7 @@ out of the resources you have. Docker has two major components: -* Docker: the open source containerization platform. +* Docker Engine: the open source containerization platform. * [Docker Hub](https://hub.docker.com): our Software-as-a-Service platform for sharing and managing Docker containers. @@ -103,11 +102,11 @@ interface to Docker. It accepts commands from the user and communicates back and forth with a Docker daemon. ### Inside Docker -To understand Docker's internals, you need to know about three components: +To understand Docker's internals, you need to know about three resources: -* Docker images. -* Docker registries. -* Docker containers. +* Docker images +* Docker registries +* Docker containers #### Docker images @@ -124,6 +123,8 @@ upload or download images. The public Docker registry is provided with the images for your use. These can be images you create yourself or you can use images that others have previously created. Docker registries are the **distribution** component of Docker. +For more information, go to [Docker Registry](https://docs.docker.com/registry/overview/) and +[Docker Trusted Registry](https://docs.docker.com/docker-trusted-registry/overview/). #### Docker containers Docker containers are similar to a directory. A Docker container holds everything that @@ -132,17 +133,6 @@ image. Docker containers can be run, started, stopped, moved, and deleted. Each container is an isolated and secure application platform. Docker containers are the **run** component of Docker. -## So how does Docker work? -So far, we've learned that: - -1. You can build Docker images that hold your applications. -2. You can create Docker containers from those Docker images to run your - applications. -3. You can share those Docker images via - [Docker Hub](https://hub.docker.com) or your own registry. - -Let's look at how these elements combine together to make Docker work. - ### How does a Docker image work? We've already seen that Docker images are read-only templates from which Docker containers are launched. Each image consists of a series of layers. Docker @@ -163,27 +153,27 @@ or `fedora`, a base Fedora image. You can also use images of your own as the basis for a new image, for example if you have a base Apache image you could use this as the base of all your web application images. -> **Note:** Docker usually gets these base images from -> [Docker Hub](https://hub.docker.com). +> **Note:** [Docker Hub](https://hub.docker.com) is a public registry and stores +images. Docker images are then built from these base images using a simple, descriptive set of steps we call *instructions*. Each instruction creates a new layer in our image. Instructions include actions like: -* Run a command. -* Add a file or directory. -* Create an environment variable. -* What process to run when launching a container from this image. +* Run a command +* Add a file or directory +* Create an environment variable +* What process to run when launching a container from this image -These instructions are stored in a file called a `Dockerfile`. Docker reads this -`Dockerfile` when you request a build of an image, executes the instructions, and -returns a final image. +These instructions are stored in a file called a `Dockerfile`. A `Dockerfile` is +a text based script that contains instructions and commands for building the image +from the base image. Docker reads this `Dockerfile` when you request a build of +an image, executes the instructions, and returns a final image. ### How does a Docker registry work? The Docker registry is the store for your Docker images. Once you build a Docker -image you can *push* it to a public registry such as the one provided by [Docker -Hub](https://hub.docker.com) or to your own registry running behind your -firewall. +image you can *push* it to a public registry such as [Docker Hub](https://hub.docker.com) +or to your own registry running behind your firewall. Using the Docker client, you can search for already published images and then pull them down to your Docker host to build containers from them. @@ -209,25 +199,24 @@ daemon to run a container. $ docker run -i -t ubuntu /bin/bash -Let's break down this command. The Docker client is launched using the `docker` -binary with the `run` option telling it to launch a new container. The bare -minimum the Docker client needs to tell the Docker daemon to run the container -is: +The Docker Engine client is launched using the `docker` binary with the `run` option +running a new container. The bare minimum the Docker client needs to tell the +Docker daemon to run the container is: -* What Docker image to build the container from, here `ubuntu`, a base Ubuntu -image; +* What Docker image to build the container from, for example, `ubuntu` * The command you want to run inside the container when it is launched, -here `/bin/bash`, to start the Bash shell inside the new container. +for example,`/bin/bash` So what happens under the hood when we run this command? -In order, Docker does the following: +In order, Docker Engine does the following: -- **Pulls the `ubuntu` image:** Docker checks for the presence of the `ubuntu` -image and, if it doesn't exist locally on the host, then Docker downloads it from -[Docker Hub](https://hub.docker.com). If the image already exists, then Docker +- **Pulls the `ubuntu` image:** Docker Engine checks for the presence of the `ubuntu` +image. If the image already exists, then Docker Engine uses it for the new container. +If it doesn't exist locally on the host, then Docker Engine pulls it from +[Docker Hub](https://hub.docker.com). If the image already exists, then Docker Engine uses it for the new container. -- **Creates a new container:** Once Docker has the image, it uses it to create a +- **Creates a new container:** Once Docker Engine has the image, it uses it to create a container. - **Allocates a filesystem and mounts a read-write _layer_:** The container is created in the file system and a read-write layer is added to the image. @@ -238,7 +227,7 @@ Docker container to talk to the local host. - **Captures and provides application output:** Connects and logs standard input, outputs and errors for you to see how your application is running. -You now have a running container! From here you can manage your container, interact with +You now have a running container! Now you can manage your container, interact with your application and then, when finished, stop and remove your container. ## The underlying technology @@ -253,40 +242,37 @@ creates a set of *namespaces* for that container. This provides a layer of isolation: each aspect of a container runs in its own namespace and does not have access outside it. -Some of the namespaces that Docker uses on Linux are: +Some of the namespaces that Docker Engine uses on Linux are: - - **The `pid` namespace:** Used for process isolation (PID: Process ID). - - **The `net` namespace:** Used for managing network interfaces (NET: + - **The `pid` namespace:** Process isolation (PID: Process ID). + - **The `net` namespace:** Managing network interfaces (NET: Networking). - - **The `ipc` namespace:** Used for managing access to IPC + - **The `ipc` namespace:** Managing access to IPC resources (IPC: InterProcess Communication). - - **The `mnt` namespace:** Used for managing mount-points (MNT: Mount). - - **The `uts` namespace:** Used for isolating kernel and version identifiers. (UTS: Unix + - **The `mnt` namespace:** Managing mount-points (MNT: Mount). + - **The `uts` namespace:** Isolating kernel and version identifiers. (UTS: Unix Timesharing System). ### Control groups -Docker on Linux also makes use of another technology called `cgroups` or control groups. +Docker Engine on Linux also makes use of another technology called `cgroups` or control groups. A key to running applications in isolation is to have them only use the resources you want. This ensures containers are good multi-tenant citizens on a -host. Control groups allow Docker to share available hardware resources to +host. Control groups allow Docker Engine to share available hardware resources to containers and, if required, set up limits and constraints. For example, limiting the memory available to a specific container. ### Union file systems Union file systems, or UnionFS, are file systems that operate by creating layers, -making them very lightweight and fast. Docker uses union file systems to provide -the building blocks for containers. Docker can make use of several union file system variants +making them very lightweight and fast. Docker Engine uses union file systems to provide +the building blocks for containers. Docker Engine can make use of several union file system variants including: AUFS, btrfs, vfs, and DeviceMapper. ### Container format -Docker combines these components into a wrapper we call a container format. The +Docker Engine combines these components into a wrapper we call a container format. The default container format is called `libcontainer`. In the future, Docker may support other container formats, for example, by integrating with BSD Jails or Solaris Zones. ## Next steps -### Installing Docker -Visit the [installation section](installation/index.md#installation). - -### The Docker user guide -[Learn Docker in depth](userguide/index.md). +Read about [Installing Docker Engine](installation/index.md#installation). +Learn about the [Docker Engine User Guide](userguide/index.md). diff --git a/components/engine/docs/userguide/containers/dockerimages.md b/components/engine/docs/userguide/containers/dockerimages.md index 74387a5166..7a8b96569e 100644 --- a/components/engine/docs/userguide/containers/dockerimages.md +++ b/components/engine/docs/userguide/containers/dockerimages.md @@ -19,14 +19,14 @@ used Docker images that already exist, for example the `ubuntu` image and the You also discovered that Docker stores downloaded images on the Docker host. If an image isn't already present on the host then it'll be downloaded from a -registry: by default the [Docker Hub Registry](https://registry.hub.docker.com). +registry: by default the [Docker Hub Registry](https://hub.docker.com). In this section you're going to explore Docker images a bit more including: * Managing and working with images locally on your Docker host. * Creating basic images. -* Uploading images to [Docker Hub Registry](https://registry.hub.docker.com). +* Uploading images to [Docker Hub Registry](https://hub.docker.com). ## Listing images on the host @@ -455,7 +455,7 @@ step-by-step. You can see that each step creates a new container, runs the instruction inside that container and then commits that change - just like the `docker commit` work flow you saw earlier. When all the instructions have executed you're left with the `97feabe5d2ed` image -(also helpfully tagged as `ouruser/sinatra:v2`) and all intermediate +(also helpfuly tagged as `ouruser/sinatra:v2`) and all intermediate containers will get removed to clean things up. > **Note:** @@ -521,7 +521,7 @@ You can also reference by digest in `create`, `run`, and `rmi` commands, as well Once you've built or created a new image you can push it to [Docker Hub](https://hub.docker.com) using the `docker push` command. This allows you to share it with others, either publicly, or push it into [a -private repository](https://registry.hub.docker.com/plans/). +private repository](https://hub.docker.com/account/billing-plans/). $ docker push ouruser/sinatra The push refers to a repository [ouruser/sinatra] (len: 1) diff --git a/components/engine/docs/userguide/containers/dockerrepos.md b/components/engine/docs/userguide/containers/dockerrepos.md index 257f87635b..9be9f53c98 100644 --- a/components/engine/docs/userguide/containers/dockerrepos.md +++ b/components/engine/docs/userguide/containers/dockerrepos.md @@ -33,15 +33,11 @@ Docker itself provides access to Docker Hub services via the `docker search`, ### Account creation and login Typically, you'll want to start by creating an account on Docker Hub (if you haven't already) and logging in. You can create your account directly on -[Docker Hub](https://hub.docker.com/account/signup/), or by running: +[Docker Hub](https://hub.docker.com/account/signup/). $ docker login -This will prompt you for a user name, which will become the public namespace for your -public repositories. -If your user name is available, Docker will prompt you to enter a password and your -e-mail address. It will then automatically log you in. You can now commit and -push your own images up to your repos on Docker Hub. +You can now commit and push your own images up to your repos on Docker Hub. > **Note:** > Your authentication credentials will be stored in the `~/.docker/config.json` @@ -124,7 +120,7 @@ information [here](https://docs.docker.com/docker-hub/). Sometimes you have images you don't want to make public and share with everyone. So Docker Hub allows you to have private repositories. You can -sign up for a plan [here](https://registry.hub.docker.com/plans/). +sign up for a plan [here](https://hub.docker.com/account/billing-plans/). ### Organizations and teams @@ -132,7 +128,7 @@ One of the useful aspects of private repositories is that you can share them only with members of your organization or team. Docker Hub lets you create organizations where you can collaborate with your colleagues and manage private repositories. You can learn how to create and manage an organization -[here](https://registry.hub.docker.com/account/organizations/). +[here](https://hub.docker.com/organizations/). ### Automated Builds @@ -144,8 +140,8 @@ triggering a build and update when you push a commit. #### To setup an Automated Build 1. Create a [Docker Hub account](https://hub.docker.com/) and login. -2. Link your GitHub or Bitbucket account through the ["Link Accounts"](https://registry.hub.docker.com/account/accounts/) menu. -3. [Configure an Automated Build](https://registry.hub.docker.com/builds/add/). +2. Link your GitHub or Bitbucket account on the ["Linked Accounts & Services"](https://hub.docker.com/account/authorized-services/) page. +3. Select "Create Automated Build" from the "Create" dropdown menu 4. Pick a GitHub or Bitbucket project that has a `Dockerfile` that you want to build. 5. Pick the branch you want to build (the default is the `master` branch). 6. Give the Automated Build a name. diff --git a/components/engine/docs/userguide/containers/dockervolumes.md b/components/engine/docs/userguide/containers/dockervolumes.md index 63d7e513d7..bdea82749b 100644 --- a/components/engine/docs/userguide/containers/dockervolumes.md +++ b/components/engine/docs/userguide/containers/dockervolumes.md @@ -67,7 +67,7 @@ The output will provide details on the container configurations including the volumes. The output should look something similar to the following: ... - Mounts": [ + "Mounts": [ { "Name": "fac362...80535", "Source": "/var/lib/docker/volumes/fac362...80535/_data", @@ -159,6 +159,48 @@ user with access to host and its mounted directory. >should be portable. A host directory wouldn't be available on all potential >hosts. +### Mount a shared-storage volume as a data volume + +In addition to mounting a host directory in your container, some Docker +[volume plugins](../../extend/plugins_volume.md) allow you to +provision and mount shared storage, such as iSCSI, NFS, or FC. + +A benefit of using shared volumes is that they are host-independent. This +means that a volume can be made available on any host that a container is +started on as long as it has access to the shared storage backend, and has +the plugin installed. + +One way to use volume drivers is through the `docker run` command. +Volume drivers create volumes by name, instead of by path like in +the other examples. + +The following command creates a named volume, called `my-named-volume`, +using the `flocker` volume driver, and makes it available within the container +at `/opt/webapp`: + +```bash +$ docker run -d -P \ + --volume-driver=flocker \ + -v my-named-volume:/opt/webapp \ + --name web training/webapp python app.py +``` + +You may also use the `docker volume create` command, to create a volume before +using it in a container. + +The following example also creates the `my-named-volume` volume, this time +using the `docker volume create` command. + +```bash +$ docker volume create -d flocker --name my-named-volume -o size=20GB +$ docker run -d -P \ + -v my-named-volume:/opt/webapp \ + --name web training/webapp python app.py +``` + +A list of available plugins, including volume plugins, is available +[here](../../extend/plugins.md). + ### Volume labels Labeling systems like SELinux require that proper labels are placed on volume diff --git a/components/engine/docs/userguide/eng-image/dockerfile_best-practices.md b/components/engine/docs/userguide/eng-image/dockerfile_best-practices.md index 1c51af7037..ea89af49d3 100644 --- a/components/engine/docs/userguide/eng-image/dockerfile_best-practices.md +++ b/components/engine/docs/userguide/eng-image/dockerfile_best-practices.md @@ -130,7 +130,7 @@ various instructions available for use in a `Dockerfile`. [Dockerfile reference for the FROM instruction](../../reference/builder.md#from) Whenever possible, use current Official Repositories as the basis for your -image. We recommend the [Debian image](https://registry.hub.docker.com/_/debian/) +image. We recommend the [Debian image](https://hub.docker.com/_/debian/) since it’s very tightly controlled and kept extremely minimal (currently under 100 mb), while still being a full distribution. @@ -365,7 +365,7 @@ The `ENTRYPOINT` instruction can also be used in combination with a helper script, allowing it to function in a similar way to the command above, even when starting the tool may require more than one step. -For example, the [Postgres Official Image](https://registry.hub.docker.com/_/postgres/) +For example, the [Postgres Official Image](https://hub.docker.com/_/postgres/) uses the following script as its `ENTRYPOINT`: ```bash @@ -481,10 +481,10 @@ allowing the `Dockerfile` author to make a choice. These Official Repositories have exemplary `Dockerfile`s: -* [Go](https://registry.hub.docker.com/_/golang/) -* [Perl](https://registry.hub.docker.com/_/perl/) -* [Hy](https://registry.hub.docker.com/_/hylang/) -* [Rails](https://registry.hub.docker.com/_/rails) +* [Go](https://hub.docker.com/_/golang/) +* [Perl](https://hub.docker.com/_/perl/) +* [Hy](https://hub.docker.com/_/hylang/) +* [Rails](https://hub.docker.com/_/rails) ## Additional resources: diff --git a/components/engine/docs/userguide/index.md b/components/engine/docs/userguide/index.md index 1bffafc4c8..2509997518 100644 --- a/components/engine/docs/userguide/index.md +++ b/components/engine/docs/userguide/index.md @@ -1,8 +1,8 @@ -# User guide +# Docker Engine user guide + +This guide helps users learn how to use Docker Engine. + +- [Introduction to Engine user guide](intro.md) + +## Learn by example + +- [Hello world in a container](containers/dockerizing.md) +- [Build your own images](containers/dockerimages.md) +- [Network containers](containers/networkingcontainers.md) +- [Run a simple application](containers/usingdocker.md) +- [Manage data in containers](containers/dockervolumes.md) +- [Store images on Docker Hub](containers/dockerrepos.md) + +## Work with images + +- [Best practices for writing Dockerfiles](eng-image/dockerfile_best-practices.md) +- [Create a base image](eng-image/baseimages.md) +- [Image management](eng-image/image_management.md) + +## Manage storage drivers + +- [Understand images, containers, and storage drivers](storagedriver/imagesandcontainers.md) +- [Select a storage driver](storagedriver/selectadriver.md) +- [AUFS storage in practice](storagedriver/aufs-driver.md) +- [Btrfs storage in practice](storagedriver/btrfs-driver.md) +- [Device Mapper storage in practice](storagedriver/device-mapper-driver.md) +- [OverlayFS storage in practice](storagedriver/overlayfs-driver.md) +- [ZFS storage in practice](storagedriver/zfs-driver.md) + +## Configure networks + +- [Understand Docker container networks](networking/dockernetworks.md) +- [Embedded DNS server in user-defined networks](networking/configure-dns.md) +- [Get started with multi-host networking](networking/get-started-overlay.md) +- [Work with network commands](networking/work-with-networks.md) + +### Work with the default network + +- [Understand container communication](networking/default_network/container-communication.md) +- [Legacy container links](networking/default_network/dockerlinks.md) +- [Binding container ports to the host](networking/default_network/binding.md) +- [Build your own bridge](networking/default_network/build-bridges.md) +- [Configure container DNS](networking/default_network/configure-dns.md) +- [Customize the docker0 bridge](networking/default_network/custom-docker0.md) +- [IPv6 with Docker](networking/default_network/ipv6.md) + +## Misc + +- [Apply custom metadata](labels-custom-metadata.md) diff --git a/components/engine/docs/userguide/networking/configure-dns.md b/components/engine/docs/userguide/networking/configure-dns.md index b87436fada..d248f4294f 100644 --- a/components/engine/docs/userguide/networking/configure-dns.md +++ b/components/engine/docs/userguide/networking/configure-dns.md @@ -16,7 +16,7 @@ user-defined networks works differently compared to the containers connected to `default bridge` network. > **Note**: In order to maintain backward compatibility, the DNS configuration -> in `default bridge` network is retained with no behaviorial change. +> in `default bridge` network is retained with no behavioral change. > Please refer to the [DNS in default bridge network](default_network/configure-dns.md) > for more information on DNS configuration in the `default bridge` network. @@ -75,7 +75,7 @@ Various container options that affect container domain name services. of the container identified by CONTAINER_NAME. When using --link the embedded DNS will guarantee that localized lookup result only on that container where the --link is used. This lets processes inside the new container - connect to container without without having to know its name or IP. + connect to container without having to know its name or IP.

diff --git a/components/engine/docs/userguide/networking/default_network/configure-dns.md b/components/engine/docs/userguide/networking/default_network/configure-dns.md index ab87c82a79..2703aca1d0 100644 --- a/components/engine/docs/userguide/networking/default_network/configure-dns.md +++ b/components/engine/docs/userguide/networking/default_network/configure-dns.md @@ -61,7 +61,7 @@ Four different options affect container domain name services. Using this option as you run a container gives the new container's /etc/hosts an extra entry named ALIAS that points to the IP address of the container - identified by CONTAINER_NAME_or_ID. This lets processes + identified by CONTAINER_NAME_or_ID. This lets processes inside the new container connect to the hostname ALIAS without having to know its IP. The --link= option is discussed in more detail below. Because Docker may assign a different IP diff --git a/components/engine/docs/userguide/networking/default_network/options.md b/components/engine/docs/userguide/networking/default_network/options.md deleted file mode 100644 index 612dffbc51..0000000000 --- a/components/engine/docs/userguide/networking/default_network/options.md +++ /dev/null @@ -1,141 +0,0 @@ - - - - -# Quick guide to the options -Here is a quick list of the networking-related Docker command-line options, in case it helps you find the section below that you are looking for. - -Some networking command-line options can only be supplied to the Docker server when it starts up, and cannot be changed once it is running: -- `-b BRIDGE` or `--bridge=BRIDGE` -- see - - [Building your own bridge](#bridge-building) - -- `--bip=CIDR` -- see - - [Customizing docker0](#docker0) - -- `--default-gateway=IP_ADDRESS` -- see - - [How Docker networks a container](#container-networking) - -- `--default-gateway-v6=IP_ADDRESS` -- see - - [IPv6](#ipv6) - -- `--fixed-cidr` -- see - - [Customizing docker0](#docker0) - -- `--fixed-cidr-v6` -- see - - [IPv6](#ipv6) - -- `-H SOCKET...` or `--host=SOCKET...` -- - - This might sound like it would affect container networking, - - but it actually faces in the other direction: - - it tells the Docker server over what channels - - it should be willing to receive commands - - like "run container" and "stop container." - -- `--icc=true|false` -- see - - [Communication between containers](#between-containers) - -- `--ip=IP_ADDRESS` -- see - - [Binding container ports](#binding-ports) - -- `--ipv6=true|false` -- see - - [IPv6](#ipv6) - -- `--ip-forward=true|false` -- see - - [Communication between containers and the wider world](#the-world) - -- `--iptables=true|false` -- see - - [Communication between containers](#between-containers) - -- `--mtu=BYTES` -- see - - [Customizing docker0](#docker0) - -- `--userland-proxy=true|false` -- see - - [Binding container ports](#binding-ports) - -There are three networking options that can be supplied either at startup or when `docker run` is invoked. When provided at startup, set the default value that `docker run` will later use if the options are not specified: -- `--dns=IP_ADDRESS...` -- see - - [Configuring DNS](#dns) - -- `--dns-search=DOMAIN...` -- see - - [Configuring DNS](#dns) - -- `--dns-opt=OPTION...` -- see - - [Configuring DNS](#dns) - -Finally, several networking options can only be provided when calling `docker run` because they specify something specific to one container: -- `-h HOSTNAME` or `--hostname=HOSTNAME` -- see - - [Configuring DNS](#dns) and - - [How Docker networks a container](#container-networking) - -- `--link=CONTAINER_NAME_or_ID:ALIAS` -- see - - [Configuring DNS](#dns) and - - [Communication between containers](#between-containers) - -- `--net=bridge|none|container:NAME_or_ID|host` -- see - - [How Docker networks a container](#container-networking) - -- `--mac-address=MACADDRESS...` -- see - - [How Docker networks a container](#container-networking) - -- `-p SPEC` or `--publish=SPEC` -- see - - [Binding container ports](#binding-ports) - -- `-P` or `--publish-all=true|false` -- see - - [Binding container ports](#binding-ports) - -To supply networking options to the Docker server at startup, use the `DOCKER_OPTS` variable in the Docker upstart configuration file. For Ubuntu, edit the variable in `/etc/default/docker` or `/etc/sysconfig/docker` for CentOS. - -The following example illustrates how to configure Docker on Ubuntu to recognize a newly built bridge. - -Edit the `/etc/default/docker` file: - -``` -$ echo 'DOCKER_OPTS="-b=bridge0"' >> /etc/default/docker -``` - -Then restart the Docker server. - -``` -$ sudo service docker start -``` - -For additional information on bridges, see [building your own bridge](#building-your-own-bridge) later on this page. diff --git a/components/engine/docs/userguide/networking/default_network/saveme.md b/components/engine/docs/userguide/networking/default_network/saveme.md deleted file mode 100644 index f0ef85e8c0..0000000000 --- a/components/engine/docs/userguide/networking/default_network/saveme.md +++ /dev/null @@ -1,28 +0,0 @@ - - - - - -## A Brief introduction to networking and docker -When Docker starts, it creates a virtual interface named `docker0` on the host machine. It randomly chooses an address and subnet from the private range defined by [RFC 1918](http://tools.ietf.org/html/rfc1918) that are not in use on the host machine, and assigns it to `docker0`. Docker made the choice `172.17.42.1/16` when I started it a few minutes ago, for example -- a 16-bit netmask providing 65,534 addresses for the host machine and its containers. The MAC address is generated using the IP address allocated to the container to avoid ARP collisions, using a range from `02:42:ac:11:00:00` to `02:42:ac:11:ff:ff`. - -> **Note:** This document discusses advanced networking configuration and options for Docker. In most cases you won't need this information. If you're looking to get started with a simpler explanation of Docker networking and an introduction to the concept of container linking see the [Docker User Guide](dockerlinks.md). - -But `docker0` is no ordinary interface. It is a virtual _Ethernet bridge_ that automatically forwards packets between any other network interfaces that are attached to it. This lets containers communicate both with the host machine and with each other. Every time Docker creates a container, it creates a pair of "peer" interfaces that are like opposite ends of a pipe -- a packet sent on one will be received on the other. It gives one of the peers to the container to become its `eth0` interface and keeps the other peer, with a unique name like `vethAQI2QT`, out in the namespace of the host machine. By binding every `veth*` interface to the `docker0` bridge, Docker creates a virtual subnet shared between the host machine and every Docker container. - -The remaining sections of this document explain all of the ways that you can use Docker options and -- in advanced cases -- raw Linux networking commands to tweak, supplement, or entirely replace Docker's default networking configuration. - -## Editing networking config files -Starting with Docker v.1.2.0, you can now edit `/etc/hosts`, `/etc/hostname` and `/etc/resolve.conf` in a running container. This is useful if you need to install bind or other services that might override one of those files. - -Note, however, that changes to these files will not be saved by `docker commit`, nor will they be saved during `docker run`. That means they won't be saved in the image, nor will they persist when a container is restarted; they will only "stick" in a running container. diff --git a/components/engine/docs/userguide/networking/default_network/tools.md b/components/engine/docs/userguide/networking/default_network/tools.md deleted file mode 100644 index 545c1e04c3..0000000000 --- a/components/engine/docs/userguide/networking/default_network/tools.md +++ /dev/null @@ -1,83 +0,0 @@ - - - - -# Tools and examples -Before diving into the following sections on custom network topologies, you might be interested in glancing at a few external tools or examples of the same kinds of configuration. Here are two: -- Jérôme Petazzoni has created a `pipework` shell script to help you - - connect together containers in arbitrarily complex scenarios: - - [https://github.com/jpetazzo/pipework](https://github.com/jpetazzo/pipework) - -- Brandon Rhodes has created a whole network topology of Docker - - containers for the next edition of Foundations of Python Network - - Programming that includes routing, NAT'd firewalls, and servers that - - offer HTTP, SMTP, POP, IMAP, Telnet, SSH, and FTP: - - [https://github.com/brandon-rhodes/fopnp/tree/m/playground](https://github.com/brandon-rhodes/fopnp/tree/m/playground) - -Both tools use networking commands very much like the ones you saw in the previous section, and will see in the following sections. - -# Building a point-to-point connection - - -By default, Docker attaches all containers to the virtual subnet implemented by `docker0`. You can create containers that are each connected to some different virtual subnet by creating your own bridge as shown in [Building your own bridge](#bridge-building), starting each container with `docker run --net=none`, and then attaching the containers to your bridge with the shell commands shown in [How Docker networks a container](#container-networking). - -But sometimes you want two particular containers to be able to communicate directly without the added complexity of both being bound to a host-wide Ethernet bridge. - -The solution is simple: when you create your pair of peer interfaces, simply throw _both_ of them into containers, and configure them as classic point-to-point links. The two containers will then be able to communicate directly (provided you manage to tell each container the other's IP address, of course). You might adjust the instructions of the previous section to go something like this: - -``` -# Start up two containers in two terminal windows - -$ docker run -i -t --rm --net=none base /bin/bash -root@1f1f4c1f931a:/# - -$ docker run -i -t --rm --net=none base /bin/bash -root@12e343489d2f:/# - -# Learn the container process IDs -# and create their namespace entries - -$ docker inspect -f '{{.State.Pid}}' 1f1f4c1f931a -2989 -$ docker inspect -f '{{.State.Pid}}' 12e343489d2f -3004 -$ sudo mkdir -p /var/run/netns -$ sudo ln -s /proc/2989/ns/net /var/run/netns/2989 -$ sudo ln -s /proc/3004/ns/net /var/run/netns/3004 - -# Create the "peer" interfaces and hand them out - -$ sudo ip link add A type veth peer name B - -$ sudo ip link set A netns 2989 -$ sudo ip netns exec 2989 ip addr add 10.1.1.1/32 dev A -$ sudo ip netns exec 2989 ip link set A up -$ sudo ip netns exec 2989 ip route add 10.1.1.2/32 dev A - -$ sudo ip link set B netns 3004 -$ sudo ip netns exec 3004 ip addr add 10.1.1.2/32 dev B -$ sudo ip netns exec 3004 ip link set B up -$ sudo ip netns exec 3004 ip route add 10.1.1.1/32 dev B -``` - -The two containers should now be able to ping each other and make connections successfully. Point-to-point links like this do not depend on a subnet nor a netmask, but on the bare assertion made by `ip route` that some other single IP address is connected to a particular network interface. - -Note that point-to-point links can be safely combined with other kinds of network connectivity -- there is no need to start the containers with `--net=none` if you want point-to-point links to be an addition to the container's normal networking instead of a replacement. - -A final permutation of this pattern is to create the point-to-point link between the Docker host and one container, which would allow the host to communicate with that one container on some single IP address and thus communicate "out-of-band" of the bridge that connects the other, more usual containers. But unless you have very specific networking needs that drive you to such a solution, it is probably far preferable to use `--icc=false` to lock down inter-container communication, as we explored earlier. diff --git a/components/engine/docs/userguide/networking/dockernetworks.md b/components/engine/docs/userguide/networking/dockernetworks.md index b9f1a63b44..1848e7a7a9 100644 --- a/components/engine/docs/userguide/networking/dockernetworks.md +++ b/components/engine/docs/userguide/networking/dockernetworks.md @@ -46,7 +46,7 @@ by default. You can see this bridge as part of a host's network stack by using the `ifconfig` command on the host. ``` -ubuntu@ip-172-31-36-118:~$ ifconfig +$ ifconfig docker0 Link encap:Ethernet HWaddr 02:42:47:bc:3a:eb inet addr:172.17.0.1 Bcast:0.0.0.0 Mask:255.255.0.0 inet6 addr: fe80::42:47ff:febc:3aeb/64 Scope:Link @@ -60,16 +60,16 @@ docker0 Link encap:Ethernet HWaddr 02:42:47:bc:3a:eb The `none` network adds a container to a container-specific network stack. That container lacks a network interface. Attaching to such a container and looking at it's stack you see this: ``` -ubuntu@ip-172-31-36-118:~$ docker attach nonenetcontainer +$ docker attach nonenetcontainer -/ # cat /etc/hosts +root@0cb243cd1293:/# cat /etc/hosts 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters -/ # ifconfig +root@0cb243cd1293:/# ifconfig lo Link encap:Local Loopback inet addr:127.0.0.1 Mask:255.0.0.0 inet6 addr: ::1/128 Scope:Host @@ -79,23 +79,24 @@ lo Link encap:Local Loopback collisions:0 txqueuelen:0 RX bytes:0 (0.0 B) TX bytes:0 (0.0 B) -/ # +root@0cb243cd1293:/# ``` >**Note**: You can detach from the container and leave it running with `CTRL-p CTRL-q`. The `host` network adds a container on the hosts network stack. You'll find the network configuration inside the container is identical to the host. -With the exception of the the `bridge` network, you really don't need to +With the exception of the `bridge` network, you really don't need to interact with these default networks. While you can list and inspect them, you cannot remove them. They are required by your Docker installation. However, you can add your own user-defined networks and these you can remove when you no longer need them. Before you learn more about creating your own networks, it is -worth looking at the `default` network a bit. +worth looking at the default `bridge` network a bit. ### The default bridge network in detail -The default bridge network is present on all Docker hosts. The `docker network inspect` +The default `bridge` network is present on all Docker hosts. The `docker network inspect` +command returns information about a network: ``` $ docker network inspect bridge @@ -189,7 +190,7 @@ You can `attach` to a running `container` and investigate its configuration: ``` $ docker attach container1 -/ # ifconfig +root@0cb243cd1293:/# ifconfig ifconfig eth0 Link encap:Ethernet HWaddr 02:42:AC:11:00:02 inet addr:172.17.0.2 Bcast:0.0.0.0 Mask:255.255.0.0 @@ -213,7 +214,7 @@ lo Link encap:Local Loopback Then use `ping` for about 3 seconds to test the connectivity of the containers on this `bridge` network. ``` -/ # ping -w3 172.17.0.3 +root@0cb243cd1293:/# ping -w3 172.17.0.3 PING 172.17.0.3 (172.17.0.3): 56 data bytes 64 bytes from 172.17.0.3: seq=0 ttl=64 time=0.096 ms 64 bytes from 172.17.0.3: seq=1 ttl=64 time=0.080 ms @@ -227,7 +228,7 @@ round-trip min/avg/max = 0.074/0.083/0.096 ms Finally, use the `cat` command to check the `container1` network configuration: ``` -/ # cat /etc/hosts +root@0cb243cd1293:/# cat /etc/hosts 172.17.0.2 3386a527aa08 127.0.0.1 localhost ::1 localhost ip6-localhost ip6-loopback @@ -241,7 +242,7 @@ To detach from a `container1` and leave it running use `CTRL-p CTRL-q`.Then, att ``` $ docker attach container2 -/ # ifconfig +root@0cb243cd1293:/# ifconfig eth0 Link encap:Ethernet HWaddr 02:42:AC:11:00:03 inet addr:172.17.0.3 Bcast:0.0.0.0 Mask:255.255.0.0 inet6 addr: fe80::42:acff:fe11:3/64 Scope:Link @@ -260,7 +261,7 @@ lo Link encap:Local Loopback collisions:0 txqueuelen:0 RX bytes:0 (0.0 B) TX bytes:0 (0.0 B) -/ # ping -w3 172.17.0.2 +root@0cb243cd1293:/# ping -w3 172.17.0.2 PING 172.17.0.2 (172.17.0.2): 56 data bytes 64 bytes from 172.17.0.2: seq=0 ttl=64 time=0.067 ms 64 bytes from 172.17.0.2: seq=1 ttl=64 time=0.075 ms @@ -279,7 +280,7 @@ ff02::1 ip6-allnodes ff02::2 ip6-allrouters ``` -The default `docker0` bridge network supports the use of port mapping and `docker run --link` to allow communications between containers in the `docker0` network. These techniques are cumbersome to set up and prone to error. While they are still available to you as techniques, it is better to avoid them and define your own bridge networks instead. +The default `docker0` bridge network supports the use of port mapping and `docker run --link` to allow communications between containers in the `docker0` network. These techniques are cumbersome to set up and prone to error. While they are still available to you as techniques, it is better to avoid them and define your own bridge networks instead. ## User-defined networks @@ -421,7 +422,7 @@ Once you have several machines provisioned, you can use Docker Swarm to quickly form them into a swarm which includes a discovery service as well. To create an overlay network, you configure options on the `daemon` on each -Docker Engine for use with `overlay` network. There are two options to set: +Docker Engine for use with `overlay` network. There are three options to set: @@ -483,7 +484,7 @@ built-in network drivers. For example: $ docker network create --driver weave mynet -You can inspect it, add containers too and from it, and so forth. Of course, +You can inspect it, add containers to and from it, and so forth. Of course, different plugins may make use of different technologies or frameworks. Custom networks can include features not present in Docker's default networks. For more information on writing plugins, see [Extending Docker](../../extend/index.md) and diff --git a/components/engine/docs/userguide/networking/get-started-overlay.md b/components/engine/docs/userguide/networking/get-started-overlay.md index 39d7da9169..89d5b2ca59 100644 --- a/components/engine/docs/userguide/networking/get-started-overlay.md +++ b/components/engine/docs/userguide/networking/get-started-overlay.md @@ -19,6 +19,7 @@ some pre-existing conditions before you can create one. These conditions are: * Access to a key-value store. Docker supports Consul, Etcd, and ZooKeeper (Distributed store) key-value stores. * A cluster of hosts with connectivity to the key-value store. * A properly configured Engine `daemon` on each host in the cluster. +* Hosts within the cluster must have unique hostnames because the key-value store uses the hostnames to identify cluster members. Though Docker Machine and Docker Swarm are not mandatory to experience Docker multi-host networking, this example uses them to illustrate how they are @@ -54,21 +55,20 @@ key-value stores. This example uses Consul. instance using the [consul image from Docker Hub](https://hub.docker.com/r/progrium/consul/). You'll do this in the next step. -3. Start a `progrium/consul` container running on the `mh-keystore` machine. +3. Set your local environment to the `mh-keystore` machine. - $ docker $(docker-machine config mh-keystore) run -d \ + $ eval "$(docker-machine env mh-keystore)" + +4. Start a `progrium/consul` container running on the `mh-keystore` machine. + + $ docker run -d \ -p "8500:8500" \ -h "consul" \ progrium/consul -server -bootstrap - A bash expansion `$(docker-machine config mh-keystore)` is used to pass the - connection configuration to the `docker run` command. The client starts a - `progrium/consul` image running in the `mh-keystore` machine. The server is - called `consul` and is listening on port `8500`. - -4. Set your local environment to the `mh-keystore` machine. - - $ eval "$(docker-machine env mh-keystore)" + The client starts a `progrium/consul` image running in the + `mh-keystore` machine. The server is called `consul` and is + listening on port `8500`. 5. Run the `docker ps` command to see the `consul` container. diff --git a/components/engine/docs/userguide/networking/work-with-networks.md b/components/engine/docs/userguide/networking/work-with-networks.md index b668bc1c77..d2ac4693af 100644 --- a/components/engine/docs/userguide/networking/work-with-networks.md +++ b/components/engine/docs/userguide/networking/work-with-networks.md @@ -62,7 +62,7 @@ $ docker network inspect simple-network Unlike `bridge` networks, `overlay` networks require some pre-existing conditions before you can create one. These conditions are: -* Access to a key-value store. Engine supports Consul Etcd, and ZooKeeper (Distributed store) key-value stores. +* Access to a key-value store. Engine supports Consul, Etcd, and ZooKeeper (Distributed store) key-value stores. * A cluster of hosts with connectivity to the key-value store. * A properly configured Engine `daemon` on each host in the swarm. @@ -111,7 +111,13 @@ The following are those options and the equivalent docker daemon flags used for | `com.docker.network.bridge.enable_icc` | `--icc` | Enable or Disable Inter Container Connectivity | | `com.docker.network.bridge.host_binding_ipv4` | `--ip` | Default IP when binding container ports | | `com.docker.network.mtu` | `--mtu` | Set the containers network MTU | -| `com.docker.network.enable_ipv6` | `--ipv6` | Enable IPv6 networking | + +The following arguments can be passed to `docker network create` for any network driver. + +| Argument | Equivalent | Description | +|--------------|------------|------------------------------------------| +| `--internal` | - | Restricts external access to the network | +| `--ipv6` | `--ipv6` | Enable IPv6 networking | For example, now let's use `-o` or `--opt` options to specify an IP address binding when publishing ports: @@ -312,6 +318,7 @@ lo Link encap:Local Loopback TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:0 (0.0 B) TX bytes:0 (0.0 B) +``` On the `isolated_nw` which was user defined, the Docker embedded DNS server enables name resolution for other containers in the network. Inside of `container2` it is possible to ping `container3` by name. @@ -376,7 +383,7 @@ You can connect both running and non-running containers to a network. However, ### Linking containers in user-defined networks -In the above example, container_2 was able to resolve container_3's name automatically +In the above example, `container2` was able to resolve `container3`'s name automatically in the user defined network `isolated_nw`, but the name resolution did not succeed automatically in the default `bridge` network. This is expected in order to maintain backward compatibility with [legacy link](default_network/dockerlinks.md). @@ -396,7 +403,7 @@ Comparing the above 4 functionalities with the non-default user-defined networks * ability to dynamically attach and detach to multiple networks * supports the `--link` option to provide name alias for the linked container -Continuing with the above example, create another container `container_4` in `isolated_nw` +Continuing with the above example, create another container `container4` in `isolated_nw` with `--link` to provide additional name resolution using alias for other containers in the same network. @@ -405,26 +412,26 @@ $ docker run --net=isolated_nw -itd --name=container4 --link container5:c5 busyb 01b5df970834b77a9eadbaff39051f237957bd35c4c56f11193e0594cfd5117c ``` -With the help of `--link` container4 will be able to reach container5 using the +With the help of `--link` `container4` will be able to reach `container5` using the aliased name `c5` as well. -Please note that while creating container4, we linked to a container named `container5` +Please note that while creating `container4`, we linked to a container named `container5` which is not created yet. That is one of the differences in behavior between the -`legacy link` in default `bridge` network and the new `link` functionality in user defined -networks. The `legacy link` is static in nature and it hard-binds the container with the -alias and it doesnt tolerate linked container restarts. While the new `link` functionality +*legacy link* in default `bridge` network and the new *link* functionality in user defined +networks. The *legacy link* is static in nature and it hard-binds the container with the +alias and it doesn't tolerate linked container restarts. While the new *link* functionality in user defined networks are dynamic in nature and supports linked container restarts including tolerating ip-address changes on the linked container. -Now let us launch another container named `container5` linking container4 to c4. +Now let us launch another container named `container5` linking `container4` to c4. ```bash $ docker run --net=isolated_nw -itd --name=container5 --link container4:c4 busybox 72eccf2208336f31e9e33ba327734125af00d1e1d2657878e2ee8154fbb23c7a ``` -As expected, container4 will be able to reach container5 by both its container name and -its alias c5 and container5 will be able to reach container4 by its container name and +As expected, `container4` will be able to reach `container5` by both its container name and +its alias c5 and `container5` will be able to reach `container4` by its container name and its alias c4. ```bash @@ -491,7 +498,7 @@ $ docker network create -d bridge --subnet 172.26.0.0/24 local_alias 76b7dc932e037589e6553f59f76008e5b76fa069638cd39776b890607f567aaa ``` -let us connect container4 and container5 to the new network `local_alias` +let us connect `container4` and `container5` to the new network `local_alias` ``` $ docker network connect --link container5:foo local_alias container4 @@ -525,7 +532,7 @@ round-trip min/avg/max = 0.070/0.081/0.097 ms ``` Note that the ping succeeds for both the aliases but on different networks. -Let us conclude this section by disconnecting container5 from the `isolated_nw` +Let us conclude this section by disconnecting `container5` from the `isolated_nw` and observe the results ``` @@ -550,9 +557,9 @@ round-trip min/avg/max = 0.070/0.081/0.097 ms ``` In conclusion, the new link functionality in user defined networks provides all the -benefits of legacy links while avoiding most of the well-known issues with `legacy links`. +benefits of legacy links while avoiding most of the well-known issues with *legacy links*. -One notable missing functionality compared to `legacy links` is the injection of +One notable missing functionality compared to *legacy links* is the injection of environment variables. Though very useful, environment variable injection is static in nature and must be injected when the container is started. One cannot inject environment variables into a running container without significant effort and hence @@ -561,10 +568,10 @@ disconnect containers to/from a network. ### Network-scoped alias -While `links` provide private name resolution that is localized within a container, +While *link*s provide private name resolution that is localized within a container, the network-scoped alias provides a way for a container to be discovered by an alternate name by any other container within the scope of a particular network. -Unlike the `link` alias, which is defined by the consumer of a service, the +Unlike the *link* alias, which is defined by the consumer of a service, the network-scoped alias is defined by the container that is offering the service to the network. diff --git a/components/engine/docs/userguide/storagedriver/aufs-driver.md b/components/engine/docs/userguide/storagedriver/aufs-driver.md index 76271cf0e1..4651e66e0c 100644 --- a/components/engine/docs/userguide/storagedriver/aufs-driver.md +++ b/components/engine/docs/userguide/storagedriver/aufs-driver.md @@ -148,7 +148,7 @@ layer IDs). Inside each file are the names of the directories that exist below it in the stack The command below shows the contents of a metadata file in -`/var/lib/docker/aufs/layers/` that lists the the three directories that are +`/var/lib/docker/aufs/layers/` that lists the three directories that are stacked below it in the union mount. Remember, these directory names do no map to image layer IDs with Docker 1.10 and higher. diff --git a/components/engine/docs/userguide/storagedriver/imagesandcontainers.md b/components/engine/docs/userguide/storagedriver/imagesandcontainers.md index e4293370f4..3f2594d3f6 100644 --- a/components/engine/docs/userguide/storagedriver/imagesandcontainers.md +++ b/components/engine/docs/userguide/storagedriver/imagesandcontainers.md @@ -110,7 +110,7 @@ single 8GB general purpose SSD EBS volume. The Docker data directory centos latest c8a648134623 4 weeks ago 196.6 MB ubuntu 15.04 c8be1ac8145a 7 weeks ago 131.3 MB - $ du -hs /var/lib/docker + $ sudo du -hs /var/lib/docker 2.0G /var/lib/docker $ time docker run --rm -v /var/lib/docker:/var/lib/docker docker/v1.10-migrator @@ -275,12 +275,12 @@ image that you just pulled, make a change to it, and build a new image based on command. 1. In an empty directory, create a simple `Dockerfile` that starts with the -2. ubuntu:15.04 image. + ubuntu:15.04 image. FROM ubuntu:15.04 2. Add a new file called "newfile" in the image's `/tmp` directory with the -3. text "Hello world" in it. + text "Hello world" in it. When you are done, the `Dockerfile` contains two lines: @@ -291,7 +291,7 @@ command. 3. Save and close the file. 4. From a terminal in the same folder as your `Dockerfile`, run the following -5. command: + command: $ docker build -t changed-ubuntu . Sending build context to Docker daemon 2.048 kB @@ -310,14 +310,14 @@ command. The output above shows a new image with image ID `94e6b7d2c720`. 5. Run the `docker images` command to verify the new `changed-ubuntu` image is -6. in the Docker host's local storage area. + in the Docker host's local storage area. REPOSITORY TAG IMAGE ID CREATED SIZE changed-ubuntu latest 03b964f68d06 33 seconds ago 131.4 MB ubuntu 15.04 013f3d01d247 6 weeks ago 131.3 MB 6. Run the `docker history` command to see which image layers were used to -7. create the new `changed-ubuntu` image. + create the new `changed-ubuntu` image. $ docker history changed-ubuntu IMAGE CREATED CREATED BY SIZE COMMENT diff --git a/components/engine/docs/userguide/storagedriver/index.md b/components/engine/docs/userguide/storagedriver/index.md index 76671c7196..60d1255d77 100644 --- a/components/engine/docs/userguide/storagedriver/index.md +++ b/components/engine/docs/userguide/storagedriver/index.md @@ -25,7 +25,7 @@ Docker relies on driver technology to manage the storage and interactions associ If you are new to Docker containers make sure you read ["Understand images, containers, and storage drivers"](imagesandcontainers.md) first. It explains key concepts and technologies that can help you when working with storage drivers. -### Acknowledgement +### Acknowledgment The Docker storage driver material was created in large part by our guest author Nigel Poulton with a bit of help from Docker's own Jérôme Petazzoni. In his diff --git a/components/engine/docs/userguide/storagedriver/overlayfs-driver.md b/components/engine/docs/userguide/storagedriver/overlayfs-driver.md index 9abc1dbe65..3f948f6bec 100644 --- a/components/engine/docs/userguide/storagedriver/overlayfs-driver.md +++ b/components/engine/docs/userguide/storagedriver/overlayfs-driver.md @@ -111,7 +111,7 @@ directories. drwxr-xr-x 4 root root 4096 Oct 28 11:06 upper drwx------ 3 root root 4096 Oct 28 11:06 work -These four filesystem objects are all artefacts of OverlayFS. The "lower-id" +These four filesystem objects are all artifacts of OverlayFS. The "lower-id" file contains the ID of the top layer of the image the container is based on. This is used by OverlayFS as the "lowerdir". diff --git a/components/engine/docs/userguide/storagedriver/zfs-driver.md b/components/engine/docs/userguide/storagedriver/zfs-driver.md index 3ecbb300d0..e55e7396f1 100644 --- a/components/engine/docs/userguide/storagedriver/zfs-driver.md +++ b/components/engine/docs/userguide/storagedriver/zfs-driver.md @@ -133,11 +133,11 @@ you should substitute your own values throughout the procedure. 1. If it is running, stop the Docker `daemon`. -1. Install `the software-properties-common` package. +1. Install the `software-properties-common` package. This is required for the `add-apt-repository` command. - $ sudo apt-get install software-properties-common + $ sudo apt-get install -y software-properties-common Reading package lists... Done Building dependency tree diff --git a/components/engine/errors/README.md b/components/engine/errors/README.md deleted file mode 100644 index 81fa04cccd..0000000000 --- a/components/engine/errors/README.md +++ /dev/null @@ -1,58 +0,0 @@ -Docker 'errors' package -======================= - -This package contains all of the error messages generated by the Docker -engine that might be exposed via the Docker engine's REST API. - -Each top-level engine package will have its own file in this directory -so that there's a clear grouping of errors, instead of just one big -file. The errors for each package are defined here instead of within -their respective package structure so that Docker CLI code that may need -to import these error definition files will not need to know or understand -the engine's package/directory structure. In other words, all they should -need to do is import `.../docker/errors` and they will automatically -pick up all Docker engine defined errors. This also gives the engine -developers the freedom to change the engine packaging structure (e.g. to -CRUD packages) without worrying about breaking existing clients. - -These errors are defined using the 'errcode' package. The `errcode` package -allows for each error to be typed and include all information necessary to -have further processing done on them if necessary. In particular, each error -includes: - -* Value - a unique string (in all caps) associated with this error. -Typically, this string is the same name as the variable name of the error -(w/o the `ErrorCode` text) but in all caps. - -* Message - the human readable sentence that will be displayed for this -error. It can contain '%s' substitutions that allows for the code generating -the error to specify values that will be inserted in the string prior to -being displayed to the end-user. The `WithArgs()` function can be used to -specify the insertion strings. Note, the evaluation of the strings will be -done at the time `WithArgs()` is called. - -* Description - additional human readable text to further explain the -circumstances of the error situation. - -* HTTPStatusCode - when the error is returned back to a CLI, this value -will be used to populate the HTTP status code. If not present the default -value will be `StatusInternalServerError`, 500. - -Not all errors generated within the engine's executable will be propagated -back to the engine's API layer. For example, it is expected that errors -generated by vendored code (under `docker/vendor`) and packaged code -(under `docker/pkg`) will be converted into errors defined by this package. - -When processing an errcode error, if you are looking for a particular -error then you can do something like: - -``` -import derr "github.com/docker/docker/errors" - -... - -err := someFunc() -if err.ErrorCode() == derr.ErrorCodeNoSuchContainer { - ... -} -``` diff --git a/components/engine/errors/builder.go b/components/engine/errors/builder.go deleted file mode 100644 index 38d0d3c391..0000000000 --- a/components/engine/errors/builder.go +++ /dev/null @@ -1,93 +0,0 @@ -package errors - -// This file contains all of the errors that can be generated from the -// docker/builder component. - -import ( - "net/http" - - "github.com/docker/distribution/registry/api/errcode" -) - -var ( - // ErrorCodeAtLeastOneArg is generated when the parser comes across a - // Dockerfile command that doesn't have any args. - ErrorCodeAtLeastOneArg = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "ATLEASTONEARG", - Message: "%s requires at least one argument", - Description: "The specified command requires at least one argument", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeExactlyOneArg is generated when the parser comes across a - // Dockerfile command that requires exactly one arg but got less/more. - ErrorCodeExactlyOneArg = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EXACTLYONEARG", - Message: "%s requires exactly one argument", - Description: "The specified command requires exactly one argument", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeAtLeastTwoArgs is generated when the parser comes across a - // Dockerfile command that requires at least two args but got less. - ErrorCodeAtLeastTwoArgs = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "ATLEASTTWOARGS", - Message: "%s requires at least two arguments", - Description: "The specified command requires at least two arguments", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeTooManyArgs is generated when the parser comes across a - // Dockerfile command that has more args than it should - ErrorCodeTooManyArgs = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "TOOMANYARGS", - Message: "Bad input to %s, too many args", - Description: "The specified command was passed too many arguments", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeChainOnBuild is generated when the parser comes across a - // Dockerfile command that is trying to chain ONBUILD commands. - ErrorCodeChainOnBuild = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CHAINONBUILD", - Message: "Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed", - Description: "ONBUILD Dockerfile commands aren't allow on ONBUILD commands", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeBadOnBuildCmd is generated when the parser comes across a - // an ONBUILD Dockerfile command with an invalid trigger/command. - ErrorCodeBadOnBuildCmd = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "BADONBUILDCMD", - Message: "%s isn't allowed as an ONBUILD trigger", - Description: "The specified ONBUILD command isn't allowed", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeMissingFrom is generated when the Dockerfile is missing - // a FROM command. - ErrorCodeMissingFrom = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "MISSINGFROM", - Message: "Please provide a source image with `from` prior to run", - Description: "The Dockerfile is missing a FROM command", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNotOnWindows is generated when the specified Dockerfile - // command is not supported on Windows. - ErrorCodeNotOnWindows = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOTONWINDOWS", - Message: "%s is not supported on Windows", - Description: "The specified Dockerfile command is not supported on Windows", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeVolumeEmpty is generated when the specified Volume string - // is empty. - ErrorCodeVolumeEmpty = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUMEEMPTY", - Message: "Volume specified can not be an empty string", - Description: "The specified volume can not be an empty string", - HTTPStatusCode: http.StatusInternalServerError, - }) -) diff --git a/components/engine/errors/daemon.go b/components/engine/errors/daemon.go deleted file mode 100644 index 1075449d2d..0000000000 --- a/components/engine/errors/daemon.go +++ /dev/null @@ -1,1013 +0,0 @@ -package errors - -// This file contains all of the errors that can be generated from the -// docker/daemon component. - -import ( - "net/http" - - "github.com/docker/distribution/registry/api/errcode" -) - -var ( - // ErrorCodeNoSuchContainer is generated when we look for a container by - // name or ID and we can't find it. - ErrorCodeNoSuchContainer = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOSUCHCONTAINER", - Message: "No such container: %s", - Description: "The specified container can not be found", - HTTPStatusCode: http.StatusNotFound, - }) - - // ErrorCodeUnregisteredContainer is generated when we try to load - // a storage driver for an unregistered container - ErrorCodeUnregisteredContainer = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "UNREGISTEREDCONTAINER", - Message: "Can't load storage driver for unregistered container %s", - Description: "An attempt was made to load the storage driver for a container that is not registered with the daemon", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeContainerBeingRemoved is generated when an attempt to start - // a container is made but its in the process of being removed, or is dead. - ErrorCodeContainerBeingRemoved = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CONTAINERBEINGREMOVED", - Message: "Container is marked for removal and cannot be started.", - Description: "An attempt was made to start a container that is in the process of being deleted", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeUnpauseContainer is generated when we attempt to stop a - // container but its paused. - ErrorCodeUnpauseContainer = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "UNPAUSECONTAINER", - Message: "Container %s is paused. Unpause the container before stopping", - Description: "The specified container is paused, before it can be stopped it must be unpaused", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeRemovalContainer is generated when we attempt to connect or disconnect a - // container but it's marked for removal. - ErrorCodeRemovalContainer = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "REMOVALCONTAINER", - Message: "Container %s is marked for removal and cannot be connected or disconnected to the network", - Description: "The specified container is marked for removal and cannot be connected or disconnected to the network", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodePausedContainer is generated when we attempt to attach a - // container but its paused. - ErrorCodePausedContainer = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CONTAINERPAUSED", - Message: "Container %s is paused. Unpause the container before attach", - Description: "The specified container is paused, unpause the container before attach", - HTTPStatusCode: http.StatusConflict, - }) - // ErrorCodeAlreadyPaused is generated when we attempt to pause a - // container when its already paused. - ErrorCodeAlreadyPaused = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "ALREADYPAUSED", - Message: "Container %s is already paused", - Description: "The specified container is already in the paused state", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNotPaused is generated when we attempt to unpause a - // container when its not paused. - ErrorCodeNotPaused = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOTPAUSED", - Message: "Container %s is not paused", - Description: "The specified container can not be unpaused because it is not in a paused state", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeImageUnregContainer is generated when we attempt to get the - // image of an unknown/unregistered container. - ErrorCodeImageUnregContainer = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "IMAGEUNREGCONTAINER", - Message: "Can't get image of unregistered container", - Description: "An attempt to retrieve the image of a container was made but the container is not registered", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeEmptyID is generated when an ID is the empty string. - ErrorCodeEmptyID = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EMPTYID", - Message: "Invalid empty id", - Description: "An attempt was made to register a container but the container's ID can not be an empty string", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeLoggingFactory is generated when we could not load the - // log driver. - ErrorCodeLoggingFactory = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "LOGGINGFACTORY", - Message: "Failed to get logging factory: %v", - Description: "An attempt was made to register a container but the container's ID can not be an empty string", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeInitLogger is generated when we could not initialize - // the logging driver. - ErrorCodeInitLogger = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "INITLOGGER", - Message: "Failed to initialize logging driver: %v", - Description: "An error occurred while trying to initialize the logging driver", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNotRunning is generated when we need to verify that - // a container is running, but its not. - ErrorCodeNotRunning = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOTRUNNING", - Message: "Container %s is not running", - Description: "The specified action can not be taken due to the container not being in a running state", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeLinkNotRunning is generated when we try to link to a - // container that is not running. - ErrorCodeLinkNotRunning = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "LINKNOTRUNNING", - Message: "Cannot link to a non running container: %s AS %s", - Description: "An attempt was made to link to a container but the container is not in a running state", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeDeviceInfo is generated when there is an error while trying - // to get info about a custom device. - // container that is not running. - ErrorCodeDeviceInfo = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "DEVICEINFO", - Message: "error gathering device information while adding custom device %q: %s", - Description: "There was an error while trying to retrieve the information about a custom device", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeEmptyEndpoint is generated when the endpoint for a port - // map is nil. - ErrorCodeEmptyEndpoint = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EMPTYENDPOINT", - Message: "invalid endpoint while building port map info", - Description: "The specified endpoint for the port mapping is empty", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeEmptyNetwork is generated when the networkSettings for a port - // map is nil. - ErrorCodeEmptyNetwork = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EMPTYNETWORK", - Message: "invalid networksettings while building port map info", - Description: "The specified endpoint for the port mapping is empty", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeParsingPort is generated when there is an error parsing - // a "port" string. - ErrorCodeParsingPort = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "PARSINGPORT", - Message: "Error parsing Port value(%v):%v", - Description: "There was an error while trying to parse the specified 'port' value", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNoSandbox is generated when we can't find the specified - // sandbox(network) by ID. - ErrorCodeNoSandbox = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOSANDBOX", - Message: "error locating sandbox id %s: %v", - Description: "There was an error trying to located the specified networking sandbox", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNetworkUpdate is generated when there is an error while - // trying update a network/sandbox config. - ErrorCodeNetworkUpdate = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NETWORKUPDATE", - Message: "Update network failed: %v", - Description: "There was an error trying to update the configuration information of the specified network sandbox", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNetworkRefresh is generated when there is an error while - // trying refresh a network/sandbox config. - ErrorCodeNetworkRefresh = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NETWORKREFRESH", - Message: "Update network failed: Failure in refresh sandbox %s: %v", - Description: "There was an error trying to refresh the configuration information of the specified network sandbox", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeHostPort is generated when there was an error while trying - // to parse a "host/port" string. - ErrorCodeHostPort = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "HOSTPORT", - Message: "Error parsing HostPort value(%s):%v", - Description: "There was an error trying to parse the specified 'HostPort' value", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNetworkConflict is generated when we try to publish a service - // in network mode. - ErrorCodeNetworkConflict = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NETWORKCONFLICT", - Message: "conflicting options: publishing a service and network mode", - Description: "It is not possible to publish a service when it is in network mode", - HTTPStatusCode: http.StatusConflict, - }) - - // ErrorCodeJoinInfo is generated when we failed to update a container's - // join info. - ErrorCodeJoinInfo = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "JOININFO", - Message: "Updating join info failed: %v", - Description: "There was an error during an attempt update a container's join information", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeIPCRunning is generated when we try to join a container's - // IPC but it's not running. - ErrorCodeIPCRunning = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "IPCRUNNING", - Message: "cannot join IPC of a non running container: %s", - Description: "An attempt was made to join the IPC of a container, but the container is not running", - HTTPStatusCode: http.StatusConflict, - }) - - // ErrorCodeNotADir is generated when we try to create a directory - // but the path isn't a dir. - ErrorCodeNotADir = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOTADIR", - Message: "Cannot mkdir: %s is not a directory", - Description: "An attempt was made create a directory, but the location in which it is being created is not a directory", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeParseContainer is generated when the reference to a - // container doesn't include a ":" (another container). - ErrorCodeParseContainer = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "PARSECONTAINER", - Message: "no container specified to join network", - Description: "The specified reference to a container is missing a ':' as a separator between 'container' and 'name'/'id'", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeJoinSelf is generated when we try to network to ourselves. - ErrorCodeJoinSelf = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "JOINSELF", - Message: "cannot join own network", - Description: "An attempt was made to have a container join its own network", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeJoinRunning is generated when we try to network to ourselves. - ErrorCodeJoinRunning = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "JOINRUNNING", - Message: "cannot join network of a non running container: %s", - Description: "An attempt to join the network of a container, but that container isn't running", - HTTPStatusCode: http.StatusConflict, - }) - - // ErrorCodeModeNotContainer is generated when we try to network to - // another container but the mode isn't 'container'. - ErrorCodeModeNotContainer = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "MODENOTCONTAINER", - Message: "network mode not set to container", - Description: "An attempt was made to connect to a container's network but the mode wasn't set to 'container'", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeRemovingVolume is generated when we try remove a mount - // point (volume) but fail. - ErrorCodeRemovingVolume = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "REMOVINGVOLUME", - Message: "Error removing volumes:\n%v", - Description: "There was an error while trying to remove the mount point (volume) of a container", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeInvalidNetworkMode is generated when an invalid network - // mode value is specified. - ErrorCodeInvalidNetworkMode = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "INVALIDNETWORKMODE", - Message: "invalid network mode: %s", - Description: "The specified networking mode is not valid", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeGetGraph is generated when there was an error while - // trying to find a graph/image. - ErrorCodeGetGraph = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "GETGRAPH", - Message: "Failed to graph.Get on ImageID %s - %s", - Description: "There was an error trying to retrieve the image for the specified image ID", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeGetLayer is generated when there was an error while - // trying to retrieve a particular layer of an image. - ErrorCodeGetLayer = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "GETLAYER", - Message: "Failed to get layer path from graphdriver %s for ImageID %s - %s", - Description: "There was an error trying to retrieve the layer of the specified image", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodePutLayer is generated when there was an error while - // trying to 'put' a particular layer of an image. - ErrorCodePutLayer = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "PUTLAYER", - Message: "Failed to put layer path from graphdriver %s for ImageID %s - %s", - Description: "There was an error trying to store a layer for the specified image", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeGetLayerMetadata is generated when there was an error while - // trying to retrieve the metadata of a layer of an image. - ErrorCodeGetLayerMetadata = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "GETLAYERMETADATA", - Message: "Failed to get layer metadata - %s", - Description: "There was an error trying to retrieve the metadata of a layer for the specified image", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeEmptyConfig is generated when the input config data - // is empty. - ErrorCodeEmptyConfig = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EMPTYCONFIG", - Message: "Config cannot be empty in order to create a container", - Description: "While trying to create a container, the specified configuration information was empty", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNoSuchImageHash is generated when we can't find the - // specified image by its hash - ErrorCodeNoSuchImageHash = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOSUCHIMAGEHASH", - Message: "No such image: %s", - Description: "An attempt was made to find an image by its hash, but the lookup failed", - HTTPStatusCode: http.StatusNotFound, - }) - - // ErrorCodeNoSuchImageTag is generated when we can't find the - // specified image byt its name/tag. - ErrorCodeNoSuchImageTag = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOSUCHIMAGETAG", - Message: "No such image: %s:%s", - Description: "An attempt was made to find an image by its name/tag, but the lookup failed", - HTTPStatusCode: http.StatusNotFound, - }) - - // ErrorCodeMountOverFile is generated when we try to mount a volume - // over an existing file (but not a dir). - ErrorCodeMountOverFile = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "MOUNTOVERFILE", - Message: "cannot mount volume over existing file, file exists %s", - Description: "An attempt was made to mount a volume at the same location as a pre-existing file", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeMountSetup is generated when we can't define a mount point - // due to the source and destination being undefined. - ErrorCodeMountSetup = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "MOUNTSETUP", - Message: "Unable to setup mount point, neither source nor volume defined", - Description: "An attempt was made to setup a mount point, but the source and destination are undefined", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeVolumeInvalidMode is generated when the mode of a volume/bind - // mount is invalid. - ErrorCodeVolumeInvalidMode = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUMEINVALIDMODE", - Message: "invalid mode: %q", - Description: "An invalid 'mode' was specified", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeVolumeInvalid is generated when the format fo the - // volume specification isn't valid. - ErrorCodeVolumeInvalid = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUMEINVALID", - Message: "Invalid volume specification: '%s'", - Description: "An invalid 'volume' was specified in the mount request", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeVolumeAbs is generated when path to a volume isn't absolute. - ErrorCodeVolumeAbs = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUMEABS", - Message: "Invalid volume destination path: '%s' mount path must be absolute.", - Description: "An invalid 'destination' path was specified in the mount request, it must be an absolute path", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeVolumeName is generated when the name of named volume isn't valid. - ErrorCodeVolumeName = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUME_NAME_INVALID", - Message: "%q includes invalid characters for a local volume name, only %q are allowed", - Description: "The name of volume is invalid", - HTTPStatusCode: http.StatusBadRequest, - }) - - // ErrorCodeVolumeSlash is generated when destination path to a volume is / - ErrorCodeVolumeSlash = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUMESLASH", - Message: "Invalid specification: destination can't be '/' in '%s'", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeVolumeDestIsC is generated the destination is c: (Windows specific) - ErrorCodeVolumeDestIsC = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUMEDESTISC", - Message: "Destination drive letter in '%s' cannot be c:", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeVolumeDestIsCRoot is generated the destination path is c:\ (Windows specific) - ErrorCodeVolumeDestIsCRoot = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUMEDESTISCROOT", - Message: `Destination path in '%s' cannot be c:\`, - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeVolumeSourceNotFound is generated the source directory could not be found (Windows specific) - ErrorCodeVolumeSourceNotFound = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUMESOURCENOTFOUND", - Message: "Source directory '%s' could not be found: %s", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeVolumeSourceNotDirectory is generated the source is not a directory (Windows specific) - ErrorCodeVolumeSourceNotDirectory = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUMESOURCENOTDIRECTORY", - Message: "Source '%s' is not a directory", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeVolumeFromBlank is generated when path to a volume is blank. - ErrorCodeVolumeFromBlank = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUMEFROMBLANK", - Message: "malformed volumes-from specification: %q", - Description: "An invalid 'destination' path was specified in the mount request, it must not be blank", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeMountDup is generated when we try to mount two mounts points - // to the same path. - ErrorCodeMountDup = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "MOUNTDUP", - Message: "Duplicate mount point '%s'", - Description: "An attempt was made to mount a content but the specified destination location is already used in a previous mount", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeVolumeNoSourceForMount is generated when no source directory - // for a volume mount was found. (Windows specific) - ErrorCodeVolumeNoSourceForMount = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUMENOSOURCEFORMOUNT", - Message: "No source for mount name '%s' driver %q destination '%s'", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeVolumeNameReservedWord is generated when the name in a volume - // uses a reserved word for filenames. (Windows specific) - ErrorCodeVolumeNameReservedWord = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUMENAMERESERVEDWORD", - Message: "Volume name %q cannot be a reserved word for Windows filenames", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeCantPause is generated when there's an error while trying - // to pause a container. - ErrorCodeCantPause = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CANTPAUSE", - Message: "Cannot pause container %s: %s", - Description: "An error occurred while trying to pause the specified container", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeCantUnpause is generated when there's an error while trying - // to unpause a container. - ErrorCodeCantUnpause = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CANTUNPAUSE", - Message: "Cannot unpause container %s: %s", - Description: "An error occurred while trying to unpause the specified container", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeCantKill is generated when there's an error while trying - // to kill a container. - ErrorCodeCantKill = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CANTKILL", - Message: "Cannot kill container %s: %s", - Description: "An error occurred while trying to kill the specified container", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeCantUpdate is generated when there's an error while trying - // to update a container. - ErrorCodeCantUpdate = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CANTUPDATE", - Message: "Cannot update container %s: %s", - Description: "An error occurred while trying to update the specified container", - HTTPStatusCode: http.StatusInternalServerError, - }) - // ErrorCodePSError is generated when trying to run 'ps'. - ErrorCodePSError = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "PSError", - Message: "Error running ps: %s", - Description: "There was an error trying to run the 'ps' command in the specified container", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNoPID is generated when looking for the PID field in the - // ps output. - ErrorCodeNoPID = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOPID", - Message: "Couldn't find PID field in ps output", - Description: "There was no 'PID' field in the output of the 'ps' command that was executed", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeBadPID is generated when we can't convert a PID to an int. - ErrorCodeBadPID = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "BADPID", - Message: "Unexpected pid '%s': %s", - Description: "While trying to parse the output of the 'ps' command, the 'PID' field was not an integer", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNoTop is generated when we try to run 'top' but can't - // because we're on windows. - ErrorCodeNoTop = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOTOP", - Message: "Top is not supported on Windows", - Description: "The 'top' command is not supported on Windows", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeStopped is generated when we try to stop a container - // that is already stopped. - ErrorCodeStopped = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "STOPPED", - Message: "Container %s is already stopped", - Description: "An attempt was made to stop a container, but the container is already stopped", - HTTPStatusCode: http.StatusNotModified, - }) - - // ErrorCodeCantStop is generated when we try to stop a container - // but failed for some reason. - ErrorCodeCantStop = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CANTSTOP", - Message: "Cannot stop container %s: %s\n", - Description: "An error occurred while tring to stop the specified container", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeBadCPUFields is generated when the number of CPU fields is - // less than 8. - ErrorCodeBadCPUFields = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "BADCPUFIELDS", - Message: "invalid number of cpu fields", - Description: "While reading the '/proc/stat' file, the number of 'cpu' fields is less than 8", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeBadCPUInt is generated the CPU field can't be parsed as an int. - ErrorCodeBadCPUInt = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "BADCPUINT", - Message: "Unable to convert value %s to int: %s", - Description: "While reading the '/proc/stat' file, the 'CPU' field could not be parsed as an integer", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeBadStatFormat is generated the output of the stat info - // isn't parseable. - ErrorCodeBadStatFormat = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "BADSTATFORMAT", - Message: "invalid stat format", - Description: "There was an error trying to parse the '/proc/stat' file", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeTimedOut is generated when a timer expires. - ErrorCodeTimedOut = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "TIMEDOUT", - Message: "Timed out: %v", - Description: "A timer expired", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeAlreadyRemoving is generated when we try to remove a - // container that is already being removed. - ErrorCodeAlreadyRemoving = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "ALREADYREMOVING", - Message: "Status is already RemovalInProgress", - Description: "An attempt to remove a container was made, but the container is already in the process of being removed", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeStartPaused is generated when we start a paused container. - ErrorCodeStartPaused = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "STARTPAUSED", - Message: "Cannot start a paused container, try unpause instead.", - Description: "An attempt to start a container was made, but the container is paused. Unpause it first", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeAlreadyStarted is generated when we try to start a container - // that is already running. - ErrorCodeAlreadyStarted = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "ALREADYSTARTED", - Message: "Container already started", - Description: "An attempt to start a container was made, but the container is already started", - HTTPStatusCode: http.StatusNotModified, - }) - - // ErrorCodeHostConfigStart is generated when a HostConfig is passed - // into the start command. - ErrorCodeHostConfigStart = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "HOSTCONFIGSTART", - Message: "Supplying a hostconfig on start is not supported. It should be supplied on create", - Description: "The 'start' command does not accept 'HostConfig' data, try using the 'create' command instead", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeCantRestart is generated when an error occurred while - // trying to restart a container. - ErrorCodeCantRestart = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CANTRESTART", - Message: "Cannot restart container %s: %s", - Description: "There was an error while trying to restart a container", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeEmptyRename is generated when one of the names on a - // rename is empty. - ErrorCodeEmptyRename = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EMPTYRENAME", - Message: "Neither old nor new names may be empty", - Description: "An attempt was made to rename a container but either the old or new names were blank", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeRenameTaken is generated when we try to rename but the - // new name isn't available. - ErrorCodeRenameTaken = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "RENAMETAKEN", - Message: "Error when allocating new name: %s", - Description: "The new name specified on the 'rename' command is already being used", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeRenameDelete is generated when we try to rename but - // failed trying to delete the old container. - ErrorCodeRenameDelete = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "RENAMEDELETE", - Message: "Failed to delete container %q: %v", - Description: "There was an error trying to delete the specified container", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodePauseError is generated when we try to pause a container - // but failed. - ErrorCodePauseError = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "PAUSEERROR", - Message: "Cannot pause container %s: %s", - Description: "There was an error trying to pause the specified container", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNeedStream is generated when we try to stream a container's - // logs but no output stream was specified. - ErrorCodeNeedStream = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NEEDSTREAM", - Message: "You must choose at least one stream", - Description: "While trying to stream a container's logs, no output stream was specified", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeDanglingOne is generated when we try to specify more than one - // 'dangling' specifier. - ErrorCodeDanglingOne = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "DANLGINGONE", - Message: "Conflict: cannot use more than 1 value for `dangling` filter", - Description: "The specified 'dangling' filter may not have more than one value", - HTTPStatusCode: http.StatusConflict, - }) - - // ErrorCodeImgDelUsed is generated when we try to delete an image - // but it is being used. - ErrorCodeImgDelUsed = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "IMGDELUSED", - Message: "conflict: unable to remove repository reference %q (must force) - container %s is using its referenced image %s", - Description: "An attempt was made to delete an image but it is currently being used", - HTTPStatusCode: http.StatusConflict, - }) - - // ErrorCodeImgNoParent is generated when we try to find an image's - // parent but its not in the graph. - ErrorCodeImgNoParent = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "IMGNOPARENT", - Message: "unable to get parent image: %v", - Description: "There was an error trying to find an image's parent, it was not in the graph", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeExportFailed is generated when an export fails. - ErrorCodeExportFailed = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EXPORTFAILED", - Message: "%s: %s", - Description: "There was an error during an export operation", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeExecResize is generated when we try to resize an exec - // but its not running. - ErrorCodeExecResize = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EXECRESIZE", - Message: "Exec %s is not running, so it can not be resized.", - Description: "An attempt was made to resize an 'exec', but the 'exec' is not running", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeContainerNotRunning is generated when we try to get the info - // on an exec but the container is not running. - ErrorCodeContainerNotRunning = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CONTAINERNOTRUNNING", - Message: "Container %s is not running: %s", - Description: "An attempt was made to retrieve the information about an 'exec' but the container is not running", - HTTPStatusCode: http.StatusConflict, - }) - - // ErrorCodeContainerRestarting is generated when an operation was made - // on a restarting container. - ErrorCodeContainerRestarting = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CONTAINERRESTARTING", - Message: "Container %s is restarting, wait until the container is running", - Description: "An operation was made on a restarting container", - HTTPStatusCode: http.StatusConflict, - }) - - // ErrorCodeNoExecID is generated when we try to get the info - // on an exec but it can't be found. - ErrorCodeNoExecID = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOEXECID", - Message: "No such exec instance '%s' found in daemon", - Description: "The specified 'exec' instance could not be found", - HTTPStatusCode: http.StatusNotFound, - }) - - // ErrorCodeExecPaused is generated when we try to start an exec - // but the container is paused. - ErrorCodeExecPaused = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EXECPAUSED", - Message: "Container %s is paused, unpause the container before exec", - Description: "An attempt to start an 'exec' was made, but the owning container is paused", - HTTPStatusCode: http.StatusConflict, - }) - - // ErrorCodeExecRunning is generated when we try to start an exec - // but its already running. - ErrorCodeExecRunning = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EXECRUNNING", - Message: "Error: Exec command %s is already running", - Description: "An attempt to start an 'exec' was made, but 'exec' is already running", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeExecExited is generated when we try to start an exec - // but its already running. - ErrorCodeExecExited = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EXECEXITED", - Message: "Error: Exec command %s has already run", - Description: "An attempt to start an 'exec' was made, but 'exec' was already run", - HTTPStatusCode: http.StatusConflict, - }) - - // ErrorCodeExecCantRun is generated when we try to start an exec - // but it failed for some reason. - ErrorCodeExecCantRun = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EXECCANTRUN", - Message: "Cannot run exec command %s in container %s: %s", - Description: "An attempt to start an 'exec' was made, but an error occurred", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeExecAttach is generated when we try to attach to an exec - // but failed. - ErrorCodeExecAttach = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EXECATTACH", - Message: "attach failed with error: %s", - Description: "There was an error while trying to attach to an 'exec'", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeExecContainerStopped is generated when we try to start - // an exec but then the container stopped. - ErrorCodeExecContainerStopped = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "EXECCONTAINERSTOPPED", - Message: "container stopped while running exec", - Description: "An attempt was made to start an 'exec' but the owning container is in the 'stopped' state", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeDefaultName is generated when we try to delete the - // default name of a container. - ErrorCodeDefaultName = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "DEFAULTNAME", - Message: "Conflict, cannot remove the default name of the container", - Description: "An attempt to delete the default name of a container was made, but that is not allowed", - HTTPStatusCode: http.StatusConflict, - }) - - // ErrorCodeNoParent is generated when we try to delete a container - // but we can't find its parent image. - ErrorCodeNoParent = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOPARENT", - Message: "Cannot get parent %s for name %s", - Description: "An attempt was made to delete a container but its parent image could not be found", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeCantDestroy is generated when we try to delete a container - // but failed for some reason. - ErrorCodeCantDestroy = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CANTDESTROY", - Message: "Cannot destroy container %s: %v", - Description: "An attempt was made to delete a container but it failed", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeRmRunning is generated when we try to delete a container - // but its still running. - ErrorCodeRmRunning = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "RMRUNNING", - Message: "You cannot remove a running container %s. Stop the container before attempting removal or use -f", - Description: "An attempt was made to delete a container but the container is still running, try to either stop it first or use '-f'", - HTTPStatusCode: http.StatusConflict, - }) - - // ErrorCodeRmFailed is generated when we try to delete a container - // but it failed for some reason. - ErrorCodeRmFailed = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "RMFAILED", - Message: "Could not kill running container %s, cannot remove - %v", - Description: "An error occurred while trying to delete a running container", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeRmNotFound is generated when we try to delete a container - // but couldn't find it. - ErrorCodeRmNotFound = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "RMNOTFOUND", - Message: "Could not kill running container, cannot remove - %v", - Description: "An attempt to delete a container was made but the container could not be found", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeRmState is generated when we try to delete a container - // but couldn't set its state to RemovalInProgress. - ErrorCodeRmState = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "RMSTATE", - Message: "Failed to set container %s state to RemovalInProgress: %s", - Description: "An attempt to delete a container was made, but there as an error trying to set its state to 'RemovalInProgress'", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeRmDriverFS is generated when we try to delete a container - // but the driver failed to delete its filesystem. - ErrorCodeRmDriverFS = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "RMDRIVERFS", - Message: "Driver %s failed to remove root filesystem %s: %s", - Description: "While trying to delete a container, the driver failed to remove the root filesystem", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeRmFS is generated when we try to delete a container - // but failed deleting its filesystem. - ErrorCodeRmFS = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "RMFS", - Message: "Unable to remove filesystem for %v: %v", - Description: "While trying to delete a container, the driver failed to remove the filesystem", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeRmExecDriver is generated when we try to delete a container - // but failed deleting its exec driver data. - ErrorCodeRmExecDriver = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "RMEXECDRIVER", - Message: "Unable to remove execdriver data for %s: %s", - Description: "While trying to delete a container, there was an error trying to remove th exec driver data", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeRmVolumeInUse is generated when we try to delete a container - // but failed deleting a volume because its being used. - ErrorCodeRmVolumeInUse = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "RMVOLUMEINUSE", - Message: "Conflict: %v", - Description: "While trying to delete a container, one of its volumes is still being used", - HTTPStatusCode: http.StatusConflict, - }) - - // ErrorCodeRmVolume is generated when we try to delete a container - // but failed deleting a volume. - ErrorCodeRmVolume = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "RMVOLUME", - Message: "Error while removing volume %s: %v", - Description: "While trying to delete a container, there was an error trying to delete one of its volumes", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeInvalidCpusetCpus is generated when user provided cpuset CPUs - // are invalid. - ErrorCodeInvalidCpusetCpus = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "INVALIDCPUSETCPUS", - Message: "Invalid value %s for cpuset cpus.", - Description: "While verifying the container's 'HostConfig', CpusetCpus value was in an incorrect format", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeInvalidCpusetMems is generated when user provided cpuset mems - // are invalid. - ErrorCodeInvalidCpusetMems = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "INVALIDCPUSETMEMS", - Message: "Invalid value %s for cpuset mems.", - Description: "While verifying the container's 'HostConfig', CpusetMems value was in an incorrect format", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNotAvailableCpusetCpus is generated when user provided cpuset - // CPUs aren't available in the container's cgroup. - ErrorCodeNotAvailableCpusetCpus = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOTAVAILABLECPUSETCPUS", - Message: "Requested CPUs are not available - requested %s, available: %s.", - Description: "While verifying the container's 'HostConfig', cpuset CPUs provided aren't available in the container's cgroup available set", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeNotAvailableCpusetMems is generated when user provided cpuset - // memory nodes aren't available in the container's cgroup. - ErrorCodeNotAvailableCpusetMems = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NOTAVAILABLECPUSETMEMS", - Message: "Requested memory nodes are not available - requested %s, available: %s.", - Description: "While verifying the container's 'HostConfig', cpuset memory nodes provided aren't available in the container's cgroup available set", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorVolumeNameTaken is generated when an error occurred while - // trying to create a volume that has existed using different driver. - ErrorVolumeNameTaken = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "VOLUME_NAME_TAKEN", - Message: "A volume named %s already exists. Choose a different volume name.", - Description: "An attempt to create a volume using a driver but the volume already exists with a different driver", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeCmdNotFound is generated when container cmd can't start, - // container command not found error, exit code 127 - ErrorCodeCmdNotFound = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CMDNOTFOUND", - Message: "Container command not found or does not exist.", - Description: "Command could not be found, command does not exist", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeCmdCouldNotBeInvoked is generated when container cmd can't start, - // container command permission denied error, exit code 126 - ErrorCodeCmdCouldNotBeInvoked = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CMDCOULDNOTBEINVOKED", - Message: "Container command could not be invoked.", - Description: "Permission denied, cannot invoke command", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeCantStart is generated when container cmd can't start, - // for any reason other than above 2 errors - ErrorCodeCantStart = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CANTSTART", - Message: "Cannot start container %s: %s", - Description: "There was an error while trying to start a container", - HTTPStatusCode: http.StatusInternalServerError, - }) - - // ErrorCodeCantDeletePredefinedNetwork is generated when one of the predefined networks - // is attempted to be deleted. - ErrorCodeCantDeletePredefinedNetwork = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CANT_DELETE_PREDEFINED_NETWORK", - Message: "%s is a pre-defined network and cannot be removed", - Description: "Engine's predefined networks cannot be deleted", - HTTPStatusCode: http.StatusForbidden, - }) - - // ErrorCodeMultipleNetworkConnect is generated when more than one network is passed - // when creating a container - ErrorCodeMultipleNetworkConnect = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "CANNOT_CONNECT_TO_MULTIPLE_NETWORKS", - Message: "Container cannot be connected to %s", - Description: "A container can only be connected to one network at the time", - HTTPStatusCode: http.StatusBadRequest, - }) -) diff --git a/components/engine/errors/error.go b/components/engine/errors/error.go deleted file mode 100644 index 37222d4438..0000000000 --- a/components/engine/errors/error.go +++ /dev/null @@ -1,6 +0,0 @@ -package errors - -// This file contains all of the errors that can be generated from the -// docker engine but are not tied to any specific top-level component. - -const errGroup = "engine" diff --git a/components/engine/errors/errors.go b/components/engine/errors/errors.go new file mode 100644 index 0000000000..8070f48fb2 --- /dev/null +++ b/components/engine/errors/errors.go @@ -0,0 +1,41 @@ +package errors + +import "net/http" + +// apiError is an error wrapper that also +// holds information about response status codes. +type apiError struct { + error + statusCode int +} + +// HTTPErrorStatusCode returns a status code. +func (e apiError) HTTPErrorStatusCode() int { + return e.statusCode +} + +// NewErrorWithStatusCode allows you to associate +// a specific HTTP Status Code to an error. +// The Server will take that code and set +// it as the response status. +func NewErrorWithStatusCode(err error, code int) error { + return apiError{err, code} +} + +// NewBadRequestError creates a new API error +// that has the 400 HTTP status code associated to it. +func NewBadRequestError(err error) error { + return NewErrorWithStatusCode(err, http.StatusBadRequest) +} + +// NewRequestNotFoundError creates a new API error +// that has the 404 HTTP status code associated to it. +func NewRequestNotFoundError(err error) error { + return NewErrorWithStatusCode(err, http.StatusNotFound) +} + +// NewRequestConflictError creates a new API error +// that has the 409 HTTP status code associated to it. +func NewRequestConflictError(err error) error { + return NewErrorWithStatusCode(err, http.StatusConflict) +} diff --git a/components/engine/errors/image.go b/components/engine/errors/image.go deleted file mode 100644 index 04efe6fcba..0000000000 --- a/components/engine/errors/image.go +++ /dev/null @@ -1,20 +0,0 @@ -package errors - -// This file contains all of the errors that can be generated from the -// docker/image component. - -import ( - "net/http" - - "github.com/docker/distribution/registry/api/errcode" -) - -var ( - // ErrorCodeInvalidImageID is generated when image id specified is incorrectly formatted. - ErrorCodeInvalidImageID = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "INVALIDIMAGEID", - Message: "image ID '%s' is invalid ", - Description: "The specified image id is incorrectly formatted", - HTTPStatusCode: http.StatusInternalServerError, - }) -) diff --git a/components/engine/errors/server.go b/components/engine/errors/server.go deleted file mode 100644 index b54d2d5436..0000000000 --- a/components/engine/errors/server.go +++ /dev/null @@ -1,45 +0,0 @@ -package errors - -import ( - "net/http" - - "github.com/docker/distribution/registry/api/errcode" -) - -var ( - // ErrorCodeNewerClientVersion is generated when a request from a client - // specifies a higher version than the server supports. - ErrorCodeNewerClientVersion = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NEWERCLIENTVERSION", - Message: "client is newer than server (client API version: %s, server API version: %s)", - Description: "The client version is higher than the server version", - HTTPStatusCode: http.StatusBadRequest, - }) - - // ErrorCodeOldClientVersion is generated when a request from a client - // specifies a version lower than the minimum version supported by the server. - ErrorCodeOldClientVersion = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "OLDCLIENTVERSION", - Message: "client version %s is too old. Minimum supported API version is %s, please upgrade your client to a newer version", - Description: "The client version is too old for the server", - HTTPStatusCode: http.StatusBadRequest, - }) - - // ErrorNetworkControllerNotEnabled is generated when the networking stack in not enabled - // for certain platforms, like windows. - ErrorNetworkControllerNotEnabled = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "NETWORK_CONTROLLER_NOT_ENABLED", - Message: "the network controller is not enabled for this platform", - Description: "Docker's networking stack is disabled for this platform", - HTTPStatusCode: http.StatusNotFound, - }) - - // ErrorCodeNoHijackConnection is generated when a request tries to attach to a container - // but the connection to hijack is not provided. - ErrorCodeNoHijackConnection = errcode.Register(errGroup, errcode.ErrorDescriptor{ - Value: "HIJACK_CONNECTION_MISSING", - Message: "error attaching to container %s, hijack connection missing", - Description: "The caller didn't provide a connection to hijack", - HTTPStatusCode: http.StatusBadRequest, - }) -) diff --git a/components/engine/hack/.vendor-helpers.sh b/components/engine/hack/.vendor-helpers.sh index fcc745c4da..a90df0496c 100755 --- a/components/engine/hack/.vendor-helpers.sh +++ b/components/engine/hack/.vendor-helpers.sh @@ -117,10 +117,19 @@ clean() { # This directory contains only .c and .h files which are necessary -path vendor/src/github.com/mattn/go-sqlite3/code ) + + # This package is required to build the Etcd client, + # but Etcd hard codes a local Godep full path. + # FIXME: fix_rewritten_imports fixes this problem in most platforms + # but it fails in very small corner cases where it makes the vendor + # script to remove this package. + # See: https://github.com/docker/docker/issues/19231 + findArgs+=( -or -path vendor/src/github.com/ugorji/go/codec ) for import in "${imports[@]}"; do [ "${#findArgs[@]}" -eq 0 ] || findArgs+=( -or ) findArgs+=( -path "vendor/src/$import" ) done + local IFS=$'\n' local prune=( $($find vendor -depth -type d -not '(' "${findArgs[@]}" ')') ) unset IFS diff --git a/components/engine/hack/Jenkins/W2L/postbuild.sh b/components/engine/hack/Jenkins/W2L/postbuild.sh new file mode 100644 index 0000000000..f228b008c6 --- /dev/null +++ b/components/engine/hack/Jenkins/W2L/postbuild.sh @@ -0,0 +1,35 @@ +set +x +set +e + +echo "" +echo "" +echo "---" +echo "Now starting POST-BUILD steps" +echo "---" +echo "" + +echo INFO: Pointing to $DOCKER_HOST + +if [ ! $(docker ps -aq | wc -l) -eq 0 ]; then + echo INFO: Removing containers... + ! docker rm -vf $(docker ps -aq) +fi + +# Remove all images which don't have docker or ubuntu in the name +if [ ! $(docker images | sed -n '1!p' | grep -v 'docker' | grep -v 'ubuntu' | awk '{ print $3 }' | wc -l) -eq 0 ]; then + echo INFO: Removing images... + ! docker rmi -f $(docker images | sed -n '1!p' | grep -v 'docker' | grep -v 'ubuntu' | awk '{ print $3 }') +fi + +# Kill off any instances of git, go and docker, just in case +! taskkill -F -IM git.exe -T >& /dev/null +! taskkill -F -IM go.exe -T >& /dev/null +! taskkill -F -IM docker.exe -T >& /dev/null + +# Remove everything +! cd /c/jenkins/gopath/src/github.com/docker/docker +! rm -rfd * >& /dev/null +! rm -rfd .* >& /dev/null + +echo INFO: Cleanup complete +exit 0 \ No newline at end of file diff --git a/components/engine/hack/Jenkins/W2L/setup.sh b/components/engine/hack/Jenkins/W2L/setup.sh index e46ffee4d5..f8d93e8c2c 100644 --- a/components/engine/hack/Jenkins/W2L/setup.sh +++ b/components/engine/hack/Jenkins/W2L/setup.sh @@ -1,9 +1,10 @@ # Jenkins CI script for Windows to Linux CI. # Heavily modified by John Howard (@jhowardmsft) December 2015 to try to make it more reliable. -set +x -SCRIPT_VER="4-Jan-2016 15:19 PST" +set +xe +SCRIPT_VER="Thu Feb 25 18:54:57 UTC 2016" # TODO to make (even) more resilient: +# - Wait for daemon to be running before executing docker commands # - Check if jq is installed # - Make sure bash is v4.3 or later. Can't do until all Azure nodes on the latest version # - Make sure we are not running as local system. Can't do until all Azure nodes are updated. @@ -18,33 +19,62 @@ SCRIPT_VER="4-Jan-2016 15:19 PST" # - Tidy up of images and containers. Either here, or in the teardown script. ec=0 +uniques=1 echo INFO: Started at `date`. Script version $SCRIPT_VER -# get the ip + +# !README! +# There are two daemons running on the remote Linux host: +# - outer: specified by DOCKER_HOST, this is the daemon that will build and run the inner docker daemon +# from the sources matching the PR. +# - inner: runs on the host network, on a port number similar to that of DOCKER_HOST but the last two digits are inverted +# (2357 if DOCKER_HOST had port 2375; and 2367 if DOCKER_HOST had port 2376). +# The windows integration tests are run against this inner daemon. + +# get the ip, inner and outer ports. ip="${DOCKER_HOST#*://}" +port_outer="${ip#*:}" +# inner port is like outer port with last two digits inverted. +port_inner=$(echo "$port_outer" | sed -E 's/(.)(.)$/\2\1/') ip="${ip%%:*}" -# make sure it is the right DOCKER_HOST. No, this is not a typo, it really -# is at port 2357. This is the daemon which is running on the Linux host. -# The way CI works is to launch a second daemon, docker-in-docker, which -# listens on port 2375 and is built from sources matching the PR. That's the -# one which is tested against. -export DOCKER_HOST="tcp://$ip:2357" +echo "INFO: IP=$ip PORT_OUTER=$port_outer PORT_INNER=$port_inner" + +# If TLS is enabled +if [ -n "$DOCKER_TLS_VERIFY" ]; then + protocol=https + if [ -z "$DOCKER_MACHINE_NAME" ]; then + ec=1 + echo "ERROR: DOCKER_MACHINE_NAME is undefined" + fi + certs=$(echo ~/.docker/machine/machines/$DOCKER_MACHINE_NAME) + curlopts="--cacert $certs/ca.pem --cert $certs/cert.pem --key $certs/key.pem" + run_extra_args="-v tlscerts:/etc/docker" + daemon_extra_args="--tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem" +else + protocol=http +fi # Save for use by make.sh and scripts it invokes -export MAIN_DOCKER_HOST="$DOCKER_HOST" - +export MAIN_DOCKER_HOST="tcp://$ip:$port_inner" # Verify we can get the remote node to respond to _ping if [ $ec -eq 0 ]; then - reply=`curl -s http://$ip:2357/_ping` + reply=`curl -s $curlopts $protocol://$ip:$port_outer/_ping` if [ "$reply" != "OK" ]; then ec=1 - echo "ERROR: Failed to get OK response from Linux node at $ip:2357. It may be down." + echo "ERROR: Failed to get an 'OK' response from the docker daemon on the Linux node" + echo " at $ip:$port_outer when called with an http request for '_ping'. This implies that" + echo " either the daemon has crashed/is not running, or the Linux node is unavailable." + echo + echo " A regular ping to the remote Linux node is below. It should reply. If not, the" + echo " machine cannot be reached at all and may have crashed. If it does reply, it is" + echo " likely a case of the Linux daemon not running or having crashed, which requires" + echo " further investigation." + echo echo " Try re-running this CI job, or ask on #docker-dev or #docker-maintainers" - echo " to see if the node is up and running." + echo " for someone to perform further diagnostics, or take this node out of rotation." echo - echo "Regular ping output for remote host below. It should reply. If not, it needs restarting." ping $ip else echo "INFO: The Linux nodes outer daemon replied to a ping. Good!" @@ -54,7 +84,7 @@ fi # Get the version from the remote node. Note this may fail if jq is not installed. # That's probably worth checking to make sure, just in case. if [ $ec -eq 0 ]; then - remoteVersion=`curl -s http://$ip:2357/version | jq -c '.Version'` + remoteVersion=`curl -s $curlopts $protocol://$ip:$port_outer/version | jq -c '.Version'` echo "INFO: Remote daemon is running docker version $remoteVersion" fi @@ -153,7 +183,8 @@ fi if [ $ec -eq 0 ]; then echo "INFO: Starting build of a Linux daemon to test against, and starting it..." set -x - docker run --pid host --privileged -d --name "docker-$COMMITHASH" --net host "docker:$COMMITHASH" bash -c 'echo "INFO: Compiling" && date && hack/make.sh binary && echo "INFO: Compile complete" && date && cp bundles/$(cat VERSION)/binary/docker /bin/docker && echo "INFO: Starting daemon" && exec docker daemon -D -H tcp://0.0.0.0:2375' + # aufs in aufs is faster than vfs in aufs + docker run $run_extra_args -e DOCKER_GRAPHDRIVER=aufs --pid host --privileged -d --name "docker-$COMMITHASH" --net host "docker:$COMMITHASH" bash -c "echo 'INFO: Compiling' && date && hack/make.sh binary && echo 'INFO: Compile complete' && date && cp bundles/$(cat VERSION)/binary/docker /bin/docker && echo 'INFO: Starting daemon' && exec docker daemon -D -H tcp://0.0.0.0:$port_inner $daemon_extra_args" ec=$? set +x if [ 0 -ne $ec ]; then @@ -166,8 +197,8 @@ if [ $ec -eq 0 ]; then echo "INFO: Starting local build of Windows binary..." set -x export TIMEOUT="120m" - export DOCKER_HOST="tcp://$ip:2375" - export DOCKER_TEST_HOST="tcp://$ip:2375" + export DOCKER_HOST="tcp://$ip:$port_inner" + export DOCKER_TEST_HOST="tcp://$ip:$port_inner" unset DOCKER_CLIENTONLY export DOCKER_REMOTE_DAEMON=1 hack/make.sh binary @@ -193,6 +224,8 @@ fi if [ $ec -eq 0 ]; then echo "INFO: Running Integration tests..." set -x + export DOCKER_TEST_TLS_VERIFY="$DOCKER_TLS_VERIFY" + export DOCKER_TEST_CERT_PATH="$DOCKER_CERT_PATH" hack/make.sh test-integration-cli ec=$? set +x @@ -212,22 +245,22 @@ fi GOVER_DOCKERFILE=`grep 'ENV GO_VERSION' Dockerfile | awk '{print $3}'` GOVER_INSTALLED=`go version | awk '{print $3}'` if [ "${GOVER_INSTALLED:2}" != "$GOVER_DOCKERFILE" ]; then - ec=1 # Uncomment to make CI fail once all nodes are updated. + #ec=1 # Uncomment to make CI fail once all nodes are updated. echo echo "---------------------------------------------------------------------------" - echo "ERROR: CI should be using go version $GOVER_DOCKERFILE, but is using ${GOVER_INSTALLED:2}" - echo " This is currently a warning, but should (will) become an error in the future." + echo "WARN: CI should be using go version $GOVER_DOCKERFILE, but is using ${GOVER_INSTALLED:2}" + echo " Please ping #docker-maintainers on IRC to get this CI server updated." echo "---------------------------------------------------------------------------" echo fi # Check the Linux box is running a matching version of docker if [ "$uniques" -ne 1 ]; then - ec=1 # Uncomment to make CI fail once all nodes are updated. + ec=0 # Uncomment to make CI fail once all nodes are updated. echo echo "---------------------------------------------------------------------------" echo "ERROR: This CI node is not running the same version of docker as the daemon." - echo " This is a CI configuration issue" + echo " This is a CI configuration issue." echo "---------------------------------------------------------------------------" echo fi diff --git a/components/engine/hack/install.sh b/components/engine/hack/install.sh index cd7b9fe1a0..218ad99c8c 100755 --- a/components/engine/hack/install.sh +++ b/components/engine/hack/install.sh @@ -28,6 +28,12 @@ apt_url="https://apt.dockerproject.org" yum_url="https://yum.dockerproject.org" gpg_fingerprint="58118E89F3A912897C070ADBF76221572C52609D" +key_servers=" +ha.pool.sks-keyservers.net +pgp.mit.edu +keyserver.ubuntu.com +" + command_exists() { command -v "$@" > /dev/null 2>&1 } @@ -102,7 +108,10 @@ rpm_import_repository_key() { local key=$1; shift local tmpdir=$(mktemp -d) chmod 600 "$tmpdir" - gpg --homedir "$tmpdir" --keyserver ha.pool.sks-keyservers.net --recv-keys "$key" + for key_server in $key_servers ; do + gpg --homedir "$tmpdir" --keyserver "$key_server" --recv-keys "$key" && break + done + gpg --homedir "$tmpdir" -k "$key" >/dev/null gpg --homedir "$tmpdir" --export --armor "$key" > "$tmpdir"/repo.key rpm --import "$tmpdir"/repo.key rm -rf "$tmpdir" @@ -414,7 +423,10 @@ do_install() { fi ( set -x - $sh_c "apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys ${gpg_fingerprint}" + for key_server in $key_servers ; do + $sh_c "apt-key adv --keyserver hkp://${key_server}:80 --recv-keys ${gpg_fingerprint}" && break + done + $sh_c "apt-key adv -k ${gpg_fingerprint} >/dev/null" $sh_c "mkdir -p /etc/apt/sources.list.d" $sh_c "echo deb [arch=$(dpkg --print-architecture)] ${apt_url}/repo ${lsb_dist}-${dist_version} ${repo} > /etc/apt/sources.list.d/docker.list" $sh_c 'sleep 3; apt-get update; apt-get install -y -q docker-engine' diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index 16b59c4584..bb3ee7204b 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -118,8 +118,10 @@ fi if [ -z "$DOCKER_CLIENTONLY" ]; then DOCKER_BUILDTAGS+=" daemon" - if pkg-config libsystemd-journal 2> /dev/null ; then + if pkg-config 'libsystemd >= 209' 2> /dev/null ; then DOCKER_BUILDTAGS+=" journald" + elif pkg-config 'libsystemd-journal' 2> /dev/null ; then + DOCKER_BUILDTAGS+=" journald journald_compat" fi fi @@ -135,7 +137,7 @@ fi # functionality. if \ command -v gcc &> /dev/null \ - && ! ( echo -e '#include \nint main() { dm_task_deferred_remove(NULL); }'| gcc -ldevmapper -xc - -o /dev/null &> /dev/null ) \ + && ! ( echo -e '#include \nint main() { dm_task_deferred_remove(NULL); }'| gcc -xc - -ldevmapper -o /dev/null &> /dev/null ) \ ; then DOCKER_BUILDTAGS+=' libdm_no_deferred_remove' fi @@ -168,6 +170,8 @@ BUILDFLAGS=( $BUILDFLAGS "${ORIG_BUILDFLAGS[@]}" ) if [ "${DOCKER_ENGINE_GOARCH}" == "arm" ]; then : ${TIMEOUT:=210m} +elif [ "${DOCKER_ENGINE_GOARCH}" == "windows" ]; then + : ${TIMEOUT:=180m} else : ${TIMEOUT:=120m} fi @@ -237,6 +241,8 @@ test_env() { # use "env -i" to tightly control the environment variables that bleed into the tests env -i \ DEST="$DEST" \ + DOCKER_TLS_VERIFY="$DOCKER_TEST_TLS_VERIFY" \ + DOCKER_CERT_PATH="$DOCKER_TEST_CERT_PATH" \ DOCKER_ENGINE_GOARCH="$DOCKER_ENGINE_GOARCH" \ DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" \ DOCKER_USERLANDPROXY="$DOCKER_USERLANDPROXY" \ diff --git a/components/engine/hack/make/.build-deb/rules b/components/engine/hack/make/.build-deb/rules index 892821509c..15b848e322 100755 --- a/components/engine/hack/make/.build-deb/rules +++ b/components/engine/hack/make/.build-deb/rules @@ -5,7 +5,7 @@ VERSION = $(shell cat VERSION) override_dh_gencontrol: # if we're on Ubuntu, we need to Recommends: apparmor echo 'apparmor:Recommends=$(shell dpkg-vendor --is Ubuntu && echo apparmor)' >> debian/docker-engine.substvars - # if we are building experimental we reccomend yubico-piv-tool + # if we are building experimental we recommend yubico-piv-tool echo 'yubico:Recommends=$(shell [ "$DOCKER_EXPERIMENTAL" ] && echo "yubico-piv-tool (>= 1.1.0~)")' >> debian/docker-engine.substvars dh_gencontrol diff --git a/components/engine/hack/make/.build-rpm/docker-engine-selinux.spec b/components/engine/hack/make/.build-rpm/docker-engine-selinux.spec index 69e5faa3f5..706af36ac6 100644 --- a/components/engine/hack/make/.build-rpm/docker-engine-selinux.spec +++ b/components/engine/hack/make/.build-rpm/docker-engine-selinux.spec @@ -44,7 +44,7 @@ Conflicts: docker-selinux # Relabel files %global relabel_files() \ - /sbin/restorecon -R %{_bindir}/docker %{_localstatedir}/run/docker.sock %{_localstatedir}/run/docker.pid %{_sharedstatedir}/docker %{_sysconfdir}/docker %{_localstatedir}/log/docker %{_localstatedir}/log/lxc %{_localstatedir}/lock/lxc %{_usr}/lib/systemd/system/docker.service /root/.docker &> /dev/null || : \ + /sbin/restorecon -R %{_bindir}/docker %{_localstatedir}/run/docker.sock %{_localstatedir}/run/docker.pid %{_sysconfdir}/docker %{_localstatedir}/log/docker %{_localstatedir}/log/lxc %{_localstatedir}/lock/lxc %{_usr}/lib/systemd/system/docker.service /root/.docker &> /dev/null || : \ %description SELinux policy modules for use with Docker @@ -83,6 +83,9 @@ fi if %{_sbindir}/selinuxenabled ; then %{_sbindir}/load_policy %relabel_files + if [ $1 -eq 1 ]; then + restorecon -R %{_sharedstatedir}/docker + fi fi %postun diff --git a/components/engine/hack/make/.ensure-httpserver b/components/engine/hack/make/.ensure-httpserver index c159fa8018..3fc84b2f26 100644 --- a/components/engine/hack/make/.ensure-httpserver +++ b/components/engine/hack/make/.ensure-httpserver @@ -8,7 +8,7 @@ dir="$DEST/httpserver" mkdir -p "$dir" ( cd "$dir" - GOOS=${DOCKER_ENGINE_GOOS:="linux"} GOARCH=${DOCKER_ENGINE_GOARCH:="amd64"} go build -o httpserver github.com/docker/docker/contrib/httpserver + GOOS=${DOCKER_ENGINE_GOOS:="linux"} GOARCH=${DOCKER_ENGINE_GOARCH:="amd64"} CGO_ENABLED=0 go build -o httpserver github.com/docker/docker/contrib/httpserver cp ../../../../contrib/httpserver/Dockerfile . docker build -qt httpserver . > /dev/null ) diff --git a/components/engine/hack/make/.ensure-nnp-test b/components/engine/hack/make/.ensure-nnp-test new file mode 100644 index 0000000000..26b11b9a5c --- /dev/null +++ b/components/engine/hack/make/.ensure-nnp-test @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + +# Build a C binary for testing no-new-privileges +# and compile it for target daemon +if [ "$DOCKER_ENGINE_GOOS" = "linux" ]; then + if [ "$DOCKER_ENGINE_OSARCH" = "$DOCKER_CLIENT_OSARCH" ]; then + tmpdir=$(mktemp -d) + gcc -g -Wall -static contrib/nnp-test/nnp-test.c -o "${tmpdir}/nnp-test" + + dockerfile="${tmpdir}/Dockerfile" + cat <<-EOF > "$dockerfile" + FROM debian:jessie + COPY . /usr/bin/ + RUN chmod +s /usr/bin/nnp-test + EOF + docker build --force-rm ${DOCKER_BUILD_ARGS} -qt nnp-test "${tmpdir}" > /dev/null + rm -rf "${tmpdir}" + else + docker build ${DOCKER_BUILD_ARGS} -qt nnp-test contrib/nnp-test > /dev/null + fi +fi diff --git a/components/engine/hack/make/.integration-daemon-setup b/components/engine/hack/make/.integration-daemon-setup index 508a9d479e..b50f945416 100644 --- a/components/engine/hack/make/.integration-daemon-setup +++ b/components/engine/hack/make/.integration-daemon-setup @@ -7,6 +7,7 @@ if [ $DOCKER_ENGINE_GOOS != "windows" ]; then bundle .ensure-frozen-images bundle .ensure-httpserver bundle .ensure-syscall-test + bundle .ensure-nnp-test else # Note this is Windows to Windows CI, not Windows to Linux CI bundle .ensure-frozen-images-windows diff --git a/components/engine/hack/make/generate-index-listing b/components/engine/hack/make/generate-index-listing index 1808ce9caa..1167ed1205 100755 --- a/components/engine/hack/make/generate-index-listing +++ b/components/engine/hack/make/generate-index-listing @@ -41,7 +41,7 @@ create_index() { IFS=$'\n'; # pretty sweet, will mimick the normal apache output - for L in $(find -L . -mount -depth -maxdepth 1 -type f ! -name 'index' -printf "%-44f@_@%Td-%Tb-%TY %Tk:%TM @%f@\n"|sort|sed 's,\([\ ]\+\)@_@,\1,g'); + for L in $(find -L . -mount -depth -maxdepth 1 -type f ! -name 'index' -printf "%f|@_@%Td-%Tb-%TY %Tk:%TM @%f@\n"|sort|column -t -s '|' | sed 's,\([\ ]\+\)@_@,\1,g'); do # file F=$(sed -e 's,^.*@\([^@]\+\)@.*$,\1,g'<<<"$L"); diff --git a/components/engine/hack/make/release-deb b/components/engine/hack/make/release-deb index a771990c4f..9b0b3ca02a 100755 --- a/components/engine/hack/make/release-deb +++ b/components/engine/hack/make/release-deb @@ -14,6 +14,8 @@ set -e # # ... and so on and so forth for the builds created by hack/make/build-deb +source "$(dirname "$BASH_SOURCE")/.detect-daemon-osarch" + : ${DOCKER_RELEASE_DIR:=$DEST} : ${GPG_KEYID:=releasedocker} APTDIR=$DOCKER_RELEASE_DIR/apt/repo @@ -47,41 +49,44 @@ if [[ ! "${components[*]}" =~ $component ]] ; then components+=( $component ) fi -# create/update apt-ftparchive file -if [ ! -f "$APTDIR/conf/apt-ftparchive.conf" ]; then - cat <<-EOF > "$APTDIR/conf/apt-ftparchive.conf" - Dir { - ArchiveDir "${APTDIR}"; - CacheDir "${APTDIR}/db"; - }; +# create apt-ftparchive file on every run. This is essential to avoid +# using stale versions of the config file that could cause unnecessary +# refreshing of bits for EOL-ed releases. +cat <<-EOF > "$APTDIR/conf/apt-ftparchive.conf" +Dir { + ArchiveDir "${APTDIR}"; + CacheDir "${APTDIR}/db"; +}; - Default { - Packages::Compress ". gzip bzip2"; - Sources::Compress ". gzip bzip2"; - Contents::Compress ". gzip bzip2"; - }; +Default { + Packages::Compress ". gzip bzip2"; + Sources::Compress ". gzip bzip2"; + Contents::Compress ". gzip bzip2"; +}; + +TreeDefault { + BinCacheDB "packages-\$(SECTION)-\$(ARCH).db"; + Directory "pool/\$(SECTION)"; + Packages "\$(DIST)/\$(SECTION)/binary-\$(ARCH)/Packages"; + SrcDirectory "pool/\$(SECTION)"; + Sources "\$(DIST)/\$(SECTION)/source/Sources"; + Contents "\$(DIST)/\$(SECTION)/Contents-\$(ARCH)"; + FileList "$APTDIR/\$(DIST)/\$(SECTION)/filelist"; +}; +EOF + +for dir in contrib/builder/deb/${PACKAGE_ARCH}/*/; do + version="$(basename "$dir")" + suite="${version//debootstrap-}" + + cat <<-EOF + Tree "dists/${suite}" { + Sections "${components[*]}"; + Architectures "${arches[*]}"; + } - TreeDefault { - BinCacheDB "packages-\$(SECTION)-\$(ARCH).db"; - Directory "pool/\$(SECTION)"; - Packages "\$(DIST)/\$(SECTION)/binary-\$(ARCH)/Packages"; - SrcDirectory "pool/\$(SECTION)"; - Sources "\$(DIST)/\$(SECTION)/source/Sources"; - Contents "\$(DIST)/\$(SECTION)/Contents-\$(ARCH)"; - FileList "$APTDIR/\$(DIST)/\$(SECTION)/filelist"; - }; EOF - - for suite in $(exec contrib/reprepro/suites.sh); do - cat <<-EOF - Tree "dists/${suite}" { - Sections "${components[*]}"; - Architectures "${arches[*]}"; - } - - EOF - done >> "$APTDIR/conf/apt-ftparchive.conf" -fi +done >> "$APTDIR/conf/apt-ftparchive.conf" if [ ! -f "$APTDIR/conf/docker-engine-release.conf" ]; then cat <<-EOF > "$APTDIR/conf/docker-engine-release.conf" @@ -116,9 +121,6 @@ for dir in contrib/builder/deb/${PACKAGE_ARCH}/*/; do -name *~${codename#*-}*.deb > "$APTDIR/dists/$codename/$component/filelist" done -# clean the databases -apt-ftparchive clean "$APTDIR/conf/apt-ftparchive.conf" - # run the apt-ftparchive commands so we can have pinning apt-ftparchive generate "$APTDIR/conf/apt-ftparchive.conf" diff --git a/components/engine/hack/make/release-rpm b/components/engine/hack/make/release-rpm index 19bdb39f34..5ed25cbe19 100755 --- a/components/engine/hack/make/release-rpm +++ b/components/engine/hack/make/release-rpm @@ -14,6 +14,8 @@ set -e # # ... and so on and so forth for the builds created by hack/make/build-rpm +source "$(dirname "$BASH_SOURCE")/.detect-daemon-osarch" + : ${DOCKER_RELEASE_DIR:=$DEST} YUMDIR=$DOCKER_RELEASE_DIR/yum/repo : ${GPG_KEYID:=releasedocker} diff --git a/components/engine/hack/make/update-apt-repo b/components/engine/hack/make/update-apt-repo index ae235c30b4..7354a2ecff 100755 --- a/components/engine/hack/make/update-apt-repo +++ b/components/engine/hack/make/update-apt-repo @@ -2,7 +2,7 @@ set -e # This script updates the apt repo in $DOCKER_RELEASE_DIR/apt/repo. -# This script is a "fix all" for any sort of problems that might have occured with +# This script is a "fix all" for any sort of problems that might have occurred with # the Release or Package files in the repo. # It should only be used in the rare case of extreme emergencies to regenerate # Release and Package files for the apt repo. diff --git a/components/engine/hack/release.sh b/components/engine/hack/release.sh index 7030ab6fad..6ad9b63571 100755 --- a/components/engine/hack/release.sh +++ b/components/engine/hack/release.sh @@ -81,7 +81,8 @@ setup_s3() { # s3cmd has no useful exit status, so we cannot check that. # Instead, we check if it outputs anything on standard output. # (When there are problems, it uses standard error instead.) - s3cmd info "s3://$BUCKET" | grep -q . + # NOTE: for some reason on debian:jessie `s3cmd info ... | grep -q .` results in a broken pipe + s3cmd info "s3://$BUCKET" | grep . >/dev/null # Make the bucket accessible through website endpoints. s3cmd ws-create --ws-index index --ws-error error "s3://$BUCKET" } diff --git a/components/engine/hack/vendor.sh b/components/engine/hack/vendor.sh index a0ea6c2996..8117d9534f 100755 --- a/components/engine/hack/vendor.sh +++ b/components/engine/hack/vendor.sh @@ -7,7 +7,8 @@ source 'hack/.vendor-helpers.sh' # the following lines are in sorted order, FYI clone git github.com/Azure/go-ansiterm 70b2c90b260171e829f1ebd7c17f600c11858dbe -clone git github.com/Microsoft/go-winio eb176a9831c54b88eaf9eb4fbc24b94080d910ad +clone git github.com/Microsoft/hcsshim 116e0e9f5ced0cec94ae46d0aa1b3002a325f532 +clone git github.com/Microsoft/go-winio c40bf24f405ab3cc8e1383542d474e813332de6d clone git github.com/Sirupsen/logrus v0.9.0 # logrus is a common dependency among multiple deps clone git github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a clone git github.com/go-check/check 11d3bc7aa68e238947792f30573146a3231fc0f1 @@ -16,27 +17,26 @@ clone git github.com/gorilla/mux e444e69cbd clone git github.com/kr/pty 5cf931ef8f clone git github.com/mattn/go-shellwords v1.0.0 clone git github.com/mattn/go-sqlite3 v1.1.0 -clone git github.com/Microsoft/hcsshim 35ad4d808a97203cb1748d7c43167e91f51e7f86 clone git github.com/mistifyio/go-zfs v2.1.1 clone git github.com/tchap/go-patricia v2.1.0 -clone git github.com/vdemeester/shakers 3c10293ce22b900c27acad7b28656196fcc2f73b +clone git github.com/vdemeester/shakers 24d7f1d6a71aa5d9cbe7390e4afb66b7eef9e1b3 clone git golang.org/x/net 47990a1ba55743e6ef1affd3a14e5bac8553615d https://github.com/golang/net.git clone git golang.org/x/sys eb2c74142fd19a79b3f237334c7384d5167b1b46 https://github.com/golang/sys.git clone git github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3 -clone git github.com/docker/go-connections v0.1.3 -clone git github.com/docker/engine-api 9a940e4ead265e18d4feb9e3c515428966a08278 +clone git github.com/docker/go-connections v0.2.0 +clone git github.com/docker/engine-api 9bab0d5b73872e53dfadfa055dcc519e57b09439 clone git github.com/RackSec/srslog 6eb773f331e46fbba8eecb8e794e635e75fc04de clone git github.com/imdario/mergo 0.2.1 #get libnetwork packages -clone git github.com/docker/libnetwork v0.6.1-rc1 +clone git github.com/docker/libnetwork v0.7.0-dev.5 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec clone git github.com/hashicorp/go-msgpack 71c2886f5a673a35f909803f38ece5810165097b clone git github.com/hashicorp/memberlist 9a1e242e454d2443df330bdd51a436d5a9058fc4 clone git github.com/hashicorp/serf 7151adcef72687bf95f451a2e0ba15cb19412bf2 clone git github.com/docker/libkv c2aac5dbbaa5c872211edea7c0f32b3bd67e7410 clone git github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25 -clone git github.com/vishvananda/netlink bfd70f556483c008636b920dda142fdaa0d59ef9 +clone git github.com/vishvananda/netlink 631962935bff4f3d20ff32a72e8944f6d2836a26 clone git github.com/BurntSushi/toml f706d00e3de6abe700c994cdd545a1a4915af060 clone git github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374 clone git github.com/deckarep/golang-set ef32fa3046d9f249d399f98ebaf9be944430fd1d @@ -48,18 +48,18 @@ clone git github.com/boltdb/bolt v1.1.0 clone git github.com/miekg/dns 75e6e86cc601825c5dbcd4e0c209eab180997cd7 # get graph and distribution packages -clone git github.com/docker/distribution ab9b433fcaf7c8319562a8b80f2720f5faca712f +clone git github.com/docker/distribution 7b66c50bb7e0e4b3b83f8fd134a9f6ea4be08b57 clone git github.com/vbatts/tar-split v0.9.11 # get desired notary commit, might also need to be updated in Dockerfile -clone git github.com/docker/notary docker-v1.10-5 +clone git github.com/docker/notary v0.2.0 clone git google.golang.org/grpc 174192fc93efcb188fc8f46ca447f0da606b6885 https://github.com/grpc/grpc-go.git -clone git github.com/miekg/pkcs11 80f102b5cac759de406949c47f0928b99bd64cdf +clone git github.com/miekg/pkcs11 df8ae6ca730422dba20c768ff38ef7d79077a59f clone git github.com/docker/go v1.5.1-1-1-gbaf439e clone git github.com/agl/ed25519 d2b94fd789ea21d12fac1a4443dd3a3f79cda72c -clone git github.com/opencontainers/runc 3d8a20bb772defc28c355534d83486416d1719b4 # libcontainer +clone git github.com/opencontainers/runc 2c3115481ee1782ad687a9e0b4834f89533c2acf # libcontainer clone git github.com/seccomp/libseccomp-golang 1b506fc7c24eec5a3693cdcbed40d9c226cfc6a1 # libcontainer deps (see src/github.com/opencontainers/runc/Godeps/Godeps.json) clone git github.com/coreos/go-systemd v4 @@ -82,4 +82,9 @@ clone git gopkg.in/fsnotify.v1 v1.2.0 clone git github.com/aws/aws-sdk-go v0.9.9 clone git github.com/vaughan0/go-ini a98ad7ee00ec53921f08832bc06ecf7fd600e6a1 +# gcplogs deps +clone git golang.org/x/oauth2 2baa8a1b9338cf13d9eeb27696d761155fa480be https://github.com/golang/oauth2.git +clone git google.golang.org/api dc6d2353af16e2a2b0ff6986af051d473a4ed468 https://code.googlesource.com/google-api-go-client +clone git google.golang.org/cloud dae7e3d993bc3812a2185af60552bb6b847e52a0 https://code.googlesource.com/gocloud + clean diff --git a/components/engine/image/spec/v1.md b/components/engine/image/spec/v1.md index f2c29155c3..57a599b8ff 100644 --- a/components/engine/image/spec/v1.md +++ b/components/engine/image/spec/v1.md @@ -176,7 +176,7 @@ Here is an example image JSON file: should be omitted. A collection of images may share many of the same ancestor layers. This organizational structure is strictly a tree with any one layer having either no parent or a single parent and zero or - more descendent layers. Cycles are not allowed and implementations + more descendant layers. Cycles are not allowed and implementations should be careful to avoid creating them or iterating through a cycle indefinitely. diff --git a/components/engine/image/store.go b/components/engine/image/store.go index 1279f59b66..92ac438db7 100644 --- a/components/engine/image/store.go +++ b/components/engine/image/store.go @@ -170,6 +170,9 @@ func (is *store) Search(term string) (ID, error) { dgst, err := is.digestSet.Lookup(term) if err != nil { + if err == digest.ErrDigestNotFound { + err = fmt.Errorf("No such image: %s", term) + } return "", err } return ID(dgst), nil diff --git a/components/engine/image/v1/imagev1.go b/components/engine/image/v1/imagev1.go index cdea0e7270..e27ebd4c0a 100644 --- a/components/engine/image/v1/imagev1.go +++ b/components/engine/image/v1/imagev1.go @@ -31,7 +31,7 @@ func HistoryFromConfig(imageJSON []byte, emptyLayer bool) (image.History, error) return image.History{ Author: v1Image.Author, Created: v1Image.Created, - CreatedBy: strings.Join(v1Image.ContainerConfig.Cmd.Slice(), " "), + CreatedBy: strings.Join(v1Image.ContainerConfig.Cmd, " "), Comment: v1Image.Comment, EmptyLayer: emptyLayer, }, nil @@ -97,7 +97,7 @@ func MakeConfigFromV1Config(imageJSON []byte, rootfs *image.RootFS, history []im delete(c, "id") delete(c, "parent") - delete(c, "Size") // Size is calculated from data on disk and is inconsitent + delete(c, "Size") // Size is calculated from data on disk and is inconsistent delete(c, "parent_id") delete(c, "layer_id") delete(c, "throwaway") @@ -142,7 +142,7 @@ func rawJSON(value interface{}) *json.RawMessage { // ValidateID checks whether an ID string is a valid image ID. func ValidateID(id string) error { if ok := validHex.MatchString(id); !ok { - return fmt.Errorf("image ID '%s' is invalid ", id) + return fmt.Errorf("image ID %q is invalid", id) } return nil } diff --git a/components/engine/integration-cli/daemon.go b/components/engine/integration-cli/daemon.go new file mode 100644 index 0000000000..3d28b709b6 --- /dev/null +++ b/components/engine/integration-cli/daemon.go @@ -0,0 +1,457 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/docker/docker/opts" + "github.com/docker/docker/pkg/integration/checker" + "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/tlsconfig" + "github.com/docker/go-connections/sockets" + "github.com/go-check/check" +) + +// Daemon represents a Docker daemon for the testing framework. +type Daemon struct { + // Defaults to "daemon" + // Useful to set to --daemon or -d for checking backwards compatibility + Command string + GlobalFlags []string + + id string + c *check.C + logFile *os.File + folder string + root string + stdin io.WriteCloser + stdout, stderr io.ReadCloser + cmd *exec.Cmd + storageDriver string + wait chan error + userlandProxy bool + useDefaultHost bool + useDefaultTLSHost bool +} + +type clientConfig struct { + transport *http.Transport + scheme string + addr string +} + +// NewDaemon returns a Daemon instance to be used for testing. +// This will create a directory such as d123456789 in the folder specified by $DEST. +// The daemon will not automatically start. +func NewDaemon(c *check.C) *Daemon { + dest := os.Getenv("DEST") + c.Assert(dest, check.Not(check.Equals), "", check.Commentf("Please set the DEST environment variable")) + + id := fmt.Sprintf("d%d", time.Now().UnixNano()%100000000) + dir := filepath.Join(dest, id) + daemonFolder, err := filepath.Abs(dir) + c.Assert(err, check.IsNil, check.Commentf("Could not make %q an absolute path", dir)) + daemonRoot := filepath.Join(daemonFolder, "root") + + c.Assert(os.MkdirAll(daemonRoot, 0755), check.IsNil, check.Commentf("Could not create daemon root %q", dir)) + + userlandProxy := true + if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { + if val, err := strconv.ParseBool(env); err != nil { + userlandProxy = val + } + } + + return &Daemon{ + Command: "daemon", + id: id, + c: c, + folder: daemonFolder, + root: daemonRoot, + storageDriver: os.Getenv("DOCKER_GRAPHDRIVER"), + userlandProxy: userlandProxy, + } +} + +func (d *Daemon) getClientConfig() (*clientConfig, error) { + var ( + transport *http.Transport + scheme string + addr string + proto string + ) + if d.useDefaultTLSHost { + option := &tlsconfig.Options{ + CAFile: "fixtures/https/ca.pem", + CertFile: "fixtures/https/client-cert.pem", + KeyFile: "fixtures/https/client-key.pem", + } + tlsConfig, err := tlsconfig.Client(*option) + if err != nil { + return nil, err + } + transport = &http.Transport{ + TLSClientConfig: tlsConfig, + } + addr = fmt.Sprintf("%s:%d", opts.DefaultHTTPHost, opts.DefaultTLSHTTPPort) + scheme = "https" + proto = "tcp" + } else if d.useDefaultHost { + addr = opts.DefaultUnixSocket + proto = "unix" + scheme = "http" + transport = &http.Transport{} + } else { + addr = filepath.Join(d.folder, "docker.sock") + proto = "unix" + scheme = "http" + transport = &http.Transport{} + } + + d.c.Assert(sockets.ConfigureTransport(transport, proto, addr), check.IsNil) + + return &clientConfig{ + transport: transport, + scheme: scheme, + addr: addr, + }, nil +} + +// Start will start the daemon and return once it is ready to receive requests. +// You can specify additional daemon flags. +func (d *Daemon) Start(args ...string) error { + logFile, err := os.OpenFile(filepath.Join(d.folder, "docker.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + d.c.Assert(err, check.IsNil, check.Commentf("[%s] Could not create %s/docker.log", d.id, d.folder)) + + return d.StartWithLogFile(logFile, args...) +} + +// StartWithLogFile will start the daemon and attach its streams to a given file. +func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { + dockerBinary, err := exec.LookPath(dockerBinary) + d.c.Assert(err, check.IsNil, check.Commentf("[%s] could not find docker binary in $PATH", d.id)) + + args := append(d.GlobalFlags, + d.Command, + "--graph", d.root, + "--pidfile", fmt.Sprintf("%s/docker.pid", d.folder), + fmt.Sprintf("--userland-proxy=%t", d.userlandProxy), + ) + if !(d.useDefaultHost || d.useDefaultTLSHost) { + args = append(args, []string{"--host", d.sock()}...) + } + if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { + args = append(args, []string{"--userns-remap", root}...) + } + + // If we don't explicitly set the log-level or debug flag(-D) then + // turn on debug mode + foundLog := false + foundSd := false + for _, a := range providedArgs { + if strings.Contains(a, "--log-level") || strings.Contains(a, "-D") || strings.Contains(a, "--debug") { + foundLog = true + } + if strings.Contains(a, "--storage-driver") { + foundSd = true + } + } + if !foundLog { + args = append(args, "--debug") + } + if d.storageDriver != "" && !foundSd { + args = append(args, "--storage-driver", d.storageDriver) + } + + args = append(args, providedArgs...) + d.cmd = exec.Command(dockerBinary, args...) + + d.cmd.Stdout = out + d.cmd.Stderr = out + d.logFile = out + + if err := d.cmd.Start(); err != nil { + return fmt.Errorf("[%s] could not start daemon container: %v", d.id, err) + } + + wait := make(chan error) + + go func() { + wait <- d.cmd.Wait() + d.c.Logf("[%s] exiting daemon", d.id) + close(wait) + }() + + d.wait = wait + + tick := time.Tick(500 * time.Millisecond) + // make sure daemon is ready to receive requests + startTime := time.Now().Unix() + for { + d.c.Logf("[%s] waiting for daemon to start", d.id) + if time.Now().Unix()-startTime > 5 { + // After 5 seconds, give up + return fmt.Errorf("[%s] Daemon exited and never started", d.id) + } + select { + case <-time.After(2 * time.Second): + return fmt.Errorf("[%s] timeout: daemon does not respond", d.id) + case <-tick: + clientConfig, err := d.getClientConfig() + if err != nil { + return err + } + + client := &http.Client{ + Transport: clientConfig.transport, + } + + req, err := http.NewRequest("GET", "/_ping", nil) + d.c.Assert(err, check.IsNil, check.Commentf("[%s] could not create new request", d.id)) + req.URL.Host = clientConfig.addr + req.URL.Scheme = clientConfig.scheme + resp, err := client.Do(req) + if err != nil { + continue + } + if resp.StatusCode != http.StatusOK { + d.c.Logf("[%s] received status != 200 OK: %s", d.id, resp.Status) + } + d.c.Logf("[%s] daemon started", d.id) + d.root, err = d.queryRootDir() + if err != nil { + return fmt.Errorf("[%s] error querying daemon for root directory: %v", d.id, err) + } + return nil + } + } +} + +// StartWithBusybox will first start the daemon with Daemon.Start() +// then save the busybox image from the main daemon and load it into this Daemon instance. +func (d *Daemon) StartWithBusybox(arg ...string) error { + if err := d.Start(arg...); err != nil { + return err + } + return d.LoadBusybox() +} + +// Stop will send a SIGINT every second and wait for the daemon to stop. +// If it timeouts, a SIGKILL is sent. +// Stop will not delete the daemon directory. If a purged daemon is needed, +// instantiate a new one with NewDaemon. +func (d *Daemon) Stop() error { + if d.cmd == nil || d.wait == nil { + return errors.New("daemon not started") + } + + defer func() { + d.logFile.Close() + d.cmd = nil + }() + + i := 1 + tick := time.Tick(time.Second) + + if err := d.cmd.Process.Signal(os.Interrupt); err != nil { + return fmt.Errorf("could not send signal: %v", err) + } +out1: + for { + select { + case err := <-d.wait: + return err + case <-time.After(15 * time.Second): + // time for stopping jobs and run onShutdown hooks + d.c.Log("timeout") + break out1 + } + } + +out2: + for { + select { + case err := <-d.wait: + return err + case <-tick: + i++ + if i > 4 { + d.c.Logf("tried to interrupt daemon for %d times, now try to kill it", i) + break out2 + } + d.c.Logf("Attempt #%d: daemon is still running with pid %d", i, d.cmd.Process.Pid) + if err := d.cmd.Process.Signal(os.Interrupt); err != nil { + return fmt.Errorf("could not send signal: %v", err) + } + } + } + + if err := d.cmd.Process.Kill(); err != nil { + d.c.Logf("Could not kill daemon: %v", err) + return err + } + + return nil +} + +// Restart will restart the daemon by first stopping it and then starting it. +func (d *Daemon) Restart(arg ...string) error { + d.Stop() + // in the case of tests running a user namespace-enabled daemon, we have resolved + // d.root to be the actual final path of the graph dir after the "uid.gid" of + // remapped root is added--we need to subtract it from the path before calling + // start or else we will continue making subdirectories rather than truly restarting + // with the same location/root: + if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { + d.root = filepath.Dir(d.root) + } + return d.Start(arg...) +} + +// LoadBusybox will load the stored busybox into a newly started daemon +func (d *Daemon) LoadBusybox() error { + bb := filepath.Join(d.folder, "busybox.tar") + if _, err := os.Stat(bb); err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("unexpected error on busybox.tar stat: %v", err) + } + // saving busybox image from main daemon + if err := exec.Command(dockerBinary, "save", "--output", bb, "busybox:latest").Run(); err != nil { + return fmt.Errorf("could not save busybox image: %v", err) + } + } + // loading busybox image to this daemon + if out, err := d.Cmd("load", "--input", bb); err != nil { + return fmt.Errorf("could not load busybox image: %s", out) + } + if err := os.Remove(bb); err != nil { + d.c.Logf("could not remove %s: %v", bb, err) + } + return nil +} + +func (d *Daemon) queryRootDir() (string, error) { + // update daemon root by asking /info endpoint (to support user + // namespaced daemon with root remapped uid.gid directory) + clientConfig, err := d.getClientConfig() + if err != nil { + return "", err + } + + client := &http.Client{ + Transport: clientConfig.transport, + } + + req, err := http.NewRequest("GET", "/info", nil) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + req.URL.Host = clientConfig.addr + req.URL.Scheme = clientConfig.scheme + + resp, err := client.Do(req) + if err != nil { + return "", err + } + body := ioutils.NewReadCloserWrapper(resp.Body, func() error { + return resp.Body.Close() + }) + + type Info struct { + DockerRootDir string + } + var b []byte + var i Info + b, err = readBody(body) + if err == nil && resp.StatusCode == 200 { + // read the docker root dir + if err = json.Unmarshal(b, &i); err == nil { + return i.DockerRootDir, nil + } + } + return "", err +} + +func (d *Daemon) sock() string { + return fmt.Sprintf("unix://%s/docker.sock", d.folder) +} + +func (d *Daemon) waitRun(contID string) error { + args := []string{"--host", d.sock()} + return waitInspectWithArgs(contID, "{{.State.Running}}", "true", 10*time.Second, args...) +} + +func (d *Daemon) getBaseDeviceSize(c *check.C) int64 { + infoCmdOutput, _, err := runCommandPipelineWithOutput( + exec.Command(dockerBinary, "-H", d.sock(), "info"), + exec.Command("grep", "Base Device Size"), + ) + c.Assert(err, checker.IsNil) + basesizeSlice := strings.Split(infoCmdOutput, ":") + basesize := strings.Trim(basesizeSlice[1], " ") + basesize = strings.Trim(basesize, "\n")[:len(basesize)-3] + basesizeFloat, err := strconv.ParseFloat(strings.Trim(basesize, " "), 64) + c.Assert(err, checker.IsNil) + basesizeBytes := int64(basesizeFloat) * (1024 * 1024 * 1024) + return basesizeBytes +} + +// Cmd will execute a docker CLI command against this Daemon. +// Example: d.Cmd("version") will run docker -H unix://path/to/unix.sock version +func (d *Daemon) Cmd(name string, arg ...string) (string, error) { + args := []string{"--host", d.sock(), name} + args = append(args, arg...) + c := exec.Command(dockerBinary, args...) + b, err := c.CombinedOutput() + return string(b), err +} + +// CmdWithArgs will execute a docker CLI command against a daemon with the +// given additional arguments +func (d *Daemon) CmdWithArgs(daemonArgs []string, name string, arg ...string) (string, error) { + args := append(daemonArgs, name) + args = append(args, arg...) + c := exec.Command(dockerBinary, args...) + b, err := c.CombinedOutput() + return string(b), err +} + +// LogFileName returns the path the the daemon's log file +func (d *Daemon) LogFileName() string { + return d.logFile.Name() +} + +func (d *Daemon) getIDByName(name string) (string, error) { + return d.inspectFieldWithError(name, "Id") +} + +func (d *Daemon) inspectFilter(name, filter string) (string, error) { + format := fmt.Sprintf("{{%s}}", filter) + out, err := d.Cmd("inspect", "-f", format, name) + if err != nil { + return "", fmt.Errorf("failed to inspect %s: %s", name, out) + } + return strings.TrimSpace(out), nil +} + +func (d *Daemon) inspectFieldWithError(name, field string) (string, error) { + return d.inspectFilter(name, fmt.Sprintf(".%s", field)) +} + +func (d *Daemon) findContainerIP(id string) string { + out, err := d.Cmd("inspect", fmt.Sprintf("--format='{{ .NetworkSettings.Networks.bridge.IPAddress }}'"), id) + if err != nil { + d.c.Log(err) + } + return strings.Trim(out, " \r\n'") +} diff --git a/components/engine/integration-cli/docker_api_containers_test.go b/components/engine/integration-cli/docker_api_containers_test.go index c20de9ad35..eb671459f2 100644 --- a/components/engine/integration-cli/docker_api_containers_test.go +++ b/components/engine/integration-cli/docker_api_containers_test.go @@ -230,7 +230,7 @@ func (s *DockerSuite) TestContainerApiStartVolumesFrom(c *check.C) { volName := "voltst" volPath := "/tmp" - dockerCmd(c, "run", "-d", "--name", volName, "-v", volPath, "busybox") + dockerCmd(c, "run", "--name", volName, "-v", volPath, "busybox") name := "TestContainerApiStartVolumesFrom" config := map[string]interface{}{ @@ -438,7 +438,7 @@ func (s *DockerSuite) TestGetStoppedContainerStats(c *check.C) { c.Assert(r.err, checker.IsNil) c.Assert(r.status, checker.Equals, http.StatusOK) case <-time.After(10 * time.Second): - c.Fatal("timeout waiting for stats reponse for stopped container") + c.Fatal("timeout waiting for stats response for stopped container") } } @@ -532,7 +532,7 @@ func (s *DockerSuite) TestContainerApiCommit(c *check.C) { c.Assert(json.Unmarshal(b, &img), checker.IsNil) cmd := inspectField(c, img.ID, "Config.Cmd") - c.Assert(cmd, checker.Equals, "{[/bin/sh -c touch /test]}", check.Commentf("got wrong Cmd from commit: %q", cmd)) + c.Assert(cmd, checker.Equals, "[/bin/sh -c touch /test]", check.Commentf("got wrong Cmd from commit: %q", cmd)) // sanity check, make sure the image is what we think it is dockerCmd(c, "run", img.ID, "ls", "/test") @@ -564,7 +564,7 @@ func (s *DockerSuite) TestContainerApiCommitWithLabelInConfig(c *check.C) { c.Assert(label2, checker.Equals, "value2") cmd := inspectField(c, img.ID, "Config.Cmd") - c.Assert(cmd, checker.Equals, "{[/bin/sh -c touch /test]}", check.Commentf("got wrong Cmd from commit: %q", cmd)) + c.Assert(cmd, checker.Equals, "[/bin/sh -c touch /test]", check.Commentf("got wrong Cmd from commit: %q", cmd)) // sanity check, make sure the image is what we think it is dockerCmd(c, "run", img.ID, "ls", "/test") @@ -643,7 +643,7 @@ func (s *DockerSuite) TestContainerApiCreateMultipleNetworksConfig(c *check.C) { c.Assert(err, checker.IsNil) c.Assert(status, checker.Equals, http.StatusBadRequest) // network name order in error message is not deterministic - c.Assert(string(b), checker.Contains, "Container cannot be connected to [") + c.Assert(string(b), checker.Contains, "Container cannot be connected to network endpoints") c.Assert(string(b), checker.Contains, "net1") c.Assert(string(b), checker.Contains, "net2") c.Assert(string(b), checker.Contains, "net3") @@ -981,7 +981,11 @@ func (s *DockerSuite) TestContainerApiStart(c *check.C) { // second call to start should give 304 status, _, err = sockRequest("POST", "/containers/"+name+"/start", conf) c.Assert(err, checker.IsNil) - c.Assert(status, checker.Equals, http.StatusNotModified) + + // TODO(tibor): figure out why this doesn't work on windows + if isLocalDaemon { + c.Assert(status, checker.Equals, http.StatusNotModified) + } } func (s *DockerSuite) TestContainerApiStop(c *check.C) { diff --git a/components/engine/integration-cli/docker_api_events_test.go b/components/engine/integration-cli/docker_api_events_test.go index 5d6e817f02..cb219fbc56 100644 --- a/components/engine/integration-cli/docker_api_events_test.go +++ b/components/engine/integration-cli/docker_api_events_test.go @@ -39,7 +39,7 @@ func (s *DockerSuite) TestEventsApiBackwardsCompatible(c *check.C) { since := daemonTime(c).Unix() ts := strconv.FormatInt(since, 10) - out, _ := dockerCmd(c, "run", "--name=foo", "-d", "busybox", "top") + out, _ := runSleepingContainer(c, "--name=foo", "-d") containerID := strings.TrimSpace(out) c.Assert(waitRun(containerID), checker.IsNil) diff --git a/components/engine/integration-cli/docker_api_exec_test.go b/components/engine/integration-cli/docker_api_exec_test.go index d3d8afd19a..f16582f40f 100644 --- a/components/engine/integration-cli/docker_api_exec_test.go +++ b/components/engine/integration-cli/docker_api_exec_test.go @@ -58,7 +58,7 @@ func (s *DockerSuite) TestExecApiCreateContainerPaused(c *check.C) { c.Assert(err, checker.IsNil) c.Assert(status, checker.Equals, http.StatusConflict) - comment := check.Commentf("Expected message when creating exec command with Container s% is paused", name) + comment := check.Commentf("Expected message when creating exec command with Container %s is paused", name) c.Assert(string(body), checker.Contains, "Container "+name+" is paused, unpause the container before exec", comment) } @@ -122,6 +122,36 @@ func (s *DockerSuite) TestExecApiStartMultipleTimesError(c *check.C) { startExec(c, execID, http.StatusConflict) } +// #20638 +func (s *DockerSuite) TestExecApiStartWithDetach(c *check.C) { + name := "foo" + runSleepingContainer(c, "-d", "-t", "--name", name) + data := map[string]interface{}{ + "cmd": []string{"true"}, + "AttachStdin": true, + } + _, b, err := sockRequest("POST", fmt.Sprintf("/containers/%s/exec", name), data) + c.Assert(err, checker.IsNil, check.Commentf(string(b))) + + createResp := struct { + ID string `json:"Id"` + }{} + c.Assert(json.Unmarshal(b, &createResp), checker.IsNil, check.Commentf(string(b))) + + _, body, err := sockRequestRaw("POST", fmt.Sprintf("/exec/%s/start", createResp.ID), strings.NewReader(`{"Detach": true}`), "application/json") + c.Assert(err, checker.IsNil) + + b, err = readBody(body) + comment := check.Commentf("response body: %s", b) + c.Assert(err, checker.IsNil, comment) + + resp, _, err := sockRequestRaw("GET", "/_ping", nil, "") + c.Assert(err, checker.IsNil) + if resp.StatusCode != http.StatusOK { + c.Fatal("daemon is down, it should alive") + } +} + func createExec(c *check.C, name string) string { _, b, err := sockRequest("POST", fmt.Sprintf("/containers/%s/exec", name), map[string]interface{}{"Cmd": []string{"true"}}) c.Assert(err, checker.IsNil, check.Commentf(string(b))) diff --git a/components/engine/integration-cli/docker_api_logs_test.go b/components/engine/integration-cli/docker_api_logs_test.go index 7c664565ed..2ff27f8ecc 100644 --- a/components/engine/integration-cli/docker_api_logs_test.go +++ b/components/engine/integration-cli/docker_api_logs_test.go @@ -13,7 +13,6 @@ import ( ) func (s *DockerSuite) TestLogsApiWithStdout(c *check.C) { - testRequires(c, DaemonIsLinux) out, _ := dockerCmd(c, "run", "-d", "-t", "busybox", "/bin/sh", "-c", "while true; do echo hello; sleep 1; done") id := strings.TrimSpace(out) c.Assert(waitRun(id), checker.IsNil) @@ -47,13 +46,12 @@ func (s *DockerSuite) TestLogsApiWithStdout(c *check.C) { if !strings.HasSuffix(l.out, "hello") { c.Fatalf("expected log output to container 'hello', but it does not") } - case <-time.After(2 * time.Second): + case <-time.After(20 * time.Second): c.Fatal("timeout waiting for logs to exit") } } func (s *DockerSuite) TestLogsApiNoStdoutNorStderr(c *check.C) { - testRequires(c, DaemonIsLinux) name := "logs_test" dockerCmd(c, "run", "-d", "-t", "--name", name, "busybox", "/bin/sh") @@ -69,7 +67,6 @@ func (s *DockerSuite) TestLogsApiNoStdoutNorStderr(c *check.C) { // Regression test for #12704 func (s *DockerSuite) TestLogsApiFollowEmptyOutput(c *check.C) { - testRequires(c, DaemonIsLinux) name := "logs_test" t0 := time.Now() dockerCmd(c, "run", "-d", "-t", "--name", name, "busybox", "sleep", "10") @@ -79,12 +76,12 @@ func (s *DockerSuite) TestLogsApiFollowEmptyOutput(c *check.C) { c.Assert(err, checker.IsNil) body.Close() elapsed := t1.Sub(t0).Seconds() - if elapsed > 5.0 { + if elapsed > 20.0 { c.Fatalf("HTTP response was not immediate (elapsed %.1fs)", elapsed) } } -func (s *DockerSuite) TestLogsAPIContainerNotFound(c *check.C) { +func (s *DockerSuite) TestLogsApiContainerNotFound(c *check.C) { name := "nonExistentContainer" resp, _, err := sockRequestRaw("GET", fmt.Sprintf("/containers/%s/logs?follow=1&stdout=1&stderr=1&tail=all", name), bytes.NewBuffer(nil), "") c.Assert(err, checker.IsNil) diff --git a/components/engine/integration-cli/docker_api_network_test.go b/components/engine/integration-cli/docker_api_network_test.go index b5748e1662..9e8056f589 100644 --- a/components/engine/integration-cli/docker_api_network_test.go +++ b/components/engine/integration-cli/docker_api_network_test.go @@ -84,6 +84,8 @@ func (s *DockerSuite) TestApiNetworkInspect(c *check.C) { nr = getNetworkResource(c, nr.ID) c.Assert(nr.Driver, checker.Equals, "bridge") c.Assert(nr.Scope, checker.Equals, "local") + c.Assert(nr.Internal, checker.Equals, false) + c.Assert(nr.EnableIPv6, checker.Equals, false) c.Assert(nr.IPAM.Driver, checker.Equals, "default") c.Assert(len(nr.Containers), checker.Equals, 1) c.Assert(nr.Containers[containerID], checker.NotNil) diff --git a/components/engine/integration-cli/docker_api_test.go b/components/engine/integration-cli/docker_api_test.go index df2ce440df..a7257779fb 100644 --- a/components/engine/integration-cli/docker_api_test.go +++ b/components/engine/integration-cli/docker_api_test.go @@ -5,7 +5,6 @@ import ( "net/http" "net/http/httptest" "net/http/httputil" - "os" "os/exec" "strconv" "strings" @@ -91,7 +90,7 @@ func (s *DockerSuite) TestApiDockerApiVersion(c *check.C) { // Test using the env var first cmd := exec.Command(dockerBinary, "-H="+server.URL[7:], "version") - cmd.Env = append([]string{"DOCKER_API_VERSION=xxx"}, os.Environ()...) + cmd.Env = appendBaseEnv(false, "DOCKER_API_VERSION=xxx") out, _, _ := runCommandWithOutput(cmd) c.Assert(svrVersion, check.Equals, "/vxxx/version") diff --git a/components/engine/integration-cli/docker_api_volumes_test.go b/components/engine/integration-cli/docker_api_volumes_test.go index 9f698eb495..732271d02d 100644 --- a/components/engine/integration-cli/docker_api_volumes_test.go +++ b/components/engine/integration-cli/docker_api_volumes_test.go @@ -12,7 +12,7 @@ import ( func (s *DockerSuite) TestVolumesApiList(c *check.C) { prefix, _ := getPrefixAndSlashFromDaemonPlatform() - dockerCmd(c, "run", "-d", "-v", prefix+"/foo", "busybox") + dockerCmd(c, "run", "-v", prefix+"/foo", "busybox") status, b, err := sockRequest("GET", "/volumes", nil) c.Assert(err, checker.IsNil) @@ -41,7 +41,7 @@ func (s *DockerSuite) TestVolumesApiCreate(c *check.C) { func (s *DockerSuite) TestVolumesApiRemove(c *check.C) { prefix, _ := getPrefixAndSlashFromDaemonPlatform() - dockerCmd(c, "run", "-d", "-v", prefix+"/foo", "--name=test", "busybox") + dockerCmd(c, "run", "-v", prefix+"/foo", "--name=test", "busybox") status, b, err := sockRequest("GET", "/volumes", nil) c.Assert(err, checker.IsNil) diff --git a/components/engine/integration-cli/docker_cli_attach_unix_test.go b/components/engine/integration-cli/docker_cli_attach_unix_test.go index 6fd7616c1c..7af761d7a3 100644 --- a/components/engine/integration-cli/docker_cli_attach_unix_test.go +++ b/components/engine/integration-cli/docker_cli_attach_unix_test.go @@ -16,6 +16,7 @@ import ( // #9860 Make sure attach ends when container ends (with no errors) func (s *DockerSuite) TestAttachClosedOnContainerStop(c *check.C) { + testRequires(c, SameHostDaemon) out, _ := dockerCmd(c, "run", "-dti", "busybox", "/bin/sh", "-c", `trap 'exit 0' SIGTERM; while true; do sleep 1; done`) diff --git a/components/engine/integration-cli/docker_cli_authz_unix_test.go b/components/engine/integration-cli/docker_cli_authz_unix_test.go index 4ab83b9179..0a208d75e1 100644 --- a/components/engine/integration-cli/docker_cli_authz_unix_test.go +++ b/components/engine/integration-cli/docker_cli_authz_unix_test.go @@ -13,13 +13,14 @@ import ( "bufio" "bytes" + "os/exec" + "strconv" + "time" + "github.com/docker/docker/pkg/authorization" "github.com/docker/docker/pkg/integration/checker" "github.com/docker/docker/pkg/plugins" "github.com/go-check/check" - "os/exec" - "strconv" - "time" ) const ( @@ -29,6 +30,10 @@ const ( containerListAPI = "/containers/json" ) +var ( + alwaysAllowed = []string{"/_ping", "/info"} +) + func init() { check.Suite(&DockerAuthzSuite{ ds: &DockerSuite{}, @@ -73,12 +78,6 @@ func (s *DockerAuthzSuite) SetUpSuite(c *check.C) { }) mux.HandleFunc("/AuthZPlugin.AuthZReq", func(w http.ResponseWriter, r *http.Request) { - if s.ctrl.reqRes.Err != "" { - w.WriteHeader(http.StatusInternalServerError) - } - b, err := json.Marshal(s.ctrl.reqRes) - c.Assert(err, check.IsNil) - w.Write(b) defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) c.Assert(err, check.IsNil) @@ -95,16 +94,20 @@ func (s *DockerAuthzSuite) SetUpSuite(c *check.C) { } s.ctrl.requestsURIs = append(s.ctrl.requestsURIs, authReq.RequestURI) + + reqRes := s.ctrl.reqRes + if isAllowed(authReq.RequestURI) { + reqRes = authorization.Response{Allow: true} + } + if reqRes.Err != "" { + w.WriteHeader(http.StatusInternalServerError) + } + b, err := json.Marshal(reqRes) + c.Assert(err, check.IsNil) + w.Write(b) }) mux.HandleFunc("/AuthZPlugin.AuthZRes", func(w http.ResponseWriter, r *http.Request) { - if s.ctrl.resRes.Err != "" { - w.WriteHeader(http.StatusInternalServerError) - } - b, err := json.Marshal(s.ctrl.resRes) - c.Assert(err, check.IsNil) - w.Write(b) - defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) c.Assert(err, check.IsNil) @@ -119,6 +122,16 @@ func (s *DockerAuthzSuite) SetUpSuite(c *check.C) { if strings.HasSuffix(authReq.RequestURI, containerListAPI) { s.ctrl.psResponseCnt++ } + resRes := s.ctrl.resRes + if isAllowed(authReq.RequestURI) { + resRes = authorization.Response{Allow: true} + } + if resRes.Err != "" { + w.WriteHeader(http.StatusInternalServerError) + } + b, err := json.Marshal(resRes) + c.Assert(err, check.IsNil) + w.Write(b) }) err := os.MkdirAll("/etc/docker/plugins", 0755) @@ -129,6 +142,16 @@ func (s *DockerAuthzSuite) SetUpSuite(c *check.C) { c.Assert(err, checker.IsNil) } +// check for always allowed endpoints to not inhibit test framework functions +func isAllowed(reqURI string) bool { + for _, endpoint := range alwaysAllowed { + if strings.HasSuffix(reqURI, endpoint) { + return true + } + } + return false +} + // assertAuthHeaders validates authentication headers are removed func assertAuthHeaders(c *check.C, headers map[string]string) error { for k := range headers { @@ -170,13 +193,10 @@ func (s *DockerAuthzSuite) TearDownSuite(c *check.C) { func (s *DockerAuthzSuite) TestAuthZPluginAllowRequest(c *check.C) { // start the daemon and load busybox, --net=none build fails otherwise // cause it needs to pull busybox - c.Assert(s.d.StartWithBusybox(), check.IsNil) - // restart the daemon and enable the plugin, otherwise busybox loading - // is blocked by the plugin itself - c.Assert(s.d.Restart("--authorization-plugin="+testAuthZPlugin), check.IsNil) - + c.Assert(s.d.Start("--authorization-plugin="+testAuthZPlugin), check.IsNil) s.ctrl.reqRes.Allow = true s.ctrl.resRes.Allow = true + c.Assert(s.d.LoadBusybox(), check.IsNil) // Ensure command successful out, err := s.d.Cmd("run", "-d", "busybox", "top") @@ -226,15 +246,15 @@ func (s *DockerAuthzSuite) TestAuthZPluginDenyResponse(c *check.C) { c.Assert(res, check.Equals, fmt.Sprintf("Error response from daemon: authorization denied by plugin %s: %s\n", testAuthZPlugin, unauthorizedMessage)) } -// TestAuthZPluginAllowEventStream verifies event stream propogates correctly after request pass through by the authorization plugin +// TestAuthZPluginAllowEventStream verifies event stream propagates correctly after request pass through by the authorization plugin func (s *DockerAuthzSuite) TestAuthZPluginAllowEventStream(c *check.C) { testRequires(c, DaemonIsLinux) - // Start the authorization plugin - err := s.d.Start("--authorization-plugin=" + testAuthZPlugin) - c.Assert(err, check.IsNil) + // start the daemon and load busybox to avoid pulling busybox from Docker Hub + c.Assert(s.d.Start("--authorization-plugin="+testAuthZPlugin), check.IsNil) s.ctrl.reqRes.Allow = true s.ctrl.resRes.Allow = true + c.Assert(s.d.LoadBusybox(), check.IsNil) startTime := strconv.FormatInt(daemonTime(c).Unix(), 10) // Add another command to to enable event pipelining @@ -256,16 +276,14 @@ func (s *DockerAuthzSuite) TestAuthZPluginAllowEventStream(c *check.C) { defer observer.Stop() // Create a container and wait for the creation events - _, err = s.d.Cmd("pull", "busybox") - c.Assert(err, check.IsNil) out, err := s.d.Cmd("run", "-d", "busybox", "top") - c.Assert(err, check.IsNil) - + c.Assert(err, check.IsNil, check.Commentf(out)) containerID := strings.TrimSpace(out) + c.Assert(s.d.waitRun(containerID), checker.IsNil) events := map[string]chan bool{ - "create": make(chan bool), - "start": make(chan bool), + "create": make(chan bool, 1), + "start": make(chan bool, 1), } matcher := matchEventLine(containerID, "container", events) @@ -276,7 +294,7 @@ func (s *DockerAuthzSuite) TestAuthZPluginAllowEventStream(c *check.C) { for event, eventChannel := range events { select { - case <-time.After(5 * time.Second): + case <-time.After(30 * time.Second): // Fail the test observer.CheckEventError(c, containerID, event, matcher) c.FailNow() diff --git a/components/engine/integration-cli/docker_cli_build_test.go b/components/engine/integration-cli/docker_cli_build_test.go index 6be7195b05..4610853f0c 100644 --- a/components/engine/integration-cli/docker_cli_build_test.go +++ b/components/engine/integration-cli/docker_cli_build_test.go @@ -824,6 +824,37 @@ RUN [ $(ls -l /exists/exists_file | awk '{print $3":"$4}') = 'dockerio:dockerio' } } +func (s *DockerSuite) TestBuildCopyToNewParentDirectory(c *check.C) { + testRequires(c, DaemonIsLinux) // Linux specific test + name := "testcopytonewdir" + ctx, err := fakeContext(`FROM busybox +COPY test_dir /new_dir +RUN [ $(ls -l / | grep new_dir | awk '{print $3":"$4}') = 'root:root' ] +RUN ls -l /new_dir`, + map[string]string{ + "test_dir/test_file": "test file", + }) + if err != nil { + c.Fatal(err) + } + defer ctx.Close() + + if _, err := buildImageFromContext(name, ctx, true); err != nil { + c.Fatal(err) + } +} + +func (s *DockerSuite) TestBuildWorkdirIsContainerRoot(c *check.C) { + testRequires(c, DaemonIsLinux) // Linux specific test + name := "testworkdirownership" + if _, err := buildImage(name, `FROM busybox +WORKDIR /new_dir +RUN ls -l / +RUN [ $(ls -l / | grep new_dir | awk '{print $3":"$4}') = 'root:root' ]`, true); err != nil { + c.Fatal(err) + } +} + func (s *DockerSuite) TestBuildAddMultipleFilesToFile(c *check.C) { name := "testaddmultiplefilestofile" @@ -1818,13 +1849,15 @@ func (s *DockerSuite) TestBuildWithInaccessibleFilesInContext(c *check.C) { } func (s *DockerSuite) TestBuildForceRm(c *check.C) { - testRequires(c, DaemonIsLinux) containerCountBefore, err := getContainerCount() if err != nil { c.Fatalf("failed to get the container count: %s", err) } name := "testbuildforcerm" - ctx, err := fakeContext("FROM scratch\nRUN true\nRUN thiswillfail", nil) + + ctx, err := fakeContext(`FROM `+minimalBaseImage()+` + RUN true + RUN thiswillfail`, nil) if err != nil { c.Fatal(err) } @@ -1844,9 +1877,11 @@ func (s *DockerSuite) TestBuildForceRm(c *check.C) { } func (s *DockerSuite) TestBuildRm(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildrm" - ctx, err := fakeContext("FROM scratch\nADD foo /\nADD foo /", map[string]string{"foo": "bar"}) + + ctx, err := fakeContext(`FROM `+minimalBaseImage()+` + ADD foo / + ADD foo /`, map[string]string{"foo": "bar"}) if err != nil { c.Fatal(err) } @@ -1924,7 +1959,7 @@ func (s *DockerSuite) TestBuildRm(c *check.C) { } func (s *DockerSuite) TestBuildWithVolumes(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Invalid volume paths on Windows var ( result map[string]map[string]struct{} name = "testbuildvolumes" @@ -1968,11 +2003,11 @@ func (s *DockerSuite) TestBuildWithVolumes(c *check.C) { } func (s *DockerSuite) TestBuildMaintainer(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildmaintainer" + expected := "dockerio" _, err := buildImage(name, - `FROM scratch + `FROM `+minimalBaseImage()+` MAINTAINER dockerio`, true) if err != nil { @@ -2004,32 +2039,58 @@ func (s *DockerSuite) TestBuildUser(c *check.C) { } func (s *DockerSuite) TestBuildRelativeWorkdir(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildrelativeworkdir" - expected := "/test2/test3" + + var ( + expected1 string + expected2 string + expected3 string + expected4 string + expectedFinal string + ) + if daemonPlatform == "windows" { + expected1 = `C:/Windows/system32` + expected2 = `C:/test1` + expected3 = `C:/test2` + expected4 = `C:/test2/test3` + expectedFinal = `\test2\test3` + } else { + expected1 = `/` + expected2 = `/test1` + expected3 = `/test2` + expected4 = `/test2/test3` + expectedFinal = `/test2/test3` + } + _, err := buildImage(name, `FROM busybox - RUN [ "$PWD" = '/' ] + RUN sh -c "[ "$PWD" = '`+expected1+`' ]" WORKDIR test1 - RUN [ "$PWD" = '/test1' ] + RUN sh -c "[ "$PWD" = '`+expected2+`' ]" WORKDIR /test2 - RUN [ "$PWD" = '/test2' ] + RUN sh -c "[ "$PWD" = '`+expected3+`' ]" WORKDIR test3 - RUN [ "$PWD" = '/test2/test3' ]`, + RUN sh -c "[ "$PWD" = '`+expected4+`' ]"`, true) if err != nil { c.Fatal(err) } res := inspectField(c, name, "Config.WorkingDir") - if res != expected { - c.Fatalf("Workdir %s, expected %s", res, expected) + if res != expectedFinal { + c.Fatalf("Workdir %s, expected %s", res, expectedFinal) } } func (s *DockerSuite) TestBuildWorkdirWithEnvVariables(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildworkdirwithenvvariables" - expected := "/test1/test2" + + var expected string + if daemonPlatform == "windows" { + expected = `\test1\test2` + } else { + expected = `/test1/test2` + } + _, err := buildImage(name, `FROM busybox ENV DIRPATH /test1 @@ -2049,30 +2110,37 @@ func (s *DockerSuite) TestBuildWorkdirWithEnvVariables(c *check.C) { func (s *DockerSuite) TestBuildRelativeCopy(c *check.C) { // cat /test1/test2/foo gets permission denied for the user testRequires(c, NotUserNamespace) - testRequires(c, DaemonIsLinux) + + var expected string + if daemonPlatform == "windows" { + expected = `C:/test1/test2` + } else { + expected = `/test1/test2` + } + name := "testbuildrelativecopy" dockerfile := ` FROM busybox WORKDIR /test1 WORKDIR test2 - RUN [ "$PWD" = '/test1/test2' ] + RUN sh -c "[ "$PWD" = '` + expected + `' ]" COPY foo ./ - RUN [ "$(cat /test1/test2/foo)" = 'hello' ] + RUN sh -c "[ $(cat /test1/test2/foo) = 'hello' ]" ADD foo ./bar/baz - RUN [ "$(cat /test1/test2/bar/baz)" = 'hello' ] + RUN sh -c "[ $(cat /test1/test2/bar/baz) = 'hello' ]" COPY foo ./bar/baz2 - RUN [ "$(cat /test1/test2/bar/baz2)" = 'hello' ] + RUN sh -c "[ $(cat /test1/test2/bar/baz2) = 'hello' ]" WORKDIR .. COPY foo ./ - RUN [ "$(cat /test1/foo)" = 'hello' ] + RUN sh -c "[ $(cat /test1/foo) = 'hello' ]" COPY foo /test3/ - RUN [ "$(cat /test3/foo)" = 'hello' ] + RUN sh -c "[ $(cat /test3/foo) = 'hello' ]" WORKDIR /test4 COPY . . - RUN [ "$(cat /test4/foo)" = 'hello' ] + RUN sh -c "[ $(cat /test4/foo) = 'hello' ]" WORKDIR /test5/test6 COPY foo ../ - RUN [ "$(cat /test5/foo)" = 'hello' ] + RUN sh -c "[ $(cat /test5/foo) = 'hello' ]" ` ctx, err := fakeContext(dockerfile, map[string]string{ "foo": "hello", @@ -2088,7 +2156,7 @@ func (s *DockerSuite) TestBuildRelativeCopy(c *check.C) { } func (s *DockerSuite) TestBuildEnv(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // ENV expansion is different in Windows name := "testbuildenv" expected := "[PATH=/test:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin PORT=2375]" _, err := buildImage(name, @@ -2107,7 +2175,7 @@ func (s *DockerSuite) TestBuildEnv(c *check.C) { } func (s *DockerSuite) TestBuildPATH(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // ENV expansion is different in Windows defPath := "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" @@ -2191,11 +2259,11 @@ func (s *DockerSuite) TestBuildContextCleanupFailedBuild(c *check.C) { } func (s *DockerSuite) TestBuildCmd(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildcmd" - expected := "{[/bin/echo Hello World]}" + + expected := "[/bin/echo Hello World]" _, err := buildImage(name, - `FROM scratch + `FROM `+minimalBaseImage()+` CMD ["/bin/echo", "Hello World"]`, true) if err != nil { @@ -2208,7 +2276,7 @@ func (s *DockerSuite) TestBuildCmd(c *check.C) { } func (s *DockerSuite) TestBuildExpose(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Expose not implemented on Windows name := "testbuildexpose" expected := "map[2375/tcp:{}]" _, err := buildImage(name, @@ -2225,7 +2293,7 @@ func (s *DockerSuite) TestBuildExpose(c *check.C) { } func (s *DockerSuite) TestBuildExposeMorePorts(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Expose not implemented on Windows // start building docker file with a large number of ports portList := make([]string, 50) line := make([]string, 100) @@ -2277,7 +2345,7 @@ func (s *DockerSuite) TestBuildExposeMorePorts(c *check.C) { } func (s *DockerSuite) TestBuildExposeOrder(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Expose not implemented on Windows buildID := func(name, exposed string) string { _, err := buildImage(name, fmt.Sprintf(`FROM scratch EXPOSE %s`, exposed), true) @@ -2296,7 +2364,7 @@ func (s *DockerSuite) TestBuildExposeOrder(c *check.C) { } func (s *DockerSuite) TestBuildExposeUpperCaseProto(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Expose not implemented on Windows name := "testbuildexposeuppercaseproto" expected := "map[5678/udp:{}]" _, err := buildImage(name, @@ -2313,7 +2381,6 @@ func (s *DockerSuite) TestBuildExposeUpperCaseProto(c *check.C) { } func (s *DockerSuite) TestBuildEmptyEntrypointInheritance(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildentrypointinheritance" name2 := "testbuildentrypointinheritance2" @@ -2326,7 +2393,7 @@ func (s *DockerSuite) TestBuildEmptyEntrypointInheritance(c *check.C) { } res := inspectField(c, name, "Config.Entrypoint") - expected := "{[/bin/echo]}" + expected := "[/bin/echo]" if res != expected { c.Fatalf("Entrypoint %s, expected %s", res, expected) } @@ -2340,7 +2407,7 @@ func (s *DockerSuite) TestBuildEmptyEntrypointInheritance(c *check.C) { } res = inspectField(c, name2, "Config.Entrypoint") - expected = "{[]}" + expected = "[]" if res != expected { c.Fatalf("Entrypoint %s, expected %s", res, expected) @@ -2349,9 +2416,8 @@ func (s *DockerSuite) TestBuildEmptyEntrypointInheritance(c *check.C) { } func (s *DockerSuite) TestBuildEmptyEntrypoint(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildentrypoint" - expected := "{[]}" + expected := "[]" _, err := buildImage(name, `FROM busybox @@ -2368,11 +2434,11 @@ func (s *DockerSuite) TestBuildEmptyEntrypoint(c *check.C) { } func (s *DockerSuite) TestBuildEntrypoint(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildentrypoint" - expected := "{[/bin/echo]}" + + expected := "[/bin/echo]" _, err := buildImage(name, - `FROM scratch + `FROM `+minimalBaseImage()+` ENTRYPOINT ["/bin/echo"]`, true) if err != nil { @@ -2387,7 +2453,6 @@ func (s *DockerSuite) TestBuildEntrypoint(c *check.C) { // #6445 ensure ONBUILD triggers aren't committed to grandchildren func (s *DockerSuite) TestBuildOnBuildLimitedInheritence(c *check.C) { - testRequires(c, DaemonIsLinux) var ( out2, out3 string ) @@ -2456,7 +2521,7 @@ func (s *DockerSuite) TestBuildOnBuildLimitedInheritence(c *check.C) { } func (s *DockerSuite) TestBuildWithCache(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Expose not implemented on Windows name := "testbuildwithcache" id1, err := buildImage(name, `FROM scratch @@ -2482,7 +2547,7 @@ func (s *DockerSuite) TestBuildWithCache(c *check.C) { } func (s *DockerSuite) TestBuildWithoutCache(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Expose not implemented on Windows name := "testbuildwithoutcache" name2 := "testbuildwithoutcache2" id1, err := buildImage(name, @@ -2510,7 +2575,6 @@ func (s *DockerSuite) TestBuildWithoutCache(c *check.C) { } func (s *DockerSuite) TestBuildConditionalCache(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildconditionalcache" dockerfile := ` @@ -2553,14 +2617,13 @@ func (s *DockerSuite) TestBuildConditionalCache(c *check.C) { func (s *DockerSuite) TestBuildAddLocalFileWithCache(c *check.C) { // local files are not owned by the correct user testRequires(c, NotUserNamespace) - testRequires(c, DaemonIsLinux) name := "testbuildaddlocalfilewithcache" name2 := "testbuildaddlocalfilewithcache2" dockerfile := ` FROM busybox MAINTAINER dockerio ADD foo /usr/lib/bla/bar - RUN [ "$(cat /usr/lib/bla/bar)" = "hello" ]` + RUN sh -c "[ $(cat /usr/lib/bla/bar) = "hello" ]"` ctx, err := fakeContext(dockerfile, map[string]string{ "foo": "hello", }) @@ -2582,14 +2645,13 @@ func (s *DockerSuite) TestBuildAddLocalFileWithCache(c *check.C) { } func (s *DockerSuite) TestBuildAddMultipleLocalFileWithCache(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildaddmultiplelocalfilewithcache" name2 := "testbuildaddmultiplelocalfilewithcache2" dockerfile := ` FROM busybox MAINTAINER dockerio ADD foo Dockerfile /usr/lib/bla/ - RUN [ "$(cat /usr/lib/bla/foo)" = "hello" ]` + RUN sh -c "[ $(cat /usr/lib/bla/foo) = "hello" ]"` ctx, err := fakeContext(dockerfile, map[string]string{ "foo": "hello", }) @@ -2613,14 +2675,13 @@ func (s *DockerSuite) TestBuildAddMultipleLocalFileWithCache(c *check.C) { func (s *DockerSuite) TestBuildAddLocalFileWithoutCache(c *check.C) { // local files are not owned by the correct user testRequires(c, NotUserNamespace) - testRequires(c, DaemonIsLinux) name := "testbuildaddlocalfilewithoutcache" name2 := "testbuildaddlocalfilewithoutcache2" dockerfile := ` FROM busybox MAINTAINER dockerio ADD foo /usr/lib/bla/bar - RUN [ "$(cat /usr/lib/bla/bar)" = "hello" ]` + RUN sh -c "[ $(cat /usr/lib/bla/bar) = "hello" ]"` ctx, err := fakeContext(dockerfile, map[string]string{ "foo": "hello", }) @@ -2642,11 +2703,11 @@ func (s *DockerSuite) TestBuildAddLocalFileWithoutCache(c *check.C) { } func (s *DockerSuite) TestBuildCopyDirButNotFile(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildcopydirbutnotfile" name2 := "testbuildcopydirbutnotfile2" + dockerfile := ` - FROM scratch + FROM ` + minimalBaseImage() + ` COPY dir /tmp/` ctx, err := fakeContext(dockerfile, map[string]string{ "dir/foo": "hello", @@ -3057,7 +3118,7 @@ func (s *DockerSuite) TestBuildEntrypointRunCleanup(c *check.C) { } res := inspectField(c, name, "Config.Cmd") // Cmd must be cleaned up - if res != "" { + if res != "[]" { c.Fatalf("Cmd %s, expected nil", res) } } @@ -3134,7 +3195,7 @@ func (s *DockerSuite) TestBuildInheritance(c *check.C) { } res := inspectField(c, name, "Config.Entrypoint") - if expected := "{[/bin/echo]}"; res != expected { + if expected := "[/bin/echo]"; res != expected { c.Fatalf("Entrypoint %s, expected %s", res, expected) } ports2 := inspectField(c, name, "Config.ExposedPorts") @@ -3644,30 +3705,25 @@ func (s *DockerSuite) TestBuildDockerignoringWildDirs(c *check.C) { FROM busybox COPY . / #RUN sh -c "[[ -e /.dockerignore ]]" - RUN sh -c "[[ -e /Dockerfile ]]" - - RUN sh -c "[[ ! -e /file0 ]]" - RUN sh -c "[[ ! -e /dir1/file0 ]]" - RUN sh -c "[[ ! -e /dir2/file0 ]]" - - RUN sh -c "[[ ! -e /file1 ]]" - RUN sh -c "[[ ! -e /dir1/file1 ]]" - RUN sh -c "[[ ! -e /dir1/dir2/file1 ]]" - - RUN sh -c "[[ ! -e /dir1/file2 ]]" - RUN sh -c "[[ -e /dir1/dir2/file2 ]]" - - RUN sh -c "[[ ! -e /dir1/dir2/file4 ]]" - RUN sh -c "[[ ! -e /dir1/dir2/file5 ]]" - RUN sh -c "[[ ! -e /dir1/dir2/file6 ]]" - RUN sh -c "[[ ! -e /dir1/dir3/file7 ]]" - RUN sh -c "[[ ! -e /dir1/dir3/file8 ]]" - RUN sh -c "[[ -e /dir1/dir3 ]]" - RUN sh -c "[[ -e /dir1/dir4 ]]" - - RUN sh -c "[[ ! -e 'dir1/dir5/fileAA' ]]" - RUN sh -c "[[ -e 'dir1/dir5/fileAB' ]]" - RUN sh -c "[[ -e 'dir1/dir5/fileB' ]]" # "." in pattern means nothing + RUN sh -c "[[ -e /Dockerfile ]] && \ + [[ ! -e /file0 ]] && \ + [[ ! -e /dir1/file0 ]] && \ + [[ ! -e /dir2/file0 ]] && \ + [[ ! -e /file1 ]] && \ + [[ ! -e /dir1/file1 ]] && \ + [[ ! -e /dir1/dir2/file1 ]] && \ + [[ ! -e /dir1/file2 ]] && \ + [[ -e /dir1/dir2/file2 ]] && \ + [[ ! -e /dir1/dir2/file4 ]] && \ + [[ ! -e /dir1/dir2/file5 ]] && \ + [[ ! -e /dir1/dir2/file6 ]] && \ + [[ ! -e /dir1/dir3/file7 ]] && \ + [[ ! -e /dir1/dir3/file8 ]] && \ + [[ -e /dir1/dir3 ]] && \ + [[ -e /dir1/dir4 ]] && \ + [[ ! -e 'dir1/dir5/fileAA' ]] && \ + [[ -e 'dir1/dir5/fileAB' ]] && \ + [[ -e 'dir1/dir5/fileB' ]]" # "." in pattern means nothing RUN echo all done!` @@ -3781,21 +3837,18 @@ USER root RUN [ "$(id -G):$(id -Gn)" = '0 10:root wheel' ] # Setup dockerio user and group -RUN echo 'dockerio:x:1001:1001::/bin:/bin/false' >> /etc/passwd -RUN echo 'dockerio:x:1001:' >> /etc/group +RUN echo 'dockerio:x:1001:1001::/bin:/bin/false' >> /etc/passwd && \ + echo 'dockerio:x:1001:' >> /etc/group # Make sure we can switch to our user and all the information is exactly as we expect it to be USER dockerio -RUN id -G -RUN id -Gn RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001:dockerio' ] # Switch back to root and double check that worked exactly as we might expect it to USER root -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '0:0/root:root/0 10:root wheel' ] - -# Add a "supplementary" group for our dockerio user -RUN echo 'supplementary:x:1002:dockerio' >> /etc/group +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '0:0/root:root/0 10:root wheel' ] && \ + # Add a "supplementary" group for our dockerio user \ + echo 'supplementary:x:1002:dockerio' >> /etc/group # ... and then go verify that we get it like we expect USER dockerio @@ -3971,7 +4024,6 @@ RUN [ "$(cat /testfile)" = 'test!' ]` func (s *DockerSuite) TestBuildAddTar(c *check.C) { // /test/foo is not owned by the correct user testRequires(c, NotUserNamespace) - testRequires(c, DaemonIsLinux) name := "testbuildaddtar" ctx := func() *FakeContext { @@ -3985,7 +4037,7 @@ ADD test.tar /unlikely-to-exist RUN cat /unlikely-to-exist/test/foo | grep Hi ADD test.tar /unlikely-to-exist-trailing-slash/ RUN cat /unlikely-to-exist-trailing-slash/test/foo | grep Hi -RUN mkdir /existing-directory +RUN sh -c "mkdir /existing-directory" #sh -c is needed on Windows to use the correct mkdir ADD test.tar /existing-directory RUN cat /existing-directory/test/foo | grep Hi ADD test.tar /existing-directory-trailing-slash/ @@ -4027,7 +4079,6 @@ RUN cat /existing-directory-trailing-slash/test/foo | grep Hi` } func (s *DockerSuite) TestBuildAddBrokenTar(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildaddbrokentar" ctx := func() *FakeContext { @@ -4079,7 +4130,6 @@ ADD test.tar /` } func (s *DockerSuite) TestBuildAddNonTar(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildaddnontar" // Should not try to extract test.tar @@ -4215,7 +4265,6 @@ func (s *DockerSuite) TestBuildAddTarXzGz(c *check.C) { } func (s *DockerSuite) TestBuildFromGIT(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildfromgit" git, err := newFakeGit("repo", map[string]string{ "Dockerfile": `FROM busybox @@ -4240,7 +4289,6 @@ func (s *DockerSuite) TestBuildFromGIT(c *check.C) { } func (s *DockerSuite) TestBuildFromGITWithContext(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildfromgit" git, err := newFakeGit("repo", map[string]string{ "docker/Dockerfile": `FROM busybox @@ -4266,7 +4314,6 @@ func (s *DockerSuite) TestBuildFromGITWithContext(c *check.C) { } func (s *DockerSuite) TestBuildFromGITwithF(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildfromgitwithf" git, err := newFakeGit("repo", map[string]string{ "myApp/myDockerfile": `FROM busybox @@ -4328,10 +4375,9 @@ func (s *DockerSuite) TestBuildFromRemoteTarball(c *check.C) { } func (s *DockerSuite) TestBuildCleanupCmdOnEntrypoint(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildcmdcleanuponentrypoint" if _, err := buildImage(name, - `FROM scratch + `FROM `+minimalBaseImage()+` CMD ["test"] ENTRYPOINT ["echo"]`, true); err != nil { @@ -4344,21 +4390,20 @@ func (s *DockerSuite) TestBuildCleanupCmdOnEntrypoint(c *check.C) { c.Fatal(err) } res := inspectField(c, name, "Config.Cmd") - if res != "" { + if res != "[]" { c.Fatalf("Cmd %s, expected nil", res) } res = inspectField(c, name, "Config.Entrypoint") - if expected := "{[cat]}"; res != expected { + if expected := "[cat]"; res != expected { c.Fatalf("Entrypoint %s, expected %s", res, expected) } } func (s *DockerSuite) TestBuildClearCmd(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildclearcmd" _, err := buildImage(name, - `From scratch + `From `+minimalBaseImage()+` ENTRYPOINT ["/bin/bash"] CMD []`, true) @@ -4372,9 +4417,8 @@ func (s *DockerSuite) TestBuildClearCmd(c *check.C) { } func (s *DockerSuite) TestBuildEmptyCmd(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildemptycmd" - if _, err := buildImage(name, "FROM scratch\nMAINTAINER quux\n", true); err != nil { + if _, err := buildImage(name, "FROM "+minimalBaseImage()+"\nMAINTAINER quux\n", true); err != nil { c.Fatal(err) } res := inspectFieldJSON(c, name, "Config.Cmd") @@ -4384,7 +4428,6 @@ func (s *DockerSuite) TestBuildEmptyCmd(c *check.C) { } func (s *DockerSuite) TestBuildOnBuildOutput(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildonbuildparent" if _, err := buildImage(name, "FROM busybox\nONBUILD RUN echo foo\n", true); err != nil { c.Fatal(err) @@ -4401,9 +4444,8 @@ func (s *DockerSuite) TestBuildOnBuildOutput(c *check.C) { } func (s *DockerSuite) TestBuildInvalidTag(c *check.C) { - testRequires(c, DaemonIsLinux) name := "abcd:" + stringutils.GenerateRandomAlphaOnlyString(200) - _, out, err := buildImageWithOut(name, "FROM scratch\nMAINTAINER quux\n", true) + _, out, err := buildImageWithOut(name, "FROM "+minimalBaseImage()+"\nMAINTAINER quux\n", true) // if the error doesn't check for illegal tag name, or the image is built // then this should fail if !strings.Contains(out, "Error parsing reference") || strings.Contains(out, "Sending build context to Docker daemon") { @@ -4412,7 +4454,6 @@ func (s *DockerSuite) TestBuildInvalidTag(c *check.C) { } func (s *DockerSuite) TestBuildCmdShDashC(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildcmdshc" if _, err := buildImage(name, "FROM busybox\nCMD echo cmd\n", true); err != nil { c.Fatal(err) @@ -4421,6 +4462,9 @@ func (s *DockerSuite) TestBuildCmdShDashC(c *check.C) { res := inspectFieldJSON(c, name, "Config.Cmd") expected := `["/bin/sh","-c","echo cmd"]` + if daemonPlatform == "windows" { + expected = `["cmd","/S","/C","echo cmd"]` + } if res != expected { c.Fatalf("Expected value %s not in Config.Cmd: %s", expected, res) @@ -4429,7 +4473,6 @@ func (s *DockerSuite) TestBuildCmdShDashC(c *check.C) { } func (s *DockerSuite) TestBuildCmdSpaces(c *check.C) { - testRequires(c, DaemonIsLinux) // Test to make sure that when we strcat arrays we take into account // the arg separator to make sure ["echo","hi"] and ["echo hi"] don't // look the same @@ -4466,7 +4509,6 @@ func (s *DockerSuite) TestBuildCmdSpaces(c *check.C) { } func (s *DockerSuite) TestBuildCmdJSONNoShDashC(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildcmdjson" if _, err := buildImage(name, "FROM busybox\nCMD [\"echo\", \"cmd\"]", true); err != nil { c.Fatal(err) @@ -4483,7 +4525,6 @@ func (s *DockerSuite) TestBuildCmdJSONNoShDashC(c *check.C) { } func (s *DockerSuite) TestBuildErrorInvalidInstruction(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildignoreinvalidinstruction" out, _, err := buildImageWithOut(name, "FROM busybox\nfoo bar", true) @@ -4494,7 +4535,6 @@ func (s *DockerSuite) TestBuildErrorInvalidInstruction(c *check.C) { } func (s *DockerSuite) TestBuildEntrypointInheritance(c *check.C) { - testRequires(c, DaemonIsLinux) if _, err := buildImage("parent", ` FROM busybox @@ -4521,13 +4561,16 @@ func (s *DockerSuite) TestBuildEntrypointInheritance(c *check.C) { } func (s *DockerSuite) TestBuildEntrypointInheritanceInspect(c *check.C) { - testRequires(c, DaemonIsLinux) var ( name = "testbuildepinherit" name2 = "testbuildepinherit2" expected = `["/bin/sh","-c","echo quux"]` ) + if daemonPlatform == "windows" { + expected = `["cmd","/S","/C","echo quux"]` + } + if _, err := buildImage(name, "FROM busybox\nENTRYPOINT /foo/bar", true); err != nil { c.Fatal(err) } @@ -4542,7 +4585,7 @@ func (s *DockerSuite) TestBuildEntrypointInheritanceInspect(c *check.C) { c.Fatalf("Expected value %s not in Config.Entrypoint: %s", expected, res) } - out, _ := dockerCmd(c, "run", "-t", name2) + out, _ := dockerCmd(c, "run", name2) expected = "quux" @@ -4553,11 +4596,10 @@ func (s *DockerSuite) TestBuildEntrypointInheritanceInspect(c *check.C) { } func (s *DockerSuite) TestBuildRunShEntrypoint(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildentrypoint" _, err := buildImage(name, `FROM busybox - ENTRYPOINT /bin/echo`, + ENTRYPOINT echo`, true) if err != nil { c.Fatal(err) @@ -4596,7 +4638,6 @@ func (s *DockerSuite) TestBuildExoticShellInterpolation(c *check.C) { } func (s *DockerSuite) TestBuildVerifySingleQuoteFails(c *check.C) { - testRequires(c, DaemonIsLinux) // This testcase is supposed to generate an error because the // JSON array we're passing in on the CMD uses single quotes instead // of double quotes (per the JSON spec). This means we interpret it @@ -4618,8 +4659,12 @@ func (s *DockerSuite) TestBuildVerifySingleQuoteFails(c *check.C) { } func (s *DockerSuite) TestBuildVerboseOut(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildverboseout" + expected := "\n123\n" + + if daemonPlatform == "windows" { + expected = "\n123\r\n" + } _, out, err := buildImageWithOut(name, `FROM busybox @@ -4629,14 +4674,13 @@ RUN echo 123`, if err != nil { c.Fatal(err) } - if !strings.Contains(out, "\n123\n") { + if !strings.Contains(out, expected) { c.Fatalf("Output should contain %q: %q", "123", out) } } func (s *DockerSuite) TestBuildWithTabs(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildwithtabs" _, err := buildImage(name, "FROM busybox\nRUN echo\tone\t\ttwo", true) @@ -4646,13 +4690,16 @@ func (s *DockerSuite) TestBuildWithTabs(c *check.C) { res := inspectFieldJSON(c, name, "ContainerConfig.Cmd") expected1 := `["/bin/sh","-c","echo\tone\t\ttwo"]` expected2 := `["/bin/sh","-c","echo\u0009one\u0009\u0009two"]` // syntactically equivalent, and what Go 1.3 generates + if daemonPlatform == "windows" { + expected1 = `["cmd","/S","/C","echo\tone\t\ttwo"]` + expected2 = `["cmd","/S","/C","echo\u0009one\u0009\u0009two"]` // syntactically equivalent, and what Go 1.3 generates + } if res != expected1 && res != expected2 { c.Fatalf("Missing tabs.\nGot: %s\nExp: %s or %s", res, expected1, expected2) } } func (s *DockerSuite) TestBuildLabels(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildlabel" expected := `{"License":"GPL","Vendor":"Acme"}` _, err := buildImage(name, @@ -4670,7 +4717,6 @@ func (s *DockerSuite) TestBuildLabels(c *check.C) { } func (s *DockerSuite) TestBuildLabelsCache(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildlabelcache" id1, err := buildImage(name, @@ -4719,7 +4765,6 @@ func (s *DockerSuite) TestBuildLabelsCache(c *check.C) { } func (s *DockerSuite) TestBuildNotVerboseSuccess(c *check.C) { - testRequires(c, DaemonIsLinux) // This test makes sure that -q works correctly when build is successful: // stdout has only the image ID (long image ID) and stderr is empty. var stdout, stderr string @@ -4799,7 +4844,6 @@ func (s *DockerSuite) TestBuildNotVerboseSuccess(c *check.C) { } func (s *DockerSuite) TestBuildNotVerboseFailure(c *check.C) { - testRequires(c, DaemonIsLinux) // This test makes sure that -q works correctly when build fails by // comparing between the stderr output in quiet mode and in stdout // and stderr output in verbose mode @@ -4819,13 +4863,12 @@ func (s *DockerSuite) TestBuildNotVerboseFailure(c *check.C) { c.Fatal(fmt.Errorf("Test [%s] expected to fail but didn't", te.TestName)) } if qstderr != vstdout+vstderr { - c.Fatal(fmt.Errorf("Test[%s] expected that quiet stderr and verbose stdout are equal; quiet [%v], verbose [%v]", te.TestName, qstderr, vstdout)) + c.Fatal(fmt.Errorf("Test[%s] expected that quiet stderr and verbose stdout are equal; quiet [%v], verbose [%v]", te.TestName, qstderr, vstdout+vstderr)) } } } func (s *DockerSuite) TestBuildNotVerboseFailureRemote(c *check.C) { - testRequires(c, DaemonIsLinux) // This test ensures that when given a wrong URL, stderr in quiet mode and // stdout and stderr in verbose mode are identical. URL := "http://bla.bla.com" @@ -4841,7 +4884,6 @@ func (s *DockerSuite) TestBuildNotVerboseFailureRemote(c *check.C) { } func (s *DockerSuite) TestBuildStderr(c *check.C) { - testRequires(c, DaemonIsLinux) // This test just makes sure that no non-error output goes // to stderr name := "testbuildstderr" @@ -4896,7 +4938,6 @@ RUN [ $(ls -l /test | awk '{print $3":"$4}') = 'root:root' ] } func (s *DockerSuite) TestBuildSymlinkBreakout(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildsymlinkbreakout" tmpdir, err := ioutil.TempDir("", name) c.Assert(err, check.IsNil) @@ -4980,15 +5021,21 @@ RUN [ ! -e /injected ]`, func (s *DockerSuite) TestBuildVolumesRetainContents(c *check.C) { // /foo/file gets permission denied for the user testRequires(c, NotUserNamespace) - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // TODO Windows: Issue #20127 var ( name = "testbuildvolumescontent" expected = "some text" + volName = "/foo" ) + + if daemonPlatform == "windows" { + volName = "C:/foo" + } + ctx, err := fakeContext(` FROM busybox COPY content /foo/file -VOLUME /foo +VOLUME `+volName+` CMD cat /foo/file`, map[string]string{ "content": expected, @@ -5010,7 +5057,6 @@ CMD cat /foo/file`, } func (s *DockerSuite) TestBuildRenamedDockerfile(c *check.C) { - testRequires(c, DaemonIsLinux) ctx, err := fakeContext(`FROM busybox RUN echo from Dockerfile`, @@ -5200,9 +5246,9 @@ RUN echo from Dockerfile`, } func (s *DockerSuite) TestBuildFromStdinWithF(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // TODO Windows: This test is flaky; no idea why ctx, err := fakeContext(`FROM busybox -RUN echo from Dockerfile`, +RUN echo "from Dockerfile"`, map[string]string{}) if err != nil { c.Fatal(err) @@ -5214,9 +5260,9 @@ RUN echo from Dockerfile`, dockerCommand := exec.Command(dockerBinary, "build", "-f", "baz", "-t", "test1", "-") dockerCommand.Dir = ctx.Dir dockerCommand.Stdin = strings.NewReader(`FROM busybox -RUN echo from baz +RUN echo "from baz" COPY * /tmp/ -RUN find /tmp/`) +RUN sh -c "find /tmp/" # sh -c is needed on Windows to use the correct find`) out, status, err := runCommandWithOutput(dockerCommand) if err != nil || status != 0 { c.Fatalf("Error building: %s", err) @@ -5231,7 +5277,6 @@ RUN find /tmp/`) } func (s *DockerSuite) TestBuildFromOfficialNames(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildfromofficial" fromNames := []string{ "busybox", @@ -5310,7 +5355,6 @@ func (s *DockerSuite) TestBuildDockerfileOutsideContext(c *check.C) { } func (s *DockerSuite) TestBuildSpaces(c *check.C) { - testRequires(c, DaemonIsLinux) // Test to make sure that leading/trailing spaces on a command // doesn't change the error msg we get var ( @@ -5403,7 +5447,7 @@ RUN echo " \ // #4393 func (s *DockerSuite) TestBuildVolumeFileExistsinContainer(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // TODO Windows: This should error out buildCmd := exec.Command(dockerBinary, "build", "-t", "docker-test-errcreatevolumewithfile", "-") buildCmd.Stdin = strings.NewReader(` FROM busybox @@ -5419,7 +5463,6 @@ func (s *DockerSuite) TestBuildVolumeFileExistsinContainer(c *check.C) { } func (s *DockerSuite) TestBuildMissingArgs(c *check.C) { - testRequires(c, DaemonIsLinux) // Test to make sure that all Dockerfile commands (except the ones listed // in skipCmds) will generate an error if no args are provided. // Note: INSERT is deprecated so we exclude it because of that. @@ -5430,6 +5473,19 @@ func (s *DockerSuite) TestBuildMissingArgs(c *check.C) { "INSERT": {}, } + if daemonPlatform == "windows" { + skipCmds = map[string]struct{}{ + "CMD": {}, + "RUN": {}, + "ENTRYPOINT": {}, + "INSERT": {}, + "STOPSIGNAL": {}, + "ARG": {}, + "USER": {}, + "EXPOSE": {}, + } + } + for cmd := range command.Commands { cmd = strings.ToUpper(cmd) if _, ok := skipCmds[cmd]; ok { @@ -5472,8 +5528,6 @@ func (s *DockerSuite) TestBuildEmptyScratch(c *check.C) { } func (s *DockerSuite) TestBuildDotDotFile(c *check.C) { - testRequires(c, DaemonIsLinux) - ctx, err := fakeContext("FROM busybox\n", map[string]string{ "..gitme": "", @@ -5489,7 +5543,7 @@ func (s *DockerSuite) TestBuildDotDotFile(c *check.C) { } func (s *DockerSuite) TestBuildRUNoneJSON(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // No hello-world Windows image name := "testbuildrunonejson" ctx, err := fakeContext(`FROM hello-world:frozen @@ -5511,7 +5565,6 @@ RUN [ "/hello" ]`, map[string]string{}) } func (s *DockerSuite) TestBuildEmptyStringVolume(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildemptystringvolume" _, err := buildImage(name, ` @@ -5557,7 +5610,6 @@ RUN cat /proc/self/cgroup } func (s *DockerSuite) TestBuildNoDupOutput(c *check.C) { - testRequires(c, DaemonIsLinux) // Check to make sure our build output prints the Dockerfile cmd // property - there was a bug that caused it to be duplicated on the // Step X line @@ -5578,7 +5630,6 @@ func (s *DockerSuite) TestBuildNoDupOutput(c *check.C) { // GH15826 func (s *DockerSuite) TestBuildStartsFromOne(c *check.C) { - testRequires(c, DaemonIsLinux) // Explicit check to ensure that build starts from step 1 rather than 0 name := "testbuildstartsfromone" @@ -5595,7 +5646,6 @@ func (s *DockerSuite) TestBuildStartsFromOne(c *check.C) { } func (s *DockerSuite) TestBuildBadCmdFlag(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildbadcmdflag" _, out, err := buildImageWithOut(name, ` @@ -5612,7 +5662,6 @@ func (s *DockerSuite) TestBuildBadCmdFlag(c *check.C) { } func (s *DockerSuite) TestBuildRUNErrMsg(c *check.C) { - testRequires(c, DaemonIsLinux) // Test to make sure the bad command is quoted with just "s and // not as a Go []string name := "testbuildbadrunerrmsg" @@ -5622,8 +5671,14 @@ func (s *DockerSuite) TestBuildRUNErrMsg(c *check.C) { if err == nil { c.Fatal("Should have failed to build") } - - exp := `The command '/bin/sh -c badEXE a1 \& a2 a3' returned a non-zero code: 127` + shell := "/bin/sh -c" + exitCode := "127" + if daemonPlatform == "windows" { + shell = "cmd /S /C" + // architectural - Windows has to start the container to determine the exe is bad, Linux does not + exitCode = "1" + } + exp := `The command '` + shell + ` badEXE a1 \& a2 a3' returned a non-zero code: ` + exitCode if !strings.Contains(out, exp) { c.Fatalf("RUN doesn't have the correct output:\nGot:%s\nExpected:%s", out, exp) } @@ -5720,15 +5775,20 @@ func (s *DockerTrustSuite) TestBuildContextDirIsSymlink(c *check.C) { // Issue #15634: COPY fails when path starts with "null" func (s *DockerSuite) TestBuildNullStringInAddCopyVolume(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildnullstringinaddcopyvolume" + volName := "nullvolume" + + if daemonPlatform == "windows" { + volName = `C:\\nullvolume` + } + ctx, err := fakeContext(` FROM busybox ADD null / COPY nullfile / - VOLUME nullvolume + VOLUME `+volName+` `, map[string]string{ "null": "test1", @@ -5743,22 +5803,29 @@ func (s *DockerSuite) TestBuildNullStringInAddCopyVolume(c *check.C) { } func (s *DockerSuite) TestBuildStopSignal(c *check.C) { - testRequires(c, DaemonIsLinux) - name := "test_build_stop_signal" - _, err := buildImage(name, + testRequires(c, DaemonIsLinux) // Windows does not support STOPSIGNAL yet + imgName := "test_build_stop_signal" + _, err := buildImage(imgName, `FROM busybox STOPSIGNAL SIGKILL`, true) c.Assert(err, check.IsNil) - res := inspectFieldJSON(c, name, "Config.StopSignal") + res := inspectFieldJSON(c, imgName, "Config.StopSignal") + if res != `"SIGKILL"` { + c.Fatalf("Signal %s, expected SIGKILL", res) + } + containerName := "test-container-stop-signal" + dockerCmd(c, "run", "-d", "--name", containerName, imgName, "top") + + res = inspectFieldJSON(c, containerName, "Config.StopSignal") if res != `"SIGKILL"` { c.Fatalf("Signal %s, expected SIGKILL", res) } } func (s *DockerSuite) TestBuildBuildTimeArg(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" envVal := "bar" @@ -5784,7 +5851,7 @@ func (s *DockerSuite) TestBuildBuildTimeArg(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgHistory(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" envVal := "bar" @@ -5810,7 +5877,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgHistory(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgCacheHit(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" envVal := "bar" @@ -5837,7 +5904,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgCacheHit(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgCacheMissExtraArg(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" envVal := "bar" @@ -5869,7 +5936,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgCacheMissExtraArg(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgCacheMissSameArgDiffVal(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" envVal := "bar" @@ -5901,7 +5968,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgCacheMissSameArgDiffVal(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgOverrideArgDefinedBeforeEnv(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" envVal := "bar" @@ -5930,7 +5997,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgOverrideArgDefinedBeforeEnv(c *check. } func (s *DockerSuite) TestBuildBuildTimeArgOverrideEnvDefinedBeforeArg(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" envVal := "bar" @@ -5959,7 +6026,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgOverrideEnvDefinedBeforeArg(c *check. } func (s *DockerSuite) TestBuildBuildTimeArgExpansion(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldvarstest" wdVar := "WDIR" @@ -6054,7 +6121,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgExpansion(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgExpansionOverride(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldvarstest" envKey := "foo" envVal := "bar" @@ -6084,7 +6151,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgExpansionOverride(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgUntrustedDefinedAfterUse(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" envVal := "bar" @@ -6110,7 +6177,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgUntrustedDefinedAfterUse(c *check.C) } func (s *DockerSuite) TestBuildBuildTimeArgBuiltinArg(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support --build-arg imgName := "bldargtest" envKey := "HTTP_PROXY" envVal := "bar" @@ -6135,7 +6202,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgBuiltinArg(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgDefaultOverride(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" envVal := "bar" @@ -6163,7 +6230,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgDefaultOverride(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgMultiArgsSameLine(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" envKey1 := "foo1" @@ -6180,7 +6247,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgMultiArgsSameLine(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgUnconsumedArg(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support --build-arg imgName := "bldargtest" envKey := "foo" envVal := "bar" @@ -6201,7 +6268,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgUnconsumedArg(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgQuotedValVariants(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" envKey1 := "foo1" @@ -6227,7 +6294,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgQuotedValVariants(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgEmptyValVariants(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" envKey1 := "foo1" @@ -6247,7 +6314,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgEmptyValVariants(c *check.C) { } func (s *DockerSuite) TestBuildBuildTimeArgDefintionWithNoEnvInjection(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux) // Windows does not support ARG imgName := "bldargtest" envKey := "foo" args := []string{} @@ -6264,11 +6331,15 @@ func (s *DockerSuite) TestBuildBuildTimeArgDefintionWithNoEnvInjection(c *check. } func (s *DockerSuite) TestBuildNoNamedVolume(c *check.C) { - testRequires(c, DaemonIsLinux) - dockerCmd(c, "run", "-v", "testname:/foo", "busybox", "sh", "-c", "touch /foo/oops") + volName := "testname:/foo" + + if daemonPlatform == "windows" { + volName = "testname:C:\\foo" + } + dockerCmd(c, "run", "-v", volName, "busybox", "sh", "-c", "touch /foo/oops") dockerFile := `FROM busybox - VOLUME testname:/foo + VOLUME ` + volName + ` RUN ls /foo/oops ` _, err := buildImage("test", dockerFile, false) @@ -6276,8 +6347,6 @@ func (s *DockerSuite) TestBuildNoNamedVolume(c *check.C) { } func (s *DockerSuite) TestBuildTagEvent(c *check.C) { - testRequires(c, DaemonIsLinux) - since := daemonTime(c).Unix() dockerFile := `FROM busybox @@ -6445,7 +6514,6 @@ func (s *DockerSuite) TestBuildSymlinkBasename(c *check.C) { // #17827 func (s *DockerSuite) TestBuildCacheRootSource(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testbuildrootsource" ctx, err := fakeContext(` FROM busybox @@ -6501,3 +6569,68 @@ func (s *DockerSuite) TestBuildWorkdirWindowsPath(c *check.C) { c.Fatal(err) } } + +func (s *DockerRegistryAuthSuite) TestBuildFromAuthenticatedRegistry(c *check.C) { + dockerCmd(c, "login", "-u", s.reg.username, "-p", s.reg.password, privateRegistryURL) + + baseImage := privateRegistryURL + "/baseimage" + + _, err := buildImage(baseImage, ` + FROM busybox + ENV env1 val1 + `, true) + + c.Assert(err, checker.IsNil) + + dockerCmd(c, "push", baseImage) + dockerCmd(c, "rmi", baseImage) + + _, err = buildImage(baseImage, fmt.Sprintf(` + FROM %s + ENV env2 val2 + `, baseImage), true) + + c.Assert(err, checker.IsNil) +} + +func (s *DockerRegistryAuthSuite) TestBuildWithExternalAuth(c *check.C) { + osPath := os.Getenv("PATH") + defer os.Setenv("PATH", osPath) + + workingDir, err := os.Getwd() + c.Assert(err, checker.IsNil) + absolute, err := filepath.Abs(filepath.Join(workingDir, "fixtures", "auth")) + c.Assert(err, checker.IsNil) + testPath := fmt.Sprintf("%s%c%s", osPath, filepath.ListSeparator, absolute) + + os.Setenv("PATH", testPath) + + repoName := fmt.Sprintf("%v/dockercli/busybox:authtest", privateRegistryURL) + + tmp, err := ioutil.TempDir("", "integration-cli-") + c.Assert(err, checker.IsNil) + + externalAuthConfig := `{ "credsStore": "shell-test" }` + + configPath := filepath.Join(tmp, "config.json") + err = ioutil.WriteFile(configPath, []byte(externalAuthConfig), 0644) + c.Assert(err, checker.IsNil) + + dockerCmd(c, "--config", tmp, "login", "-u", s.reg.username, "-p", s.reg.password, privateRegistryURL) + + b, err := ioutil.ReadFile(configPath) + c.Assert(err, checker.IsNil) + c.Assert(string(b), checker.Not(checker.Contains), "\"auth\":") + + dockerCmd(c, "--config", tmp, "tag", "busybox", repoName) + dockerCmd(c, "--config", tmp, "push", repoName) + + // make sure the image is pulled when building + dockerCmd(c, "rmi", repoName) + + buildCmd := exec.Command(dockerBinary, "--config", tmp, "build", "-") + buildCmd.Stdin = strings.NewReader(fmt.Sprintf("FROM %s", repoName)) + + out, _, err := runCommandWithOutput(buildCmd) + c.Assert(err, check.IsNil, check.Commentf(out)) +} diff --git a/components/engine/integration-cli/docker_cli_build_unix_test.go b/components/engine/integration-cli/docker_cli_build_unix_test.go index a519d34f3b..56ab66efae 100644 --- a/components/engine/integration-cli/docker_cli_build_unix_test.go +++ b/components/engine/integration-cli/docker_cli_build_unix_test.go @@ -171,8 +171,8 @@ func (s *DockerSuite) TestBuildCancellationKillsSleep(c *check.C) { } testActions := map[string]chan bool{ - "start": make(chan bool), - "die": make(chan bool), + "start": make(chan bool, 1), + "die": make(chan bool, 1), } matcher := matchEventLine(buildID, "container", testActions) diff --git a/components/engine/integration-cli/docker_cli_by_digest_test.go b/components/engine/integration-cli/docker_cli_by_digest_test.go index 0d5ae482a2..f3948d67cc 100644 --- a/components/engine/integration-cli/docker_cli_by_digest_test.go +++ b/components/engine/integration-cli/docker_cli_by_digest_test.go @@ -31,7 +31,7 @@ func setupImage(c *check.C) (digest.Digest, error) { func setupImageWithTag(c *check.C, tag string) (digest.Digest, error) { containerName := "busyboxbydigest" - dockerCmd(c, "run", "-d", "-e", "digest=1", "--name", containerName, "busybox") + dockerCmd(c, "run", "-e", "digest=1", "--name", containerName, "busybox") // tag the image to upload it to the private registry repoAndTag := repoName + ":" + tag @@ -354,17 +354,19 @@ func (s *DockerRegistrySuite) TestPsListContainersFilterAncestorImageByDigest(c c.Assert(err, checker.IsNil) // run a container based on that - out, _ := dockerCmd(c, "run", "-d", imageReference, "echo", "hello") - expectedID := strings.TrimSpace(out) + dockerCmd(c, "run", "--name=test1", imageReference, "echo", "hello") + expectedID, err := getIDByName("test1") + c.Assert(err, check.IsNil) // run a container based on the a descendant of that too - out, _ = dockerCmd(c, "run", "-d", imageName1, "echo", "hello") - expectedID1 := strings.TrimSpace(out) + dockerCmd(c, "run", "--name=test2", imageName1, "echo", "hello") + expectedID1, err := getIDByName("test2") + c.Assert(err, check.IsNil) expectedIDs := []string{expectedID, expectedID1} // Invalid imageReference - out, _ = dockerCmd(c, "ps", "-a", "-q", "--no-trunc", fmt.Sprintf("--filter=ancestor=busybox@%s", digest)) + out, _ := dockerCmd(c, "ps", "-a", "-q", "--no-trunc", fmt.Sprintf("--filter=ancestor=busybox@%s", digest)) // Filter container for ancestor filter should be empty c.Assert(strings.TrimSpace(out), checker.Equals, "") diff --git a/components/engine/integration-cli/docker_cli_commit_test.go b/components/engine/integration-cli/docker_cli_commit_test.go index 9a4520bba4..086a203124 100644 --- a/components/engine/integration-cli/docker_cli_commit_test.go +++ b/components/engine/integration-cli/docker_cli_commit_test.go @@ -129,9 +129,9 @@ func (s *DockerSuite) TestCommitChange(c *check.C) { "Config.ExposedPorts": "map[8080/tcp:{}]", "Config.Env": "[DEBUG=true test=1 PATH=/foo]", "Config.Labels": "map[foo:bar]", - "Config.Cmd": "{[/bin/sh]}", + "Config.Cmd": "[/bin/sh]", "Config.WorkingDir": "/opt", - "Config.Entrypoint": "{[/bin/sh]}", + "Config.Entrypoint": "[/bin/sh]", "Config.User": "testuser", "Config.Volumes": "map[/var/lib/docker:{}]", "Config.OnBuild": "[/usr/local/bin/python-build --dir /app/src]", diff --git a/components/engine/integration-cli/docker_cli_config_test.go b/components/engine/integration-cli/docker_cli_config_test.go index 969ec389fb..6015231065 100644 --- a/components/engine/integration-cli/docker_cli_config_test.go +++ b/components/engine/integration-cli/docker_cli_config_test.go @@ -71,7 +71,7 @@ func (s *DockerSuite) TestConfigDir(c *check.C) { // Test with env var too cmd := exec.Command(dockerBinary, "ps") - cmd.Env = append(os.Environ(), "DOCKER_CONFIG="+cDir) + cmd.Env = appendBaseEnv(true, "DOCKER_CONFIG="+cDir) out, _, err := runCommandWithOutput(cmd) c.Assert(err, checker.IsNil, check.Commentf("ps2 didn't work,out:%v", out)) @@ -95,7 +95,10 @@ func (s *DockerSuite) TestConfigDir(c *check.C) { err = ioutil.WriteFile(tmpCfg, []byte(data), 0600) c.Assert(err, checker.IsNil, check.Commentf("Err creating file")) + env := appendBaseEnv(false) + cmd = exec.Command(dockerBinary, "--config", cDir, "-H="+server.URL[7:], "ps") + cmd.Env = env out, _, err = runCommandWithOutput(cmd) c.Assert(err, checker.NotNil, check.Commentf("out:%v", out)) @@ -105,7 +108,7 @@ func (s *DockerSuite) TestConfigDir(c *check.C) { // Reset headers and try again using env var this time headers = map[string][]string{} cmd = exec.Command(dockerBinary, "-H="+server.URL[7:], "ps") - cmd.Env = append(os.Environ(), "DOCKER_CONFIG="+cDir) + cmd.Env = append(env, "DOCKER_CONFIG="+cDir) out, _, err = runCommandWithOutput(cmd) c.Assert(err, checker.NotNil, check.Commentf("%v", out)) @@ -115,7 +118,7 @@ func (s *DockerSuite) TestConfigDir(c *check.C) { // Reset headers and make sure flag overrides the env var headers = map[string][]string{} cmd = exec.Command(dockerBinary, "--config", cDir, "-H="+server.URL[7:], "ps") - cmd.Env = append(os.Environ(), "DOCKER_CONFIG=MissingDir") + cmd.Env = append(env, "DOCKER_CONFIG=MissingDir") out, _, err = runCommandWithOutput(cmd) c.Assert(err, checker.NotNil, check.Commentf("out:%v", out)) @@ -127,10 +130,9 @@ func (s *DockerSuite) TestConfigDir(c *check.C) { // ignore - we don't want to default back to the env var. headers = map[string][]string{} cmd = exec.Command(dockerBinary, "--config", "MissingDir", "-H="+server.URL[7:], "ps") - cmd.Env = append(os.Environ(), "DOCKER_CONFIG="+cDir) + cmd.Env = append(env, "DOCKER_CONFIG="+cDir) out, _, err = runCommandWithOutput(cmd) c.Assert(err, checker.NotNil, check.Commentf("out:%v", out)) c.Assert(headers["Myheader"], checker.IsNil, check.Commentf("ps6 - Headers shouldn't be the expected value,out:%v", out)) - } diff --git a/components/engine/integration-cli/docker_cli_cp_to_container_unix_test.go b/components/engine/integration-cli/docker_cli_cp_to_container_unix_test.go new file mode 100644 index 0000000000..45d85ba5d1 --- /dev/null +++ b/components/engine/integration-cli/docker_cli_cp_to_container_unix_test.go @@ -0,0 +1,39 @@ +// +build !windows + +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/docker/docker/pkg/integration/checker" + "github.com/docker/docker/pkg/system" + "github.com/go-check/check" +) + +// Check ownership is root, both in non-userns and userns enabled modes +func (s *DockerSuite) TestCpCheckDestOwnership(c *check.C) { + testRequires(c, DaemonIsLinux, SameHostDaemon) + tmpVolDir := getTestDir(c, "test-cp-tmpvol") + containerID := makeTestContainer(c, + testContainerOptions{volumes: []string{fmt.Sprintf("%s:/tmpvol", tmpVolDir)}}) + + tmpDir := getTestDir(c, "test-cp-to-check-ownership") + defer os.RemoveAll(tmpDir) + + makeTestContentInDir(c, tmpDir) + + srcPath := cpPath(tmpDir, "file1") + dstPath := containerCpPath(containerID, "/tmpvol", "file1") + + err := runDockerCp(c, srcPath, dstPath) + c.Assert(err, checker.IsNil) + + stat, err := system.Stat(filepath.Join(tmpVolDir, "file1")) + c.Assert(err, checker.IsNil) + uid, gid, err := getRootUIDGID() + c.Assert(err, checker.IsNil) + c.Assert(stat.UID(), checker.Equals, uint32(uid), check.Commentf("Copied file not owned by container root UID")) + c.Assert(stat.GID(), checker.Equals, uint32(gid), check.Commentf("Copied file not owned by container root GID")) +} diff --git a/components/engine/integration-cli/docker_cli_create_test.go b/components/engine/integration-cli/docker_cli_create_test.go index 657c3fd6a2..c0f1854178 100644 --- a/components/engine/integration-cli/docker_cli_create_test.go +++ b/components/engine/integration-cli/docker_cli_create_test.go @@ -365,7 +365,7 @@ func (s *DockerTrustSuite) TestCreateWhenCertExpired(c *check.C) { func (s *DockerTrustSuite) TestTrustedCreateFromBadTrustServer(c *check.C) { repoName := fmt.Sprintf("%v/dockerclievilcreate/trusted:latest", privateRegistryURL) - evilLocalConfigDir, err := ioutil.TempDir("", "evil-local-config-dir") + evilLocalConfigDir, err := ioutil.TempDir("", "evilcreate-local-config-dir") c.Assert(err, check.IsNil) // tag the image and upload it to the private registry @@ -404,12 +404,16 @@ func (s *DockerTrustSuite) TestTrustedCreateFromBadTrustServer(c *check.C) { c.Assert(err, check.IsNil) c.Assert(string(out), checker.Contains, "Signing and pushing trust metadata", check.Commentf("Missing expected output on trusted push:\n%s", out)) - // Now, try creating with the original client from this new trust server. This should fail. + // Now, try creating with the original client from this new trust server. This should fallback to our cached timestamp and metadata. createCmd = exec.Command(dockerBinary, "create", repoName) s.trustedCmd(createCmd) out, _, err = runCommandWithOutput(createCmd) - c.Assert(err, check.Not(check.IsNil)) - c.Assert(string(out), checker.Contains, "valid signatures did not meet threshold", check.Commentf("Missing expected output on trusted push:\n%s", out)) + if err != nil { + c.Fatalf("Error falling back to cached trust data: %s\n%s", err, out) + } + if !strings.Contains(string(out), "Error while downloading remote metadata, using cached timestamp") { + c.Fatalf("Missing expected output on trusted create:\n%s", out) + } } @@ -436,3 +440,13 @@ func (s *DockerSuite) TestCreateWithWorkdir(c *check.C) { dockerCmd(c, "create", "--name", name, "-w", dir, "busybox") dockerCmd(c, "cp", fmt.Sprintf("%s:%s", name, dir), prefix+slash+"tmp") } + +func (s *DockerSuite) TestCreateWithInvalidLogOpts(c *check.C) { + name := "test-invalidate-log-opts" + out, _, err := dockerCmdWithError("create", "--name", name, "--log-opt", "invalid=true", "busybox") + c.Assert(err, checker.NotNil) + c.Assert(out, checker.Contains, "unknown log opt") + + out, _ = dockerCmd(c, "ps", "-a") + c.Assert(out, checker.Not(checker.Contains), name) +} diff --git a/components/engine/integration-cli/docker_cli_daemon_test.go b/components/engine/integration-cli/docker_cli_daemon_test.go index 42ffe6cd4d..483ff78b03 100644 --- a/components/engine/integration-cli/docker_cli_daemon_test.go +++ b/components/engine/integration-cli/docker_cli_daemon_test.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "sync" + "syscall" "time" "github.com/docker/docker/pkg/integration/checker" @@ -71,7 +72,7 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithVolumesRefs(c *check.C) { c.Fatal(err) } - if out, err := s.d.Cmd("run", "-d", "--name", "volrestarttest1", "-v", "/foo", "busybox"); err != nil { + if out, err := s.d.Cmd("run", "--name", "volrestarttest1", "-v", "/foo", "busybox"); err != nil { c.Fatal(err, out) } @@ -377,6 +378,8 @@ func (s *DockerSuite) TestDaemonIPv6Enabled(c *check.C) { // TestDaemonIPv6FixedCIDR checks that when the daemon is started with --ipv6=true and a fixed CIDR // that running containers are given a link-local and global IPv6 address func (s *DockerDaemonSuite) TestDaemonIPv6FixedCIDR(c *check.C) { + // IPv6 setup is messing with local bridge address. + testRequires(c, SameHostDaemon) err := setupV6() c.Assert(err, checker.IsNil, check.Commentf("Could not set up host for IPv6 tests")) @@ -406,6 +409,8 @@ func (s *DockerDaemonSuite) TestDaemonIPv6FixedCIDR(c *check.C) { // TestDaemonIPv6FixedCIDRAndMac checks that when the daemon is started with ipv6 fixed CIDR // the running containers are given a an IPv6 address derived from the MAC address and the ipv6 fixed CIDR func (s *DockerDaemonSuite) TestDaemonIPv6FixedCIDRAndMac(c *check.C) { + // IPv6 setup is messing with local bridge address. + testRequires(c, SameHostDaemon) err := setupV6() c.Assert(err, checker.IsNil) @@ -967,6 +972,7 @@ func (s *DockerDaemonSuite) TestDaemonIP(c *check.C) { } func (s *DockerDaemonSuite) TestDaemonICCPing(c *check.C) { + testRequires(c, bridgeNfIptables) d := s.d bridgeName := "external-bridge" @@ -1151,15 +1157,11 @@ func (s *DockerDaemonSuite) TestDaemonLoggingDriverDefault(c *check.C) { c.Fatal(err) } - out, err := s.d.Cmd("run", "-d", "busybox", "echo", "testline") - if err != nil { - c.Fatal(out, err) - } - id := strings.TrimSpace(out) + out, err := s.d.Cmd("run", "--name=test", "busybox", "echo", "testline") + c.Assert(err, check.IsNil, check.Commentf(out)) + id, err := s.d.getIDByName("test") + c.Assert(err, check.IsNil) - if out, err := s.d.Cmd("wait", id); err != nil { - c.Fatal(out, err) - } logPath := filepath.Join(s.d.root, "containers", id, id+"-json.log") if _, err := os.Stat(logPath); err != nil { @@ -1193,15 +1195,13 @@ func (s *DockerDaemonSuite) TestDaemonLoggingDriverDefaultOverride(c *check.C) { c.Fatal(err) } - out, err := s.d.Cmd("run", "-d", "--log-driver=none", "busybox", "echo", "testline") + out, err := s.d.Cmd("run", "--name=test", "--log-driver=none", "busybox", "echo", "testline") if err != nil { c.Fatal(out, err) } - id := strings.TrimSpace(out) + id, err := s.d.getIDByName("test") + c.Assert(err, check.IsNil) - if out, err := s.d.Cmd("wait", id); err != nil { - c.Fatal(out, err) - } logPath := filepath.Join(s.d.root, "containers", id, id+"-json.log") if _, err := os.Stat(logPath); err == nil || !os.IsNotExist(err) { @@ -1214,14 +1214,12 @@ func (s *DockerDaemonSuite) TestDaemonLoggingDriverNone(c *check.C) { c.Fatal(err) } - out, err := s.d.Cmd("run", "-d", "busybox", "echo", "testline") + out, err := s.d.Cmd("run", "--name=test", "busybox", "echo", "testline") if err != nil { c.Fatal(out, err) } - id := strings.TrimSpace(out) - if out, err := s.d.Cmd("wait", id); err != nil { - c.Fatal(out, err) - } + id, err := s.d.getIDByName("test") + c.Assert(err, check.IsNil) logPath := filepath.Join(s.d.folder, "graph", "containers", id, id+"-json.log") @@ -1235,15 +1233,13 @@ func (s *DockerDaemonSuite) TestDaemonLoggingDriverNoneOverride(c *check.C) { c.Fatal(err) } - out, err := s.d.Cmd("run", "-d", "--log-driver=json-file", "busybox", "echo", "testline") + out, err := s.d.Cmd("run", "--name=test", "--log-driver=json-file", "busybox", "echo", "testline") if err != nil { c.Fatal(out, err) } - id := strings.TrimSpace(out) + id, err := s.d.getIDByName("test") + c.Assert(err, check.IsNil) - if out, err := s.d.Cmd("wait", id); err != nil { - c.Fatal(out, err) - } logPath := filepath.Join(s.d.root, "containers", id, id+"-json.log") if _, err := os.Stat(logPath); err != nil { @@ -1273,22 +1269,15 @@ func (s *DockerDaemonSuite) TestDaemonLoggingDriverNoneOverride(c *check.C) { } func (s *DockerDaemonSuite) TestDaemonLoggingDriverNoneLogsError(c *check.C) { - if err := s.d.StartWithBusybox("--log-driver=none"); err != nil { - c.Fatal(err) - } + c.Assert(s.d.StartWithBusybox("--log-driver=none"), checker.IsNil) - out, err := s.d.Cmd("run", "-d", "busybox", "echo", "testline") - if err != nil { - c.Fatal(out, err) - } - id := strings.TrimSpace(out) - out, err = s.d.Cmd("logs", id) - if err == nil { - c.Fatalf("Logs should fail with 'none' driver") - } - if !strings.Contains(out, `"logs" command is supported only for "json-file" and "journald" logging drivers (got: none)`) { - c.Fatalf("There should be an error about none not being a recognized log driver, got: %s", out) - } + out, err := s.d.Cmd("run", "--name=test", "busybox", "echo", "testline") + c.Assert(err, checker.IsNil, check.Commentf(out)) + + out, err = s.d.Cmd("logs", "test") + c.Assert(err, check.NotNil, check.Commentf("Logs should fail with 'none' driver")) + expected := `"logs" command is supported only for "json-file" and "journald" logging drivers (got: none)` + c.Assert(out, checker.Contains, expected) } func (s *DockerDaemonSuite) TestDaemonDots(c *check.C) { @@ -1570,7 +1559,7 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithSocketAsVolume(c *check.C) { socket := filepath.Join(s.d.folder, "docker.sock") - out, err := s.d.Cmd("run", "-d", "--restart=always", "-v", socket+":/sock", "busybox") + out, err := s.d.Cmd("run", "--restart=always", "-v", socket+":/sock", "busybox") c.Assert(err, check.IsNil, check.Commentf("Output: %s", out)) c.Assert(s.d.Restart(), check.IsNil) } @@ -1690,13 +1679,11 @@ func (s *DockerDaemonSuite) TestDaemonNoTlsCliTlsVerifyWithEnv(c *check.C) { func setupV6() error { // Hack to get the right IPv6 address on docker0, which has already been created - err := exec.Command("ip", "addr", "add", "fe80::1/64", "dev", "docker0").Run() - return err + return exec.Command("ip", "addr", "add", "fe80::1/64", "dev", "docker0").Run() } func teardownV6() error { - err := exec.Command("ip", "addr", "del", "fe80::1/64", "dev", "docker0").Run() - return err + return exec.Command("ip", "addr", "del", "fe80::1/64", "dev", "docker0").Run() } func (s *DockerDaemonSuite) TestDaemonRestartWithContainerWithRestartPolicyAlways(c *check.C) { @@ -1886,7 +1873,7 @@ func (s *DockerDaemonSuite) TestBridgeIPIsExcludedFromAllocatorPool(c *check.C) // Test daemon for no space left on device error func (s *DockerDaemonSuite) TestDaemonNoSpaceleftOnDeviceError(c *check.C) { - testRequires(c, SameHostDaemon, DaemonIsLinux) + testRequires(c, SameHostDaemon, DaemonIsLinux, Network) // create a 2MiB image and mount it as graph root cmd := exec.Command("dd", "of=/tmp/testfs.img", "bs=1M", "seek=2", "count=0") @@ -1910,8 +1897,7 @@ func (s *DockerDaemonSuite) TestDaemonNoSpaceleftOnDeviceError(c *check.C) { // pull a repository large enough to fill the mount point out, err := s.d.Cmd("pull", "registry:2") - - c.Assert(strings.Contains(out, "no space left on device"), check.Equals, true) + c.Assert(out, checker.Contains, "no space left on device") } // Test daemon restart with container links + auto restart @@ -2107,7 +2093,7 @@ func (s *DockerDaemonSuite) TestRunLinksChanged(c *check.C) { } func (s *DockerDaemonSuite) TestDaemonStartWithoutColors(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux, NotPpc64le) newD := NewDaemon(c) infoLog := "\x1b[34mINFO\x1b" @@ -2136,7 +2122,7 @@ func (s *DockerDaemonSuite) TestDaemonStartWithoutColors(c *check.C) { } func (s *DockerDaemonSuite) TestDaemonDebugLog(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux, NotPpc64le) newD := NewDaemon(c) debugLog := "\x1b[37mDEBU\x1b" @@ -2155,3 +2141,45 @@ func (s *DockerDaemonSuite) TestDaemonDebugLog(c *check.C) { newD.Stop() c.Assert(b.String(), checker.Contains, debugLog) } + +func (s *DockerSuite) TestDaemonDiscoveryBackendConfigReload(c *check.C) { + testRequires(c, SameHostDaemon, DaemonIsLinux) + + // daemon config file + daemonConfig := `{ "debug" : false }` + configFilePath := "test.json" + + configFile, err := os.Create(configFilePath) + c.Assert(err, checker.IsNil) + fmt.Fprintf(configFile, "%s", daemonConfig) + + d := NewDaemon(c) + err = d.Start(fmt.Sprintf("--config-file=%s", configFilePath)) + c.Assert(err, checker.IsNil) + defer d.Stop() + + // daemon config file + daemonConfig = `{ + "cluster-store": "consul://consuladdr:consulport/some/path", + "cluster-advertise": "192.168.56.100:0", + "debug" : false + }` + + configFile.Close() + os.Remove(configFilePath) + + configFile, err = os.Create(configFilePath) + c.Assert(err, checker.IsNil) + defer os.Remove(configFilePath) + fmt.Fprintf(configFile, "%s", daemonConfig) + configFile.Close() + + syscall.Kill(d.cmd.Process.Pid, syscall.SIGHUP) + + time.Sleep(3 * time.Second) + + out, err := d.Cmd("info") + c.Assert(err, checker.IsNil) + c.Assert(out, checker.Contains, fmt.Sprintf("Cluster store: consul://consuladdr:consulport/some/path")) + c.Assert(out, checker.Contains, fmt.Sprintf("Cluster advertise: 192.168.56.100:0")) +} diff --git a/components/engine/integration-cli/docker_cli_events_test.go b/components/engine/integration-cli/docker_cli_events_test.go index 11b1ec78d9..f426650c2b 100644 --- a/components/engine/integration-cli/docker_cli_events_test.go +++ b/components/engine/integration-cli/docker_cli_events_test.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/docker/docker/daemon/events/testutils" "github.com/docker/docker/pkg/integration/checker" "github.com/go-check/check" ) @@ -20,7 +21,7 @@ func (s *DockerSuite) TestEventsTimestampFormats(c *check.C) { image := "busybox" // Start stopwatch, generate an event - time.Sleep(1 * time.Second) // so that we don't grab events from previous test occured in the same second + time.Sleep(1 * time.Second) // so that we don't grab events from previous test occurred in the same second start := daemonTime(c) dockerCmd(c, "tag", image, "timestamptest:1") dockerCmd(c, "rmi", "timestamptest:1") @@ -152,7 +153,7 @@ func (s *DockerSuite) TestEventsContainerEventsAttrSort(c *check.C) { c.Assert(nEvents, checker.GreaterOrEqualThan, 3) //Missing expected event matchedEvents := 0 for _, event := range events { - matches := parseEventText(event) + matches := eventstestutils.ScanMap(event) if matches["id"] != containerID { continue } @@ -201,7 +202,7 @@ func (s *DockerSuite) TestEventsImageTag(c *check.C) { c.Assert(events, checker.HasLen, 1, check.Commentf("was expecting 1 event. out=%s", out)) event := strings.TrimSpace(events[0]) - matches := parseEventText(event) + matches := eventstestutils.ScanMap(event) c.Assert(matchEventID(matches, image), checker.True, check.Commentf("matches: %v\nout:\n%s", matches, out)) c.Assert(matches["action"], checker.Equals, "tag") } @@ -220,7 +221,7 @@ func (s *DockerSuite) TestEventsImagePull(c *check.C) { events := strings.Split(strings.TrimSpace(out), "\n") event := strings.TrimSpace(events[len(events)-1]) - matches := parseEventText(event) + matches := eventstestutils.ScanMap(event) c.Assert(matches["id"], checker.Equals, "hello-world:latest") c.Assert(matches["action"], checker.Equals, "pull") @@ -245,7 +246,7 @@ func (s *DockerSuite) TestEventsImageImport(c *check.C) { out, _ = dockerCmd(c, "events", fmt.Sprintf("--since=%d", since), fmt.Sprintf("--until=%d", daemonTime(c).Unix()), "--filter", "event=import") events := strings.Split(strings.TrimSpace(out), "\n") c.Assert(events, checker.HasLen, 1) - matches := parseEventText(events[0]) + matches := eventstestutils.ScanMap(events[0]) c.Assert(matches["id"], checker.Equals, imageRef, check.Commentf("matches: %v\nout:\n%s\n", matches, out)) c.Assert(matches["action"], checker.Equals, "import", check.Commentf("matches: %v\nout:\n%s\n", matches, out)) } @@ -370,7 +371,7 @@ func (s *DockerSuite) TestEventsFilterContainer(c *check.C) { return fmt.Errorf("expected 4 events, got %v", events) } for _, event := range events { - matches := parseEventText(event) + matches := eventstestutils.ScanMap(event) if !matchEventID(matches, id) { return fmt.Errorf("expected event for container id %s: %s - parsed container id: %s", id, event, matches["id"]) } @@ -439,7 +440,7 @@ func (s *DockerSuite) TestEventsCopy(c *check.C) { func (s *DockerSuite) TestEventsResize(c *check.C) { since := daemonTime(c).Unix() - out, _ := dockerCmd(c, "run", "-d", "busybox", "top") + out, _ := runSleepingContainer(c, "-d") cID := strings.TrimSpace(out) c.Assert(waitRun(cID), checker.IsNil) diff --git a/components/engine/integration-cli/docker_cli_events_unix_test.go b/components/engine/integration-cli/docker_cli_events_unix_test.go index 8af0b77c17..c5e48f2197 100644 --- a/components/engine/integration-cli/docker_cli_events_unix_test.go +++ b/components/engine/integration-cli/docker_cli_events_unix_test.go @@ -46,7 +46,7 @@ func (s *DockerSuite) TestEventsRedirectStdout(c *check.C) { } func (s *DockerSuite) TestEventsOOMDisableFalse(c *check.C) { - testRequires(c, DaemonIsLinux, oomControl, memoryLimitSupport, NotGCCGO) + testRequires(c, DaemonIsLinux, oomControl, memoryLimitSupport, NotGCCGO, swapMemorySupport) errChan := make(chan error) go func() { @@ -76,7 +76,7 @@ func (s *DockerSuite) TestEventsOOMDisableFalse(c *check.C) { } func (s *DockerSuite) TestEventsOOMDisableTrue(c *check.C) { - testRequires(c, DaemonIsLinux, oomControl, memoryLimitSupport, NotGCCGO, NotArm) + testRequires(c, DaemonIsLinux, oomControl, memoryLimitSupport, NotGCCGO, NotArm, swapMemorySupport) errChan := make(chan error) observer, err := newEventObserver(c) @@ -232,10 +232,10 @@ func (s *DockerSuite) TestEventsStreaming(c *check.C) { containerID := strings.TrimSpace(out) testActions := map[string]chan bool{ - "create": make(chan bool), - "start": make(chan bool), - "die": make(chan bool), - "destroy": make(chan bool), + "create": make(chan bool, 1), + "start": make(chan bool, 1), + "die": make(chan bool, 1), + "destroy": make(chan bool, 1), } matcher := matchEventLine(containerID, "container", testActions) @@ -291,8 +291,8 @@ func (s *DockerSuite) TestEventsImageUntagDelete(c *check.C) { c.Assert(deleteImages(name), checker.IsNil) testActions := map[string]chan bool{ - "untag": make(chan bool), - "delete": make(chan bool), + "untag": make(chan bool, 1), + "delete": make(chan bool, 1), } matcher := matchEventLine(imageID, "image", testActions) diff --git a/components/engine/integration-cli/docker_cli_exec_test.go b/components/engine/integration-cli/docker_cli_exec_test.go index 8b5ef48bd5..4d04126350 100644 --- a/components/engine/integration-cli/docker_cli_exec_test.go +++ b/components/engine/integration-cli/docker_cli_exec_test.go @@ -21,9 +21,10 @@ import ( func (s *DockerSuite) TestExec(c *check.C) { testRequires(c, DaemonIsLinux) - dockerCmd(c, "run", "-d", "--name", "testing", "busybox", "sh", "-c", "echo test > /tmp/file && top") + out, _ := dockerCmd(c, "run", "-d", "--name", "testing", "busybox", "sh", "-c", "echo test > /tmp/file && top") + c.Assert(waitRun(strings.TrimSpace(out)), check.IsNil) - out, _ := dockerCmd(c, "exec", "testing", "cat", "/tmp/file") + out, _ = dockerCmd(c, "exec", "testing", "cat", "/tmp/file") out = strings.Trim(out, "\r\n") c.Assert(out, checker.Equals, "test") @@ -66,7 +67,8 @@ func (s *DockerSuite) TestExecInteractive(c *check.C) { } func (s *DockerSuite) TestExecAfterContainerRestart(c *check.C) { - out, _ := runSleepingContainer(c, "-d") + testRequires(c, DaemonIsLinux) + out, _ := runSleepingContainer(c) cleanedContainerID := strings.TrimSpace(out) c.Assert(waitRun(cleanedContainerID), check.IsNil) dockerCmd(c, "restart", cleanedContainerID) @@ -311,13 +313,12 @@ func (s *DockerSuite) TestExecInspectID(c *check.C) { tries := 10 for i := 0; i < tries; i++ { // Since its still running we should see exec as part of the container - out = inspectField(c, id, "ExecIDs") + out = strings.TrimSpace(inspectField(c, id, "ExecIDs")) - out = strings.TrimSuffix(out, "\n") if out != "[]" && out != "" { break } - c.Assert(i+1, checker.Not(checker.Equals), tries, check.Commentf("ExecIDs should be empty, got: %s", out)) + c.Assert(i+1, checker.Not(checker.Equals), tries, check.Commentf("ExecIDs still empty after 10 second")) time.Sleep(1 * time.Second) } @@ -334,11 +335,17 @@ func (s *DockerSuite) TestExecInspectID(c *check.C) { // Wait for 1st exec to complete cmd.Wait() - // All execs for the container should be gone now - out = inspectField(c, id, "ExecIDs") + // Give the exec 10 chances/seconds to stop then give up and stop the test + for i := 0; i < tries; i++ { + // Since its still running we should see exec as part of the container + out = strings.TrimSpace(inspectField(c, id, "ExecIDs")) - out = strings.TrimSuffix(out, "\n") - c.Assert(out == "[]" || out == "", checker.True) + if out == "[]" { + break + } + c.Assert(i+1, checker.Not(checker.Equals), tries, check.Commentf("ExecIDs still not empty after 10 second")) + time.Sleep(1 * time.Second) + } // But we should still be able to query the execID sc, body, err := sockRequest("GET", "/exec/"+execID+"/json", nil) diff --git a/components/engine/integration-cli/docker_cli_exec_unix_test.go b/components/engine/integration-cli/docker_cli_exec_unix_test.go index a50d580de3..9363092652 100644 --- a/components/engine/integration-cli/docker_cli_exec_unix_test.go +++ b/components/engine/integration-cli/docker_cli_exec_unix_test.go @@ -41,7 +41,7 @@ func (s *DockerSuite) TestExecInteractiveStdinClose(c *check.C) { } func (s *DockerSuite) TestExecTTY(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux, SameHostDaemon) dockerCmd(c, "run", "-d", "--name=test", "busybox", "sh", "-c", "echo hello > /foo && top") cmd := exec.Command(dockerBinary, "exec", "-it", "test", "sh") diff --git a/components/engine/integration-cli/docker_cli_help_test.go b/components/engine/integration-cli/docker_cli_help_test.go index c8ebfd3d18..93ccbeb8fd 100644 --- a/components/engine/integration-cli/docker_cli_help_test.go +++ b/components/engine/integration-cli/docker_cli_help_test.go @@ -1,7 +1,6 @@ package main import ( - "os" "os/exec" "runtime" "strings" @@ -30,7 +29,7 @@ func (s *DockerSuite) TestHelpTextVerify(c *check.C) { } homeKey := homedir.Key() - baseEnvs := os.Environ() + baseEnvs := appendBaseEnv(true) // Remove HOME env var from list so we can add a new value later. for i, env := range baseEnvs { @@ -54,9 +53,12 @@ func (s *DockerSuite) TestHelpTextVerify(c *check.C) { out, _, err := runCommandWithOutput(helpCmd) c.Assert(err, checker.IsNil, check.Commentf(out)) lines := strings.Split(out, "\n") + foundTooLongLine := false for _, line := range lines { - c.Assert(len(line), checker.LessOrEqualThan, 80, check.Commentf("Line is too long:\n%s", line)) - + if !foundTooLongLine && len(line) > 80 { + c.Logf("Line is too long:\n%s", line) + foundTooLongLine = true + } // All lines should not end with a space c.Assert(line, checker.Not(checker.HasSuffix), " ", check.Commentf("Line should not end with a space")) diff --git a/components/engine/integration-cli/docker_cli_history_test.go b/components/engine/integration-cli/docker_cli_history_test.go index 55b789c54d..0ee1c46be0 100644 --- a/components/engine/integration-cli/docker_cli_history_test.go +++ b/components/engine/integration-cli/docker_cli_history_test.go @@ -18,33 +18,33 @@ func (s *DockerSuite) TestBuildHistory(c *check.C) { // Removing it from Windows CI for now, but this will be revisited in the // TP5 timeframe when perf is better. name := "testbuildhistory" - _, err := buildImage(name, `FROM busybox -RUN echo "A" -RUN echo "B" -RUN echo "C" -RUN echo "D" -RUN echo "E" -RUN echo "F" -RUN echo "G" -RUN echo "H" -RUN echo "I" -RUN echo "J" -RUN echo "K" -RUN echo "L" -RUN echo "M" -RUN echo "N" -RUN echo "O" -RUN echo "P" -RUN echo "Q" -RUN echo "R" -RUN echo "S" -RUN echo "T" -RUN echo "U" -RUN echo "V" -RUN echo "W" -RUN echo "X" -RUN echo "Y" -RUN echo "Z"`, + _, err := buildImage(name, `FROM `+minimalBaseImage()+` +LABEL label.A="A" +LABEL label.B="B" +LABEL label.C="C" +LABEL label.D="D" +LABEL label.E="E" +LABEL label.F="F" +LABEL label.G="G" +LABEL label.H="H" +LABEL label.I="I" +LABEL label.J="J" +LABEL label.K="K" +LABEL label.L="L" +LABEL label.M="M" +LABEL label.N="N" +LABEL label.O="O" +LABEL label.P="P" +LABEL label.Q="Q" +LABEL label.R="R" +LABEL label.S="S" +LABEL label.T="T" +LABEL label.U="U" +LABEL label.V="V" +LABEL label.W="W" +LABEL label.X="X" +LABEL label.Y="Y" +LABEL label.Z="Z"`, true) c.Assert(err, checker.IsNil) @@ -54,7 +54,7 @@ RUN echo "Z"`, expectedValues := [26]string{"Z", "Y", "X", "W", "V", "U", "T", "S", "R", "Q", "P", "O", "N", "M", "L", "K", "J", "I", "H", "G", "F", "E", "D", "C", "B", "A"} for i := 0; i < 26; i++ { - echoValue := fmt.Sprintf("echo \"%s\"", expectedValues[i]) + echoValue := fmt.Sprintf("LABEL label.%s=%s", expectedValues[i], expectedValues[i]) actualValue := actualValues[i] c.Assert(actualValue, checker.Contains, echoValue) } diff --git a/components/engine/integration-cli/docker_cli_import_test.go b/components/engine/integration-cli/docker_cli_import_test.go index 4352817285..9420dafa57 100644 --- a/components/engine/integration-cli/docker_cli_import_test.go +++ b/components/engine/integration-cli/docker_cli_import_test.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "compress/gzip" "io/ioutil" "os" "os/exec" @@ -59,6 +60,31 @@ func (s *DockerSuite) TestImportFile(c *check.C) { c.Assert(out, checker.Equals, "", check.Commentf("command output should've been nothing.")) } +func (s *DockerSuite) TestImportGzipped(c *check.C) { + testRequires(c, DaemonIsLinux) + dockerCmd(c, "run", "--name", "test-import", "busybox", "true") + + temporaryFile, err := ioutil.TempFile("", "exportImportTest") + c.Assert(err, checker.IsNil, check.Commentf("failed to create temporary file")) + defer os.Remove(temporaryFile.Name()) + + runCmd := exec.Command(dockerBinary, "export", "test-import") + w := gzip.NewWriter(temporaryFile) + runCmd.Stdout = w + + _, err = runCommand(runCmd) + c.Assert(err, checker.IsNil, check.Commentf("failed to export a container")) + err = w.Close() + c.Assert(err, checker.IsNil, check.Commentf("failed to close gzip writer")) + temporaryFile.Close() + out, _ := dockerCmd(c, "import", temporaryFile.Name()) + c.Assert(out, checker.Count, "\n", 1, check.Commentf("display is expected 1 '\\n' but didn't")) + image := strings.TrimSpace(out) + + out, _ = dockerCmd(c, "run", "--rm", image, "true") + c.Assert(out, checker.Equals, "", check.Commentf("command output should've been nothing.")) +} + func (s *DockerSuite) TestImportFileWithMessage(c *check.C) { testRequires(c, DaemonIsLinux) dockerCmd(c, "run", "--name", "test-import", "busybox", "true") diff --git a/components/engine/integration-cli/docker_cli_inspect_test.go b/components/engine/integration-cli/docker_cli_inspect_test.go index b40d2d9a7b..e365350c47 100644 --- a/components/engine/integration-cli/docker_cli_inspect_test.go +++ b/components/engine/integration-cli/docker_cli_inspect_test.go @@ -166,16 +166,12 @@ func (s *DockerSuite) TestInspectContainerFilterInt(c *check.C) { } func (s *DockerSuite) TestInspectImageGraphDriver(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux, Devicemapper) imageTest := "emptyfs" name := inspectField(c, imageTest, "GraphDriver.Name") checkValidGraphDriver(c, name) - if name != "devicemapper" { - c.Skip("requires devicemapper graphdriver") - } - deviceID := inspectField(c, imageTest, "GraphDriver.Data.DeviceId") _, err := strconv.Atoi(deviceID) @@ -188,7 +184,8 @@ func (s *DockerSuite) TestInspectImageGraphDriver(c *check.C) { } func (s *DockerSuite) TestInspectContainerGraphDriver(c *check.C) { - testRequires(c, DaemonIsLinux) + testRequires(c, DaemonIsLinux, Devicemapper) + out, _ := dockerCmd(c, "run", "-d", "busybox", "true") out = strings.TrimSpace(out) @@ -196,10 +193,6 @@ func (s *DockerSuite) TestInspectContainerGraphDriver(c *check.C) { checkValidGraphDriver(c, name) - if name != "devicemapper" { - return - } - imageDeviceID := inspectField(c, "busybox", "GraphDriver.Data.DeviceId") deviceID := inspectField(c, out, "GraphDriver.Data.DeviceId") @@ -280,7 +273,7 @@ func (s *DockerSuite) TestInspectNoSizeFlagContainer(c *check.C) { //Both the container and image are named busybox. docker inspect will fetch container //JSON SizeRw and SizeRootFs field. If there is no flag --size/-s, there are no size fields. - dockerCmd(c, "run", "--name=busybox", "-d", "busybox", "top") + runSleepingContainer(c, "--name=busybox", "-d") formatStr := "--format='{{.SizeRw}},{{.SizeRootFs}}'" out, _ := dockerCmd(c, "inspect", "--type=container", formatStr, "busybox") @@ -288,7 +281,7 @@ func (s *DockerSuite) TestInspectNoSizeFlagContainer(c *check.C) { } func (s *DockerSuite) TestInspectSizeFlagContainer(c *check.C) { - dockerCmd(c, "run", "--name=busybox", "-d", "busybox", "top") + runSleepingContainer(c, "--name=busybox", "-d") formatStr := "--format='{{.SizeRw}},{{.SizeRootFs}}'" out, _ := dockerCmd(c, "inspect", "-s", "--type=container", formatStr, "busybox") @@ -299,7 +292,7 @@ func (s *DockerSuite) TestInspectSizeFlagContainer(c *check.C) { } func (s *DockerSuite) TestInspectSizeFlagImage(c *check.C) { - dockerCmd(c, "run", "--name=busybox", "-d", "busybox", "top") + runSleepingContainer(c, "-d") formatStr := "--format='{{.SizeRw}},{{.SizeRootFs}}'" out, _, err := dockerCmdWithError("inspect", "-s", "--type=image", formatStr, "busybox") @@ -310,10 +303,10 @@ func (s *DockerSuite) TestInspectSizeFlagImage(c *check.C) { c.Assert(out, checker.Contains, "Template parsing error") } -func (s *DockerSuite) TestInspectTempateError(c *check.C) { +func (s *DockerSuite) TestInspectTemplateError(c *check.C) { // Template parsing error for both the container and image. - dockerCmd(c, "run", "--name=container1", "-d", "busybox", "top") + runSleepingContainer(c, "--name=container1", "-d") out, _, err := dockerCmdWithError("inspect", "--type=container", "--format='Format container: {{.ThisDoesNotExist}}'", "container1") c.Assert(err, check.Not(check.IsNil)) @@ -325,7 +318,7 @@ func (s *DockerSuite) TestInspectTempateError(c *check.C) { } func (s *DockerSuite) TestInspectJSONFields(c *check.C) { - dockerCmd(c, "run", "--name=busybox", "-d", "busybox", "top") + runSleepingContainer(c, "--name=busybox", "-d") out, _, err := dockerCmdWithError("inspect", "--type=container", "--format='{{.HostConfig.Dns}}'", "busybox") c.Assert(err, check.IsNil) @@ -344,8 +337,8 @@ func (s *DockerSuite) TestInspectByPrefix(c *check.C) { } func (s *DockerSuite) TestInspectStopWhenNotFound(c *check.C) { - dockerCmd(c, "run", "--name=busybox", "-d", "busybox", "top") - dockerCmd(c, "run", "--name=not-shown", "-d", "busybox", "top") + runSleepingContainer(c, "--name=busybox", "-d") + runSleepingContainer(c, "--name=not-shown", "-d") out, _, err := dockerCmdWithError("inspect", "--type=container", "--format='{{.Name}}'", "busybox", "missing", "not-shown") c.Assert(err, checker.Not(check.IsNil)) diff --git a/components/engine/integration-cli/docker_cli_kill_test.go b/components/engine/integration-cli/docker_cli_kill_test.go index f1a39e954c..05d9a55879 100644 --- a/components/engine/integration-cli/docker_cli_kill_test.go +++ b/components/engine/integration-cli/docker_cli_kill_test.go @@ -10,8 +10,7 @@ import ( ) func (s *DockerSuite) TestKillContainer(c *check.C) { - testRequires(c, DaemonIsLinux) - out, _ := dockerCmd(c, "run", "-d", "busybox", "top") + out, _ := runSleepingContainer(c, "-d") cleanedContainerID := strings.TrimSpace(out) c.Assert(waitRun(cleanedContainerID), check.IsNil) @@ -22,9 +21,8 @@ func (s *DockerSuite) TestKillContainer(c *check.C) { } -func (s *DockerSuite) TestKillofStoppedContainer(c *check.C) { - testRequires(c, DaemonIsLinux) - out, _ := dockerCmd(c, "run", "-d", "busybox", "top") +func (s *DockerSuite) TestKillOffStoppedContainer(c *check.C) { + out, _ := runSleepingContainer(c, "-d") cleanedContainerID := strings.TrimSpace(out) dockerCmd(c, "stop", cleanedContainerID) @@ -34,6 +32,7 @@ func (s *DockerSuite) TestKillofStoppedContainer(c *check.C) { } func (s *DockerSuite) TestKillDifferentUserContainer(c *check.C) { + // TODO Windows: Windows does not yet support -u (Feb 2016). testRequires(c, DaemonIsLinux) out, _ := dockerCmd(c, "run", "-u", "daemon", "-d", "busybox", "top") cleanedContainerID := strings.TrimSpace(out) @@ -48,6 +47,7 @@ func (s *DockerSuite) TestKillDifferentUserContainer(c *check.C) { // regression test about correct signal parsing see #13665 func (s *DockerSuite) TestKillWithSignal(c *check.C) { + // Cannot port to Windows - does not support signals in the same was a Linux does testRequires(c, DaemonIsLinux) out, _ := dockerCmd(c, "run", "-d", "busybox", "top") cid := strings.TrimSpace(out) @@ -61,8 +61,7 @@ func (s *DockerSuite) TestKillWithSignal(c *check.C) { } func (s *DockerSuite) TestKillWithInvalidSignal(c *check.C) { - testRequires(c, DaemonIsLinux) - out, _ := dockerCmd(c, "run", "-d", "busybox", "top") + out, _ := runSleepingContainer(c, "-d") cid := strings.TrimSpace(out) c.Assert(waitRun(cid), check.IsNil) @@ -73,7 +72,7 @@ func (s *DockerSuite) TestKillWithInvalidSignal(c *check.C) { running := inspectField(c, cid, "State.Running") c.Assert(running, checker.Equals, "true", check.Commentf("Container should be in running state after an invalid signal")) - out, _ = dockerCmd(c, "run", "-d", "busybox", "top") + out, _ = runSleepingContainer(c, "-d") cid = strings.TrimSpace(out) c.Assert(waitRun(cid), check.IsNil) @@ -87,8 +86,7 @@ func (s *DockerSuite) TestKillWithInvalidSignal(c *check.C) { } func (s *DockerSuite) TestKillStoppedContainerAPIPre120(c *check.C) { - testRequires(c, DaemonIsLinux) - dockerCmd(c, "run", "--name", "docker-kill-test-api", "-d", "busybox", "top") + runSleepingContainer(c, "--name", "docker-kill-test-api", "-d") dockerCmd(c, "stop", "docker-kill-test-api") status, _, err := sockRequest("POST", fmt.Sprintf("/v1.19/containers/%s/kill", "docker-kill-test-api"), nil) diff --git a/components/engine/integration-cli/docker_cli_login_test.go b/components/engine/integration-cli/docker_cli_login_test.go index ab6294092d..204d032e10 100644 --- a/components/engine/integration-cli/docker_cli_login_test.go +++ b/components/engine/integration-cli/docker_cli_login_test.go @@ -20,11 +20,25 @@ func (s *DockerSuite) TestLoginWithoutTTY(c *check.C) { } func (s *DockerRegistryAuthSuite) TestLoginToPrivateRegistry(c *check.C) { + // wrong credentials + out, _, err := dockerCmdWithError("login", "-u", s.reg.username, "-p", "WRONGPASSWORD", privateRegistryURL) + c.Assert(err, checker.NotNil, check.Commentf(out)) + c.Assert(out, checker.Contains, "401 Unauthorized") + + // now it's fine + dockerCmd(c, "login", "-u", s.reg.username, "-p", s.reg.password, privateRegistryURL) +} + +func (s *DockerRegistryAuthSuite) TestLoginToPrivateRegistryDeprecatedEmailFlag(c *check.C) { + // Test to make sure login still works with the deprecated -e and --email flags // wrong credentials out, _, err := dockerCmdWithError("login", "-u", s.reg.username, "-p", "WRONGPASSWORD", "-e", s.reg.email, privateRegistryURL) c.Assert(err, checker.NotNil, check.Commentf(out)) c.Assert(out, checker.Contains, "401 Unauthorized") // now it's fine + // -e flag dockerCmd(c, "login", "-u", s.reg.username, "-p", s.reg.password, "-e", s.reg.email, privateRegistryURL) + // --email flag + dockerCmd(c, "login", "-u", s.reg.username, "-p", s.reg.password, "--email", s.reg.email, privateRegistryURL) } diff --git a/components/engine/integration-cli/docker_cli_logout_test.go b/components/engine/integration-cli/docker_cli_logout_test.go new file mode 100644 index 0000000000..8b756146f1 --- /dev/null +++ b/components/engine/integration-cli/docker_cli_logout_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/docker/docker/pkg/integration/checker" + "github.com/go-check/check" +) + +func (s *DockerRegistryAuthSuite) TestLogoutWithExternalAuth(c *check.C) { + osPath := os.Getenv("PATH") + defer os.Setenv("PATH", osPath) + + workingDir, err := os.Getwd() + c.Assert(err, checker.IsNil) + absolute, err := filepath.Abs(filepath.Join(workingDir, "fixtures", "auth")) + c.Assert(err, checker.IsNil) + testPath := fmt.Sprintf("%s%c%s", osPath, filepath.ListSeparator, absolute) + + os.Setenv("PATH", testPath) + + repoName := fmt.Sprintf("%v/dockercli/busybox:authtest", privateRegistryURL) + + tmp, err := ioutil.TempDir("", "integration-cli-") + c.Assert(err, checker.IsNil) + + externalAuthConfig := `{ "credsStore": "shell-test" }` + + configPath := filepath.Join(tmp, "config.json") + err = ioutil.WriteFile(configPath, []byte(externalAuthConfig), 0644) + c.Assert(err, checker.IsNil) + + dockerCmd(c, "--config", tmp, "login", "-u", s.reg.username, "-p", s.reg.password, privateRegistryURL) + + b, err := ioutil.ReadFile(configPath) + c.Assert(err, checker.IsNil) + c.Assert(string(b), checker.Not(checker.Contains), "\"auth\":") + c.Assert(string(b), checker.Contains, privateRegistryURL) + + dockerCmd(c, "--config", tmp, "tag", "busybox", repoName) + dockerCmd(c, "--config", tmp, "push", repoName) + + dockerCmd(c, "--config", tmp, "logout", privateRegistryURL) + + b, err = ioutil.ReadFile(configPath) + c.Assert(err, checker.IsNil) + c.Assert(string(b), checker.Not(checker.Contains), privateRegistryURL) + + // check I cannot pull anymore + out, _, err := dockerCmdWithError("--config", tmp, "pull", repoName) + c.Assert(err, check.NotNil, check.Commentf(out)) + c.Assert(out, checker.Contains, fmt.Sprintf("Error: image dockercli/busybox not found")) +} diff --git a/components/engine/integration-cli/docker_cli_logs_test.go b/components/engine/integration-cli/docker_cli_logs_test.go index 1b4f0ddb62..95964c14e7 100644 --- a/components/engine/integration-cli/docker_cli_logs_test.go +++ b/components/engine/integration-cli/docker_cli_logs_test.go @@ -6,7 +6,6 @@ import ( "io" "os/exec" "regexp" - "strconv" "strings" "time" @@ -17,7 +16,6 @@ import ( // This used to work, it test a log of PageSize-1 (gh#4851) func (s *DockerSuite) TestLogsContainerSmallerThanPage(c *check.C) { - testRequires(c, DaemonIsLinux) testLen := 32767 out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", fmt.Sprintf("for i in $(seq 1 %d); do echo -n =; done; echo", testLen)) @@ -31,7 +29,6 @@ func (s *DockerSuite) TestLogsContainerSmallerThanPage(c *check.C) { // Regression test: When going over the PageSize, it used to panic (gh#4851) func (s *DockerSuite) TestLogsContainerBiggerThanPage(c *check.C) { - testRequires(c, DaemonIsLinux) testLen := 32768 out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", fmt.Sprintf("for i in $(seq 1 %d); do echo -n =; done; echo", testLen)) @@ -45,7 +42,6 @@ func (s *DockerSuite) TestLogsContainerBiggerThanPage(c *check.C) { // Regression test: When going much over the PageSize, it used to block (gh#4851) func (s *DockerSuite) TestLogsContainerMuchBiggerThanPage(c *check.C) { - testRequires(c, DaemonIsLinux) testLen := 33000 out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", fmt.Sprintf("for i in $(seq 1 %d); do echo -n =; done; echo", testLen)) @@ -58,7 +54,6 @@ func (s *DockerSuite) TestLogsContainerMuchBiggerThanPage(c *check.C) { } func (s *DockerSuite) TestLogsTimestamps(c *check.C) { - testRequires(c, DaemonIsLinux) testLen := 100 out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", fmt.Sprintf("for i in $(seq 1 %d); do echo =; done;", testLen)) @@ -84,7 +79,6 @@ func (s *DockerSuite) TestLogsTimestamps(c *check.C) { } func (s *DockerSuite) TestLogsSeparateStderr(c *check.C) { - testRequires(c, DaemonIsLinux) msg := "stderr_log" out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", fmt.Sprintf("echo %s 1>&2", msg)) @@ -101,6 +95,8 @@ func (s *DockerSuite) TestLogsSeparateStderr(c *check.C) { } func (s *DockerSuite) TestLogsStderrInStdout(c *check.C) { + // TODO Windows: Needs investigation why this fails. Obtained string includes + // a bunch of ANSI escape sequences before the "stderr_log" message. testRequires(c, DaemonIsLinux) msg := "stderr_log" out, _ := dockerCmd(c, "run", "-d", "-t", "busybox", "sh", "-c", fmt.Sprintf("echo %s 1>&2", msg)) @@ -116,7 +112,6 @@ func (s *DockerSuite) TestLogsStderrInStdout(c *check.C) { } func (s *DockerSuite) TestLogsTail(c *check.C) { - testRequires(c, DaemonIsLinux) testLen := 100 out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", fmt.Sprintf("for i in $(seq 1 %d); do echo =; done;", testLen)) @@ -143,11 +138,9 @@ func (s *DockerSuite) TestLogsTail(c *check.C) { } func (s *DockerSuite) TestLogsFollowStopped(c *check.C) { - testRequires(c, DaemonIsLinux) - out, _ := dockerCmd(c, "run", "-d", "busybox", "echo", "hello") - - id := strings.TrimSpace(out) - dockerCmd(c, "wait", id) + dockerCmd(c, "run", "--name=test", "busybox", "echo", "hello") + id, err := getIDByName("test") + c.Assert(err, check.IsNil) logsCmd := exec.Command(dockerBinary, "logs", "-f", id) c.Assert(logsCmd.Start(), checker.IsNil) @@ -161,13 +154,12 @@ func (s *DockerSuite) TestLogsFollowStopped(c *check.C) { select { case err := <-errChan: c.Assert(err, checker.IsNil) - case <-time.After(1 * time.Second): + case <-time.After(30 * time.Second): c.Fatal("Following logs is hanged") } } func (s *DockerSuite) TestLogsSince(c *check.C) { - testRequires(c, DaemonIsLinux) name := "testlogssince" dockerCmd(c, "run", "--name="+name, "busybox", "/bin/sh", "-c", "for i in $(seq 1 3); do sleep 2; echo log$i; done") out, _ := dockerCmd(c, "logs", "-t", name) @@ -203,24 +195,41 @@ func (s *DockerSuite) TestLogsSince(c *check.C) { } func (s *DockerSuite) TestLogsSinceFutureFollow(c *check.C) { + // TODO Windows: Flakey on TP4. Enable for next technical preview. testRequires(c, DaemonIsLinux) - out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", `for i in $(seq 1 5); do date +%s; sleep 1; done`) - id := strings.TrimSpace(out) + name := "testlogssincefuturefollow" + out, _ := dockerCmd(c, "run", "-d", "--name", name, "busybox", "/bin/sh", "-c", `for i in $(seq 1 5); do echo log$i; sleep 1; done`) - now := daemonTime(c).Unix() - since := now + 2 - out, _ = dockerCmd(c, "logs", "-f", fmt.Sprintf("--since=%v", since), id) + // Extract one timestamp from the log file to give us a starting point for + // our `--since` argument. Because the log producer runs in the background, + // we need to check repeatedly for some output to be produced. + var timestamp string + for i := 0; i != 100 && timestamp == ""; i++ { + if out, _ = dockerCmd(c, "logs", "-t", name); out == "" { + time.Sleep(time.Millisecond * 100) // Retry + } else { + timestamp = strings.Split(strings.Split(out, "\n")[0], " ")[0] + } + } + + c.Assert(timestamp, checker.Not(checker.Equals), "") + t, err := time.Parse(time.RFC3339Nano, timestamp) + c.Assert(err, check.IsNil) + + since := t.Unix() + 2 + out, _ = dockerCmd(c, "logs", "-t", "-f", fmt.Sprintf("--since=%v", since), name) lines := strings.Split(strings.TrimSpace(out), "\n") c.Assert(lines, checker.Not(checker.HasLen), 0) for _, v := range lines { - ts, err := strconv.ParseInt(v, 10, 64) - c.Assert(err, checker.IsNil, check.Commentf("cannot parse timestamp output from log: '%v'\nout=%s", v, out)) - c.Assert(ts >= since, checker.Equals, true, check.Commentf("earlier log found. since=%v logdate=%v", since, ts)) + ts, err := time.Parse(time.RFC3339Nano, strings.Split(v, " ")[0]) + c.Assert(err, checker.IsNil, check.Commentf("cannot parse timestamp output from log: '%v'", v)) + c.Assert(ts.Unix() >= since, checker.Equals, true, check.Commentf("earlier log found. since=%v logdate=%v", since, ts)) } } // Regression test for #8832 func (s *DockerSuite) TestLogsFollowSlowStdoutConsumer(c *check.C) { + // TODO Windows: Consider enabling post-TP4. Too expensive to run on TP4 testRequires(c, DaemonIsLinux) out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", `usleep 600000;yes X | head -c 200000`) @@ -249,11 +258,9 @@ func (s *DockerSuite) TestLogsFollowSlowStdoutConsumer(c *check.C) { actual := bytes1 + bytes2 expected := 200000 c.Assert(actual, checker.Equals, expected) - } func (s *DockerSuite) TestLogsFollowGoroutinesWithStdout(c *check.C) { - testRequires(c, DaemonIsLinux) out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", "while true; do echo hello; sleep 2; done") id := strings.TrimSpace(out) c.Assert(waitRun(id), checker.IsNil) @@ -305,7 +312,6 @@ func (s *DockerSuite) TestLogsFollowGoroutinesWithStdout(c *check.C) { } func (s *DockerSuite) TestLogsFollowGoroutinesNoOutput(c *check.C) { - testRequires(c, DaemonIsLinux) out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", "while true; do sleep 2; done") id := strings.TrimSpace(out) c.Assert(waitRun(id), checker.IsNil) diff --git a/components/engine/integration-cli/docker_cli_network_unix_test.go b/components/engine/integration-cli/docker_cli_network_unix_test.go index 59ed572612..fd6da66883 100644 --- a/components/engine/integration-cli/docker_cli_network_unix_test.go +++ b/components/engine/integration-cli/docker_cli_network_unix_test.go @@ -10,7 +10,6 @@ import ( "net/http" "net/http/httptest" "os" - "sort" "strings" "time" @@ -248,7 +247,7 @@ func isNwPresent(c *check.C, name string) bool { return false } -// assertNwList checks network list retrived with ls command +// assertNwList checks network list retrieved with ls command // equals to expected network list // note: out should be `network ls [option]` result func assertNwList(c *check.C, out string, expectNws []string) { @@ -259,9 +258,6 @@ func assertNwList(c *check.C, out string, expectNws []string) { // wrap all network name in nwList nwList = append(nwList, netFields[1]) } - // first need to sort out and expected - sort.StringSlice(nwList).Sort() - sort.StringSlice(expectNws).Sort() // network ls should contains all expected networks c.Assert(nwList, checker.DeepEquals, expectNws) @@ -321,11 +317,11 @@ func (s *DockerNetworkSuite) TestDockerNetworkLsFilter(c *check.C) { // filter with partial ID and partial name // only show 'bridge' and 'dev' network out, _ = dockerCmd(c, "network", "ls", "-f", "id="+networkID[0:5], "-f", "name=dge") - assertNwList(c, out, []string{"dev", "bridge"}) + assertNwList(c, out, []string{"bridge", "dev"}) // only show built-in network (bridge, none, host) out, _ = dockerCmd(c, "network", "ls", "-f", "type=builtin") - assertNwList(c, out, []string{"bridge", "none", "host"}) + assertNwList(c, out, []string{"bridge", "host", "none"}) // only show custom networks (dev) out, _ = dockerCmd(c, "network", "ls", "-f", "type=custom") @@ -334,7 +330,7 @@ func (s *DockerNetworkSuite) TestDockerNetworkLsFilter(c *check.C) { // show all networks with filter // it should be equivalent of ls without option out, _ = dockerCmd(c, "network", "ls", "-f", "type=custom", "-f", "type=builtin") - assertNwList(c, out, []string{"dev", "bridge", "host", "none"}) + assertNwList(c, out, []string{"bridge", "dev", "host", "none"}) } func (s *DockerNetworkSuite) TestDockerNetworkCreateDelete(c *check.C) { @@ -574,18 +570,24 @@ func (s *DockerNetworkSuite) TestDockerNetworkInspectDefault(c *check.C) { nr := getNetworkResource(c, "none") c.Assert(nr.Driver, checker.Equals, "null") c.Assert(nr.Scope, checker.Equals, "local") + c.Assert(nr.Internal, checker.Equals, false) + c.Assert(nr.EnableIPv6, checker.Equals, false) c.Assert(nr.IPAM.Driver, checker.Equals, "default") c.Assert(len(nr.IPAM.Config), checker.Equals, 0) nr = getNetworkResource(c, "host") c.Assert(nr.Driver, checker.Equals, "host") c.Assert(nr.Scope, checker.Equals, "local") + c.Assert(nr.Internal, checker.Equals, false) + c.Assert(nr.EnableIPv6, checker.Equals, false) c.Assert(nr.IPAM.Driver, checker.Equals, "default") c.Assert(len(nr.IPAM.Config), checker.Equals, 0) nr = getNetworkResource(c, "bridge") c.Assert(nr.Driver, checker.Equals, "bridge") c.Assert(nr.Scope, checker.Equals, "local") + c.Assert(nr.Internal, checker.Equals, false) + c.Assert(nr.EnableIPv6, checker.Equals, false) c.Assert(nr.IPAM.Driver, checker.Equals, "default") c.Assert(len(nr.IPAM.Config), checker.Equals, 1) c.Assert(nr.IPAM.Config[0].Subnet, checker.NotNil) @@ -600,6 +602,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkInspectCustomUnspecified(c *check. nr := getNetworkResource(c, "test01") c.Assert(nr.Driver, checker.Equals, "bridge") c.Assert(nr.Scope, checker.Equals, "local") + c.Assert(nr.Internal, checker.Equals, false) + c.Assert(nr.EnableIPv6, checker.Equals, false) c.Assert(nr.IPAM.Driver, checker.Equals, "default") c.Assert(len(nr.IPAM.Config), checker.Equals, 1) c.Assert(nr.IPAM.Config[0].Subnet, checker.NotNil) @@ -610,12 +614,14 @@ func (s *DockerNetworkSuite) TestDockerNetworkInspectCustomUnspecified(c *check. } func (s *DockerNetworkSuite) TestDockerNetworkInspectCustomSpecified(c *check.C) { - dockerCmd(c, "network", "create", "--driver=bridge", "--subnet=172.28.0.0/16", "--ip-range=172.28.5.0/24", "--gateway=172.28.5.254", "br0") + dockerCmd(c, "network", "create", "--driver=bridge", "--ipv6", "--subnet=172.28.0.0/16", "--ip-range=172.28.5.0/24", "--gateway=172.28.5.254", "br0") assertNwIsAvailable(c, "br0") nr := getNetworkResource(c, "br0") c.Assert(nr.Driver, checker.Equals, "bridge") c.Assert(nr.Scope, checker.Equals, "local") + c.Assert(nr.Internal, checker.Equals, false) + c.Assert(nr.EnableIPv6, checker.Equals, true) c.Assert(nr.IPAM.Driver, checker.Equals, "default") c.Assert(len(nr.IPAM.Config), checker.Equals, 1) c.Assert(nr.IPAM.Config[0].Subnet, checker.Equals, "172.28.0.0/16") @@ -795,8 +801,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkAnonymousEndpoint(c *check.C) { c.Assert(err, check.NotNil) } -func (s *DockerNetworkSuite) TestDockerNetworkLinkOndefaultNetworkOnly(c *check.C) { - // Link feature must work only on default network, and not across networks +func (s *DockerNetworkSuite) TestDockerNetworkLinkOnDefaultNetworkOnly(c *check.C) { + // Legacy Link feature must work only on default network, and not across networks cnt1 := "container1" cnt2 := "container2" network := "anotherbridge" @@ -1012,6 +1018,10 @@ func (s *DockerNetworkSuite) TestDockerNetworkHostModeUngracefulDaemonRestart(c cName := fmt.Sprintf("hostc-%d", i) out, err := s.d.Cmd("run", "-d", "--name", cName, "--net=host", "--restart=always", "busybox", "top") c.Assert(err, checker.IsNil, check.Commentf(out)) + + // verfiy container has finished starting before killing daemon + err = s.d.waitRun(cName) + c.Assert(err, checker.IsNil) } // Kill daemon ungracefully and restart @@ -1232,7 +1242,7 @@ func (s *DockerSuite) TestUserDefinedNetworkConnectDisconnectLink(c *check.C) { c.Assert(waitRun("first"), check.IsNil) // run a container in a user-defined network with a link for an existing container - // and a link for a container that doesnt exist + // and a link for a container that doesn't exist dockerCmd(c, "run", "-d", "--net=foo1", "--name=second", "--link=first:FirstInFoo1", "--link=third:bar", "busybox", "top") c.Assert(waitRun("second"), check.IsNil) @@ -1304,6 +1314,19 @@ func (s *DockerNetworkSuite) TestDockerNetworkDisconnectDefault(c *check.C) { c.Assert(networks, checker.Not(checker.Contains), "bridge", check.Commentf("Should not contain 'bridge' network")) } +func (s *DockerNetworkSuite) TestDockerNetworkConnectWithAliasOnDefaultNetworks(c *check.C) { + testRequires(c, DaemonIsLinux, NotUserNamespace, NotArm) + + defaults := []string{"bridge", "host", "none"} + out, _ := dockerCmd(c, "run", "-d", "--net=none", "busybox", "top") + containerID := strings.TrimSpace(out) + for _, net := range defaults { + res, _, err := dockerCmdWithError("network", "connect", "--alias", "alias"+net, net, containerID) + c.Assert(err, checker.NotNil) + c.Assert(res, checker.Contains, runconfig.ErrUnsupportedNetworkAndAlias.Error()) + } +} + func (s *DockerSuite) TestUserDefinedNetworkConnectDisconnectAlias(c *check.C) { testRequires(c, DaemonIsLinux, NotUserNamespace, NotArm) dockerCmd(c, "network", "create", "-d", "bridge", "net1") @@ -1375,6 +1398,14 @@ func (s *DockerSuite) TestUserDefinedNetworkConnectivity(c *check.C) { c.Assert(err, check.NotNil) } +func (s *DockerSuite) TestEmbeddedDNSInvalidInput(c *check.C) { + testRequires(c, DaemonIsLinux, NotUserNamespace) + dockerCmd(c, "network", "create", "-d", "bridge", "nw1") + + // Sending garbage to embedded DNS shouldn't crash the daemon + dockerCmd(c, "run", "-i", "--net=nw1", "--name=c1", "debian:jessie", "bash", "-c", "echo InvalidQuery > /dev/udp/127.0.0.11/53") +} + func (s *DockerSuite) TestDockerNetworkConnectFailsNoInspectChange(c *check.C) { dockerCmd(c, "run", "-d", "--name=bb", "busybox", "top") c.Assert(waitRun("bb"), check.IsNil) @@ -1389,7 +1420,7 @@ func (s *DockerSuite) TestDockerNetworkConnectFailsNoInspectChange(c *check.C) { c.Assert(ns1, check.Equals, ns0) } -func (s *DockerNetworkSuite) TestDockerNetworkInternalMode(c *check.C) { +func (s *DockerSuite) TestDockerNetworkInternalMode(c *check.C) { dockerCmd(c, "network", "create", "--driver=bridge", "--internal", "internal") assertNwIsAvailable(c, "internal") nr := getNetworkResource(c, "internal") @@ -1399,8 +1430,9 @@ func (s *DockerNetworkSuite) TestDockerNetworkInternalMode(c *check.C) { c.Assert(waitRun("first"), check.IsNil) dockerCmd(c, "run", "-d", "--net=internal", "--name=second", "busybox", "top") c.Assert(waitRun("second"), check.IsNil) - _, _, err := dockerCmdWithError("exec", "first", "ping", "-c", "1", "www.google.com") + out, _, err := dockerCmdWithError("exec", "first", "ping", "-W", "4", "-c", "1", "www.google.com") c.Assert(err, check.NotNil) + c.Assert(out, checker.Contains, "ping: bad address") _, _, err = dockerCmdWithError("exec", "second", "ping", "-c", "1", "first") c.Assert(err, check.IsNil) } diff --git a/components/engine/integration-cli/docker_cli_port_test.go b/components/engine/integration-cli/docker_cli_port_test.go index a4361f2eaa..80b00fe93e 100644 --- a/components/engine/integration-cli/docker_cli_port_test.go +++ b/components/engine/integration-cli/docker_cli_port_test.go @@ -297,6 +297,9 @@ func (s *DockerSuite) TestPortExposeHostBinding(c *check.C) { func (s *DockerSuite) TestPortBindingOnSandbox(c *check.C) { testRequires(c, DaemonIsLinux, NotUserNamespace) dockerCmd(c, "network", "create", "--internal", "-d", "bridge", "internal-net") + nr := getNetworkResource(c, "internal-net") + c.Assert(nr.Internal, checker.Equals, true) + dockerCmd(c, "run", "--net", "internal-net", "-d", "--name", "c1", "-p", "8080:8080", "busybox", "nc", "-l", "-p", "8080") c.Assert(waitRun("c1"), check.IsNil) diff --git a/components/engine/integration-cli/docker_cli_proxy_test.go b/components/engine/integration-cli/docker_cli_proxy_test.go index e0b60fcc59..e5699ca52c 100644 --- a/components/engine/integration-cli/docker_cli_proxy_test.go +++ b/components/engine/integration-cli/docker_cli_proxy_test.go @@ -14,7 +14,7 @@ func (s *DockerSuite) TestCliProxyDisableProxyUnixSock(c *check.C) { testRequires(c, SameHostDaemon) // test is valid when DOCKER_HOST=unix://.. cmd := exec.Command(dockerBinary, "info") - cmd.Env = appendBaseEnv([]string{"HTTP_PROXY=http://127.0.0.1:9999"}) + cmd.Env = appendBaseEnv(false, "HTTP_PROXY=http://127.0.0.1:9999") out, _, err := runCommandWithOutput(cmd) c.Assert(err, checker.IsNil, check.Commentf("%v", out)) diff --git a/components/engine/integration-cli/docker_cli_ps_test.go b/components/engine/integration-cli/docker_cli_ps_test.go index f72bf6b21d..51cc4349ce 100644 --- a/components/engine/integration-cli/docker_cli_ps_test.go +++ b/components/engine/integration-cli/docker_cli_ps_test.go @@ -64,6 +64,10 @@ func (s *DockerSuite) TestPsListContainersBase(c *check.C) { expected = []string{fourthID, secondID} c.Assert(assertContainerList(out, expected), checker.Equals, true, check.Commentf("SINCE filter: Container list is not in the correct order: \n%s", out)) + out, _ = dockerCmd(c, "ps", "-f", "since="+thirdID) + expected = []string{fourthID} + c.Assert(assertContainerList(out, expected), checker.Equals, true, check.Commentf("SINCE filter: Container list is not in the correct order: \n%s", out)) + // filter before out, _ = dockerCmd(c, "ps", "-f", "before="+fourthID, "-a") expected = []string{thirdID, secondID, firstID} @@ -73,6 +77,10 @@ func (s *DockerSuite) TestPsListContainersBase(c *check.C) { expected = []string{secondID, firstID} c.Assert(assertContainerList(out, expected), checker.Equals, true, check.Commentf("BEFORE filter: Container list is not in the correct order: \n%s", out)) + out, _ = dockerCmd(c, "ps", "-f", "before="+thirdID) + expected = []string{secondID, firstID} + c.Assert(assertContainerList(out, expected), checker.Equals, true, check.Commentf("SINCE filter: Container list is not in the correct order: \n%s", out)) + // filter since & before out, _ = dockerCmd(c, "ps", "-f", "since="+firstID, "-f", "before="+fourthID, "-a") expected = []string{thirdID, secondID} @@ -208,7 +216,7 @@ func assertContainerList(out string, expected []string) bool { func (s *DockerSuite) TestPsListContainersSize(c *check.C) { // Problematic on Windows as it doesn't report the size correctly @swernli testRequires(c, DaemonIsLinux) - dockerCmd(c, "run", "-d", "busybox", "echo", "hello") + dockerCmd(c, "run", "-d", "busybox") baseOut, _ := dockerCmd(c, "ps", "-s", "-n=1") baseLines := strings.Split(strings.Trim(baseOut, "\n "), "\n") @@ -218,11 +226,12 @@ func (s *DockerSuite) TestPsListContainersSize(c *check.C) { c.Assert(err, checker.IsNil) name := "test_size" - out, _ := dockerCmd(c, "run", "--name", name, "busybox", "sh", "-c", "echo 1 > test") + dockerCmd(c, "run", "--name", name, "busybox", "sh", "-c", "echo 1 > test") id, err := getIDByName(name) c.Assert(err, checker.IsNil) runCmd := exec.Command(dockerBinary, "ps", "-s", "-n=1") + var out string wait := make(chan struct{}) go func() { @@ -244,7 +253,6 @@ func (s *DockerSuite) TestPsListContainersSize(c *check.C) { expectedSize := fmt.Sprintf("%d B", (2 + baseBytes)) foundSize := lines[1][sizeIndex:] c.Assert(foundSize, checker.Contains, expectedSize, check.Commentf("Expected size %q, got %q", expectedSize, foundSize)) - } func (s *DockerSuite) TestPsListContainersFilterStatus(c *check.C) { @@ -303,17 +311,17 @@ func (s *DockerSuite) TestPsListContainersFilterID(c *check.C) { func (s *DockerSuite) TestPsListContainersFilterName(c *check.C) { // start container - out, _ := dockerCmd(c, "run", "-d", "--name=a_name_to_match", "busybox") - firstID := strings.TrimSpace(out) + dockerCmd(c, "run", "--name=a_name_to_match", "busybox") + id, err := getIDByName("a_name_to_match") + c.Assert(err, check.IsNil) // start another container runSleepingContainer(c, "--name=b_name_to_match") // filter containers by name - out, _ = dockerCmd(c, "ps", "-a", "-q", "--filter=name=a_name_to_match") + out, _ := dockerCmd(c, "ps", "-a", "-q", "--filter=name=a_name_to_match") containerOut := strings.TrimSpace(out) - c.Assert(containerOut, checker.Equals, firstID[:12], check.Commentf("Expected id %s, got %s for exited filter, output: %q", firstID[:12], containerOut, out)) - + c.Assert(containerOut, checker.Equals, id[:12], check.Commentf("Expected id %s, got %s for exited filter, output: %q", id[:12], containerOut, out)) } // Test for the ancestor filter for ps. @@ -345,24 +353,29 @@ func (s *DockerSuite) TestPsListContainersFilterAncestorImage(c *check.C) { c.Assert(err, checker.IsNil) // start containers - out, _ := dockerCmd(c, "run", "-d", "busybox", "echo", "hello") - firstID := strings.TrimSpace(out) + dockerCmd(c, "run", "--name=first", "busybox", "echo", "hello") + firstID, err := getIDByName("first") + c.Assert(err, check.IsNil) // start another container - out, _ = dockerCmd(c, "run", "-d", "busybox", "echo", "hello") - secondID := strings.TrimSpace(out) + dockerCmd(c, "run", "--name=second", "busybox", "echo", "hello") + secondID, err := getIDByName("second") + c.Assert(err, check.IsNil) // start third container - out, _ = dockerCmd(c, "run", "-d", imageName1, "echo", "hello") - thirdID := strings.TrimSpace(out) + dockerCmd(c, "run", "--name=third", imageName1, "echo", "hello") + thirdID, err := getIDByName("third") + c.Assert(err, check.IsNil) // start fourth container - out, _ = dockerCmd(c, "run", "-d", imageName1Tagged, "echo", "hello") - fourthID := strings.TrimSpace(out) + dockerCmd(c, "run", "--name=fourth", imageName1Tagged, "echo", "hello") + fourthID, err := getIDByName("fourth") + c.Assert(err, check.IsNil) // start fifth container - out, _ = dockerCmd(c, "run", "-d", imageName2, "echo", "hello") - fifthID := strings.TrimSpace(out) + dockerCmd(c, "run", "--name=fifth", imageName2, "echo", "hello") + fifthID, err := getIDByName("fifth") + c.Assert(err, check.IsNil) var filterTestSuite = []struct { filterName string @@ -387,6 +400,7 @@ func (s *DockerSuite) TestPsListContainersFilterAncestorImage(c *check.C) { {imageID2, []string{fifthID}}, } + var out string for _, filter := range filterTestSuite { out, _ = dockerCmd(c, "ps", "-a", "-q", "--no-trunc", "--filter=ancestor="+filter.filterName) checkPsAncestorFilterOutput(c, out, filter.filterName, filter.expectedIDs) @@ -421,19 +435,22 @@ func checkPsAncestorFilterOutput(c *check.C, out string, filterName string, expe func (s *DockerSuite) TestPsListContainersFilterLabel(c *check.C) { // start container - out, _ := dockerCmd(c, "run", "-d", "-l", "match=me", "-l", "second=tag", "busybox") - firstID := strings.TrimSpace(out) + dockerCmd(c, "run", "--name=first", "-l", "match=me", "-l", "second=tag", "busybox") + firstID, err := getIDByName("first") + c.Assert(err, check.IsNil) // start another container - out, _ = dockerCmd(c, "run", "-d", "-l", "match=me too", "busybox") - secondID := strings.TrimSpace(out) + dockerCmd(c, "run", "--name=second", "-l", "match=me too", "busybox") + secondID, err := getIDByName("second") + c.Assert(err, check.IsNil) // start third container - out, _ = dockerCmd(c, "run", "-d", "-l", "nomatch=me", "busybox") - thirdID := strings.TrimSpace(out) + dockerCmd(c, "run", "--name=third", "-l", "nomatch=me", "busybox") + thirdID, err := getIDByName("third") + c.Assert(err, check.IsNil) // filter containers by exact match - out, _ = dockerCmd(c, "ps", "-a", "-q", "--no-trunc", "--filter=label=match=me") + out, _ := dockerCmd(c, "ps", "-a", "-q", "--no-trunc", "--filter=label=match=me") containerOut := strings.TrimSpace(out) c.Assert(containerOut, checker.Equals, firstID, check.Commentf("Expected id %s, got %s for exited filter, output: %q", firstID, containerOut, out)) @@ -734,3 +751,56 @@ func (s *DockerSuite) TestPsNotShowPortsOfStoppedContainer(c *check.C) { fields = strings.Fields(lines[1]) c.Assert(fields[len(fields)-2], checker.Not(checker.Equals), expected, check.Commentf("Should not got %v", expected)) } + +func (s *DockerSuite) TestPsShowMounts(c *check.C) { + prefix, slash := getPrefixAndSlashFromDaemonPlatform() + + mp := prefix + slash + "test" + + dockerCmd(c, "volume", "create", "--name", "ps-volume-test") + runSleepingContainer(c, "--name=volume-test-1", "--volume", "ps-volume-test:"+mp) + c.Assert(waitRun("volume-test-1"), checker.IsNil) + runSleepingContainer(c, "--name=volume-test-2", "--volume", mp) + c.Assert(waitRun("volume-test-2"), checker.IsNil) + + out, _ := dockerCmd(c, "ps", "--format", "{{.Names}} {{.Mounts}}") + + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + c.Assert(lines, checker.HasLen, 2) + + fields := strings.Fields(lines[0]) + c.Assert(fields, checker.HasLen, 2) + + annonymounsVolumeID := fields[1] + + fields = strings.Fields(lines[1]) + c.Assert(fields[1], checker.Equals, "ps-volume-test") + + // filter by volume name + out, _ = dockerCmd(c, "ps", "--format", "{{.Names}} {{.Mounts}}", "--filter", "volume=ps-volume-test") + + lines = strings.Split(strings.TrimSpace(string(out)), "\n") + c.Assert(lines, checker.HasLen, 1) + + fields = strings.Fields(lines[0]) + c.Assert(fields[1], checker.Equals, "ps-volume-test") + + // empty results filtering by unknown volume + out, _ = dockerCmd(c, "ps", "--format", "{{.Names}} {{.Mounts}}", "--filter", "volume=this-volume-should-not-exist") + c.Assert(strings.TrimSpace(string(out)), checker.HasLen, 0) + + // filter by mount destination + out, _ = dockerCmd(c, "ps", "--format", "{{.Names}} {{.Mounts}}", "--filter", "volume="+mp) + + lines = strings.Split(strings.TrimSpace(string(out)), "\n") + c.Assert(lines, checker.HasLen, 2) + + fields = strings.Fields(lines[0]) + c.Assert(fields[1], checker.Equals, annonymounsVolumeID) + fields = strings.Fields(lines[1]) + c.Assert(fields[1], checker.Equals, "ps-volume-test") + + // empty results filtering by unknown mount point + out, _ = dockerCmd(c, "ps", "--format", "{{.Names}} {{.Mounts}}", "--filter", "volume="+prefix+slash+"this-path-was-never-mounted") + c.Assert(strings.TrimSpace(string(out)), checker.HasLen, 0) +} diff --git a/components/engine/integration-cli/docker_cli_pull_local_test.go b/components/engine/integration-cli/docker_cli_pull_local_test.go index 8846fecea2..54b5b9ef90 100644 --- a/components/engine/integration-cli/docker_cli_pull_local_test.go +++ b/components/engine/integration-cli/docker_cli_pull_local_test.go @@ -361,3 +361,38 @@ func (s *DockerRegistrySuite) TestPullManifestList(c *check.C) { dockerCmd(c, "rmi", repoName) } + +func (s *DockerRegistryAuthSuite) TestPullWithExternalAuth(c *check.C) { + osPath := os.Getenv("PATH") + defer os.Setenv("PATH", osPath) + + workingDir, err := os.Getwd() + c.Assert(err, checker.IsNil) + absolute, err := filepath.Abs(filepath.Join(workingDir, "fixtures", "auth")) + c.Assert(err, checker.IsNil) + testPath := fmt.Sprintf("%s%c%s", osPath, filepath.ListSeparator, absolute) + + os.Setenv("PATH", testPath) + + repoName := fmt.Sprintf("%v/dockercli/busybox:authtest", privateRegistryURL) + + tmp, err := ioutil.TempDir("", "integration-cli-") + c.Assert(err, checker.IsNil) + + externalAuthConfig := `{ "credsStore": "shell-test" }` + + configPath := filepath.Join(tmp, "config.json") + err = ioutil.WriteFile(configPath, []byte(externalAuthConfig), 0644) + c.Assert(err, checker.IsNil) + + dockerCmd(c, "--config", tmp, "login", "-u", s.reg.username, "-p", s.reg.password, privateRegistryURL) + + b, err := ioutil.ReadFile(configPath) + c.Assert(err, checker.IsNil) + c.Assert(string(b), checker.Not(checker.Contains), "\"auth\":") + + dockerCmd(c, "--config", tmp, "tag", "busybox", repoName) + dockerCmd(c, "--config", tmp, "push", repoName) + + dockerCmd(c, "--config", tmp, "pull", repoName) +} diff --git a/components/engine/integration-cli/docker_cli_pull_test.go b/components/engine/integration-cli/docker_cli_pull_test.go index 9d36296091..43369c8ed6 100644 --- a/components/engine/integration-cli/docker_cli_pull_test.go +++ b/components/engine/integration-cli/docker_cli_pull_test.go @@ -4,6 +4,7 @@ import ( "fmt" "regexp" "strings" + "sync" "time" "github.com/docker/distribution/digest" @@ -39,30 +40,68 @@ func (s *DockerHubPullSuite) TestPullFromCentralRegistry(c *check.C) { // combinations of implicit tag and library prefix. func (s *DockerHubPullSuite) TestPullNonExistingImage(c *check.C) { testRequires(c, DaemonIsLinux) - for _, e := range []struct { + + type entry struct { Repo string Alias string - }{ + } + + entries := []entry{ {"library/asdfasdf", "asdfasdf:foobar"}, {"library/asdfasdf", "library/asdfasdf:foobar"}, {"library/asdfasdf", "asdfasdf"}, {"library/asdfasdf", "asdfasdf:latest"}, {"library/asdfasdf", "library/asdfasdf"}, {"library/asdfasdf", "library/asdfasdf:latest"}, - } { - out, err := s.CmdWithError("pull", e.Alias) - c.Assert(err, checker.NotNil, check.Commentf("expected non-zero exit status when pulling non-existing image: %s", out)) - // Hub returns 401 rather than 404 for nonexistent repos over - // the v2 protocol - but we should end up falling back to v1, - // which does return a 404. - c.Assert(out, checker.Contains, fmt.Sprintf("Error: image %s not found", e.Repo), check.Commentf("expected image not found error messages")) + } - // pull -a on a nonexistent registry should fall back as well + // The option field indicates "-a" or not. + type record struct { + e entry + option string + out string + err error + } + + // Execute 'docker pull' in parallel, pass results (out, err) and + // necessary information ("-a" or not, and the image name) to channel. + var group sync.WaitGroup + recordChan := make(chan record, len(entries)*2) + for _, e := range entries { + group.Add(1) + go func(e entry) { + defer group.Done() + out, err := s.CmdWithError("pull", e.Alias) + recordChan <- record{e, "", out, err} + }(e) if !strings.ContainsRune(e.Alias, ':') { - out, err := s.CmdWithError("pull", "-a", e.Alias) - c.Assert(err, checker.NotNil, check.Commentf("expected non-zero exit status when pulling non-existing image: %s", out)) - c.Assert(out, checker.Contains, fmt.Sprintf("Error: image %s not found", e.Repo), check.Commentf("expected image not found error messages")) - c.Assert(out, checker.Not(checker.Contains), "unauthorized", check.Commentf(`message should not contain "unauthorized"`)) + // pull -a on a nonexistent registry should fall back as well + group.Add(1) + go func(e entry) { + defer group.Done() + out, err := s.CmdWithError("pull", "-a", e.Alias) + recordChan <- record{e, "-a", out, err} + }(e) + } + } + + // Wait for completion + group.Wait() + close(recordChan) + + // Process the results (out, err). + for record := range recordChan { + if len(record.option) == 0 { + c.Assert(record.err, checker.NotNil, check.Commentf("expected non-zero exit status when pulling non-existing image: %s", record.out)) + // Hub returns 401 rather than 404 for nonexistent repos over + // the v2 protocol - but we should end up falling back to v1, + // which does return a 404. + c.Assert(record.out, checker.Contains, fmt.Sprintf("Error: image %s not found", record.e.Repo), check.Commentf("expected image not found error messages")) + } else { + // pull -a on a nonexistent registry should fall back as well + c.Assert(record.err, checker.NotNil, check.Commentf("expected non-zero exit status when pulling non-existing image: %s", record.out)) + c.Assert(record.out, checker.Contains, fmt.Sprintf("Error: image %s not found", record.e.Repo), check.Commentf("expected image not found error messages")) + c.Assert(record.out, checker.Not(checker.Contains), "unauthorized", check.Commentf(`message should not contain "unauthorized"`)) } } @@ -74,10 +113,37 @@ func (s *DockerHubPullSuite) TestPullNonExistingImage(c *check.C) { // multiple images. func (s *DockerHubPullSuite) TestPullFromCentralRegistryImplicitRefParts(c *check.C) { testRequires(c, DaemonIsLinux) - s.Cmd(c, "pull", "hello-world") + + // Pull hello-world from v2 + pullFromV2 := func(ref string) (int, string) { + out := s.Cmd(c, "pull", "hello-world") + v1Retries := 0 + for strings.Contains(out, "this image was pulled from a legacy registry") { + // Some network errors may cause fallbacks to the v1 + // protocol, which would violate the test's assumption + // that it will get the same images. To make the test + // more robust against these network glitches, allow a + // few retries if we end up with a v1 pull. + + if v1Retries > 2 { + c.Fatalf("too many v1 fallback incidents when pulling %s", ref) + } + + s.Cmd(c, "rmi", ref) + out = s.Cmd(c, "pull", ref) + + v1Retries++ + } + + return v1Retries, out + } + + pullFromV2("hello-world") defer deleteImages("hello-world") - for _, i := range []string{ + s.Cmd(c, "tag", "hello-world", "hello-world-backup") + + for _, ref := range []string{ "hello-world", "hello-world:latest", "library/hello-world", @@ -85,10 +151,26 @@ func (s *DockerHubPullSuite) TestPullFromCentralRegistryImplicitRefParts(c *chec "docker.io/library/hello-world", "index.docker.io/library/hello-world", } { - out := s.Cmd(c, "pull", i) + var out string + for { + var v1Retries int + v1Retries, out = pullFromV2(ref) + + // Keep repeating the test case until we don't hit a v1 + // fallback case. We won't get the right "Image is up + // to date" message if the local image was replaced + // with one pulled from v1. + if v1Retries == 0 { + break + } + s.Cmd(c, "rmi", ref) + s.Cmd(c, "tag", "hello-world-backup", "hello-world") + } c.Assert(out, checker.Contains, "Image is up to date for hello-world:latest") } + s.Cmd(c, "rmi", "hello-world-backup") + // We should have a single entry in images. img := strings.TrimSpace(s.Cmd(c, "images")) splitImg := strings.Split(img, "\n") diff --git a/components/engine/integration-cli/docker_cli_pull_trusted_test.go b/components/engine/integration-cli/docker_cli_pull_trusted_test.go index 2f194d08a4..fdaaaa1a82 100644 --- a/components/engine/integration-cli/docker_cli_pull_trusted_test.go +++ b/components/engine/integration-cli/docker_cli_pull_trusted_test.go @@ -135,13 +135,16 @@ func (s *DockerTrustSuite) TestTrustedPullFromBadTrustServer(c *check.C) { c.Assert(err, check.IsNil, check.Commentf(out)) c.Assert(string(out), checker.Contains, "Signing and pushing trust metadata", check.Commentf(out)) - // Now, try pulling with the original client from this new trust server. This should fail. + // Now, try pulling with the original client from this new trust server. This should fall back to cached metadata. pullCmd = exec.Command(dockerBinary, "pull", repoName) s.trustedCmd(pullCmd) out, _, err = runCommandWithOutput(pullCmd) - - c.Assert(err, check.NotNil, check.Commentf(out)) - c.Assert(string(out), checker.Contains, "valid signatures did not meet threshold", check.Commentf(out)) + if err != nil { + c.Fatalf("Error falling back to cached trust data: %s\n%s", err, out) + } + if !strings.Contains(string(out), "Error while downloading remote metadata, using cached timestamp") { + c.Fatalf("Missing expected output on trusted pull:\n%s", out) + } } func (s *DockerTrustSuite) TestTrustedPullWithExpiredSnapshot(c *check.C) { diff --git a/components/engine/integration-cli/docker_cli_push_test.go b/components/engine/integration-cli/docker_cli_push_test.go index 0e6717644d..ee91abfb2f 100644 --- a/components/engine/integration-cli/docker_cli_push_test.go +++ b/components/engine/integration-cli/docker_cli_push_test.go @@ -148,6 +148,61 @@ func (s *DockerSchema1RegistrySuite) TestPushEmptyLayer(c *check.C) { testPushEmptyLayer(c) } +// testConcurrentPush pushes multiple tags to the same repo +// concurrently. +func testConcurrentPush(c *check.C) { + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + + repos := []string{} + for _, tag := range []string{"push1", "push2", "push3"} { + repo := fmt.Sprintf("%v:%v", repoName, tag) + _, err := buildImage(repo, fmt.Sprintf(` + FROM busybox + ENTRYPOINT ["/bin/echo"] + ENV FOO foo + ENV BAR bar + CMD echo %s +`, repo), true) + c.Assert(err, checker.IsNil) + repos = append(repos, repo) + } + + // Push tags, in parallel + results := make(chan error) + + for _, repo := range repos { + go func(repo string) { + _, _, err := runCommandWithOutput(exec.Command(dockerBinary, "push", repo)) + results <- err + }(repo) + } + + for range repos { + err := <-results + c.Assert(err, checker.IsNil, check.Commentf("concurrent push failed with error: %v", err)) + } + + // Clear local images store. + args := append([]string{"rmi"}, repos...) + dockerCmd(c, args...) + + // Re-pull and run individual tags, to make sure pushes succeeded + for _, repo := range repos { + dockerCmd(c, "pull", repo) + dockerCmd(c, "inspect", repo) + out, _ := dockerCmd(c, "run", "--rm", repo) + c.Assert(strings.TrimSpace(out), checker.Equals, "/bin/sh -c echo "+repo) + } +} + +func (s *DockerRegistrySuite) TestConcurrentPush(c *check.C) { + testConcurrentPush(c) +} + +func (s *DockerSchema1RegistrySuite) TestConcurrentPush(c *check.C) { + testConcurrentPush(c) +} + func (s *DockerRegistrySuite) TestCrossRepositoryLayerPush(c *check.C) { sourceRepoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) // tag the image to upload it to the private registry @@ -201,7 +256,7 @@ func (s *DockerSchema1RegistrySuite) TestCrossRepositoryLayerPushNotSupported(c out2, _, err := dockerCmdWithError("push", destRepoName) c.Assert(err, check.IsNil, check.Commentf("pushing the image to the private registry has failed: %s", out2)) // schema1 registry should not support cross-repo layer mounts, so ensure that this does not happen - c.Assert(strings.Contains(out2, "Mounted from dockercli/busybox"), check.Equals, false) + c.Assert(strings.Contains(out2, "Mounted from"), check.Equals, false) digest2 := digest.DigestRegexp.FindString(out2) c.Assert(len(digest2), checker.GreaterThan, 0, check.Commentf("no digest found for pushed manifest")) @@ -231,6 +286,12 @@ func (s *DockerTrustSuite) TestTrustedPush(c *check.C) { out, _, err = runCommandWithOutput(pullCmd) c.Assert(err, check.IsNil, check.Commentf(out)) c.Assert(string(out), checker.Contains, "Status: Downloaded", check.Commentf(out)) + + // Assert that we rotated the snapshot key to the server by checking our local keystore + contents, err := ioutil.ReadDir(filepath.Join(cliconfig.ConfigDir(), "trust/private/tuf_keys", privateRegistryURL, "dockerclitrusted/pushtest")) + c.Assert(err, check.IsNil, check.Commentf("Unable to read local tuf key files")) + // Check that we only have 1 key (targets key) + c.Assert(contents, checker.HasLen, 1) } func (s *DockerTrustSuite) TestTrustedPushWithEnvPasswords(c *check.C) { diff --git a/components/engine/integration-cli/docker_cli_restart_test.go b/components/engine/integration-cli/docker_cli_restart_test.go index f2e0662db5..2de3d12d93 100644 --- a/components/engine/integration-cli/docker_cli_restart_test.go +++ b/components/engine/integration-cli/docker_cli_restart_test.go @@ -11,13 +11,11 @@ import ( ) func (s *DockerSuite) TestRestartStoppedContainer(c *check.C) { - testRequires(c, DaemonIsLinux) - out, _ := dockerCmd(c, "run", "-d", "busybox", "echo", "foobar") + dockerCmd(c, "run", "--name=test", "busybox", "echo", "foobar") + cleanedContainerID, err := getIDByName("test") + c.Assert(err, check.IsNil) - cleanedContainerID := strings.TrimSpace(out) - dockerCmd(c, "wait", cleanedContainerID) - - out, _ = dockerCmd(c, "logs", cleanedContainerID) + out, _ := dockerCmd(c, "logs", cleanedContainerID) c.Assert(out, checker.Equals, "foobar\n") dockerCmd(c, "restart", cleanedContainerID) @@ -27,7 +25,6 @@ func (s *DockerSuite) TestRestartStoppedContainer(c *check.C) { } func (s *DockerSuite) TestRestartRunningContainer(c *check.C) { - testRequires(c, DaemonIsLinux) out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", "echo foobar && sleep 30 && echo 'should not print this'") cleanedContainerID := strings.TrimSpace(out) @@ -48,8 +45,8 @@ func (s *DockerSuite) TestRestartRunningContainer(c *check.C) { // Test that restarting a container with a volume does not create a new volume on restart. Regression test for #819. func (s *DockerSuite) TestRestartWithVolumes(c *check.C) { - testRequires(c, DaemonIsLinux) - out, _ := dockerCmd(c, "run", "-d", "-v", "/test", "busybox", "top") + prefix, slash := getPrefixAndSlashFromDaemonPlatform() + out, _ := runSleepingContainer(c, "-d", "-v", prefix+slash+"test") cleanedContainerID := strings.TrimSpace(out) out, err := inspectFilter(cleanedContainerID, "len .Mounts") @@ -57,7 +54,7 @@ func (s *DockerSuite) TestRestartWithVolumes(c *check.C) { out = strings.Trim(out, " \n\r") c.Assert(out, checker.Equals, "1") - source, err := inspectMountSourceField(cleanedContainerID, "/test") + source, err := inspectMountSourceField(cleanedContainerID, prefix+slash+"test") c.Assert(err, checker.IsNil) dockerCmd(c, "restart", cleanedContainerID) @@ -67,13 +64,12 @@ func (s *DockerSuite) TestRestartWithVolumes(c *check.C) { out = strings.Trim(out, " \n\r") c.Assert(out, checker.Equals, "1") - sourceAfterRestart, err := inspectMountSourceField(cleanedContainerID, "/test") + sourceAfterRestart, err := inspectMountSourceField(cleanedContainerID, prefix+slash+"test") c.Assert(err, checker.IsNil) c.Assert(source, checker.Equals, sourceAfterRestart) } func (s *DockerSuite) TestRestartPolicyNO(c *check.C) { - testRequires(c, DaemonIsLinux) out, _ := dockerCmd(c, "run", "-d", "--restart=no", "busybox", "false") id := strings.TrimSpace(string(out)) @@ -82,7 +78,6 @@ func (s *DockerSuite) TestRestartPolicyNO(c *check.C) { } func (s *DockerSuite) TestRestartPolicyAlways(c *check.C) { - testRequires(c, DaemonIsLinux) out, _ := dockerCmd(c, "run", "-d", "--restart=always", "busybox", "false") id := strings.TrimSpace(string(out)) @@ -96,7 +91,6 @@ func (s *DockerSuite) TestRestartPolicyAlways(c *check.C) { } func (s *DockerSuite) TestRestartPolicyOnFailure(c *check.C) { - testRequires(c, DaemonIsLinux) out, _ := dockerCmd(c, "run", "-d", "--restart=on-failure:1", "busybox", "false") id := strings.TrimSpace(string(out)) @@ -107,12 +101,11 @@ func (s *DockerSuite) TestRestartPolicyOnFailure(c *check.C) { // a good container with --restart=on-failure:3 // MaximumRetryCount!=0; RestartCount=0 -func (s *DockerSuite) TestContainerRestartwithGoodContainer(c *check.C) { - testRequires(c, DaemonIsLinux) +func (s *DockerSuite) TestRestartContainerwithGoodContainer(c *check.C) { out, _ := dockerCmd(c, "run", "-d", "--restart=on-failure:3", "busybox", "true") id := strings.TrimSpace(string(out)) - err := waitInspect(id, "{{ .State.Restarting }} {{ .State.Running }}", "false false", 5*time.Second) + err := waitInspect(id, "{{ .State.Restarting }} {{ .State.Running }}", "false false", 30*time.Second) c.Assert(err, checker.IsNil) count := inspectField(c, id, "RestartCount") @@ -123,10 +116,10 @@ func (s *DockerSuite) TestContainerRestartwithGoodContainer(c *check.C) { } -func (s *DockerSuite) TestContainerRestartSuccess(c *check.C) { - testRequires(c, DaemonIsLinux, SameHostDaemon) +func (s *DockerSuite) TestRestartContainerSuccess(c *check.C) { + testRequires(c, SameHostDaemon) - out, _ := dockerCmd(c, "run", "-d", "--restart=always", "busybox", "top") + out, _ := runSleepingContainer(c, "-d", "--restart=always") id := strings.TrimSpace(out) c.Assert(waitRun(id), check.IsNil) @@ -142,14 +135,15 @@ func (s *DockerSuite) TestContainerRestartSuccess(c *check.C) { err = p.Kill() c.Assert(err, check.IsNil) - err = waitInspect(id, "{{.RestartCount}}", "1", 5*time.Second) + err = waitInspect(id, "{{.RestartCount}}", "1", 30*time.Second) c.Assert(err, check.IsNil) - err = waitInspect(id, "{{.State.Status}}", "running", 5*time.Second) + err = waitInspect(id, "{{.State.Status}}", "running", 30*time.Second) c.Assert(err, check.IsNil) } -func (s *DockerSuite) TestUserDefinedNetworkWithRestartPolicy(c *check.C) { +func (s *DockerSuite) TestRestartWithPolicyUserDefinedNetwork(c *check.C) { + // TODO Windows. This may be portable following HNS integration post TP5. testRequires(c, DaemonIsLinux, SameHostDaemon, NotUserNamespace, NotArm) dockerCmd(c, "network", "create", "-d", "bridge", "udNet") diff --git a/components/engine/integration-cli/docker_cli_run_test.go b/components/engine/integration-cli/docker_cli_run_test.go index 66cb72ee87..efbb25e617 100644 --- a/components/engine/integration-cli/docker_cli_run_test.go +++ b/components/engine/integration-cli/docker_cli_run_test.go @@ -20,6 +20,7 @@ import ( "github.com/docker/docker/pkg/integration/checker" "github.com/docker/docker/pkg/mount" + "github.com/docker/docker/pkg/stringutils" "github.com/docker/docker/runconfig" "github.com/docker/go-connections/nat" "github.com/docker/libnetwork/netutils" @@ -206,7 +207,7 @@ func (s *DockerSuite) TestUserDefinedNetworkLinks(c *check.C) { c.Assert(waitRun("first"), check.IsNil) // run a container in user-defined network udlinkNet with a link for an existing container - // and a link for a container that doesnt exist + // and a link for a container that doesn't exist dockerCmd(c, "run", "-d", "--net=udlinkNet", "--name=second", "--link=first:foo", "--link=third:bar", "busybox", "top") c.Assert(waitRun("second"), check.IsNil) @@ -272,6 +273,17 @@ func (s *DockerSuite) TestUserDefinedNetworkLinksWithRestart(c *check.C) { c.Assert(err, check.IsNil) } +func (s *DockerSuite) TestRunWithNetAliasOnDefaultNetworks(c *check.C) { + testRequires(c, DaemonIsLinux, NotUserNamespace, NotArm) + + defaults := []string{"bridge", "host", "none"} + for _, net := range defaults { + out, _, err := dockerCmdWithError("run", "-d", "--net", net, "--net-alias", "alias_"+net, "busybox", "top") + c.Assert(err, checker.NotNil) + c.Assert(out, checker.Contains, runconfig.ErrUnsupportedNetworkAndAlias.Error()) + } +} + func (s *DockerSuite) TestUserDefinedNetworkAlias(c *check.C) { testRequires(c, DaemonIsLinux, NotUserNamespace, NotArm) dockerCmd(c, "network", "create", "-d", "bridge", "net1") @@ -463,7 +475,7 @@ func (s *DockerSuite) TestRunVolumesFromInReadWriteMode(c *check.C) { dockerCmd(c, "run", "--name", "parent", "-v", volumeDir, "busybox", "true") dockerCmd(c, "run", "--volumes-from", "parent:rw", "busybox", "touch", fileInVol) - if out, _, err := dockerCmdWithError("run", "--volumes-from", "parent:bar", "busybox", "touch", fileInVol); err == nil || !strings.Contains(out, `invalid mode: "bar"`) { + if out, _, err := dockerCmdWithError("run", "--volumes-from", "parent:bar", "busybox", "touch", fileInVol); err == nil || !strings.Contains(out, `invalid mode: bar`) { c.Fatalf("running --volumes-from parent:bar should have failed with invalid mode: %q", out) } @@ -658,7 +670,12 @@ func (s *DockerSuite) TestRunExitCode(c *check.C) { func (s *DockerSuite) TestRunUserDefaults(c *check.C) { expected := "uid=0(root) gid=0(root)" if daemonPlatform == "windows" { - expected = "uid=1000(SYSTEM) gid=1000(SYSTEM)" + // TODO Windows: Remove this check once TP4 is no longer supported. + if windowsDaemonKV < 14250 { + expected = "uid=1000(SYSTEM) gid=1000(SYSTEM)" + } else { + expected = "uid=1000(ContainerAdministrator) gid=1000(ContainerAdministrator)" + } } out, _ := dockerCmd(c, "run", "busybox", "id") if !strings.Contains(out, expected) { @@ -818,7 +835,7 @@ func (s *DockerSuite) TestRunEnvironmentErase(c *check.C) { // the container cmd := exec.Command(dockerBinary, "run", "-e", "FOO", "-e", "HOSTNAME", "busybox", "env") - cmd.Env = appendBaseEnv([]string{}) + cmd.Env = appendBaseEnv(true) out, _, err := runCommandWithOutput(cmd) if err != nil { @@ -852,7 +869,7 @@ func (s *DockerSuite) TestRunEnvironmentOverride(c *check.C) { // already in the env that we're overriding them cmd := exec.Command(dockerBinary, "run", "-e", "HOSTNAME", "-e", "HOME=/root2", "busybox", "env") - cmd.Env = appendBaseEnv([]string{"HOSTNAME=bar"}) + cmd.Env = appendBaseEnv(true, "HOSTNAME=bar") out, _, err := runCommandWithOutput(cmd) if err != nil { @@ -1710,7 +1727,12 @@ func (s *DockerSuite) TestRunCleanupCmdOnEntrypoint(c *check.C) { out = strings.TrimSpace(out) expected := "root" if daemonPlatform == "windows" { - expected = `nt authority\system` + // TODO Windows: Remove this check once TP4 is no longer supported. + if windowsDaemonKV < 14250 { + expected = `nt authority\system` + } else { + expected = `user manager\containeradministrator` + } } if out != expected { c.Fatalf("Expected output %s, got %q", expected, out) @@ -1720,7 +1742,7 @@ func (s *DockerSuite) TestRunCleanupCmdOnEntrypoint(c *check.C) { // TestRunWorkdirExistsAndIsFile checks that if 'docker run -w' with existing file can be detected func (s *DockerSuite) TestRunWorkdirExistsAndIsFile(c *check.C) { existingFile := "/bin/cat" - expected := "Cannot mkdir: /bin/cat is not a directory" + expected := "not a directory" if daemonPlatform == "windows" { existingFile = `\windows\system32\ntdll.dll` expected = `Cannot mkdir: \windows\system32\ntdll.dll is not a directory.` @@ -1728,7 +1750,7 @@ func (s *DockerSuite) TestRunWorkdirExistsAndIsFile(c *check.C) { out, exitCode, err := dockerCmdWithError("run", "-w", existingFile, "busybox") if !(err != nil && exitCode == 125 && strings.Contains(out, expected)) { - c.Fatalf("Docker must complains about making dir with exitCode 125 but we got out: %s, exitCode: %d", out, exitCode) + c.Fatalf("Existing binary as a directory should error out with exitCode 125; we got: %s, exitCode: %d", out, exitCode) } } @@ -1806,8 +1828,10 @@ func (s *DockerSuite) TestRunWriteHostsFileAndNotCommit(c *check.C) { } func eqToBaseDiff(out string, c *check.C) bool { - out1, _ := dockerCmd(c, "run", "-d", "busybox", "echo", "hello") - cID := strings.TrimSpace(out1) + name := "eqToBaseDiff" + stringutils.GenerateRandomAlphaOnlyString(32) + dockerCmd(c, "run", "--name", name, "busybox", "echo", "hello") + cID, err := getIDByName(name) + c.Assert(err, check.IsNil) baseDiff, _ := dockerCmd(c, "diff", cID) baseArr := strings.Split(baseDiff, "\n") @@ -2029,10 +2053,10 @@ func (s *DockerSuite) TestRunInspectMacAddress(c *check.C) { } } -// test docker run use a invalid mac address +// test docker run use an invalid mac address func (s *DockerSuite) TestRunWithInvalidMacAddress(c *check.C) { out, _, err := dockerCmdWithError("run", "--mac-address", "92:d0:c6:0a:29", "busybox") - //use a invalid mac address should with a error out + //use an invalid mac address should with an error out if err == nil || !strings.Contains(out, "is not a valid mac address") { c.Fatalf("run with an invalid --mac-address should with error out") } @@ -2312,13 +2336,10 @@ func (s *DockerSuite) TestRunAllowPortRangeThroughExpose(c *check.C) { } } -// test docker run expose a invalid port func (s *DockerSuite) TestRunExposePort(c *check.C) { out, _, err := dockerCmdWithError("run", "--expose", "80000", "busybox") - //expose a invalid port should with a error out - if err == nil || !strings.Contains(out, "Invalid range format for --expose") { - c.Fatalf("run --expose a invalid port should with error out") - } + c.Assert(err, checker.NotNil, check.Commentf("--expose with an invalid port should error out")) + c.Assert(out, checker.Contains, "invalid range format for --expose") } func (s *DockerSuite) TestRunUnknownCommand(c *check.C) { @@ -2521,6 +2542,8 @@ func (s *DockerSuite) TestRunModeUTSHost(c *check.C) { } func (s *DockerSuite) TestRunTLSverify(c *check.C) { + // Remote daemons use TLS and this test is not applicable when TLS is required. + testRequires(c, SameHostDaemon) if out, code, err := dockerCmdWithError("ps"); err != nil || code != 0 { c.Fatalf("Should have worked: %v:\n%v", err, out) } @@ -2921,7 +2944,7 @@ func (s *DockerSuite) TestRunReadProcLatency(c *check.C) { // some kernels don't have this configured so skip the test if this file is not found // on the host running the tests. if _, err := os.Stat("/proc/latency_stats"); err != nil { - c.Skip("kernel doesnt have latency_stats configured") + c.Skip("kernel doesn't have latency_stats configured") return } out, code, err := dockerCmdWithError("run", "busybox", "cat", "/proc/latency_stats") @@ -2979,29 +3002,51 @@ func (s *DockerSuite) TestRunUnshareProc(c *check.C) { // Not applicable on Windows as uses Unix specific functionality testRequires(c, Apparmor, DaemonIsLinux, NotUserNamespace) - name := "acidburn" - out, _, err := dockerCmdWithError("run", "--name", name, "--security-opt", "seccomp:unconfined", "debian:jessie", "unshare", "-p", "-m", "-f", "-r", "--mount-proc=/proc", "mount") - if err == nil || - !(strings.Contains(strings.ToLower(out), "permission denied") || - strings.Contains(strings.ToLower(out), "operation not permitted")) { - c.Fatalf("unshare with --mount-proc should have failed with 'permission denied' or 'operation not permitted', got: %s, %v", out, err) - } + // In this test goroutines are used to run test cases in parallel to prevent the test from taking a long time to run. + errChan := make(chan error) - name = "cereal" - out, _, err = dockerCmdWithError("run", "--name", name, "--security-opt", "seccomp:unconfined", "debian:jessie", "unshare", "-p", "-m", "-f", "-r", "mount", "-t", "proc", "none", "/proc") - if err == nil || - !(strings.Contains(strings.ToLower(out), "mount: cannot mount none") || - strings.Contains(strings.ToLower(out), "permission denied")) { - c.Fatalf("unshare and mount of /proc should have failed with 'mount: cannot mount none' or 'permission denied', got: %s, %v", out, err) - } + go func() { + name := "acidburn" + out, _, err := dockerCmdWithError("run", "--name", name, "--security-opt", "seccomp:unconfined", "debian:jessie", "unshare", "-p", "-m", "-f", "-r", "--mount-proc=/proc", "mount") + if err == nil || + !(strings.Contains(strings.ToLower(out), "permission denied") || + strings.Contains(strings.ToLower(out), "operation not permitted")) { + errChan <- fmt.Errorf("unshare with --mount-proc should have failed with 'permission denied' or 'operation not permitted', got: %s, %v", out, err) + } else { + errChan <- nil + } + }() + + go func() { + name := "cereal" + out, _, err := dockerCmdWithError("run", "--name", name, "--security-opt", "seccomp:unconfined", "debian:jessie", "unshare", "-p", "-m", "-f", "-r", "mount", "-t", "proc", "none", "/proc") + if err == nil || + !(strings.Contains(strings.ToLower(out), "mount: cannot mount none") || + strings.Contains(strings.ToLower(out), "permission denied")) { + errChan <- fmt.Errorf("unshare and mount of /proc should have failed with 'mount: cannot mount none' or 'permission denied', got: %s, %v", out, err) + } else { + errChan <- nil + } + }() /* Ensure still fails if running privileged with the default policy */ - name = "crashoverride" - out, _, err = dockerCmdWithError("run", "--privileged", "--security-opt", "seccomp:unconfined", "--security-opt", "apparmor:docker-default", "--name", name, "debian:jessie", "unshare", "-p", "-m", "-f", "-r", "mount", "-t", "proc", "none", "/proc") - if err == nil || - !(strings.Contains(strings.ToLower(out), "mount: cannot mount none") || - strings.Contains(strings.ToLower(out), "permission denied")) { - c.Fatalf("privileged unshare with apparmor should have failed with 'mount: cannot mount none' or 'permission denied', got: %s, %v", out, err) + go func() { + name := "crashoverride" + out, _, err := dockerCmdWithError("run", "--privileged", "--security-opt", "seccomp:unconfined", "--security-opt", "apparmor:docker-default", "--name", name, "debian:jessie", "unshare", "-p", "-m", "-f", "-r", "mount", "-t", "proc", "none", "/proc") + if err == nil || + !(strings.Contains(strings.ToLower(out), "mount: cannot mount none") || + strings.Contains(strings.ToLower(out), "permission denied")) { + errChan <- fmt.Errorf("privileged unshare with apparmor should have failed with 'mount: cannot mount none' or 'permission denied', got: %s, %v", out, err) + } else { + errChan <- nil + } + }() + + for i := 0; i < 3; i++ { + err := <-errChan + if err != nil { + c.Fatal(err) + } } } @@ -3251,7 +3296,7 @@ func (s *DockerTrustSuite) TestTrustedRunFromBadTrustServer(c *check.C) { // Windows does not support this functionality testRequires(c, DaemonIsLinux) repoName := fmt.Sprintf("%v/dockerclievilrun/trusted:latest", privateRegistryURL) - evilLocalConfigDir, err := ioutil.TempDir("", "evil-local-config-dir") + evilLocalConfigDir, err := ioutil.TempDir("", "evilrun-local-config-dir") if err != nil { c.Fatalf("Failed to create local temp dir") } @@ -3307,15 +3352,15 @@ func (s *DockerTrustSuite) TestTrustedRunFromBadTrustServer(c *check.C) { c.Fatalf("Missing expected output on trusted push:\n%s", out) } - // Now, try running with the original client from this new trust server. This should fail. + // Now, try running with the original client from this new trust server. This should fallback to our cached timestamp and metadata. runCmd = exec.Command(dockerBinary, "run", repoName) s.trustedCmd(runCmd) out, _, err = runCommandWithOutput(runCmd) - if err == nil { - c.Fatalf("Expected to fail on this run due to different remote data: %s\n%s", err, out) - } - if !strings.Contains(string(out), "valid signatures did not meet threshold") { + if err != nil { + c.Fatalf("Error falling back to cached trust data: %s\n%s", err, out) + } + if !strings.Contains(string(out), "Error while downloading remote metadata, using cached timestamp") { c.Fatalf("Missing expected output on trusted push:\n%s", out) } } diff --git a/components/engine/integration-cli/docker_cli_run_unix_test.go b/components/engine/integration-cli/docker_cli_run_unix_test.go index 428e21be74..fc54556ef1 100644 --- a/components/engine/integration-cli/docker_cli_run_unix_test.go +++ b/components/engine/integration-cli/docker_cli_run_unix_test.go @@ -55,7 +55,7 @@ func (s *DockerSuite) TestRunRedirectStdout(c *check.C) { // Test recursive bind mount works by default func (s *DockerSuite) TestRunWithVolumesIsRecursive(c *check.C) { // /tmp gets permission denied - testRequires(c, NotUserNamespace) + testRequires(c, NotUserNamespace, SameHostDaemon) tmpDir, err := ioutil.TempDir("", "docker_recursive_mount_test") c.Assert(err, checker.IsNil) @@ -607,7 +607,7 @@ func (s *DockerSuite) TestRunSwapLessThanMemoryLimit(c *check.C) { } func (s *DockerSuite) TestRunInvalidCpusetCpusFlagValue(c *check.C) { - testRequires(c, cgroupCpuset) + testRequires(c, cgroupCpuset, SameHostDaemon) sysInfo := sysinfo.New(true) cpus, err := parsers.ParseUintList(sysInfo.Cpus) @@ -817,7 +817,7 @@ func (s *DockerSuite) TestRunSeccompProfileDenyCloneUserns(c *check.C) { // TestRunSeccompUnconfinedCloneUserns checks that // 'docker run --security-opt seccomp:unconfined syscall-test' allows creating a userns. func (s *DockerSuite) TestRunSeccompUnconfinedCloneUserns(c *check.C) { - testRequires(c, SameHostDaemon, seccompEnabled, NotUserNamespace) + testRequires(c, SameHostDaemon, seccompEnabled, UserNamespaceInKernel, NotUserNamespace) // make sure running w privileged is ok runCmd := exec.Command(dockerBinary, "run", "--security-opt", "seccomp:unconfined", "syscall-test", "userns-test", "id") @@ -829,7 +829,7 @@ func (s *DockerSuite) TestRunSeccompUnconfinedCloneUserns(c *check.C) { // TestRunSeccompAllowPrivCloneUserns checks that 'docker run --privileged syscall-test' // allows creating a userns. func (s *DockerSuite) TestRunSeccompAllowPrivCloneUserns(c *check.C) { - testRequires(c, SameHostDaemon, seccompEnabled, NotUserNamespace) + testRequires(c, SameHostDaemon, seccompEnabled, UserNamespaceInKernel, NotUserNamespace) // make sure running w privileged is ok runCmd := exec.Command(dockerBinary, "run", "--privileged", "syscall-test", "userns-test", "id") @@ -838,14 +838,14 @@ func (s *DockerSuite) TestRunSeccompAllowPrivCloneUserns(c *check.C) { } } -// TestRunSeccompAllowAptKey checks that 'docker run debian:jessie apt-key' succeeds. -func (s *DockerSuite) TestRunSeccompAllowAptKey(c *check.C) { - testRequires(c, SameHostDaemon, seccompEnabled, Network) +// TestRunSeccompAllowSetrlimit checks that 'docker run debian:jessie ulimit -v 1048510' succeeds. +func (s *DockerSuite) TestRunSeccompAllowSetrlimit(c *check.C) { + testRequires(c, SameHostDaemon, seccompEnabled) - // apt-key uses setrlimit & getrlimit, so we want to make sure we don't break it - runCmd := exec.Command(dockerBinary, "run", "debian:jessie", "apt-key", "adv", "--keyserver", "hkp://p80.pool.sks-keyservers.net:80", "--recv-keys", "E871F18B51E0147C77796AC81196BA81F6B0FC61") + // ulimit uses setrlimit, so we want to make sure we don't break it + runCmd := exec.Command(dockerBinary, "run", "debian:jessie", "bash", "-c", "ulimit -v 1048510") if out, _, err := runCommandWithOutput(runCmd); err != nil { - c.Fatalf("expected apt-key with seccomp to succeed, got %s: %v", out, err) + c.Fatalf("expected ulimit with seccomp to succeed, got %s: %v", out, err) } } @@ -895,6 +895,18 @@ func (s *DockerSuite) TestRunSeccompDefaultProfile(c *check.C) { } } +// TestRunNoNewPrivSetuid checks that --security-opt=no-new-privileges prevents +// effective uid transtions on executing setuid binaries. +func (s *DockerSuite) TestRunNoNewPrivSetuid(c *check.C) { + testRequires(c, DaemonIsLinux, NotUserNamespace, SameHostDaemon) + + // test that running a setuid binary results in no effective uid transition + runCmd := exec.Command(dockerBinary, "run", "--security-opt", "no-new-privileges", "--user", "1000", "nnp-test", "/usr/bin/nnp-test") + if out, _, err := runCommandWithOutput(runCmd); err != nil || !strings.Contains(out, "EUID=1000") { + c.Fatalf("expected output to contain EUID=1000, got %s: %v", out, err) + } +} + func (s *DockerSuite) TestRunApparmorProcDirectory(c *check.C) { testRequires(c, SameHostDaemon, Apparmor) @@ -909,3 +921,62 @@ func (s *DockerSuite) TestRunApparmorProcDirectory(c *check.C) { c.Fatalf("expected chmod 777 /proc/1/attr/current to fail, got %s: %v", out, err) } } + +// make sure the default profile can be successfully parsed (using unshare as it is +// something which we know is blocked in the default profile) +func (s *DockerSuite) TestRunSeccompWithDefaultProfile(c *check.C) { + testRequires(c, SameHostDaemon, seccompEnabled) + + out, _, err := dockerCmdWithError("run", "--security-opt", "seccomp:../profiles/seccomp/default.json", "debian:jessie", "unshare", "--map-root-user", "--user", "sh", "-c", "whoami") + c.Assert(err, checker.NotNil, check.Commentf(out)) + c.Assert(strings.TrimSpace(out), checker.Equals, "unshare: unshare failed: Operation not permitted") +} + +// TestRunDeviceSymlink checks run with device that follows symlink (#13840) +func (s *DockerSuite) TestRunDeviceSymlink(c *check.C) { + testRequires(c, DaemonIsLinux, NotUserNamespace, NotArm, SameHostDaemon) + if _, err := os.Stat("/dev/zero"); err != nil { + c.Skip("Host does not have /dev/zero") + } + + // Create a temporary directory to create symlink + tmpDir, err := ioutil.TempDir("", "docker_device_follow_symlink_tests") + c.Assert(err, checker.IsNil) + + defer os.RemoveAll(tmpDir) + + // Create a symbolic link to /dev/zero + symZero := filepath.Join(tmpDir, "zero") + err = os.Symlink("/dev/zero", symZero) + c.Assert(err, checker.IsNil) + + // Create a temporary file "temp" inside tmpDir, write some data to "tmpDir/temp", + // then create a symlink "tmpDir/file" to the temporary file "tmpDir/temp". + tmpFile := filepath.Join(tmpDir, "temp") + err = ioutil.WriteFile(tmpFile, []byte("temp"), 0666) + c.Assert(err, checker.IsNil) + symFile := filepath.Join(tmpDir, "file") + err = os.Symlink(tmpFile, symFile) + c.Assert(err, checker.IsNil) + + // md5sum of 'dd if=/dev/zero bs=4K count=8' is bb7df04e1b0a2570657527a7e108ae23 + out, _ := dockerCmd(c, "run", "--device", symZero+":/dev/symzero", "busybox", "sh", "-c", "dd if=/dev/symzero bs=4K count=8 | md5sum") + c.Assert(strings.Trim(out, "\r\n"), checker.Contains, "bb7df04e1b0a2570657527a7e108ae23", check.Commentf("expected output bb7df04e1b0a2570657527a7e108ae23")) + + // symlink "tmpDir/file" to a file "tmpDir/temp" will result in an error as it is not a device. + out, _, err = dockerCmdWithError("run", "--device", symFile+":/dev/symzero", "busybox", "sh", "-c", "dd if=/dev/symzero bs=4K count=8 | md5sum") + c.Assert(err, check.NotNil) + c.Assert(strings.Trim(out, "\r\n"), checker.Contains, "not a device node", check.Commentf("expected output 'not a device node'")) +} + +// TestRunPidsLimit makes sure the pids cgroup is set with --pids-limit +func (s *DockerSuite) TestRunPidsLimit(c *check.C) { + testRequires(c, pidsLimit) + + file := "/sys/fs/cgroup/pids/pids.max" + out, _ := dockerCmd(c, "run", "--name", "skittles", "--pids-limit", "2", "busybox", "cat", file) + c.Assert(strings.TrimSpace(out), checker.Equals, "2") + + out = inspectField(c, "skittles", "HostConfig.PidsLimit") + c.Assert(out, checker.Equals, "2", check.Commentf("setting the pids limit failed")) +} diff --git a/components/engine/integration-cli/docker_cli_save_load_test.go b/components/engine/integration-cli/docker_cli_save_load_test.go index f023ec51e5..4479588ef3 100644 --- a/components/engine/integration-cli/docker_cli_save_load_test.go +++ b/components/engine/integration-cli/docker_cli_save_load_test.go @@ -167,6 +167,16 @@ func (s *DockerSuite) TestSaveAndLoadRepoFlags(c *check.C) { c.Assert(before, checker.Equals, after, check.Commentf("inspect is not the same after a save / load")) } +func (s *DockerSuite) TestSaveWithNoExistImage(c *check.C) { + testRequires(c, DaemonIsLinux) + + imgName := "foobar-non-existing-image" + + out, _, err := dockerCmdWithError("save", "-o", "test-img.tar", imgName) + c.Assert(err, checker.NotNil, check.Commentf("save image should fail for non-existing image")) + c.Assert(out, checker.Contains, fmt.Sprintf("No such image: %s", imgName)) +} + func (s *DockerSuite) TestSaveMultipleNames(c *check.C) { testRequires(c, DaemonIsLinux) repoName := "foobar-save-multi-name-test" diff --git a/components/engine/integration-cli/docker_cli_start_test.go b/components/engine/integration-cli/docker_cli_start_test.go index dcb983a0ac..43342191c2 100644 --- a/components/engine/integration-cli/docker_cli_start_test.go +++ b/components/engine/integration-cli/docker_cli_start_test.go @@ -13,11 +13,10 @@ import ( func (s *DockerSuite) TestStartAttachReturnsOnError(c *check.C) { // Windows does not support link testRequires(c, DaemonIsLinux) - dockerCmd(c, "run", "-d", "--name", "test", "busybox") - dockerCmd(c, "wait", "test") + dockerCmd(c, "run", "--name", "test", "busybox") // Expect this to fail because the above container is stopped, this is what we want - out, _, err := dockerCmdWithError("run", "-d", "--name", "test2", "--link", "test:test", "busybox") + out, _, err := dockerCmdWithError("run", "--name", "test2", "--link", "test:test", "busybox") // err shouldn't be nil because container test2 try to link to stopped container c.Assert(err, checker.NotNil, check.Commentf("out: %s", out)) @@ -148,7 +147,7 @@ func (s *DockerSuite) TestStartMultipleContainers(c *check.C) { func (s *DockerSuite) TestStartAttachMultipleContainers(c *check.C) { // run multiple containers to test for _, container := range []string{"test1", "test2", "test3"} { - dockerCmd(c, "run", "-d", "--name", container, "busybox", "top") + runSleepingContainer(c, "--name", container) } // stop all the containers diff --git a/components/engine/integration-cli/docker_cli_start_volume_driver_unix_test.go b/components/engine/integration-cli/docker_cli_start_volume_driver_unix_test.go index 3945d30f3a..cc5d0b11e6 100644 --- a/components/engine/integration-cli/docker_cli_start_volume_driver_unix_test.go +++ b/components/engine/integration-cli/docker_cli_start_volume_driver_unix_test.go @@ -16,6 +16,7 @@ import ( "time" "github.com/docker/docker/pkg/integration/checker" + "github.com/docker/engine-api/types" "github.com/go-check/check" ) @@ -59,6 +60,7 @@ func (s *DockerExternalVolumeSuite) SetUpSuite(c *check.C) { type pluginRequest struct { Name string + Opts map[string]string } type pluginResp struct { @@ -69,6 +71,7 @@ func (s *DockerExternalVolumeSuite) SetUpSuite(c *check.C) { type vol struct { Name string Mountpoint string + Ninja bool // hack used to trigger an null volume return on `Get` } var volList []vol @@ -106,13 +109,21 @@ func (s *DockerExternalVolumeSuite) SetUpSuite(c *check.C) { send(w, err) return } - volList = append(volList, vol{Name: pr.Name}) + _, isNinja := pr.Opts["ninja"] + volList = append(volList, vol{Name: pr.Name, Ninja: isNinja}) send(w, nil) }) mux.HandleFunc("/VolumeDriver.List", func(w http.ResponseWriter, r *http.Request) { s.ec.lists++ - send(w, map[string][]vol{"Volumes": volList}) + vols := []vol{} + for _, v := range volList { + if v.Ninja { + continue + } + vols = append(vols, v) + } + send(w, map[string][]vol{"Volumes": vols}) }) mux.HandleFunc("/VolumeDriver.Get", func(w http.ResponseWriter, r *http.Request) { @@ -125,6 +136,10 @@ func (s *DockerExternalVolumeSuite) SetUpSuite(c *check.C) { for _, v := range volList { if v.Name == pr.Name { + if v.Ninja { + send(w, map[string]vol{}) + return + } v.Mountpoint = hostVolumePath(pr.Name) send(w, map[string]vol{"Volume": v}) return @@ -141,15 +156,10 @@ func (s *DockerExternalVolumeSuite) SetUpSuite(c *check.C) { return } - if err := os.RemoveAll(hostVolumePath(pr.Name)); err != nil { - send(w, &pluginResp{Err: err.Error()}) - return - } - for i, v := range volList { if v.Name == pr.Name { if err := os.RemoveAll(hostVolumePath(v.Name)); err != nil { - send(w, fmt.Sprintf(`{"Err": "%v"}`, err)) + send(w, &pluginResp{Err: err.Error()}) return } volList = append(volList[:i], volList[i+1:]...) @@ -258,11 +268,11 @@ func (s *DockerExternalVolumeSuite) TestExternalVolumeDriverUnnamed(c *check.C) c.Assert(s.ec.unmounts, checker.Equals, 1) } -func (s DockerExternalVolumeSuite) TestExternalVolumeDriverVolumesFrom(c *check.C) { +func (s *DockerExternalVolumeSuite) TestExternalVolumeDriverVolumesFrom(c *check.C) { err := s.d.StartWithBusybox() c.Assert(err, checker.IsNil) - out, err := s.d.Cmd("run", "-d", "--name", "vol-test1", "-v", "/foo", "--volume-driver", "test-external-volume-driver", "busybox:latest") + out, err := s.d.Cmd("run", "--name", "vol-test1", "-v", "/foo", "--volume-driver", "test-external-volume-driver", "busybox:latest") c.Assert(err, checker.IsNil, check.Commentf(out)) out, err = s.d.Cmd("run", "--rm", "--volumes-from", "vol-test1", "--name", "vol-test2", "busybox", "ls", "/tmp") @@ -278,11 +288,11 @@ func (s DockerExternalVolumeSuite) TestExternalVolumeDriverVolumesFrom(c *check. c.Assert(s.ec.unmounts, checker.Equals, 2) } -func (s DockerExternalVolumeSuite) TestExternalVolumeDriverDeleteContainer(c *check.C) { +func (s *DockerExternalVolumeSuite) TestExternalVolumeDriverDeleteContainer(c *check.C) { err := s.d.StartWithBusybox() c.Assert(err, checker.IsNil) - out, err := s.d.Cmd("run", "-d", "--name", "vol-test1", "-v", "/foo", "--volume-driver", "test-external-volume-driver", "busybox:latest") + out, err := s.d.Cmd("run", "--name", "vol-test1", "-v", "/foo", "--volume-driver", "test-external-volume-driver", "busybox:latest") c.Assert(err, checker.IsNil, check.Commentf(out)) out, err = s.d.Cmd("rm", "-fv", "vol-test1") @@ -390,8 +400,8 @@ func (s *DockerExternalVolumeSuite) TestExternalVolumeDriverBindExternalVolume(c c.Assert(mounts[0].Driver, checker.Equals, "test-external-volume-driver") } -func (s *DockerExternalVolumeSuite) TesttExternalVolumeDriverList(c *check.C) { - dockerCmd(c, "volume", "create", "-d", "test-external-volume-driver", "--name", "abc") +func (s *DockerExternalVolumeSuite) TestExternalVolumeDriverList(c *check.C) { + dockerCmd(c, "volume", "create", "-d", "test-external-volume-driver", "--name", "abc3") out, _ := dockerCmd(c, "volume", "ls") ls := strings.Split(strings.TrimSpace(out), "\n") c.Assert(len(ls), check.Equals, 2, check.Commentf("\n%s", out)) @@ -399,7 +409,7 @@ func (s *DockerExternalVolumeSuite) TesttExternalVolumeDriverList(c *check.C) { vol := strings.Fields(ls[len(ls)-1]) c.Assert(len(vol), check.Equals, 2, check.Commentf("%v", vol)) c.Assert(vol[0], check.Equals, "test-external-volume-driver") - c.Assert(vol[1], check.Equals, "abc") + c.Assert(vol[1], check.Equals, "abc3") c.Assert(s.ec.lists, check.Equals, 1) } @@ -410,3 +420,24 @@ func (s *DockerExternalVolumeSuite) TestExternalVolumeDriverGet(c *check.C) { c.Assert(s.ec.gets, check.Equals, 1) c.Assert(out, checker.Contains, "No such volume") } + +func (s *DockerExternalVolumeSuite) TestExternalVolumeDriverWithDaemnRestart(c *check.C) { + dockerCmd(c, "volume", "create", "-d", "test-external-volume-driver", "--name", "abc1") + err := s.d.Restart() + c.Assert(err, checker.IsNil) + + dockerCmd(c, "run", "--name=test", "-v", "abc1:/foo", "busybox", "true") + var mounts []types.MountPoint + inspectFieldAndMarshall(c, "test", "Mounts", &mounts) + c.Assert(mounts, checker.HasLen, 1) + c.Assert(mounts[0].Driver, checker.Equals, "test-external-volume-driver") +} + +// Ensures that the daemon handles when the plugin responds to a `Get` request with a null volume and a null error. +// Prior the daemon would panic in this scenario. +func (s *DockerExternalVolumeSuite) TestExternalVolumeDriverGetEmptyResponse(c *check.C) { + dockerCmd(c, "volume", "create", "-d", "test-external-volume-driver", "--name", "abc2", "--opt", "ninja=1") + out, _, err := dockerCmdWithError("volume", "inspect", "abc2") + c.Assert(err, checker.NotNil, check.Commentf(out)) + c.Assert(out, checker.Contains, "No such volume") +} diff --git a/components/engine/integration-cli/docker_cli_stats_test.go b/components/engine/integration-cli/docker_cli_stats_test.go index cabc03e9be..e3c7a3e2e7 100644 --- a/components/engine/integration-cli/docker_cli_stats_test.go +++ b/components/engine/integration-cli/docker_cli_stats_test.go @@ -75,6 +75,18 @@ func (s *DockerSuite) TestStatsAllRunningNoStream(c *check.C) { if strings.Contains(out, id3) { c.Fatalf("Did not expect %s in stats, got %s", id3, out) } + + // check output contains real data, but not all zeros + reg, _ := regexp.Compile("[1-9]+") + // split output with "\n", outLines[1] is id2's output + // outLines[2] is id1's output + outLines := strings.Split(out, "\n") + // check stat result of id2 contains real data + realData := reg.Find([]byte(outLines[1][12:])) + c.Assert(realData, checker.NotNil, check.Commentf("stat result are empty: %s", out)) + // check stat result of id1 contains real data + realData = reg.Find([]byte(outLines[2][12:])) + c.Assert(realData, checker.NotNil, check.Commentf("stat result are empty: %s", out)) } func (s *DockerSuite) TestStatsAllNoStream(c *check.C) { @@ -93,16 +105,30 @@ func (s *DockerSuite) TestStatsAllNoStream(c *check.C) { if !strings.Contains(out, id1) || !strings.Contains(out, id2) { c.Fatalf("Expected stats output to contain both %s and %s, got %s", id1, id2, out) } + + // check output contains real data, but not all zeros + reg, _ := regexp.Compile("[1-9]+") + // split output with "\n", outLines[1] is id2's output + outLines := strings.Split(out, "\n") + // check stat result of id2 contains real data + realData := reg.Find([]byte(outLines[1][12:])) + c.Assert(realData, checker.NotNil, check.Commentf("stat result of %s is empty: %s", id2, out)) + // check stat result of id1 contains all zero + realData = reg.Find([]byte(outLines[2][12:])) + c.Assert(realData, checker.IsNil, check.Commentf("stat result of %s should be empty : %s", id1, out)) } func (s *DockerSuite) TestStatsAllNewContainersAdded(c *check.C) { // Windows does not support stats - testRequires(c, DaemonIsLinux) + // TODO: remove SameHostDaemon + // The reason it was added is because, there seems to be some race that makes this test fail + // for remote daemons (namely in the win2lin CI). We highly welcome contributions to fix this. + testRequires(c, DaemonIsLinux, SameHostDaemon) id := make(chan string) addedChan := make(chan struct{}) - dockerCmd(c, "run", "-d", "busybox", "top") + runSleepingContainer(c, "-d") statsCmd := exec.Command(dockerBinary, "stats") stdout, err := statsCmd.StdoutPipe() c.Assert(err, check.IsNil) @@ -118,16 +144,17 @@ func (s *DockerSuite) TestStatsAllNewContainersAdded(c *check.C) { switch { case matchID.MatchString(scanner.Text()): close(addedChan) + return } } }() - out, _ := dockerCmd(c, "run", "-d", "busybox", "top") + out, _ := runSleepingContainer(c, "-d") c.Assert(waitRun(strings.TrimSpace(out)), check.IsNil) id <- strings.TrimSpace(out)[:12] select { - case <-time.After(5 * time.Second): + case <-time.After(30 * time.Second): c.Fatal("failed to observe new container created added to stats") case <-addedChan: // ignore, done diff --git a/components/engine/integration-cli/docker_cli_tag_test.go b/components/engine/integration-cli/docker_cli_tag_test.go index 9bacb46d6b..1e601527e6 100644 --- a/components/engine/integration-cli/docker_cli_tag_test.go +++ b/components/engine/integration-cli/docker_cli_tag_test.go @@ -31,7 +31,7 @@ func (s *DockerSuite) TestTagUnprefixedRepoByID(c *check.C) { // ensure we don't allow the use of invalid repository names; these tag operations should fail func (s *DockerSuite) TestTagInvalidUnprefixedRepo(c *check.C) { - invalidRepos := []string{"fo$z$", "Foo@3cc", "Foo$3", "Foo*3", "Fo^3", "Foo!3", "F)xcz(", "fo%asd"} + invalidRepos := []string{"fo$z$", "Foo@3cc", "Foo$3", "Foo*3", "Fo^3", "Foo!3", "F)xcz(", "fo%asd", "FOO/bar"} for _, repo := range invalidRepos { out, _, err := dockerCmdWithError("tag", "busybox", repo) @@ -61,7 +61,7 @@ func (s *DockerSuite) TestTagValidPrefixedRepo(c *check.C) { } } - validRepos := []string{"fooo/bar", "fooaa/test", "foooo:t"} + validRepos := []string{"fooo/bar", "fooaa/test", "foooo:t", "HOSTNAME.DOMAIN.COM:443/foo/bar"} for _, repo := range validRepos { _, _, err := dockerCmdWithError("tag", "busybox:latest", repo) diff --git a/components/engine/integration-cli/docker_cli_update_test.go b/components/engine/integration-cli/docker_cli_update_test.go new file mode 100644 index 0000000000..466cc88248 --- /dev/null +++ b/components/engine/integration-cli/docker_cli_update_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "strings" + "time" + + "github.com/docker/docker/pkg/integration/checker" + "github.com/go-check/check" +) + +func (s *DockerSuite) TestUpdateRestartPolicy(c *check.C) { + out, _ := dockerCmd(c, "run", "-d", "--restart=on-failure:3", "busybox", "sh", "-c", "sleep 1 && false") + timeout := 60 * time.Second + if daemonPlatform == "windows" { + timeout = 150 * time.Second + } + + id := strings.TrimSpace(string(out)) + + // update restart policy to on-failure:5 + dockerCmd(c, "update", "--restart=on-failure:5", id) + + err := waitExited(id, timeout) + c.Assert(err, checker.IsNil) + + count := inspectField(c, id, "RestartCount") + c.Assert(count, checker.Equals, "5") + + maximumRetryCount := inspectField(c, id, "HostConfig.RestartPolicy.MaximumRetryCount") + c.Assert(maximumRetryCount, checker.Equals, "5") +} diff --git a/components/engine/integration-cli/docker_cli_update_unix_test.go b/components/engine/integration-cli/docker_cli_update_unix_test.go index bb261e8bec..c40ad3ee12 100644 --- a/components/engine/integration-cli/docker_cli_update_unix_test.go +++ b/components/engine/integration-cli/docker_cli_update_unix_test.go @@ -139,6 +139,47 @@ func (s *DockerSuite) TestUpdateKernelMemory(c *check.C) { c.Assert(strings.TrimSpace(out), checker.Equals, "104857600") } +func (s *DockerSuite) TestUpdateSwapMemoryOnly(c *check.C) { + testRequires(c, DaemonIsLinux) + testRequires(c, memoryLimitSupport) + testRequires(c, swapMemorySupport) + + name := "test-update-container" + dockerCmd(c, "run", "-d", "--name", name, "--memory", "300M", "--memory-swap", "500M", "busybox", "top") + dockerCmd(c, "update", "--memory-swap", "600M", name) + + c.Assert(inspectField(c, name, "HostConfig.MemorySwap"), checker.Equals, "629145600") + + file := "/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes" + out, _ := dockerCmd(c, "exec", name, "cat", file) + c.Assert(strings.TrimSpace(out), checker.Equals, "629145600") +} + +func (s *DockerSuite) TestUpdateInvalidSwapMemory(c *check.C) { + testRequires(c, DaemonIsLinux) + testRequires(c, memoryLimitSupport) + testRequires(c, swapMemorySupport) + + name := "test-update-container" + dockerCmd(c, "run", "-d", "--name", name, "--memory", "300M", "--memory-swap", "500M", "busybox", "top") + _, _, err := dockerCmdWithError("update", "--memory-swap", "200M", name) + // Update invalid swap memory should fail. + // This will pass docker config validation, but failed at kernel validation + c.Assert(err, check.NotNil) + + // Update invalid swap memory with failure should not change HostConfig + c.Assert(inspectField(c, name, "HostConfig.Memory"), checker.Equals, "314572800") + c.Assert(inspectField(c, name, "HostConfig.MemorySwap"), checker.Equals, "524288000") + + dockerCmd(c, "update", "--memory-swap", "600M", name) + + c.Assert(inspectField(c, name, "HostConfig.MemorySwap"), checker.Equals, "629145600") + + file := "/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes" + out, _ := dockerCmd(c, "exec", name, "cat", file) + c.Assert(strings.TrimSpace(out), checker.Equals, "629145600") +} + func (s *DockerSuite) TestUpdateStats(c *check.C) { testRequires(c, DaemonIsLinux) testRequires(c, memoryLimitSupport) diff --git a/components/engine/integration-cli/docker_cli_userns_test.go b/components/engine/integration-cli/docker_cli_userns_test.go index ab503b4038..967debd581 100644 --- a/components/engine/integration-cli/docker_cli_userns_test.go +++ b/components/engine/integration-cli/docker_cli_userns_test.go @@ -20,7 +20,7 @@ import ( // 1. validate uid/gid maps are set properly // 2. verify that files created are owned by remapped root func (s *DockerDaemonSuite) TestDaemonUserNamespaceRootSetting(c *check.C) { - testRequires(c, DaemonIsLinux, SameHostDaemon) + testRequires(c, DaemonIsLinux, SameHostDaemon, UserNamespaceInKernel) c.Assert(s.d.StartWithBusybox("--userns-remap", "default"), checker.IsNil) diff --git a/components/engine/integration-cli/docker_cli_v2_only_test.go b/components/engine/integration-cli/docker_cli_v2_only_test.go index b0bfb6f424..889936a062 100644 --- a/components/engine/integration-cli/docker_cli_v2_only_test.go +++ b/components/engine/integration-cli/docker_cli_v2_only_test.go @@ -106,20 +106,19 @@ func (s *DockerRegistrySuite) TestV1(c *check.C) { defer cleanup() s.d.Cmd("build", "--file", dockerfileName, ".") - c.Assert(v1Repo, check.Not(check.Equals), 0, check.Commentf("Expected v1 repository access after build")) + c.Assert(v1Repo, check.Equals, 1, check.Commentf("Expected v1 repository access after build")) repoName := fmt.Sprintf("%s/busybox", reg.hostport) s.d.Cmd("run", repoName) - c.Assert(v1Repo, check.Not(check.Equals), 1, check.Commentf("Expected v1 repository access after run")) + c.Assert(v1Repo, check.Equals, 2, check.Commentf("Expected v1 repository access after run")) - s.d.Cmd("login", "-u", "richard", "-p", "testtest", "-e", "testuser@testdomain.com", reg.hostport) - c.Assert(v1Logins, check.Not(check.Equals), 0, check.Commentf("Expected v1 login attempt")) + s.d.Cmd("login", "-u", "richard", "-p", "testtest", reg.hostport) + c.Assert(v1Logins, check.Equals, 1, check.Commentf("Expected v1 login attempt")) s.d.Cmd("tag", "busybox", repoName) s.d.Cmd("push", repoName) c.Assert(v1Repo, check.Equals, 2) - c.Assert(v1Pings, check.Equals, 1) s.d.Cmd("pull", repoName) c.Assert(v1Repo, check.Equals, 3, check.Commentf("Expected v1 repository access after pull")) diff --git a/components/engine/integration-cli/docker_cli_volume_driver_compat_unix_test.go b/components/engine/integration-cli/docker_cli_volume_driver_compat_unix_test.go deleted file mode 100644 index 2207822f76..0000000000 --- a/components/engine/integration-cli/docker_cli_volume_driver_compat_unix_test.go +++ /dev/null @@ -1,234 +0,0 @@ -// +build !windows - -package main - -import ( - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - - "github.com/docker/docker/pkg/integration/checker" - - "github.com/go-check/check" -) - -func init() { - check.Suite(&DockerExternalVolumeSuiteCompatV1_1{ - ds: &DockerSuite{}, - }) -} - -type vol struct { - Name string - Mountpoint string - Opts map[string]string -} - -type DockerExternalVolumeSuiteCompatV1_1 struct { - server *httptest.Server - ds *DockerSuite - d *Daemon - ec *eventCounter - volList []vol -} - -func (s *DockerExternalVolumeSuiteCompatV1_1) SetUpTest(c *check.C) { - s.d = NewDaemon(c) - s.ec = &eventCounter{} -} - -func (s *DockerExternalVolumeSuiteCompatV1_1) TearDownTest(c *check.C) { - s.d.Stop() - s.ds.TearDownTest(c) -} - -func (s *DockerExternalVolumeSuiteCompatV1_1) SetUpSuite(c *check.C) { - mux := http.NewServeMux() - s.server = httptest.NewServer(mux) - - type pluginRequest struct { - Name string - Opts map[string]string - } - - type pluginResp struct { - Mountpoint string `json:",omitempty"` - Err string `json:",omitempty"` - } - - read := func(b io.ReadCloser) (pluginRequest, error) { - defer b.Close() - var pr pluginRequest - if err := json.NewDecoder(b).Decode(&pr); err != nil { - return pr, err - } - return pr, nil - } - - send := func(w http.ResponseWriter, data interface{}) { - switch t := data.(type) { - case error: - http.Error(w, t.Error(), 500) - case string: - w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") - fmt.Fprintln(w, t) - default: - w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") - json.NewEncoder(w).Encode(&data) - } - } - - mux.HandleFunc("/Plugin.Activate", func(w http.ResponseWriter, r *http.Request) { - s.ec.activations++ - send(w, `{"Implements": ["VolumeDriver"]}`) - }) - - mux.HandleFunc("/VolumeDriver.Create", func(w http.ResponseWriter, r *http.Request) { - s.ec.creations++ - pr, err := read(r.Body) - if err != nil { - send(w, err) - return - } - s.volList = append(s.volList, vol{Name: pr.Name, Opts: pr.Opts}) - send(w, nil) - }) - - mux.HandleFunc("/VolumeDriver.Remove", func(w http.ResponseWriter, r *http.Request) { - s.ec.removals++ - pr, err := read(r.Body) - if err != nil { - send(w, err) - return - } - - if err := os.RemoveAll(hostVolumePath(pr.Name)); err != nil { - send(w, &pluginResp{Err: err.Error()}) - return - } - - for i, v := range s.volList { - if v.Name == pr.Name { - if err := os.RemoveAll(hostVolumePath(v.Name)); err != nil { - send(w, fmt.Sprintf(`{"Err": "%v"}`, err)) - return - } - s.volList = append(s.volList[:i], s.volList[i+1:]...) - break - } - } - send(w, nil) - }) - - mux.HandleFunc("/VolumeDriver.Path", func(w http.ResponseWriter, r *http.Request) { - s.ec.paths++ - - pr, err := read(r.Body) - if err != nil { - send(w, err) - return - } - p := hostVolumePath(pr.Name) - send(w, &pluginResp{Mountpoint: p}) - }) - - mux.HandleFunc("/VolumeDriver.Mount", func(w http.ResponseWriter, r *http.Request) { - s.ec.mounts++ - - pr, err := read(r.Body) - if err != nil { - send(w, err) - return - } - - p := hostVolumePath(pr.Name) - if err := os.MkdirAll(p, 0755); err != nil { - send(w, &pluginResp{Err: err.Error()}) - return - } - - if err := ioutil.WriteFile(filepath.Join(p, "test"), []byte(s.server.URL), 0644); err != nil { - send(w, err) - return - } - - send(w, &pluginResp{Mountpoint: p}) - }) - - mux.HandleFunc("/VolumeDriver.Unmount", func(w http.ResponseWriter, r *http.Request) { - s.ec.unmounts++ - - _, err := read(r.Body) - if err != nil { - send(w, err) - return - } - - send(w, nil) - }) - - err := os.MkdirAll("/etc/docker/plugins", 0755) - c.Assert(err, checker.IsNil) - - err = ioutil.WriteFile("/etc/docker/plugins/test-external-volume-driver.spec", []byte(s.server.URL), 0644) - c.Assert(err, checker.IsNil) -} - -func (s *DockerExternalVolumeSuiteCompatV1_1) TearDownSuite(c *check.C) { - s.server.Close() - - err := os.RemoveAll("/etc/docker/plugins") - c.Assert(err, checker.IsNil) -} - -func (s *DockerExternalVolumeSuiteCompatV1_1) TestExternalVolumeDriverCompatV1_1(c *check.C) { - err := s.d.StartWithBusybox() - c.Assert(err, checker.IsNil) - - out, err := s.d.Cmd("run", "--name=test", "-v", "foo:/bar", "--volume-driver", "test-external-volume-driver", "busybox", "sh", "-c", "echo hello > /bar/hello") - c.Assert(err, checker.IsNil, check.Commentf(out)) - out, err = s.d.Cmd("rm", "test") - c.Assert(err, checker.IsNil, check.Commentf(out)) - - out, err = s.d.Cmd("run", "--name=test2", "-v", "foo:/bar", "busybox", "cat", "/bar/hello") - c.Assert(err, checker.IsNil, check.Commentf(out)) - c.Assert(strings.TrimSpace(out), checker.Equals, "hello") - - err = s.d.Restart() - c.Assert(err, checker.IsNil) - - out, err = s.d.Cmd("start", "-a", "test2") - c.Assert(strings.TrimSpace(out), checker.Equals, "hello") - - out, err = s.d.Cmd("rm", "test2") - c.Assert(err, checker.IsNil, check.Commentf(out)) - - out, err = s.d.Cmd("volume", "inspect", "foo") - c.Assert(err, checker.IsNil, check.Commentf(out)) - - out, err = s.d.Cmd("volume", "rm", "foo") - c.Assert(err, checker.IsNil, check.Commentf(out)) -} - -func (s *DockerExternalVolumeSuiteCompatV1_1) TestExternalVolumeDriverCompatOptionsV1_1(c *check.C) { - err := s.d.StartWithBusybox() - c.Assert(err, checker.IsNil) - - out, err := s.d.Cmd("volume", "create", "--name", "optvol", "--driver", "test-external-volume-driver", "--opt", "opt1=opt1val", "--opt", "opt2=opt2val") - c.Assert(err, checker.IsNil, check.Commentf(out)) - - out, err = s.d.Cmd("volume", "inspect", "optvol") - c.Assert(err, checker.IsNil, check.Commentf(out)) - - c.Assert(s.volList[0].Opts["opt1"], checker.Equals, "opt1val") - c.Assert(s.volList[0].Opts["opt2"], checker.Equals, "opt2val") - - out, err = s.d.Cmd("volume", "rm", "optvol") - c.Assert(err, checker.IsNil, check.Commentf(out)) -} diff --git a/components/engine/integration-cli/docker_cli_volume_test.go b/components/engine/integration-cli/docker_cli_volume_test.go index bd84becb66..ddbba0db2d 100644 --- a/components/engine/integration-cli/docker_cli_volume_test.go +++ b/components/engine/integration-cli/docker_cli_volume_test.go @@ -65,19 +65,34 @@ func (s *DockerSuite) TestVolumeCliInspectMulti(c *check.C) { func (s *DockerSuite) TestVolumeCliLs(c *check.C) { prefix, _ := getPrefixAndSlashFromDaemonPlatform() - out, _ := dockerCmd(c, "volume", "create") - id := strings.TrimSpace(out) + out, _ := dockerCmd(c, "volume", "create", "--name", "aaa") dockerCmd(c, "volume", "create", "--name", "test") - dockerCmd(c, "run", "-v", prefix+"/foo", "busybox", "ls", "/") + + dockerCmd(c, "volume", "create", "--name", "soo") + dockerCmd(c, "run", "-v", "soo:"+prefix+"/foo", "busybox", "ls", "/") out, _ = dockerCmd(c, "volume", "ls") outArr := strings.Split(strings.TrimSpace(out), "\n") c.Assert(len(outArr), check.Equals, 4, check.Commentf("\n%s", out)) - // Since there is no guarantee of ordering of volumes, we just make sure the names are in the output - c.Assert(strings.Contains(out, id+"\n"), check.Equals, true) - c.Assert(strings.Contains(out, "test\n"), check.Equals, true) + assertVolList(c, out, []string{"aaa", "soo", "test"}) +} + +// assertVolList checks volume retrieved with ls command +// equals to expected volume list +// note: out should be `volume ls [option]` result +func assertVolList(c *check.C, out string, expectVols []string) { + lines := strings.Split(out, "\n") + var volList []string + for _, line := range lines[1 : len(lines)-1] { + volFields := strings.Fields(line) + // wrap all volume name in volList + volList = append(volList, volFields[1]) + } + + // volume ls should contains all expected volumes + c.Assert(volList, checker.DeepEquals, expectVols) } func (s *DockerSuite) TestVolumeCliLsFilterDangling(c *check.C) { @@ -203,3 +218,26 @@ func (s *DockerSuite) TestVolumeCliInspectTmplError(c *check.C) { c.Assert(exitCode, checker.Equals, 1, check.Commentf("Output: %s", out)) c.Assert(out, checker.Contains, "Template parsing error") } + +func (s *DockerSuite) TestVolumeCliCreateWithOpts(c *check.C) { + testRequires(c, DaemonIsLinux) + + dockerCmd(c, "volume", "create", "-d", "local", "--name", "test", "--opt=type=tmpfs", "--opt=device=tmpfs", "--opt=o=size=1m,uid=1000") + out, _ := dockerCmd(c, "run", "-v", "test:/foo", "busybox", "mount") + + mounts := strings.Split(out, "\n") + var found bool + for _, m := range mounts { + if strings.Contains(m, "/foo") { + found = true + info := strings.Fields(m) + // tmpfs on type tmpfs (rw,relatime,size=1024k,uid=1000) + c.Assert(info[0], checker.Equals, "tmpfs") + c.Assert(info[2], checker.Equals, "/foo") + c.Assert(info[4], checker.Equals, "tmpfs") + c.Assert(info[5], checker.Contains, "uid=1000") + c.Assert(info[5], checker.Contains, "size=1024k") + } + } + c.Assert(found, checker.Equals, true) +} diff --git a/components/engine/integration-cli/docker_utils.go b/components/engine/integration-cli/docker_utils.go index 78d530c7a5..c1bb8915f6 100644 --- a/components/engine/integration-cli/docker_utils.go +++ b/components/engine/integration-cli/docker_utils.go @@ -25,20 +25,20 @@ import ( "github.com/docker/docker/opts" "github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/integration" - "github.com/docker/docker/pkg/integration/checker" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/stringutils" "github.com/docker/engine-api/types" - "github.com/docker/go-connections/sockets" "github.com/docker/go-connections/tlsconfig" "github.com/docker/go-units" "github.com/go-check/check" ) func init() { - out, err := exec.Command(dockerBinary, "images").CombinedOutput() + cmd := exec.Command(dockerBinary, "images") + cmd.Env = appendBaseEnv(true) + out, err := cmd.CombinedOutput() if err != nil { - panic(err) + panic(fmt.Errorf("err=%v\nout=%s\n", err, out)) } lines := strings.Split(string(out), "\n")[1:] for _, l := range lines { @@ -100,387 +100,6 @@ func init() { } } -// Daemon represents a Docker daemon for the testing framework. -type Daemon struct { - // Defaults to "daemon" - // Useful to set to --daemon or -d for checking backwards compatibility - Command string - GlobalFlags []string - - id string - c *check.C - logFile *os.File - folder string - root string - stdin io.WriteCloser - stdout, stderr io.ReadCloser - cmd *exec.Cmd - storageDriver string - wait chan error - userlandProxy bool - useDefaultHost bool - useDefaultTLSHost bool -} - -type clientConfig struct { - transport *http.Transport - scheme string - addr string -} - -// NewDaemon returns a Daemon instance to be used for testing. -// This will create a directory such as d123456789 in the folder specified by $DEST. -// The daemon will not automatically start. -func NewDaemon(c *check.C) *Daemon { - dest := os.Getenv("DEST") - c.Assert(dest, check.Not(check.Equals), "", check.Commentf("Please set the DEST environment variable")) - - id := fmt.Sprintf("d%d", time.Now().UnixNano()%100000000) - dir := filepath.Join(dest, id) - daemonFolder, err := filepath.Abs(dir) - c.Assert(err, check.IsNil, check.Commentf("Could not make %q an absolute path", dir)) - daemonRoot := filepath.Join(daemonFolder, "root") - - c.Assert(os.MkdirAll(daemonRoot, 0755), check.IsNil, check.Commentf("Could not create daemon root %q", dir)) - - userlandProxy := true - if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { - if val, err := strconv.ParseBool(env); err != nil { - userlandProxy = val - } - } - - return &Daemon{ - Command: "daemon", - id: id, - c: c, - folder: daemonFolder, - root: daemonRoot, - storageDriver: os.Getenv("DOCKER_GRAPHDRIVER"), - userlandProxy: userlandProxy, - } -} - -func (d *Daemon) getClientConfig() (*clientConfig, error) { - var ( - transport *http.Transport - scheme string - addr string - proto string - ) - if d.useDefaultTLSHost { - option := &tlsconfig.Options{ - CAFile: "fixtures/https/ca.pem", - CertFile: "fixtures/https/client-cert.pem", - KeyFile: "fixtures/https/client-key.pem", - } - tlsConfig, err := tlsconfig.Client(*option) - if err != nil { - return nil, err - } - transport = &http.Transport{ - TLSClientConfig: tlsConfig, - } - addr = fmt.Sprintf("%s:%d", opts.DefaultHTTPHost, opts.DefaultTLSHTTPPort) - scheme = "https" - proto = "tcp" - } else if d.useDefaultHost { - addr = opts.DefaultUnixSocket - proto = "unix" - scheme = "http" - transport = &http.Transport{} - } else { - addr = filepath.Join(d.folder, "docker.sock") - proto = "unix" - scheme = "http" - transport = &http.Transport{} - } - - sockets.ConfigureTransport(transport, proto, addr) - - return &clientConfig{ - transport: transport, - scheme: scheme, - addr: addr, - }, nil -} - -// Start will start the daemon and return once it is ready to receive requests. -// You can specify additional daemon flags. -func (d *Daemon) Start(args ...string) error { - logFile, err := os.OpenFile(filepath.Join(d.folder, "docker.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - d.c.Assert(err, check.IsNil, check.Commentf("[%s] Could not create %s/docker.log", d.id, d.folder)) - - return d.StartWithLogFile(logFile, args...) -} - -// StartWithLogFile will start the daemon and attach its streams to a given file. -func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { - dockerBinary, err := exec.LookPath(dockerBinary) - d.c.Assert(err, check.IsNil, check.Commentf("[%s] could not find docker binary in $PATH", d.id)) - - args := append(d.GlobalFlags, - d.Command, - "--graph", d.root, - "--pidfile", fmt.Sprintf("%s/docker.pid", d.folder), - fmt.Sprintf("--userland-proxy=%t", d.userlandProxy), - ) - if !(d.useDefaultHost || d.useDefaultTLSHost) { - args = append(args, []string{"--host", d.sock()}...) - } - if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { - args = append(args, []string{"--userns-remap", root}...) - } - - // If we don't explicitly set the log-level or debug flag(-D) then - // turn on debug mode - foundLog := false - foundSd := false - for _, a := range providedArgs { - if strings.Contains(a, "--log-level") || strings.Contains(a, "-D") || strings.Contains(a, "--debug") { - foundLog = true - } - if strings.Contains(a, "--storage-driver") { - foundSd = true - } - } - if !foundLog { - args = append(args, "--debug") - } - if d.storageDriver != "" && !foundSd { - args = append(args, "--storage-driver", d.storageDriver) - } - - args = append(args, providedArgs...) - d.cmd = exec.Command(dockerBinary, args...) - - d.cmd.Stdout = out - d.cmd.Stderr = out - d.logFile = out - - if err := d.cmd.Start(); err != nil { - return fmt.Errorf("[%s] could not start daemon container: %v", d.id, err) - } - - wait := make(chan error) - - go func() { - wait <- d.cmd.Wait() - d.c.Logf("[%s] exiting daemon", d.id) - close(wait) - }() - - d.wait = wait - - tick := time.Tick(500 * time.Millisecond) - // make sure daemon is ready to receive requests - startTime := time.Now().Unix() - for { - d.c.Logf("[%s] waiting for daemon to start", d.id) - if time.Now().Unix()-startTime > 5 { - // After 5 seconds, give up - return fmt.Errorf("[%s] Daemon exited and never started", d.id) - } - select { - case <-time.After(2 * time.Second): - return fmt.Errorf("[%s] timeout: daemon does not respond", d.id) - case <-tick: - clientConfig, err := d.getClientConfig() - if err != nil { - return err - } - - client := &http.Client{ - Transport: clientConfig.transport, - } - - req, err := http.NewRequest("GET", "/_ping", nil) - d.c.Assert(err, check.IsNil, check.Commentf("[%s] could not create new request", d.id)) - req.URL.Host = clientConfig.addr - req.URL.Scheme = clientConfig.scheme - resp, err := client.Do(req) - if err != nil { - continue - } - if resp.StatusCode != http.StatusOK { - d.c.Logf("[%s] received status != 200 OK: %s", d.id, resp.Status) - } - d.c.Logf("[%s] daemon started", d.id) - d.root, err = d.queryRootDir() - if err != nil { - return fmt.Errorf("[%s] error querying daemon for root directory: %v", d.id, err) - } - return nil - } - } -} - -// StartWithBusybox will first start the daemon with Daemon.Start() -// then save the busybox image from the main daemon and load it into this Daemon instance. -func (d *Daemon) StartWithBusybox(arg ...string) error { - if err := d.Start(arg...); err != nil { - return err - } - bb := filepath.Join(d.folder, "busybox.tar") - if _, err := os.Stat(bb); err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("unexpected error on busybox.tar stat: %v", err) - } - // saving busybox image from main daemon - if err := exec.Command(dockerBinary, "save", "--output", bb, "busybox:latest").Run(); err != nil { - return fmt.Errorf("could not save busybox image: %v", err) - } - } - // loading busybox image to this daemon - if out, err := d.Cmd("load", "--input", bb); err != nil { - return fmt.Errorf("could not load busybox image: %s", out) - } - if err := os.Remove(bb); err != nil { - d.c.Logf("could not remove %s: %v", bb, err) - } - return nil -} - -// Stop will send a SIGINT every second and wait for the daemon to stop. -// If it timeouts, a SIGKILL is sent. -// Stop will not delete the daemon directory. If a purged daemon is needed, -// instantiate a new one with NewDaemon. -func (d *Daemon) Stop() error { - if d.cmd == nil || d.wait == nil { - return errors.New("daemon not started") - } - - defer func() { - d.logFile.Close() - d.cmd = nil - }() - - i := 1 - tick := time.Tick(time.Second) - - if err := d.cmd.Process.Signal(os.Interrupt); err != nil { - return fmt.Errorf("could not send signal: %v", err) - } -out1: - for { - select { - case err := <-d.wait: - return err - case <-time.After(15 * time.Second): - // time for stopping jobs and run onShutdown hooks - d.c.Log("timeout") - break out1 - } - } - -out2: - for { - select { - case err := <-d.wait: - return err - case <-tick: - i++ - if i > 4 { - d.c.Logf("tried to interrupt daemon for %d times, now try to kill it", i) - break out2 - } - d.c.Logf("Attempt #%d: daemon is still running with pid %d", i, d.cmd.Process.Pid) - if err := d.cmd.Process.Signal(os.Interrupt); err != nil { - return fmt.Errorf("could not send signal: %v", err) - } - } - } - - if err := d.cmd.Process.Kill(); err != nil { - d.c.Logf("Could not kill daemon: %v", err) - return err - } - - return nil -} - -// Restart will restart the daemon by first stopping it and then starting it. -func (d *Daemon) Restart(arg ...string) error { - d.Stop() - // in the case of tests running a user namespace-enabled daemon, we have resolved - // d.root to be the actual final path of the graph dir after the "uid.gid" of - // remapped root is added--we need to subtract it from the path before calling - // start or else we will continue making subdirectories rather than truly restarting - // with the same location/root: - if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { - d.root = filepath.Dir(d.root) - } - return d.Start(arg...) -} - -func (d *Daemon) queryRootDir() (string, error) { - // update daemon root by asking /info endpoint (to support user - // namespaced daemon with root remapped uid.gid directory) - clientConfig, err := d.getClientConfig() - if err != nil { - return "", err - } - - client := &http.Client{ - Transport: clientConfig.transport, - } - - req, err := http.NewRequest("GET", "/info", nil) - if err != nil { - return "", err - } - req.Header.Set("Content-Type", "application/json") - req.URL.Host = clientConfig.addr - req.URL.Scheme = clientConfig.scheme - - resp, err := client.Do(req) - if err != nil { - return "", err - } - body := ioutils.NewReadCloserWrapper(resp.Body, func() error { - return resp.Body.Close() - }) - - type Info struct { - DockerRootDir string - } - var b []byte - var i Info - b, err = readBody(body) - if err == nil && resp.StatusCode == 200 { - // read the docker root dir - if err = json.Unmarshal(b, &i); err == nil { - return i.DockerRootDir, nil - } - } - return "", err -} - -func (d *Daemon) sock() string { - return fmt.Sprintf("unix://%s/docker.sock", d.folder) -} - -func (d *Daemon) waitRun(contID string) error { - args := []string{"--host", d.sock()} - return waitInspectWithArgs(contID, "{{.State.Running}}", "true", 10*time.Second, args...) -} - -func (d *Daemon) getBaseDeviceSize(c *check.C) int64 { - - infoCmdOutput, _, err := runCommandPipelineWithOutput( - exec.Command(dockerBinary, "-H", d.sock(), "info"), - exec.Command("grep", "Base Device Size"), - ) - c.Assert(err, checker.IsNil) - basesizeSlice := strings.Split(infoCmdOutput, ":") - basesize := strings.Trim(basesizeSlice[1], " ") - basesize = strings.Trim(basesize, "\n")[:len(basesize)-3] - basesizeFloat, err := strconv.ParseFloat(strings.Trim(basesize, " "), 64) - c.Assert(err, checker.IsNil) - basesizeBytes := int64(basesizeFloat) * (1024 * 1024 * 1024) - return basesizeBytes -} - func convertBasesize(basesizeBytes int64) (int64, error) { basesize := units.HumanSize(float64(basesizeBytes)) basesize = strings.Trim(basesize, " ")[:len(basesize)-3] @@ -491,31 +110,6 @@ func convertBasesize(basesizeBytes int64) (int64, error) { return int64(basesizeFloat) * 1024 * 1024 * 1024, nil } -// Cmd will execute a docker CLI command against this Daemon. -// Example: d.Cmd("version") will run docker -H unix://path/to/unix.sock version -func (d *Daemon) Cmd(name string, arg ...string) (string, error) { - args := []string{"--host", d.sock(), name} - args = append(args, arg...) - c := exec.Command(dockerBinary, args...) - b, err := c.CombinedOutput() - return string(b), err -} - -// CmdWithArgs will execute a docker CLI command against a daemon with the -// given additional arguments -func (d *Daemon) CmdWithArgs(daemonArgs []string, name string, arg ...string) (string, error) { - args := append(daemonArgs, name) - args = append(args, arg...) - c := exec.Command(dockerBinary, args...) - b, err := c.CombinedOutput() - return string(b), err -} - -// LogFileName returns the path the the daemon's log file -func (d *Daemon) LogFileName() string { - return d.logFile.Name() -} - func daemonHost() string { daemonURLStr := "unix://" + opts.DefaultUnixSocket if daemonHostVar := os.Getenv("DOCKER_HOST"); daemonHostVar != "" { @@ -553,6 +147,8 @@ func sockConn(timeout time.Duration) (net.Conn, error) { var c net.Conn switch daemonURL.Scheme { + case "npipe": + return npipeDial(daemonURL.Path, timeout) case "unix": return net.DialTimeout(daemonURL.Scheme, daemonURL.Path, timeout) case "tcp": @@ -751,7 +347,9 @@ func getAllVolumes() ([]*types.Volume, error) { var protectedImages = map[string]struct{}{} func deleteAllImages() error { - out, err := exec.Command(dockerBinary, "images").CombinedOutput() + cmd := exec.Command(dockerBinary, "images") + cmd.Env = appendBaseEnv(true) + out, err := cmd.CombinedOutput() if err != nil { return err } @@ -867,32 +465,68 @@ func pullImageIfNotExist(image string) error { } func dockerCmdWithError(args ...string) (string, int, error) { + if err := validateArgs(args...); err != nil { + return "", 0, err + } return integration.DockerCmdWithError(dockerBinary, args...) } func dockerCmdWithStdoutStderr(c *check.C, args ...string) (string, string, int) { + if err := validateArgs(args...); err != nil { + c.Fatalf(err.Error()) + } return integration.DockerCmdWithStdoutStderr(dockerBinary, c, args...) } func dockerCmd(c *check.C, args ...string) (string, int) { + if err := validateArgs(args...); err != nil { + c.Fatalf(err.Error()) + } return integration.DockerCmd(dockerBinary, c, args...) } // execute a docker command with a timeout func dockerCmdWithTimeout(timeout time.Duration, args ...string) (string, int, error) { + if err := validateArgs(args...); err != nil { + return "", 0, err + } return integration.DockerCmdWithTimeout(dockerBinary, timeout, args...) } // execute a docker command in a directory func dockerCmdInDir(c *check.C, path string, args ...string) (string, int, error) { + if err := validateArgs(args...); err != nil { + c.Fatalf(err.Error()) + } return integration.DockerCmdInDir(dockerBinary, path, args...) } // execute a docker command in a directory with a timeout func dockerCmdInDirWithTimeout(timeout time.Duration, path string, args ...string) (string, int, error) { + if err := validateArgs(args...); err != nil { + return "", 0, err + } return integration.DockerCmdInDirWithTimeout(dockerBinary, timeout, path, args...) } +// validateArgs is a checker to ensure tests are not running commands which are +// not supported on platforms. Specifically on Windows this is 'busybox top'. +func validateArgs(args ...string) error { + if daemonPlatform != "windows" { + return nil + } + foundBusybox := -1 + for key, value := range args { + if strings.ToLower(value) == "busybox" { + foundBusybox = key + } + if (foundBusybox != -1) && (key == foundBusybox+1) && (strings.ToLower(value) == "top") { + return errors.New("Cannot use 'busybox top' in tests on Windows. Use runSleepingContainer()") + } + } + return nil +} + // find the State.ExitCode in container metadata func findContainerExitCode(c *check.C, name string, vargs ...string) string { args := append(vargs, "inspect", "--format='{{ .State.ExitCode }} {{ .State.Error }}'", name) @@ -909,14 +543,6 @@ func findContainerIP(c *check.C, id string, network string) string { return strings.Trim(out, " \r\n'") } -func (d *Daemon) findContainerIP(id string) string { - out, err := d.Cmd("inspect", fmt.Sprintf("--format='{{ .NetworkSettings.Networks.bridge.IPAddress }}'"), id) - if err != nil { - d.c.Log(err) - } - return strings.Trim(out, " \r\n'") -} - func getContainerCount() (int, error) { const containers = "Containers:" @@ -1295,7 +921,7 @@ func getContainerState(c *check.C, id string) (int, bool, error) { } func buildImageCmd(name, dockerfile string, useCache bool, buildFlags ...string) *exec.Cmd { - args := []string{"-D", "build", "-t", name} + args := []string{"build", "-t", name} if !useCache { args = append(args, "--no-cache") } @@ -1637,7 +1263,7 @@ func setupNotary(c *check.C) *testNotary { // appendBaseEnv appends the minimum set of environment variables to exec the // docker cli binary for testing with correct configuration to the given env // list. -func appendBaseEnv(env []string) []string { +func appendBaseEnv(isTLS bool, env ...string) []string { preserveList := []string{ // preserve remote test host "DOCKER_HOST", @@ -1646,6 +1272,9 @@ func appendBaseEnv(env []string) []string { // with "GetAddrInfoW: A non-recoverable error occurred during a database lookup." "SystemRoot", } + if isTLS { + preserveList = append(preserveList, "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH") + } for _, key := range preserveList { if val := os.Getenv(key); val != "" { @@ -1781,6 +1410,23 @@ func runSleepingContainerInImage(c *check.C, image string, extraArgs ...string) return dockerCmd(c, args...) } +func getRootUIDGID() (int, int, error) { + uidgid := strings.Split(filepath.Base(dockerBasePath), ".") + if len(uidgid) == 1 { + //user namespace remapping is not turned on; return 0 + return 0, 0, nil + } + uid, err := strconv.Atoi(uidgid[0]) + if err != nil { + return 0, 0, err + } + gid, err := strconv.Atoi(uidgid[1]) + if err != nil { + return 0, 0, err + } + return uid, gid, nil +} + // minimalBaseImage returns the name of the minimal base image for the current // daemon platform. func minimalBaseImage() string { diff --git a/components/engine/integration-cli/events_utils.go b/components/engine/integration-cli/events_utils.go index 77ec33dfd0..cdd3106b8d 100644 --- a/components/engine/integration-cli/events_utils.go +++ b/components/engine/integration-cli/events_utils.go @@ -3,7 +3,6 @@ package main import ( "bufio" "bytes" - "fmt" "io" "os/exec" "regexp" @@ -11,22 +10,11 @@ import ( "strings" "github.com/Sirupsen/logrus" + "github.com/docker/docker/daemon/events/testutils" "github.com/docker/docker/pkg/integration/checker" "github.com/go-check/check" ) -var ( - reTimestamp = `\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{9}(:?(:?(:?-|\+)\d{2}:\d{2})|Z)` - reEventType = `(?P\w+)` - reAction = `(?P\w+)` - reID = `(?P[^\s]+)` - reAttributes = `(\s\((?P[^\)]+)\))?` - reString = fmt.Sprintf(`\A%s\s%s\s%s\s%s%s\z`, reTimestamp, reEventType, reAction, reID, reAttributes) - - // eventCliRegexp is a regular expression that matches all possible event outputs in the cli - eventCliRegexp = regexp.MustCompile(reString) -) - // eventMatcher is a function that tries to match an event input. // It returns true if the event matches and a map with // a set of key/value to identify the match. @@ -131,7 +119,7 @@ func (e *eventObserver) CheckEventError(c *check.C, id, event string, match even // It returns an empty map and false if there is no match. func matchEventLine(id, eventType string, actions map[string]chan bool) eventMatcher { return func(text string) (map[string]string, bool) { - matches := parseEventText(text) + matches := eventstestutils.ScanMap(text) if len(matches) == 0 { return matches, false } @@ -149,31 +137,15 @@ func matchEventLine(id, eventType string, actions map[string]chan bool) eventMat func processEventMatch(actions map[string]chan bool) eventMatchProcessor { return func(matches map[string]string) { if ch, ok := actions[matches["action"]]; ok { - close(ch) + ch <- true } } } -// parseEventText parses a line of events coming from the cli and returns -// the matchers in a map. -func parseEventText(text string) map[string]string { - matches := eventCliRegexp.FindAllStringSubmatch(text, -1) - md := map[string]string{} - if len(matches) == 0 { - return md - } - - names := eventCliRegexp.SubexpNames() - for i, n := range matches[0] { - md[names[i]] = n - } - return md -} - // parseEventAction parses an event text and returns the action. // It fails if the text is not in the event format. func parseEventAction(c *check.C, text string) string { - matches := parseEventText(text) + matches := eventstestutils.ScanMap(text) return matches["action"] } @@ -182,7 +154,7 @@ func parseEventAction(c *check.C, text string) string { func eventActionsByIDAndType(c *check.C, events []string, id, eventType string) []string { var filtered []string for _, event := range events { - matches := parseEventText(event) + matches := eventstestutils.ScanMap(event) c.Assert(matches, checker.Not(checker.IsNil)) if matchIDAndEventType(matches, id, eventType) { filtered = append(filtered, matches["action"]) @@ -214,7 +186,7 @@ func matchEventID(matches map[string]string, id string) bool { func parseEvents(c *check.C, out, match string) { events := strings.Split(strings.TrimSpace(out), "\n") for _, event := range events { - matches := parseEventText(event) + matches := eventstestutils.ScanMap(event) matched, err := regexp.MatchString(match, matches["action"]) c.Assert(err, checker.IsNil) c.Assert(matched, checker.True, check.Commentf("Matcher: %s did not match %s", match, matches["action"])) @@ -224,7 +196,7 @@ func parseEvents(c *check.C, out, match string) { func parseEventsWithID(c *check.C, out, match, id string) { events := strings.Split(strings.TrimSpace(out), "\n") for _, event := range events { - matches := parseEventText(event) + matches := eventstestutils.ScanMap(event) c.Assert(matchEventID(matches, id), checker.True) matched, err := regexp.MatchString(match, matches["action"]) diff --git a/components/engine/integration-cli/fixtures/auth/docker-credential-shell-test b/components/engine/integration-cli/fixtures/auth/docker-credential-shell-test new file mode 100755 index 0000000000..0c94bcd216 --- /dev/null +++ b/components/engine/integration-cli/fixtures/auth/docker-credential-shell-test @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +case $1 in + "store") + in=$( $TEMP/$server + ;; + "get") + in=$( 0 { + if n, err := vrc.verifier.Write(p[:n]); err != nil { + return n, err + } + } + if err == io.EOF { + if !vrc.verifier.Verified() { + err = fmt.Errorf("could not verify layer data for: %s. This may be because internal files in the layer store were modified. Re-pulling or rebuilding this image may resolve the issue", vrc.dgst) + } + } + return +} +func (vrc *verifiedReadCloser) Close() error { + return vrc.rc.Close() +} diff --git a/components/engine/man/docker-cp.1.md b/components/engine/man/docker-cp.1.md index bd15625e1b..84d64c2688 100644 --- a/components/engine/man/docker-cp.1.md +++ b/components/engine/man/docker-cp.1.md @@ -20,7 +20,7 @@ You can copy from the container's file system to the local machine or the reverse, from the local filesystem to the container. If `-` is specified for either the `SRC_PATH` or `DEST_PATH`, you can also stream a tar archive from `STDIN` or to `STDOUT`. The `CONTAINER` can be a running or stopped container. -The `SRC_PATH` or `DEST_PATH` be a file or directory. +The `SRC_PATH` or `DEST_PATH` can be a file or directory. The `docker cp` command assumes container paths are relative to the container's `/` (root) directory. This means supplying the initial forward slash is optional; @@ -36,7 +36,8 @@ the user and primary group at the destination. For example, files copied to a container are created with `UID:GID` of the root user. Files copied to the local machine are created with the `UID:GID` of the user which invoked the `docker cp` command. If you specify the `-L` option, `docker cp` follows any symbolic link -in the `SRC_PATH`. +in the `SRC_PATH`. `docker cp` does *not* create parent directories for +`DEST_PATH` if they do not exist. Assuming a path separator of `/`, a first argument of `SRC_PATH` and second argument of `DEST_PATH`, the behavior is as follows: @@ -82,7 +83,7 @@ It is not possible to copy certain system files such as resources under Using `-` as the `SRC_PATH` streams the contents of `STDIN` as a tar archive. The command extracts the content of the tar to the `DEST_PATH` in container's filesystem. In this case, `DEST_PATH` must specify a directory. Using `-` as -`DEST_PATH` streams the contents of the resource as a tar archive to `STDOUT`. +the `DEST_PATH` streams the contents of the resource as a tar archive to `STDOUT`. # OPTIONS **-L**, **--follow-link**=*true*|*false* diff --git a/components/engine/man/docker-create.1.md b/components/engine/man/docker-create.1.md index a45eef7d63..16f70a958d 100644 --- a/components/engine/man/docker-create.1.md +++ b/components/engine/man/docker-create.1.md @@ -58,6 +58,7 @@ docker-create - Create a new container [**-P**|**--publish-all**] [**-p**|**--publish**[=*[]*]] [**--pid**[=*[]*]] +[**--pids-limit**[=*PIDS_LIMIT*]] [**--privileged**] [**--read-only**] [**--restart**[=*RESTART*]] @@ -214,7 +215,7 @@ millions of trillions. Add link to another container in the form of :alias or just in which case the alias will match the name. -**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*none*" +**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*gcplogs*|*none*" Logging driver for container. Default is defined by daemon `--log-driver` flag. **Warning**: the `docker logs` command works only for the `json-file` and `journald` logging drivers. @@ -290,6 +291,9 @@ unit, `b` is used. Set LIMIT to `-1` to enable unlimited swap. **host**: use the host's PID namespace inside the container. Note: the host mode gives the container full access to local PID and is therefore considered insecure. +**--pids-limit**="" + Tune the container's pids limit. Set `-1` to have unlimited pids for the container. + **--privileged**=*true*|*false* Give extended privileges to this container. The default is *false*. diff --git a/components/engine/man/docker-daemon.8.md b/components/engine/man/docker-daemon.8.md index 051c9e0748..b3fc64dcc5 100644 --- a/components/engine/man/docker-daemon.8.md +++ b/components/engine/man/docker-daemon.8.md @@ -185,7 +185,7 @@ unix://[/path/to/socket] to use. **--label**="[]" Set key=value labels to the daemon (displayed in `docker info`) -**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*none*" +**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*gcplogs*|*none*" Default driver for container logs. Default is `json-file`. **Warning**: `docker logs` command works only for `json-file` logging driver. @@ -244,9 +244,10 @@ internals) to create writable containers from images. Many of these backends use operating system level technologies and can be configured. -Specify options to the storage backend with **--storage-opt** flags. The only -backend that currently takes options is *devicemapper*. Therefore use these -flags with **-s=**devicemapper. +Specify options to the storage backend with **--storage-opt** flags. The +backends that currently take options are *devicemapper* and *zfs*. +Options for *devicemapper* are prefixed with *dm* and options for *zfs* +start with *zfs*. Specifically for devicemapper, the default is a "loopback" model which requires no pre-configuration, but is extremely inefficient. Do not @@ -258,7 +259,7 @@ more information see `man lvmthin`. Then, use `--storage-opt dm.thinpooldev` to tell the Docker engine to use that pool for allocating images and container snapshots. -Here is the list of *devicemapper* options: +## Devicemapper options #### dm.thinpooldev @@ -464,6 +465,16 @@ this topic, see Otherwise, set this flag for migrating existing Docker daemons to a daemon with a supported environment. +## ZFS options + +#### zfs.fsname + +Set zfs filesystem under which docker will create its own datasets. +By default docker will pick up the zfs filesystem where docker graph +(`/var/lib/docker`) is located. + +Example use: `docker daemon -s zfs --storage-opt zfs.fsname=zroot/docker` + # CLUSTER STORE OPTIONS The daemon uses libkv to advertise @@ -510,7 +521,7 @@ multiple plugins installed, at least one must allow the request for it to complete. For information about how to create an authorization plugin, see [authorization -plugin](https://docs.docker.com/engine/extend/authorization.md) section in the +plugin](https://docs.docker.com/engine/extend/plugins_authorization.md) section in the Docker extend section of this documentation. diff --git a/components/engine/man/docker-events.1.md b/components/engine/man/docker-events.1.md index fb8d7b00b8..4d0bff25c5 100644 --- a/components/engine/man/docker-events.1.md +++ b/components/engine/man/docker-events.1.md @@ -39,7 +39,7 @@ and Docker images will report: The `--since` and `--until` parameters can be Unix timestamps, date formatted timestamps, or Go duration strings (e.g. `10m`, `1h30m`) computed -relative to the client machine’s time. If you do not provide the --since option, +relative to the client machine’s time. If you do not provide the `--since` option, the command returns only new and/or live events. Supported formats for date formatted time stamps include RFC3339Nano, RFC3339, `2006-01-02T15:04:05`, `2006-01-02T15:04:05.999999999`, `2006-01-02Z07:00`, and `2006-01-02`. The local diff --git a/components/engine/man/docker-exec.1.md b/components/engine/man/docker-exec.1.md index 49f6dbc286..16a061d069 100644 --- a/components/engine/man/docker-exec.1.md +++ b/components/engine/man/docker-exec.1.md @@ -27,10 +27,10 @@ container is unpaused, and then run # OPTIONS **-d**, **--detach**=*true*|*false* - Override the key sequence for detaching a container. Format is a single character `[a-Z]` or `ctrl-` where `` is one of: `a-z`, `@`, `^`, `[`, `,` or `_`. + Detached mode: run command in the background. The default is *false*. **--detach-keys**="" - Define the key sequence which detaches the container. + Override the key sequence for detaching a container. Format is a single character `[a-Z]` or `ctrl-` where `` is one of: `a-z`, `@`, `^`, `[`, `,` or `_`. **--help** Print usage statement diff --git a/components/engine/man/docker-import.1.md b/components/engine/man/docker-import.1.md index 0509dd0d67..43d65efe6a 100644 --- a/components/engine/man/docker-import.1.md +++ b/components/engine/man/docker-import.1.md @@ -39,7 +39,7 @@ Import to docker via pipe and stdin: # cat exampleimage.tgz | docker import - example/imagelocal -Import with a commit message +Import with a commit message. # cat exampleimage.tgz | docker import --message "New image imported from tarball" - exampleimagelocal:new diff --git a/components/engine/man/docker-info.1.md b/components/engine/man/docker-info.1.md index 0dc46c14ed..1ebcd1a419 100644 --- a/components/engine/man/docker-info.1.md +++ b/components/engine/man/docker-info.1.md @@ -42,6 +42,7 @@ Here is a sample output: Dirs: 80 Execution Driver: native-0.2 Logging Driver: json-file + Cgroup Driver: cgroupfs Plugins: Volume: local Network: bridge null host @@ -51,7 +52,14 @@ Here is a sample output: Architecture: x86_64 CPUs: 1 Total Memory: 2 GiB - + Name: docker + ID: I54V:OLXT:HVMM:TPKO:JPHQ:CQCD:JNLC:O3BZ:4ZVJ:43XJ:PFHZ:6N2S + Docker Root Dir: /var/lib/docker + Debug mode (client): false + Debug mode (server): false + Username: xyz + Registry: https://index.docker.io/v1/ + # HISTORY April 2014, Originally compiled by William Henry (whenry at redhat dot com) based on docker.com source material and internal work. diff --git a/components/engine/man/docker-login.1.md b/components/engine/man/docker-login.1.md index c32a49b075..ea89937644 100644 --- a/components/engine/man/docker-login.1.md +++ b/components/engine/man/docker-login.1.md @@ -2,26 +2,25 @@ % Docker Community % JUNE 2014 # NAME -docker-login - Register or log in to a Docker registry. +docker-login - Log in to a Docker registry. # SYNOPSIS **docker login** -[**-e**|**--email**[=*EMAIL*]] [**--help**] [**-p**|**--password**[=*PASSWORD*]] [**-u**|**--username**[=*USERNAME*]] [SERVER] # DESCRIPTION -Register or log in to a Docker Registry located on the specified +Log in to a Docker Registry located on the specified `SERVER`. You can specify a URL or a `hostname` for the `SERVER` value. If you do not specify a `SERVER`, the command uses Docker's public registry located at `https://registry-1.docker.io/` by default. To get a username/password for Docker's public registry, create an account on Docker Hub. -`docker login` requires user to use `sudo` or be `root`, except when: +`docker login` requires user to use `sudo` or be `root`, except when: 1. connecting to a remote daemon, such as a `docker-machine` provisioned `docker engine`. -2. user is added to the `docker` group. This will impact the security of your system; the `docker` group is `root` equivalent. See [Docker Daemon Attack Surface](https://docs.docker.com/articles/security/#docker-daemon-attack-surface) for details. +2. user is added to the `docker` group. This will impact the security of your system; the `docker` group is `root` equivalent. See [Docker Daemon Attack Surface](https://docs.docker.com/articles/security/#docker-daemon-attack-surface) for details. You can log into any public or private repository for which you have credentials. When you log in, the command stores encoded credentials in @@ -31,9 +30,6 @@ credentials. When you log in, the command stores encoded credentials in > # OPTIONS -**-e**, **--email**="" - Email - **--help** Print usage statement diff --git a/components/engine/man/docker-logout.1.md b/components/engine/man/docker-logout.1.md index d116986798..a8a4b7c3c0 100644 --- a/components/engine/man/docker-logout.1.md +++ b/components/engine/man/docker-logout.1.md @@ -24,7 +24,7 @@ There are no available options. # docker logout localhost:8080 # See also -**docker-login(1)** to register or log in to a Docker registry server. +**docker-login(1)** to log in to a Docker registry server. # HISTORY June 2014, Originally compiled by Daniel, Dao Quang Minh (daniel at nitrous dot io) diff --git a/components/engine/man/docker-logs.1.md b/components/engine/man/docker-logs.1.md index 21501dc51d..f910b53574 100644 --- a/components/engine/man/docker-logs.1.md +++ b/components/engine/man/docker-logs.1.md @@ -42,9 +42,9 @@ logging drivers. **--tail**="*all*" Output the specified number of lines at the end of logs (defaults to all logs) -The `--since` option can be Unix timestamps, date formated timestamps, or Go +The `--since` option can be Unix timestamps, date formatted timestamps, or Go duration strings (e.g. `10m`, `1h30m`) computed relative to the client machine’s -time. Supported formats for date formated time stamps include RFC3339Nano, +time. Supported formats for date formatted time stamps include RFC3339Nano, RFC3339, `2006-01-02T15:04:05`, `2006-01-02T15:04:05.999999999`, `2006-01-02Z07:00`, and `2006-01-02`. The local timezone on the client will be used if you do not provide either a `Z` or a `+-00:00` timezone offset at the diff --git a/components/engine/man/docker-network-create.1.md b/components/engine/man/docker-network-create.1.md index e1fea9f367..e2c34bff17 100644 --- a/components/engine/man/docker-network-create.1.md +++ b/components/engine/man/docker-network-create.1.md @@ -14,6 +14,7 @@ docker-network-create - create a new network [**--ip-range**=*[]*] [**--ipam-driver**=*default*] [**--ipam-opt**=*map[]*] +[**--ipv6**] [**-o**|**--opt**=*map[]*] [**--subnet**=*[]*] NETWORK-NAME @@ -152,6 +153,9 @@ If you want to create an externally isolated `overlay` network, you can specify **--ipam-opt**=map[] Set custom IPAM driver options +**--ipv6** + Enable IPv6 networking + **-o**, **--opt**=map[] Set custom driver options diff --git a/components/engine/man/docker-network-ls.1.md b/components/engine/man/docker-network-ls.1.md index ceca40573c..56a8334ae4 100644 --- a/components/engine/man/docker-network-ls.1.md +++ b/components/engine/man/docker-network-ls.1.md @@ -89,7 +89,7 @@ NETWORK ID NAME DRIVER You can also filter for a substring in a name as this shows: ```bash -$ docker ps --filter name=foo +$ docker network ls --filter name=foo NETWORK ID NAME DRIVER 95e74588f40d foo bridge 06e7eef0a170 foobar bridge @@ -99,8 +99,8 @@ NETWORK ID NAME DRIVER The `id` filter matches on all or part of a network's ID. -The following filter matches all networks with a name containing the -`06e7eef01700` string. +The following filter matches all networks with an ID containing the +`63d1ff1f77b0...` string. ```bash $ docker network ls --filter id=63d1ff1f77b07ca51070a8c227e962238358bd310bde1529cf62e6c307ade161 @@ -108,14 +108,14 @@ NETWORK ID NAME DRIVER 63d1ff1f77b0 dev bridge ``` -You can also filter for a substring in a ID as this shows: +You can also filter for a substring in an ID as this shows: ```bash -$ docker ps --filter id=95e74588f40d +$ docker network ls --filter id=95e74588f40d NETWORK ID NAME DRIVER 95e74588f40d foo bridge -$ docker ps --filter id=95e +$ docker network ls --filter id=95e NETWORK ID NAME DRIVER 95e74588f40d foo bridge ``` diff --git a/components/engine/man/docker-network-rm.1.md b/components/engine/man/docker-network-rm.1.md index 7f8e3dae53..c094a15286 100644 --- a/components/engine/man/docker-network-rm.1.md +++ b/components/engine/man/docker-network-rm.1.md @@ -20,7 +20,7 @@ To remove the network named 'my-network': ``` To delete multiple networks in a single `docker network rm` command, provide -multiple network names or id's. The following example deletes a network with id +multiple network names or ids. The following example deletes a network with id `3695c422697f` and a network named `my-network`: ```bash diff --git a/components/engine/man/docker-ps.1.md b/components/engine/man/docker-ps.1.md index 91d1b21733..f567966487 100644 --- a/components/engine/man/docker-ps.1.md +++ b/components/engine/man/docker-ps.1.md @@ -35,6 +35,7 @@ the running containers. - before=(|) - since=(|) - ancestor=([:tag]||) - containers created from an image or a descendant. + - volume=(|) **--format**="*TEMPLATE*" Pretty-print containers using a Go template. @@ -47,8 +48,10 @@ the running containers. .Ports - Exposed ports. .Status - Container status. .Size - Container disk size. + .Names - Container names. .Labels - All labels assigned to the container. .Label - Value of a specific label for this container. For example `{{.Label "com.docker.swarm.cpu"}}` + .Mounts - Names of the volumes mounted in this container. **--help** Print usage statement @@ -117,6 +120,18 @@ the running containers. c1d3b0166030 debian 41d50ecd2f57 fedora +# Display containers with `remote-volume` mounted + + $ docker ps --filter volume=remote-volume --format "table {{.ID}}\t{{.Mounts}}" + CONTAINER ID MOUNTS + 9c3527ed70ce remote-volume + +# Display containers with a volume mounted in `/data` + + $ docker ps --filter volume=/data --format "table {{.ID}}\t{{.Mounts}}" + CONTAINER ID MOUNTS + 9c3527ed70ce remote-volume + # HISTORY April 2014, Originally compiled by William Henry (whenry at redhat dot com) based on docker.com source material and internal work. diff --git a/components/engine/man/docker-rm.1.md b/components/engine/man/docker-rm.1.md index 9ae3142a6c..2105288d0d 100644 --- a/components/engine/man/docker-rm.1.md +++ b/components/engine/man/docker-rm.1.md @@ -50,15 +50,15 @@ command. The use that name as follows: ## Removing a container and all associated volumes - $ docker rm -v redis - redis + $ docker rm -v redis + redis This command will remove the container and any volumes associated with it. Note that if a volume was specified with a name, it will not be removed. - $ docker create -v awesome:/foo -v /bar --name hello redis - hello - $ docker rm -v hello + $ docker create -v awesome:/foo -v /bar --name hello redis + hello + $ docker rm -v hello In this example, the volume for `/foo` will remain in tact, but the volume for `/bar` will be removed. The same behavior holds for volumes inherited with diff --git a/components/engine/man/docker-run.1.md b/components/engine/man/docker-run.1.md index 210343e3e4..a22d43762d 100644 --- a/components/engine/man/docker-run.1.md +++ b/components/engine/man/docker-run.1.md @@ -60,6 +60,7 @@ docker-run - Run a command in a new container [**-P**|**--publish-all**] [**-p**|**--publish**[=*[]*]] [**--pid**[=*[]*]] +[**--pids-limit**[=*PIDS_LIMIT*]] [**--privileged**] [**--read-only**] [**--restart**[=*RESTART*]] @@ -320,7 +321,7 @@ container can access the exposed port via a private networking interface. Docker will set some environment variables in the client container to help indicate which interface and port to use. -**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*none*" +**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*gcplogs*|*none*" Logging driver for container. Default is defined by daemon `--log-driver` flag. **Warning**: the `docker logs` command works only for the `json-file` and `journald` logging drivers. @@ -420,6 +421,9 @@ Use `docker port` to see the actual mapping: `docker port CONTAINER $CONTAINERPO **host**: use the host's PID namespace inside the container. Note: the host mode gives the container full access to local PID and is therefore considered insecure. +**--pids-limit**="" + Tune the container's pids limit. Set `-1` to have unlimited pids for the container. + **--uts**=*host* Set the UTS mode for the container **host**: use the host's UTS namespace inside the container. @@ -459,6 +463,8 @@ its root filesystem mounted as read only prohibiting any writes. "label:type:TYPE" : Set the label type for the container "label:level:LEVEL" : Set the label level for the container "label:disable" : Turn off label confinement for the container + "no-new-privileges" : Disable container processes from gaining additional privileges + **--stop-signal**=*SIGTERM* Signal to stop a container. Default is SIGTERM. diff --git a/components/engine/man/docker-update.1.md b/components/engine/man/docker-update.1.md index a49fbd83d3..87849ef8d5 100644 --- a/components/engine/man/docker-update.1.md +++ b/components/engine/man/docker-update.1.md @@ -2,7 +2,7 @@ % Docker Community % JUNE 2014 # NAME -docker-update - Update resource configs of one or more containers +docker-update - Update configuration of one or more containers # SYNOPSIS **docker update** @@ -17,15 +17,16 @@ docker-update - Update resource configs of one or more containers [**-m**|**--memory**[=*MEMORY*]] [**--memory-reservation**[=*MEMORY-RESERVATION*]] [**--memory-swap**[=*MEMORY-SWAP*]] +[**--restart**[=*""*]] CONTAINER [CONTAINER...] # DESCRIPTION -The `docker update` command dynamically updates container resources. Use this -command to prevent containers from consuming too many resources from their -Docker host. With a single command, you can place limits on a single -container or on many. To specify more than one container, provide -space-separated list of container names or IDs. +The `docker update` command dynamically updates container configuration. +you can Use this command to prevent containers from consuming too many +resources from their Docker host. With a single command, you can place +limits on a single container or on many. To specify more than one container, +provide space-separated list of container names or IDs. With the exception of the `--kernel-memory` value, you can specify these options on a running or a stopped container. You can only update @@ -33,6 +34,10 @@ options on a running or a stopped container. You can only update stopped container, the next time you restart it, the container uses those values. +Another configuration you can change with this command is restart policy, +new restart policy will take effect instantly after you run `docker update` +on a container. + # OPTIONS **--blkio-weight**=0 Block IO weight (relative weight) accepts a weight value between 10 and 1000. @@ -70,6 +75,9 @@ be updated to a stopped container, and affect after it's started. **--memory-swap**="" Total memory limit (memory + swap) +**--restart**="" + Restart policy to apply when a container exits (no, on-failure[:max-retry], always, unless-stopped). + # EXAMPLES The following sections illustrate ways to use this command. @@ -91,3 +99,10 @@ To update multiple resource configurations for multiple containers: ```bash $ docker update --cpu-shares 512 -m 300M abebf7571666 hopeful_morse ``` + +### Update a container's restart policy + +To update restart policy for one or more containers: +```bash +$ docker update --restart=on-failure:3 abebf7571666 hopeful_morse +``` diff --git a/components/engine/man/docker-volume-create.1.md b/components/engine/man/docker-volume-create.1.md index 24b39bc5a2..43338095c7 100644 --- a/components/engine/man/docker-volume-create.1.md +++ b/components/engine/man/docker-volume-create.1.md @@ -15,11 +15,9 @@ docker-volume-create - Create a new volume Creates a new volume that containers can consume and store data in. If a name is not specified, Docker generates a random name. You create a volume and then configure the container to use it, for example: - ``` - $ docker volume create --name hello - hello - $ docker run -d -v hello:/world busybox ls /world - ``` + $ docker volume create --name hello + hello + $ docker run -d -v hello:/world busybox ls /world The mount is created inside the container's `/src` directory. Docker doesn't not support relative paths for mount points inside the container. @@ -29,14 +27,22 @@ Multiple containers can use the same volume in the same time period. This is use Some volume drivers may take options to customize the volume creation. Use the `-o` or `--opt` flags to pass driver options: - ``` - $ docker volume create --driver fake --opt tardis=blue --opt timey=wimey - ``` + $ docker volume create --driver fake --opt tardis=blue --opt timey=wimey These options are passed directly to the volume driver. Options for different volume drivers may do different things (or nothing at all). -*Note*: The built-in `local` volume driver does not currently accept any options. +The built-in `local` driver on Windows does not support any options. + +The built-in `local` driver on Linux accepts options similar to the linux `mount` +command: + + $ docker volume create --driver local --opt type=tmpfs --opt device=tmpfs --opt o=size=100m,uid=1000 + +Another example: + + $ docker volume create --driver local --opt type=btrfs --opt device=/dev/sda2 + # OPTIONS **-d**, **--driver**="*local*" diff --git a/components/engine/man/docker-volume.1.md b/components/engine/man/docker-volume.1.md index c4569bf01a..5ba8fc4ba3 100644 --- a/components/engine/man/docker-volume.1.md +++ b/components/engine/man/docker-volume.1.md @@ -10,32 +10,21 @@ docker-volume - Create a new volume # DESCRIPTION -docker volume command manages content volumes for docker containers. +docker volume has subcommands for managing data volumes. ## Data volumes -A *data volume* is a specially-designated directory within one or more -containers. +The `docker volume` command has subcommands for managing data volumes. A data volume is a specially-designated directory that by-passes storage driver management. -Data volumes provide several useful features for persistent or shared data: +Data volumes persist data independent of a container's life cycle. When you delete a container, the Engine daemon does not delete any data volumes. You can share volumes across multiple containers. Moreover, you can share data volumes with other computing resources in your system. -Volumes are initialized when a container is created. If the container's -base image contains data at the specified mount point, that existing data is -copied into the new volume upon volume initialization. (Note that this does -not apply when [mounting a host directory](#mount-a-host-directory-as-a-data-volume).) +To see help for a subcommand, use: -Data volumes can be shared and reused among containers. +``` +docker volume CMD help +``` -Changes to a data volume are made directly. - -Changes to a data volume will not be included when you update an image. - -Data volumes persist even if the container itself is deleted. - -Data volumes are designed to persist data, independent of the container's life -cycle. Docker therefore *never* automatically deletes volumes when you remove -a container, nor will it "garbage collect" volumes that are no longer -referenced by a container. +For full details on using docker volume visit Docker's online documentation. # OPTIONS **--help** diff --git a/components/engine/man/docker.1.md b/components/engine/man/docker.1.md index f2bb68f2ce..f59f98abdf 100644 --- a/components/engine/man/docker.1.md +++ b/components/engine/man/docker.1.md @@ -132,7 +132,7 @@ inside it) See **docker-load(1)** for full documentation on the **load** command. **login** - Register or login to a Docker Registry + Log in to a Docker Registry See **docker-login(1)** for full documentation on the **login** command. **logout** diff --git a/components/engine/migrate/v1/migratev1.go b/components/engine/migrate/v1/migratev1.go index 9243c5a42a..aa9d48cb32 100644 --- a/components/engine/migrate/v1/migratev1.go +++ b/components/engine/migrate/v1/migratev1.go @@ -160,7 +160,12 @@ func calculateLayerChecksum(graphDir, id string, ls checksumCalculator) error { return err } - if err := ioutil.WriteFile(filepath.Join(graphDir, id, migrationDiffIDFileName), []byte(diffID), 0600); err != nil { + tmpFile := filepath.Join(graphDir, id, migrationDiffIDFileName+".tmp") + if err := ioutil.WriteFile(tmpFile, []byte(diffID), 0600); err != nil { + return err + } + + if err := os.Rename(tmpFile, filepath.Join(graphDir, id, migrationDiffIDFileName)); err != nil { return err } @@ -282,7 +287,8 @@ func migrateContainers(root string, ls graphIDMounter, is image.Store, imageMapp } if err := ls.CreateRWLayerByGraphID(id, id, img.RootFS.ChainID()); err != nil { - return err + logrus.Errorf("migrate container error: %v", err) + continue } logrus.Infof("migrated container %s to point to %s", id, imageID) @@ -422,7 +428,11 @@ func migrateImage(id, root string, ls graphIDRegistrar, is image.Store, ms metad history = parentImg.History } - diffID, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationDiffIDFileName)) + diffIDData, err := ioutil.ReadFile(filepath.Join(root, graphDirName, id, migrationDiffIDFileName)) + if err != nil { + return err + } + diffID, err := digest.ParseDigest(string(diffIDData)) if err != nil { return err } diff --git a/components/engine/migrate/v1/migratev1_test.go b/components/engine/migrate/v1/migratev1_test.go index 6e8af7fdc8..73878f11e9 100644 --- a/components/engine/migrate/v1/migratev1_test.go +++ b/components/engine/migrate/v1/migratev1_test.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "testing" "github.com/docker/distribution/digest" @@ -62,6 +63,10 @@ func TestMigrateRefs(t *testing.T) { } func TestMigrateContainers(t *testing.T) { + // TODO Windows: Figure out why this is failing + if runtime.GOOS == "windows" { + t.Skip("Failing on Windows") + } tmpdir, err := ioutil.TempDir("", "migrate-containers") if err != nil { t.Fatal(err) @@ -133,6 +138,10 @@ func TestMigrateContainers(t *testing.T) { } func TestMigrateImages(t *testing.T) { + // TODO Windows: Figure out why this is failing + if runtime.GOOS == "windows" { + t.Skip("Failing on Windows") + } tmpdir, err := ioutil.TempDir("", "migrate-images") if err != nil { t.Fatal(err) diff --git a/components/engine/opts/hosts_test.go b/components/engine/opts/hosts_test.go index 8856f95fed..dc527e6388 100644 --- a/components/engine/opts/hosts_test.go +++ b/components/engine/opts/hosts_test.go @@ -6,15 +6,16 @@ import ( ) func TestParseHost(t *testing.T) { - invalid := map[string]string{ - "anything": "Invalid bind address format: anything", - "something with spaces": "Invalid bind address format: something with spaces", - "://": "Invalid bind address format: ://", - "unknown://": "Invalid bind address format: unknown://", - "tcp://:port": "Invalid bind address format: :port", - "tcp://invalid": "Invalid bind address format: invalid", - "tcp://invalid:port": "Invalid bind address format: invalid:port", + invalid := []string{ + "anything", + "something with spaces", + "://", + "unknown://", + "tcp://:port", + "tcp://invalid", + "tcp://invalid:port", } + valid := map[string]string{ "": DefaultHost, " ": DefaultHost, @@ -37,11 +38,12 @@ func TestParseHost(t *testing.T) { "npipe:////./pipe/foo": "npipe:////./pipe/foo", } - for value, errorMessage := range invalid { - if _, err := ParseHost(false, value); err == nil || err.Error() != errorMessage { - t.Errorf("Expected an error for %v with [%v], got [%v]", value, errorMessage, err) + for _, value := range invalid { + if _, err := ParseHost(false, value); err == nil { + t.Errorf("Expected an error for %v, got [nil]", value) } } + for value, expected := range valid { if actual, err := ParseHost(false, value); err != nil || actual != expected { t.Errorf("Expected for %v [%v], got [%v, %v]", value, expected, actual, err) diff --git a/components/engine/opts/hosts_windows.go b/components/engine/opts/hosts_windows.go index ec52e9a70a..7c239e00f1 100644 --- a/components/engine/opts/hosts_windows.go +++ b/components/engine/opts/hosts_windows.go @@ -3,4 +3,4 @@ package opts // DefaultHost constant defines the default host string used by docker on Windows -var DefaultHost = DefaultTCPHost +var DefaultHost = "npipe://" + DefaultNamedPipe diff --git a/components/engine/pkg/aaparser/aaparser.go b/components/engine/pkg/aaparser/aaparser.go index 2d34643a7d..507298f42a 100644 --- a/components/engine/pkg/aaparser/aaparser.go +++ b/components/engine/pkg/aaparser/aaparser.go @@ -14,13 +14,13 @@ const ( ) // GetVersion returns the major and minor version of apparmor_parser. -func GetVersion() (int, int, error) { +func GetVersion() (int, error) { output, err := cmd("", "--version") if err != nil { - return -1, -1, err + return -1, err } - return parseVersion(string(output)) + return parseVersion(output) } // LoadProfile runs `apparmor_parser -r -W` on a specified apparmor profile to @@ -47,30 +47,46 @@ func cmd(dir string, arg ...string) (string, error) { } // parseVersion takes the output from `apparmor_parser --version` and returns -// the major and minor version for `apparor_parser`. -func parseVersion(output string) (int, int, error) { +// a representation of the {major, minor, patch} version as a single number of +// the form MMmmPPP {major, minor, patch}. +func parseVersion(output string) (int, error) { // output is in the form of the following: // AppArmor parser version 2.9.1 // Copyright (C) 1999-2008 Novell Inc. // Copyright 2009-2012 Canonical Ltd. + lines := strings.SplitN(output, "\n", 2) words := strings.Split(lines[0], " ") version := words[len(words)-1] // split by major minor version v := strings.Split(version, ".") - if len(v) < 2 { - return -1, -1, fmt.Errorf("parsing major minor version failed for output: `%s`", output) + if len(v) == 0 || len(v) > 3 { + return -1, fmt.Errorf("parsing version failed for output: `%s`", output) } + // Default the versions to 0. + var majorVersion, minorVersion, patchLevel int + majorVersion, err := strconv.Atoi(v[0]) if err != nil { - return -1, -1, err - } - minorVersion, err := strconv.Atoi(v[1]) - if err != nil { - return -1, -1, err + return -1, err } - return majorVersion, minorVersion, nil + if len(v) > 1 { + minorVersion, err = strconv.Atoi(v[1]) + if err != nil { + return -1, err + } + } + if len(v) > 2 { + patchLevel, err = strconv.Atoi(v[2]) + if err != nil { + return -1, err + } + } + + // major*10^5 + minor*10^3 + patch*10^0 + numericVersion := majorVersion*1e5 + minorVersion*1e3 + patchLevel + return numericVersion, nil } diff --git a/components/engine/pkg/aaparser/aaparser_test.go b/components/engine/pkg/aaparser/aaparser_test.go index 4befa0ac91..69bc8d2fd8 100644 --- a/components/engine/pkg/aaparser/aaparser_test.go +++ b/components/engine/pkg/aaparser/aaparser_test.go @@ -5,9 +5,8 @@ import ( ) type versionExpected struct { - output string - major int - minor int + output string + version int } func TestParseVersion(t *testing.T) { @@ -18,8 +17,7 @@ Copyright (C) 1999-2008 Novell Inc. Copyright 2009-2012 Canonical Ltd. `, - major: 2, - minor: 10, + version: 210000, }, { output: `AppArmor parser version 2.8 @@ -27,8 +25,7 @@ Copyright (C) 1999-2008 Novell Inc. Copyright 2009-2012 Canonical Ltd. `, - major: 2, - minor: 8, + version: 208000, }, { output: `AppArmor parser version 2.20 @@ -36,8 +33,7 @@ Copyright (C) 1999-2008 Novell Inc. Copyright 2009-2012 Canonical Ltd. `, - major: 2, - minor: 20, + version: 220000, }, { output: `AppArmor parser version 2.05 @@ -45,21 +41,33 @@ Copyright (C) 1999-2008 Novell Inc. Copyright 2009-2012 Canonical Ltd. `, - major: 2, - minor: 5, + version: 205000, + }, + { + output: `AppArmor parser version 2.9.95 +Copyright (C) 1999-2008 Novell Inc. +Copyright 2009-2012 Canonical Ltd. + +`, + version: 209095, + }, + { + output: `AppArmor parser version 3.14.159 +Copyright (C) 1999-2008 Novell Inc. +Copyright 2009-2012 Canonical Ltd. + +`, + version: 314159, }, } for _, v := range versions { - major, minor, err := parseVersion(v.output) + version, err := parseVersion(v.output) if err != nil { t.Fatalf("expected error to be nil for %#v, got: %v", v, err) } - if major != v.major { - t.Fatalf("expected major version to be %d, was %d, for: %#v\n", v.major, major, v) - } - if minor != v.minor { - t.Fatalf("expected minor version to be %d, was %d, for: %#v\n", v.minor, minor, v) + if version != v.version { + t.Fatalf("expected version to be %d, was %d, for: %#v\n", v.version, version, v) } } } diff --git a/components/engine/pkg/archive/archive.go b/components/engine/pkg/archive/archive.go index 1281683ee4..358ab097b2 100644 --- a/components/engine/pkg/archive/archive.go +++ b/components/engine/pkg/archive/archive.go @@ -502,13 +502,13 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) defer func() { // Make sure to check the error on Close. if err := ta.TarWriter.Close(); err != nil { - logrus.Debugf("Can't close tar writer: %s", err) + logrus.Errorf("Can't close tar writer: %s", err) } if err := compressWriter.Close(); err != nil { - logrus.Debugf("Can't close compress writer: %s", err) + logrus.Errorf("Can't close compress writer: %s", err) } if err := pipeWriter.Close(); err != nil { - logrus.Debugf("Can't close pipe writer: %s", err) + logrus.Errorf("Can't close pipe writer: %s", err) } }() @@ -551,7 +551,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) walkRoot := getWalkRoot(srcPath, include) filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error { if err != nil { - logrus.Debugf("Tar: Can't stat file %s to tar: %s", srcPath, err) + logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err) return nil } @@ -576,16 +576,42 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) if include != relFilePath { skip, err = fileutils.OptimizedMatches(relFilePath, patterns, patDirs) if err != nil { - logrus.Debugf("Error matching %s: %v", relFilePath, err) + logrus.Errorf("Error matching %s: %v", relFilePath, err) return err } } if skip { - if !exceptions && f.IsDir() { + // If we want to skip this file and its a directory + // then we should first check to see if there's an + // excludes pattern (eg !dir/file) that starts with this + // dir. If so then we can't skip this dir. + + // Its not a dir then so we can just return/skip. + if !f.IsDir() { + return nil + } + + // No exceptions (!...) in patterns so just skip dir + if !exceptions { return filepath.SkipDir } - return nil + + dirSlash := relFilePath + string(filepath.Separator) + + for _, pat := range patterns { + if pat[0] != '!' { + continue + } + pat = pat[1:] + string(filepath.Separator) + if strings.HasPrefix(pat, dirSlash) { + // found a match - so can't skip this dir + return nil + } + } + + // No matching exclusion dir so just skip dir + return filepath.SkipDir } if seen[relFilePath] { @@ -607,7 +633,11 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) } if err := ta.addTarFile(filePath, relFilePath); err != nil { - logrus.Debugf("Can't add file %s to tar: %s", filePath, err) + logrus.Errorf("Can't add file %s to tar: %s", filePath, err) + // if pipe is broken, stop writting tar stream to it + if err == io.ErrClosedPipe { + return err + } } return nil }) @@ -660,7 +690,7 @@ loop: parent := filepath.Dir(hdr.Name) parentPath := filepath.Join(dest, parent) if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { - err = system.MkdirAll(parentPath, 0777) + err = idtools.MkdirAllNewAs(parentPath, 0777, remappedRootUID, remappedRootGID) if err != nil { return err } diff --git a/components/engine/pkg/archive/archive_test.go b/components/engine/pkg/archive/archive_test.go index 0a899023a8..495cac832c 100644 --- a/components/engine/pkg/archive/archive_test.go +++ b/components/engine/pkg/archive/archive_test.go @@ -8,16 +8,22 @@ import ( "io/ioutil" "os" "os/exec" - "path" "path/filepath" + "runtime" "strings" - "syscall" "testing" "time" - - "github.com/docker/docker/pkg/system" ) +var tmp string + +func init() { + tmp = "/tmp/" + if runtime.GOOS == "windows" { + tmp = os.Getenv("TEMP") + `\` + } +} + func TestIsArchiveNilHeader(t *testing.T) { out := IsArchive(nil) if out { @@ -50,51 +56,51 @@ func TestIsArchive7zip(t *testing.T) { } func TestIsArchivePathDir(t *testing.T) { - cmd := exec.Command("/bin/sh", "-c", "mkdir -p /tmp/archivedir") + cmd := exec.Command("sh", "-c", "mkdir -p /tmp/archivedir") output, err := cmd.CombinedOutput() if err != nil { t.Fatalf("Fail to create an archive file for test : %s.", output) } - if IsArchivePath("/tmp/archivedir") { + if IsArchivePath(tmp + "archivedir") { t.Fatalf("Incorrectly recognised directory as an archive") } } func TestIsArchivePathInvalidFile(t *testing.T) { - cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1K count=1 of=/tmp/archive && gzip --stdout /tmp/archive > /tmp/archive.gz") + cmd := exec.Command("sh", "-c", "dd if=/dev/zero bs=1K count=1 of=/tmp/archive && gzip --stdout /tmp/archive > /tmp/archive.gz") output, err := cmd.CombinedOutput() if err != nil { t.Fatalf("Fail to create an archive file for test : %s.", output) } - if IsArchivePath("/tmp/archive") { + if IsArchivePath(tmp + "archive") { t.Fatalf("Incorrectly recognised invalid tar path as archive") } - if IsArchivePath("/tmp/archive.gz") { + if IsArchivePath(tmp + "archive.gz") { t.Fatalf("Incorrectly recognised invalid compressed tar path as archive") } } func TestIsArchivePathTar(t *testing.T) { - cmd := exec.Command("/bin/sh", "-c", "touch /tmp/archivedata && tar -cf /tmp/archive /tmp/archivedata && gzip --stdout /tmp/archive > /tmp/archive.gz") + cmd := exec.Command("sh", "-c", "touch /tmp/archivedata && tar -cf /tmp/archive /tmp/archivedata && gzip --stdout /tmp/archive > /tmp/archive.gz") output, err := cmd.CombinedOutput() if err != nil { t.Fatalf("Fail to create an archive file for test : %s.", output) } - if !IsArchivePath("/tmp/archive") { + if !IsArchivePath(tmp + "/archive") { t.Fatalf("Did not recognise valid tar path as archive") } - if !IsArchivePath("/tmp/archive.gz") { + if !IsArchivePath(tmp + "archive.gz") { t.Fatalf("Did not recognise valid compressed tar path as archive") } } func TestDecompressStreamGzip(t *testing.T) { - cmd := exec.Command("/bin/sh", "-c", "touch /tmp/archive && gzip -f /tmp/archive") + cmd := exec.Command("sh", "-c", "touch /tmp/archive && gzip -f /tmp/archive") output, err := cmd.CombinedOutput() if err != nil { t.Fatalf("Fail to create an archive file for test : %s.", output) } - archive, err := os.Open("/tmp/archive.gz") + archive, err := os.Open(tmp + "archive.gz") _, err = DecompressStream(archive) if err != nil { t.Fatalf("Failed to decompress a gzip file.") @@ -102,12 +108,12 @@ func TestDecompressStreamGzip(t *testing.T) { } func TestDecompressStreamBzip2(t *testing.T) { - cmd := exec.Command("/bin/sh", "-c", "touch /tmp/archive && bzip2 -f /tmp/archive") + cmd := exec.Command("sh", "-c", "touch /tmp/archive && bzip2 -f /tmp/archive") output, err := cmd.CombinedOutput() if err != nil { t.Fatalf("Fail to create an archive file for test : %s.", output) } - archive, err := os.Open("/tmp/archive.bz2") + archive, err := os.Open(tmp + "archive.bz2") _, err = DecompressStream(archive) if err != nil { t.Fatalf("Failed to decompress a bzip2 file.") @@ -115,12 +121,15 @@ func TestDecompressStreamBzip2(t *testing.T) { } func TestDecompressStreamXz(t *testing.T) { - cmd := exec.Command("/bin/sh", "-c", "touch /tmp/archive && xz -f /tmp/archive") + if runtime.GOOS == "windows" { + t.Skip("Xz not present in msys2") + } + cmd := exec.Command("sh", "-c", "touch /tmp/archive && xz -f /tmp/archive") output, err := cmd.CombinedOutput() if err != nil { t.Fatalf("Fail to create an archive file for test : %s.", output) } - archive, err := os.Open("/tmp/archive.xz") + archive, err := os.Open(tmp + "archive.xz") _, err = DecompressStream(archive) if err != nil { t.Fatalf("Failed to decompress a xz file.") @@ -128,7 +137,7 @@ func TestDecompressStreamXz(t *testing.T) { } func TestCompressStreamXzUnsuported(t *testing.T) { - dest, err := os.Create("/tmp/dest") + dest, err := os.Create(tmp + "dest") if err != nil { t.Fatalf("Fail to create the destination file") } @@ -139,7 +148,7 @@ func TestCompressStreamXzUnsuported(t *testing.T) { } func TestCompressStreamBzip2Unsupported(t *testing.T) { - dest, err := os.Create("/tmp/dest") + dest, err := os.Create(tmp + "dest") if err != nil { t.Fatalf("Fail to create the destination file") } @@ -150,7 +159,7 @@ func TestCompressStreamBzip2Unsupported(t *testing.T) { } func TestCompressStreamInvalid(t *testing.T) { - dest, err := os.Create("/tmp/dest") + dest, err := os.Create(tmp + "dest") if err != nil { t.Fatalf("Fail to create the destination file") } @@ -198,7 +207,7 @@ func TestExtensionXz(t *testing.T) { } func TestCmdStreamLargeStderr(t *testing.T) { - cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello") + cmd := exec.Command("sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello") out, _, err := cmdStream(cmd, nil) if err != nil { t.Fatalf("Failed to start command: %s", err) @@ -219,7 +228,11 @@ func TestCmdStreamLargeStderr(t *testing.T) { } func TestCmdStreamBad(t *testing.T) { - badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1") + // TODO Windows: Figure out why this is failing in CI but not locally + if runtime.GOOS == "windows" { + t.Skip("Failing on Windows CI machines") + } + badCmd := exec.Command("sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1") out, _, err := cmdStream(badCmd, nil) if err != nil { t.Fatalf("Failed to start command: %s", err) @@ -234,7 +247,7 @@ func TestCmdStreamBad(t *testing.T) { } func TestCmdStreamGood(t *testing.T) { - cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0") + cmd := exec.Command("sh", "-c", "echo hello; exit 0") out, _, err := cmdStream(cmd, nil) if err != nil { t.Fatal(err) @@ -252,13 +265,22 @@ func TestUntarPathWithInvalidDest(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tempFolder) - invalidDestFolder := path.Join(tempFolder, "invalidDest") + invalidDestFolder := filepath.Join(tempFolder, "invalidDest") // Create a src file - srcFile := path.Join(tempFolder, "src") - tarFile := path.Join(tempFolder, "src.tar") + srcFile := filepath.Join(tempFolder, "src") + tarFile := filepath.Join(tempFolder, "src.tar") os.Create(srcFile) os.Create(invalidDestFolder) // being a file (not dir) should cause an error - cmd := exec.Command("/bin/sh", "-c", "tar cf "+tarFile+" "+srcFile) + + // Translate back to Unix semantics as next exec.Command is run under sh + srcFileU := srcFile + tarFileU := tarFile + if runtime.GOOS == "windows" { + tarFileU = "/tmp/" + filepath.Base(filepath.Dir(tarFile)) + "/src.tar" + srcFileU = "/tmp/" + filepath.Base(filepath.Dir(srcFile)) + "/src" + } + + cmd := exec.Command("sh", "-c", "tar cf "+tarFileU+" "+srcFileU) _, err = cmd.CombinedOutput() if err != nil { t.Fatal(err) @@ -288,24 +310,34 @@ func TestUntarPath(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmpFolder) - srcFile := path.Join(tmpFolder, "src") - tarFile := path.Join(tmpFolder, "src.tar") - os.Create(path.Join(tmpFolder, "src")) - cmd := exec.Command("/bin/sh", "-c", "tar cf "+tarFile+" "+srcFile) - _, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(err) - } - destFolder := path.Join(tmpFolder, "dest") + srcFile := filepath.Join(tmpFolder, "src") + tarFile := filepath.Join(tmpFolder, "src.tar") + os.Create(filepath.Join(tmpFolder, "src")) + + destFolder := filepath.Join(tmpFolder, "dest") err = os.MkdirAll(destFolder, 0740) if err != nil { t.Fatalf("Fail to create the destination file") } + + // Translate back to Unix semantics as next exec.Command is run under sh + srcFileU := srcFile + tarFileU := tarFile + if runtime.GOOS == "windows" { + tarFileU = "/tmp/" + filepath.Base(filepath.Dir(tarFile)) + "/src.tar" + srcFileU = "/tmp/" + filepath.Base(filepath.Dir(srcFile)) + "/src" + } + cmd := exec.Command("sh", "-c", "tar cf "+tarFileU+" "+srcFileU) + _, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(err) + } + err = UntarPath(tarFile, destFolder) if err != nil { t.Fatalf("UntarPath shouldn't throw an error, %s.", err) } - expectedFile := path.Join(destFolder, srcFile) + expectedFile := filepath.Join(destFolder, srcFileU) _, err = os.Stat(expectedFile) if err != nil { t.Fatalf("Destination folder should contain the source file but did not.") @@ -319,15 +351,23 @@ func TestUntarPathWithDestinationFile(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmpFolder) - srcFile := path.Join(tmpFolder, "src") - tarFile := path.Join(tmpFolder, "src.tar") - os.Create(path.Join(tmpFolder, "src")) - cmd := exec.Command("/bin/sh", "-c", "tar cf "+tarFile+" "+srcFile) + srcFile := filepath.Join(tmpFolder, "src") + tarFile := filepath.Join(tmpFolder, "src.tar") + os.Create(filepath.Join(tmpFolder, "src")) + + // Translate back to Unix semantics as next exec.Command is run under sh + srcFileU := srcFile + tarFileU := tarFile + if runtime.GOOS == "windows" { + tarFileU = "/tmp/" + filepath.Base(filepath.Dir(tarFile)) + "/src.tar" + srcFileU = "/tmp/" + filepath.Base(filepath.Dir(srcFile)) + "/src" + } + cmd := exec.Command("sh", "-c", "tar cf "+tarFileU+" "+srcFileU) _, err = cmd.CombinedOutput() if err != nil { t.Fatal(err) } - destFile := path.Join(tmpFolder, "dest") + destFile := filepath.Join(tmpFolder, "dest") _, err = os.Create(destFile) if err != nil { t.Fatalf("Fail to create the destination file") @@ -347,21 +387,30 @@ func TestUntarPathWithDestinationSrcFileAsFolder(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmpFolder) - srcFile := path.Join(tmpFolder, "src") - tarFile := path.Join(tmpFolder, "src.tar") + srcFile := filepath.Join(tmpFolder, "src") + tarFile := filepath.Join(tmpFolder, "src.tar") os.Create(srcFile) - cmd := exec.Command("/bin/sh", "-c", "tar cf "+tarFile+" "+srcFile) + + // Translate back to Unix semantics as next exec.Command is run under sh + srcFileU := srcFile + tarFileU := tarFile + if runtime.GOOS == "windows" { + tarFileU = "/tmp/" + filepath.Base(filepath.Dir(tarFile)) + "/src.tar" + srcFileU = "/tmp/" + filepath.Base(filepath.Dir(srcFile)) + "/src" + } + + cmd := exec.Command("sh", "-c", "tar cf "+tarFileU+" "+srcFileU) _, err = cmd.CombinedOutput() if err != nil { t.Fatal(err) } - destFolder := path.Join(tmpFolder, "dest") + destFolder := filepath.Join(tmpFolder, "dest") err = os.MkdirAll(destFolder, 0740) if err != nil { t.Fatalf("Fail to create the destination folder") } // Let's create a folder that will has the same path as the extracted file (from tar) - destSrcFileAsFolder := path.Join(destFolder, srcFile) + destSrcFileAsFolder := filepath.Join(destFolder, srcFileU) err = os.MkdirAll(destSrcFileAsFolder, 0740) if err != nil { t.Fatal(err) @@ -377,8 +426,8 @@ func TestCopyWithTarInvalidSrc(t *testing.T) { if err != nil { t.Fatal(nil) } - destFolder := path.Join(tempFolder, "dest") - invalidSrc := path.Join(tempFolder, "doesnotexists") + destFolder := filepath.Join(tempFolder, "dest") + invalidSrc := filepath.Join(tempFolder, "doesnotexists") err = os.MkdirAll(destFolder, 0740) if err != nil { t.Fatal(err) @@ -394,8 +443,8 @@ func TestCopyWithTarInexistentDestWillCreateIt(t *testing.T) { if err != nil { t.Fatal(nil) } - srcFolder := path.Join(tempFolder, "src") - inexistentDestFolder := path.Join(tempFolder, "doesnotexists") + srcFolder := filepath.Join(tempFolder, "src") + inexistentDestFolder := filepath.Join(tempFolder, "doesnotexists") err = os.MkdirAll(srcFolder, 0740) if err != nil { t.Fatal(err) @@ -417,9 +466,9 @@ func TestCopyWithTarSrcFile(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(folder) - dest := path.Join(folder, "dest") - srcFolder := path.Join(folder, "src") - src := path.Join(folder, path.Join("src", "src")) + dest := filepath.Join(folder, "dest") + srcFolder := filepath.Join(folder, "src") + src := filepath.Join(folder, filepath.Join("src", "src")) err = os.MkdirAll(srcFolder, 0740) if err != nil { t.Fatal(err) @@ -447,8 +496,8 @@ func TestCopyWithTarSrcFolder(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(folder) - dest := path.Join(folder, "dest") - src := path.Join(folder, path.Join("src", "folder")) + dest := filepath.Join(folder, "dest") + src := filepath.Join(folder, filepath.Join("src", "folder")) err = os.MkdirAll(src, 0740) if err != nil { t.Fatal(err) @@ -457,7 +506,7 @@ func TestCopyWithTarSrcFolder(t *testing.T) { if err != nil { t.Fatal(err) } - ioutil.WriteFile(path.Join(src, "file"), []byte("content"), 0777) + ioutil.WriteFile(filepath.Join(src, "file"), []byte("content"), 0777) err = CopyWithTar(src, dest) if err != nil { t.Fatalf("archiver.CopyWithTar shouldn't throw an error, %s.", err) @@ -475,12 +524,12 @@ func TestCopyFileWithTarInvalidSrc(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tempFolder) - destFolder := path.Join(tempFolder, "dest") + destFolder := filepath.Join(tempFolder, "dest") err = os.MkdirAll(destFolder, 0740) if err != nil { t.Fatal(err) } - invalidFile := path.Join(tempFolder, "doesnotexists") + invalidFile := filepath.Join(tempFolder, "doesnotexists") err = CopyFileWithTar(invalidFile, destFolder) if err == nil { t.Fatalf("archiver.CopyWithTar with invalid src path should throw an error.") @@ -493,8 +542,8 @@ func TestCopyFileWithTarInexistentDestWillCreateIt(t *testing.T) { t.Fatal(nil) } defer os.RemoveAll(tempFolder) - srcFile := path.Join(tempFolder, "src") - inexistentDestFolder := path.Join(tempFolder, "doesnotexists") + srcFile := filepath.Join(tempFolder, "src") + inexistentDestFolder := filepath.Join(tempFolder, "doesnotexists") _, err = os.Create(srcFile) if err != nil { t.Fatal(err) @@ -516,8 +565,8 @@ func TestCopyFileWithTarSrcFolder(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(folder) - dest := path.Join(folder, "dest") - src := path.Join(folder, "srcfolder") + dest := filepath.Join(folder, "dest") + src := filepath.Join(folder, "srcfolder") err = os.MkdirAll(src, 0740) if err != nil { t.Fatal(err) @@ -538,9 +587,9 @@ func TestCopyFileWithTarSrcFile(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(folder) - dest := path.Join(folder, "dest") - srcFolder := path.Join(folder, "src") - src := path.Join(folder, path.Join("src", "src")) + dest := filepath.Join(folder, "dest") + srcFolder := filepath.Join(folder, "src") + src := filepath.Join(folder, filepath.Join("src", "src")) err = os.MkdirAll(srcFolder, 0740) if err != nil { t.Fatal(err) @@ -561,6 +610,10 @@ func TestCopyFileWithTarSrcFile(t *testing.T) { } func TestTarFiles(t *testing.T) { + // TODO Windows: Figure out how to port this test. + if runtime.GOOS == "windows" { + t.Skip("Failing on Windows") + } // try without hardlinks if err := checkNoChanges(1000, false); err != nil { t.Fatal(err) @@ -639,18 +692,22 @@ func tarUntar(t *testing.T, origin string, options *TarOptions) ([]Change, error } func TestTarUntar(t *testing.T) { + // TODO Windows: Figure out how to fix this test. + if runtime.GOOS == "windows" { + t.Skip("Failing on Windows") + } origin, err := ioutil.TempDir("", "docker-test-untar-origin") if err != nil { t.Fatal(err) } defer os.RemoveAll(origin) - if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil { + if err := ioutil.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0700); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil { + if err := ioutil.WriteFile(filepath.Join(origin, "2"), []byte("welcome!"), 0700); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(path.Join(origin, "3"), []byte("will be ignored"), 0700); err != nil { + if err := ioutil.WriteFile(filepath.Join(origin, "3"), []byte("will be ignored"), 0700); err != nil { t.Fatal(err) } @@ -673,49 +730,11 @@ func TestTarUntar(t *testing.T) { } } -func TestTarUntarWithXattr(t *testing.T) { - origin, err := ioutil.TempDir("", "docker-test-untar-origin") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(origin) - if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil { - t.Fatal(err) - } - if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil { - t.Fatal(err) - } - if err := ioutil.WriteFile(path.Join(origin, "3"), []byte("will be ignored"), 0700); err != nil { - t.Fatal(err) - } - if err := system.Lsetxattr(path.Join(origin, "2"), "security.capability", []byte{0x00}, 0); err != nil { - t.Fatal(err) - } - - for _, c := range []Compression{ - Uncompressed, - Gzip, - } { - changes, err := tarUntar(t, origin, &TarOptions{ - Compression: c, - ExcludePatterns: []string{"3"}, - }) - - if err != nil { - t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err) - } - - if len(changes) != 1 || changes[0].Path != "/3" { - t.Fatalf("Unexpected differences after tarUntar: %v", changes) - } - capability, _ := system.Lgetxattr(path.Join(origin, "2"), "security.capability") - if capability == nil && capability[0] != 0x00 { - t.Fatalf("Untar should have kept the 'security.capability' xattr.") - } - } -} - func TestTarWithOptions(t *testing.T) { + // TODO Windows: Figure out how to fix this test. + if runtime.GOOS == "windows" { + t.Skip("Failing on Windows") + } origin, err := ioutil.TempDir("", "docker-test-untar-origin") if err != nil { t.Fatal(err) @@ -724,10 +743,10 @@ func TestTarWithOptions(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(origin) - if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil { + if err := ioutil.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0700); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil { + if err := ioutil.WriteFile(filepath.Join(origin, "2"), []byte("welcome!"), 0700); err != nil { t.Fatal(err) } @@ -798,150 +817,15 @@ func TestUntarUstarGnuConflict(t *testing.T) { } } -func TestTarWithBlockCharFifo(t *testing.T) { - origin, err := ioutil.TempDir("", "docker-test-tar-hardlink") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(origin) - if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil { - t.Fatal(err) - } - if err := system.Mknod(path.Join(origin, "2"), syscall.S_IFBLK, int(system.Mkdev(int64(12), int64(5)))); err != nil { - t.Fatal(err) - } - if err := system.Mknod(path.Join(origin, "3"), syscall.S_IFCHR, int(system.Mkdev(int64(12), int64(5)))); err != nil { - t.Fatal(err) - } - if err := system.Mknod(path.Join(origin, "4"), syscall.S_IFIFO, int(system.Mkdev(int64(12), int64(5)))); err != nil { - t.Fatal(err) - } - - dest, err := ioutil.TempDir("", "docker-test-tar-hardlink-dest") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dest) - - // we'll do this in two steps to separate failure - fh, err := Tar(origin, Uncompressed) - if err != nil { - t.Fatal(err) - } - - // ensure we can read the whole thing with no error, before writing back out - buf, err := ioutil.ReadAll(fh) - if err != nil { - t.Fatal(err) - } - - bRdr := bytes.NewReader(buf) - err = Untar(bRdr, dest, &TarOptions{Compression: Uncompressed}) - if err != nil { - t.Fatal(err) - } - - changes, err := ChangesDirs(origin, dest) - if err != nil { - t.Fatal(err) - } - if len(changes) > 0 { - t.Fatalf("Tar with special device (block, char, fifo) should keep them (recreate them when untar) : %v", changes) - } -} - -func TestTarWithHardLink(t *testing.T) { - origin, err := ioutil.TempDir("", "docker-test-tar-hardlink") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(origin) - if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil { - t.Fatal(err) - } - if err := os.Link(path.Join(origin, "1"), path.Join(origin, "2")); err != nil { - t.Fatal(err) - } - - var i1, i2 uint64 - if i1, err = getNlink(path.Join(origin, "1")); err != nil { - t.Fatal(err) - } - // sanity check that we can hardlink - if i1 != 2 { - t.Skipf("skipping since hardlinks don't work here; expected 2 links, got %d", i1) - } - - dest, err := ioutil.TempDir("", "docker-test-tar-hardlink-dest") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dest) - - // we'll do this in two steps to separate failure - fh, err := Tar(origin, Uncompressed) - if err != nil { - t.Fatal(err) - } - - // ensure we can read the whole thing with no error, before writing back out - buf, err := ioutil.ReadAll(fh) - if err != nil { - t.Fatal(err) - } - - bRdr := bytes.NewReader(buf) - err = Untar(bRdr, dest, &TarOptions{Compression: Uncompressed}) - if err != nil { - t.Fatal(err) - } - - if i1, err = getInode(path.Join(dest, "1")); err != nil { - t.Fatal(err) - } - if i2, err = getInode(path.Join(dest, "2")); err != nil { - t.Fatal(err) - } - - if i1 != i2 { - t.Errorf("expected matching inodes, but got %d and %d", i1, i2) - } -} - -func getNlink(path string) (uint64, error) { - stat, err := os.Stat(path) - if err != nil { - return 0, err - } - statT, ok := stat.Sys().(*syscall.Stat_t) - if !ok { - return 0, fmt.Errorf("expected type *syscall.Stat_t, got %t", stat.Sys()) - } - // We need this conversion on ARM64 - return uint64(statT.Nlink), nil -} - -func getInode(path string) (uint64, error) { - stat, err := os.Stat(path) - if err != nil { - return 0, err - } - statT, ok := stat.Sys().(*syscall.Stat_t) - if !ok { - return 0, fmt.Errorf("expected type *syscall.Stat_t, got %t", stat.Sys()) - } - return statT.Ino, nil -} - func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) { fileData := []byte("fooo") for n := 0; n < numberOfFiles; n++ { fileName := fmt.Sprintf("file-%d", n) - if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil { + if err := ioutil.WriteFile(filepath.Join(targetPath, fileName), fileData, 0700); err != nil { return 0, err } if makeLinks { - if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil { + if err := os.Link(filepath.Join(targetPath, fileName), filepath.Join(targetPath, fileName+"-link")); err != nil { return 0, err } } @@ -959,7 +843,7 @@ func BenchmarkTarUntar(b *testing.B) { if err != nil { b.Fatal(err) } - target := path.Join(tempDir, "dest") + target := filepath.Join(tempDir, "dest") n, err := prepareUntarSourceDirectory(100, origin, false) if err != nil { b.Fatal(err) @@ -987,7 +871,7 @@ func BenchmarkTarUntarWithLinks(b *testing.B) { if err != nil { b.Fatal(err) } - target := path.Join(tempDir, "dest") + target := filepath.Join(tempDir, "dest") n, err := prepareUntarSourceDirectory(100, origin, true) if err != nil { b.Fatal(err) @@ -1007,6 +891,10 @@ func BenchmarkTarUntarWithLinks(b *testing.B) { } func TestUntarInvalidFilenames(t *testing.T) { + // TODO Windows: Figure out how to fix this test. + if runtime.GOOS == "windows" { + t.Skip("Passes but hits breakoutError: platform and architecture is not supported") + } for i, headers := range [][]*tar.Header{ { { @@ -1031,6 +919,10 @@ func TestUntarInvalidFilenames(t *testing.T) { } func TestUntarHardlinkToSymlink(t *testing.T) { + // TODO Windows. There may be a way of running this, but turning off for now + if runtime.GOOS == "windows" { + t.Skip("hardlinks on Windows") + } for i, headers := range [][]*tar.Header{ { { @@ -1059,6 +951,10 @@ func TestUntarHardlinkToSymlink(t *testing.T) { } func TestUntarInvalidHardlink(t *testing.T) { + // TODO Windows. There may be a way of running this, but turning off for now + if runtime.GOOS == "windows" { + t.Skip("hardlinks on Windows") + } for i, headers := range [][]*tar.Header{ { // try reading victim/hello (../) { @@ -1139,6 +1035,10 @@ func TestUntarInvalidHardlink(t *testing.T) { } func TestUntarInvalidSymlink(t *testing.T) { + // TODO Windows. There may be a way of running this, but turning off for now + if runtime.GOOS == "windows" { + t.Skip("hardlinks on Windows") + } for i, headers := range [][]*tar.Header{ { // try reading victim/hello (../) { diff --git a/components/engine/pkg/archive/archive_unix_test.go b/components/engine/pkg/archive/archive_unix_test.go index 18f45c480f..548391b35d 100644 --- a/components/engine/pkg/archive/archive_unix_test.go +++ b/components/engine/pkg/archive/archive_unix_test.go @@ -3,8 +3,15 @@ package archive import ( + "bytes" + "fmt" + "io/ioutil" "os" + "path/filepath" + "syscall" "testing" + + "github.com/docker/docker/pkg/system" ) func TestCanonicalTarNameForPath(t *testing.T) { @@ -58,3 +65,181 @@ func TestChmodTarEntry(t *testing.T) { } } } + +func TestTarWithHardLink(t *testing.T) { + origin, err := ioutil.TempDir("", "docker-test-tar-hardlink") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(origin) + if err := ioutil.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0700); err != nil { + t.Fatal(err) + } + if err := os.Link(filepath.Join(origin, "1"), filepath.Join(origin, "2")); err != nil { + t.Fatal(err) + } + + var i1, i2 uint64 + if i1, err = getNlink(filepath.Join(origin, "1")); err != nil { + t.Fatal(err) + } + // sanity check that we can hardlink + if i1 != 2 { + t.Skipf("skipping since hardlinks don't work here; expected 2 links, got %d", i1) + } + + dest, err := ioutil.TempDir("", "docker-test-tar-hardlink-dest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dest) + + // we'll do this in two steps to separate failure + fh, err := Tar(origin, Uncompressed) + if err != nil { + t.Fatal(err) + } + + // ensure we can read the whole thing with no error, before writing back out + buf, err := ioutil.ReadAll(fh) + if err != nil { + t.Fatal(err) + } + + bRdr := bytes.NewReader(buf) + err = Untar(bRdr, dest, &TarOptions{Compression: Uncompressed}) + if err != nil { + t.Fatal(err) + } + + if i1, err = getInode(filepath.Join(dest, "1")); err != nil { + t.Fatal(err) + } + if i2, err = getInode(filepath.Join(dest, "2")); err != nil { + t.Fatal(err) + } + + if i1 != i2 { + t.Errorf("expected matching inodes, but got %d and %d", i1, i2) + } +} + +func getNlink(path string) (uint64, error) { + stat, err := os.Stat(path) + if err != nil { + return 0, err + } + statT, ok := stat.Sys().(*syscall.Stat_t) + if !ok { + return 0, fmt.Errorf("expected type *syscall.Stat_t, got %t", stat.Sys()) + } + // We need this conversion on ARM64 + return uint64(statT.Nlink), nil +} + +func getInode(path string) (uint64, error) { + stat, err := os.Stat(path) + if err != nil { + return 0, err + } + statT, ok := stat.Sys().(*syscall.Stat_t) + if !ok { + return 0, fmt.Errorf("expected type *syscall.Stat_t, got %t", stat.Sys()) + } + return statT.Ino, nil +} + +func TestTarWithBlockCharFifo(t *testing.T) { + origin, err := ioutil.TempDir("", "docker-test-tar-hardlink") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(origin) + if err := ioutil.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0700); err != nil { + t.Fatal(err) + } + if err := system.Mknod(filepath.Join(origin, "2"), syscall.S_IFBLK, int(system.Mkdev(int64(12), int64(5)))); err != nil { + t.Fatal(err) + } + if err := system.Mknod(filepath.Join(origin, "3"), syscall.S_IFCHR, int(system.Mkdev(int64(12), int64(5)))); err != nil { + t.Fatal(err) + } + if err := system.Mknod(filepath.Join(origin, "4"), syscall.S_IFIFO, int(system.Mkdev(int64(12), int64(5)))); err != nil { + t.Fatal(err) + } + + dest, err := ioutil.TempDir("", "docker-test-tar-hardlink-dest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dest) + + // we'll do this in two steps to separate failure + fh, err := Tar(origin, Uncompressed) + if err != nil { + t.Fatal(err) + } + + // ensure we can read the whole thing with no error, before writing back out + buf, err := ioutil.ReadAll(fh) + if err != nil { + t.Fatal(err) + } + + bRdr := bytes.NewReader(buf) + err = Untar(bRdr, dest, &TarOptions{Compression: Uncompressed}) + if err != nil { + t.Fatal(err) + } + + changes, err := ChangesDirs(origin, dest) + if err != nil { + t.Fatal(err) + } + if len(changes) > 0 { + t.Fatalf("Tar with special device (block, char, fifo) should keep them (recreate them when untar) : %v", changes) + } +} + +// TestTarUntarWithXattr is Unix as Lsetxattr is not supported on Windows +func TestTarUntarWithXattr(t *testing.T) { + origin, err := ioutil.TempDir("", "docker-test-untar-origin") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(origin) + if err := ioutil.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0700); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(filepath.Join(origin, "2"), []byte("welcome!"), 0700); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(filepath.Join(origin, "3"), []byte("will be ignored"), 0700); err != nil { + t.Fatal(err) + } + if err := system.Lsetxattr(filepath.Join(origin, "2"), "security.capability", []byte{0x00}, 0); err != nil { + t.Fatal(err) + } + + for _, c := range []Compression{ + Uncompressed, + Gzip, + } { + changes, err := tarUntar(t, origin, &TarOptions{ + Compression: c, + ExcludePatterns: []string{"3"}, + }) + + if err != nil { + t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err) + } + + if len(changes) != 1 || changes[0].Path != "/3" { + t.Fatalf("Unexpected differences after tarUntar: %v", changes) + } + capability, _ := system.Lgetxattr(filepath.Join(origin, "2"), "security.capability") + if capability == nil && capability[0] != 0x00 { + t.Fatalf("Untar should have kept the 'security.capability' xattr.") + } + } +} diff --git a/components/engine/pkg/archive/archive_windows_test.go b/components/engine/pkg/archive/archive_windows_test.go index b7abc40223..0c6733d6bd 100644 --- a/components/engine/pkg/archive/archive_windows_test.go +++ b/components/engine/pkg/archive/archive_windows_test.go @@ -10,6 +10,10 @@ import ( ) func TestCopyFileWithInvalidDest(t *testing.T) { + // TODO Windows: This is currently failing. Not sure what has + // recently changed in CopyWithTar as used to pass. Further investigation + // is required. + t.Skip("Currently fails") folder, err := ioutil.TempDir("", "docker-archive-test") if err != nil { t.Fatal(err) diff --git a/components/engine/pkg/archive/changes_test.go b/components/engine/pkg/archive/changes_test.go index 00bd69f31e..bca682502d 100644 --- a/components/engine/pkg/archive/changes_test.go +++ b/components/engine/pkg/archive/changes_test.go @@ -5,6 +5,7 @@ import ( "os" "os/exec" "path" + "runtime" "sort" "testing" "time" @@ -115,6 +116,11 @@ func TestChangeString(t *testing.T) { } func TestChangesWithNoChanges(t *testing.T) { + // TODO Windows. There may be a way of running this, but turning off for now + // as createSampleDir uses symlinks. + if runtime.GOOS == "windows" { + t.Skip("symlinks on Windows") + } rwLayer, err := ioutil.TempDir("", "docker-changes-test") if err != nil { t.Fatal(err) @@ -136,6 +142,11 @@ func TestChangesWithNoChanges(t *testing.T) { } func TestChangesWithChanges(t *testing.T) { + // TODO Windows. There may be a way of running this, but turning off for now + // as createSampleDir uses symlinks. + if runtime.GOOS == "windows" { + t.Skip("symlinks on Windows") + } // Mock the readonly layer layer, err := ioutil.TempDir("", "docker-changes-test-layer") if err != nil { @@ -182,6 +193,11 @@ func TestChangesWithChanges(t *testing.T) { // See https://github.com/docker/docker/pull/13590 func TestChangesWithChangesGH13590(t *testing.T) { + // TODO Windows. There may be a way of running this, but turning off for now + // as createSampleDir uses symlinks. + if runtime.GOOS == "windows" { + t.Skip("symlinks on Windows") + } baseLayer, err := ioutil.TempDir("", "docker-changes-test.") defer os.RemoveAll(baseLayer) @@ -238,6 +254,11 @@ func TestChangesWithChangesGH13590(t *testing.T) { // Create an directory, copy it, make sure we report no changes between the two func TestChangesDirsEmpty(t *testing.T) { + // TODO Windows. There may be a way of running this, but turning off for now + // as createSampleDir uses symlinks. + if runtime.GOOS == "windows" { + t.Skip("symlinks on Windows") + } src, err := ioutil.TempDir("", "docker-changes-test") if err != nil { t.Fatal(err) @@ -341,6 +362,11 @@ func mutateSampleDir(t *testing.T, root string) { } func TestChangesDirsMutated(t *testing.T) { + // TODO Windows. There may be a way of running this, but turning off for now + // as createSampleDir uses symlinks. + if runtime.GOOS == "windows" { + t.Skip("symlinks on Windows") + } src, err := ioutil.TempDir("", "docker-changes-test") if err != nil { t.Fatal(err) @@ -397,6 +423,11 @@ func TestChangesDirsMutated(t *testing.T) { } func TestApplyLayer(t *testing.T) { + // TODO Windows. There may be a way of running this, but turning off for now + // as createSampleDir uses symlinks. + if runtime.GOOS == "windows" { + t.Skip("symlinks on Windows") + } src, err := ioutil.TempDir("", "docker-changes-test") if err != nil { t.Fatal(err) @@ -440,6 +471,11 @@ func TestApplyLayer(t *testing.T) { } func TestChangesSizeWithHardlinks(t *testing.T) { + // TODO Windows. There may be a way of running this, but turning off for now + // as createSampleDir uses symlinks. + if runtime.GOOS == "windows" { + t.Skip("hardlinks on Windows") + } srcDir, err := ioutil.TempDir("", "docker-test-srcDir") if err != nil { t.Fatal(err) diff --git a/components/engine/pkg/archive/copy_test.go b/components/engine/pkg/archive/copy_unix_test.go similarity index 99% rename from components/engine/pkg/archive/copy_test.go rename to components/engine/pkg/archive/copy_unix_test.go index f1dc23824d..ecbfc172b0 100644 --- a/components/engine/pkg/archive/copy_test.go +++ b/components/engine/pkg/archive/copy_unix_test.go @@ -1,3 +1,7 @@ +// +build !windows + +// TODO Windows: Some of these tests may be salvagable and portable to Windows. + package archive import ( diff --git a/components/engine/pkg/archive/diff_test.go b/components/engine/pkg/archive/diff_test.go index 2b29992db5..8167941ac0 100644 --- a/components/engine/pkg/archive/diff_test.go +++ b/components/engine/pkg/archive/diff_test.go @@ -7,12 +7,17 @@ import ( "os" "path/filepath" "reflect" + "runtime" "testing" "github.com/docker/docker/pkg/ioutils" ) func TestApplyLayerInvalidFilenames(t *testing.T) { + // TODO Windows: Figure out how to fix this test. + if runtime.GOOS == "windows" { + t.Skip("Passes but hits breakoutError: platform and architecture is not supported") + } for i, headers := range [][]*tar.Header{ { { @@ -37,6 +42,9 @@ func TestApplyLayerInvalidFilenames(t *testing.T) { } func TestApplyLayerInvalidHardlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("TypeLink support on Windows") + } for i, headers := range [][]*tar.Header{ { // try reading victim/hello (../) { @@ -117,6 +125,9 @@ func TestApplyLayerInvalidHardlink(t *testing.T) { } func TestApplyLayerInvalidSymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("TypeSymLink support on Windows") + } for i, headers := range [][]*tar.Header{ { // try reading victim/hello (../) { @@ -197,6 +208,11 @@ func TestApplyLayerInvalidSymlink(t *testing.T) { } func TestApplyLayerWhiteouts(t *testing.T) { + // TODO Windows: Figure out why this test fails + if runtime.GOOS == "windows" { + t.Skip("Failing on Windows") + } + wd, err := ioutil.TempDir("", "graphdriver-test-whiteouts") if err != nil { return diff --git a/components/engine/pkg/authorization/authz.go b/components/engine/pkg/authorization/authz.go index 8a15b2b27e..f703908649 100644 --- a/components/engine/pkg/authorization/authz.go +++ b/components/engine/pkg/authorization/authz.go @@ -54,13 +54,11 @@ type Ctx struct { // AuthZRequest authorized the request to the docker daemon using authZ plugins func (ctx *Ctx) AuthZRequest(w http.ResponseWriter, r *http.Request) error { var body []byte - if sendBody(ctx.requestURI, r.Header) { - if r.ContentLength < maxBodySize { - var err error - body, r.Body, err = drainBody(r.Body) - if err != nil { - return err - } + if sendBody(ctx.requestURI, r.Header) && r.ContentLength > 0 && r.ContentLength < maxBodySize { + var err error + body, r.Body, err = drainBody(r.Body) + if err != nil { + return err } } @@ -121,23 +119,23 @@ func (ctx *Ctx) AuthZResponse(rm ResponseModifier, r *http.Request) error { return nil } -// drainBody dump the body, it reads the body data into memory and -// see go sources /go/src/net/http/httputil/dump.go +// drainBody dump the body (if it's length is less than 1MB) without modifying the request state func drainBody(body io.ReadCloser) ([]byte, io.ReadCloser, error) { bufReader := bufio.NewReaderSize(body, maxBodySize) newBody := ioutils.NewReadCloserWrapper(bufReader, func() error { return body.Close() }) data, err := bufReader.Peek(maxBodySize) - if err != io.EOF { - // This means the request is larger than our max - if err == bufio.ErrBufferFull { - return nil, newBody, nil - } - // This means we had an error reading - return nil, nil, err + // Body size exceeds max body size + if err == nil { + logrus.Warnf("Request body is larger than: '%d' skipping body", maxBodySize) + return nil, newBody, nil } - - return data, newBody, nil + // Body size is less than maximum size + if err == io.EOF { + return data, newBody, nil + } + // Unknown error + return nil, newBody, err } // sendBody returns true when request/response body should be sent to AuthZPlugin @@ -148,8 +146,7 @@ func sendBody(url string, header http.Header) bool { } // body is sent only for text or json messages - v := header.Get("Content-Type") - return strings.HasPrefix(v, "text/") || v == "application/json" + return header.Get("Content-Type") == "application/json" } // headers returns flatten version of the http headers excluding authorization diff --git a/components/engine/pkg/authorization/authz_test.go b/components/engine/pkg/authorization/authz_unix_test.go similarity index 82% rename from components/engine/pkg/authorization/authz_test.go rename to components/engine/pkg/authorization/authz_unix_test.go index 3a6a991511..7d673fe518 100644 --- a/components/engine/pkg/authorization/authz_test.go +++ b/components/engine/pkg/authorization/authz_unix_test.go @@ -1,3 +1,8 @@ +// +build !windows + +// TODO Windows: This uses a Unix socket for testing. This might be possible +// to port to Windows using a named pipe instead. + package authorization import ( @@ -12,6 +17,9 @@ import ( "reflect" "testing" + "bytes" + "strings" + "github.com/docker/docker/pkg/plugins" "github.com/docker/go-connections/tlsconfig" "github.com/gorilla/mux" @@ -130,6 +138,41 @@ func TestResponseModifier(t *testing.T) { } } +func TestDrainBody(t *testing.T) { + + tests := []struct { + length int // length is the message length send to drainBody + expectedBodyLength int // expectedBodyLength is the expected body length after drainBody is called + }{ + {10, 10}, // Small message size + {maxBodySize - 1, maxBodySize - 1}, // Max message size + {maxBodySize * 2, 0}, // Large message size (skip copying body) + + } + + for _, test := range tests { + + msg := strings.Repeat("a", test.length) + body, closer, err := drainBody(ioutil.NopCloser(bytes.NewReader([]byte(msg)))) + if len(body) != test.expectedBodyLength { + t.Fatalf("Body must be copied, actual length: '%d'", len(body)) + } + if closer == nil { + t.Fatalf("Closer must not be nil") + } + if err != nil { + t.Fatalf("Error must not be nil: '%v'", err) + } + modified, err := ioutil.ReadAll(closer) + if err != nil { + t.Fatalf("Error must not be nil: '%v'", err) + } + if len(modified) != len(msg) { + t.Fatalf("Result should not be truncated. Original length: '%d', new length: '%d'", len(msg), len(modified)) + } + } +} + func TestResponseModifierOverride(t *testing.T) { r := httptest.NewRecorder() m := NewResponseModifier(r) diff --git a/components/engine/pkg/authorization/response.go b/components/engine/pkg/authorization/response.go index abe3c9f471..245a0ef7fd 100644 --- a/components/engine/pkg/authorization/response.go +++ b/components/engine/pkg/authorization/response.go @@ -148,7 +148,7 @@ func (rm *responseModifier) Hijack() (net.Conn, *bufio.ReadWriter, error) { hijacker, ok := rm.rw.(http.Hijacker) if !ok { - return nil, nil, fmt.Errorf("Internal reponse writer doesn't support the Hijacker interface") + return nil, nil, fmt.Errorf("Internal response writer doesn't support the Hijacker interface") } return hijacker.Hijack() } @@ -157,7 +157,7 @@ func (rm *responseModifier) Hijack() (net.Conn, *bufio.ReadWriter, error) { func (rm *responseModifier) CloseNotify() <-chan bool { closeNotifier, ok := rm.rw.(http.CloseNotifier) if !ok { - logrus.Errorf("Internal reponse writer doesn't support the CloseNotifier interface") + logrus.Errorf("Internal response writer doesn't support the CloseNotifier interface") return nil } return closeNotifier.CloseNotify() @@ -167,7 +167,7 @@ func (rm *responseModifier) CloseNotify() <-chan bool { func (rm *responseModifier) Flush() { flusher, ok := rm.rw.(http.Flusher) if !ok { - logrus.Errorf("Internal reponse writer doesn't support the Flusher interface") + logrus.Errorf("Internal response writer doesn't support the Flusher interface") return } diff --git a/components/engine/pkg/chrootarchive/archive_test.go b/components/engine/pkg/chrootarchive/archive_test.go index 1d6c2b9220..5fbe20843f 100644 --- a/components/engine/pkg/chrootarchive/archive_test.go +++ b/components/engine/pkg/chrootarchive/archive_test.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -151,6 +152,10 @@ func compareFiles(src string, dest string) error { } func TestChrootTarUntarWithSymlink(t *testing.T) { + // TODO Windows: Figure out why this is failing + if runtime.GOOS == "windows" { + t.Skip("Failing on Windows") + } tmpdir, err := ioutil.TempDir("", "docker-TestChrootTarUntarWithSymlink") if err != nil { t.Fatal(err) @@ -173,6 +178,10 @@ func TestChrootTarUntarWithSymlink(t *testing.T) { } func TestChrootCopyWithTar(t *testing.T) { + // TODO Windows: Figure out why this is failing + if runtime.GOOS == "windows" { + t.Skip("Failing on Windows") + } tmpdir, err := ioutil.TempDir("", "docker-TestChrootCopyWithTar") if err != nil { t.Fatal(err) @@ -262,6 +271,10 @@ func TestChrootCopyFileWithTar(t *testing.T) { } func TestChrootUntarPath(t *testing.T) { + // TODO Windows: Figure out why this is failing + if runtime.GOOS == "windows" { + t.Skip("Failing on Windows") + } tmpdir, err := ioutil.TempDir("", "docker-TestChrootUntarPath") if err != nil { t.Fatal(err) diff --git a/components/engine/pkg/discovery/backends.go b/components/engine/pkg/discovery/backends.go index f150115a1d..65364c9ae8 100644 --- a/components/engine/pkg/discovery/backends.go +++ b/components/engine/pkg/discovery/backends.go @@ -89,7 +89,7 @@ func ParseAdvertise(advertise string) (string, error) { return "", fmt.Errorf("couldnt find a valid ip-address in interface %s", advertise) } - addr = fmt.Sprintf("%s:%s", addr, port) + addr = net.JoinHostPort(addr, port) return addr, nil } diff --git a/components/engine/pkg/discovery/discovery_test.go b/components/engine/pkg/discovery/discovery_test.go index ed0964dc14..6084f3ef0d 100644 --- a/components/engine/pkg/discovery/discovery_test.go +++ b/components/engine/pkg/discovery/discovery_test.go @@ -19,6 +19,11 @@ func (s *DiscoverySuite) TestNewEntry(c *check.C) { c.Assert(entry.Equals(&Entry{Host: "127.0.0.1", Port: "2375"}), check.Equals, true) c.Assert(entry.String(), check.Equals, "127.0.0.1:2375") + entry, err = NewEntry("[2001:db8:0:f101::2]:2375") + c.Assert(err, check.IsNil) + c.Assert(entry.Equals(&Entry{Host: "2001:db8:0:f101::2", Port: "2375"}), check.Equals, true) + c.Assert(entry.String(), check.Equals, "[2001:db8:0:f101::2]:2375") + _, err = NewEntry("127.0.0.1") c.Assert(err, check.NotNil) } @@ -50,11 +55,12 @@ func (s *DiscoverySuite) TestCreateEntries(c *check.C) { c.Assert(entries, check.DeepEquals, Entries{}) c.Assert(err, check.IsNil) - entries, err = CreateEntries([]string{"127.0.0.1:2375", "127.0.0.2:2375", ""}) + entries, err = CreateEntries([]string{"127.0.0.1:2375", "127.0.0.2:2375", "[2001:db8:0:f101::2]:2375", ""}) c.Assert(err, check.IsNil) expected := Entries{ &Entry{Host: "127.0.0.1", Port: "2375"}, &Entry{Host: "127.0.0.2", Port: "2375"}, + &Entry{Host: "2001:db8:0:f101::2", Port: "2375"}, } c.Assert(entries.Equals(expected), check.Equals, true) diff --git a/components/engine/pkg/discovery/entry.go b/components/engine/pkg/discovery/entry.go index e9cee26ee1..ce23bbf89b 100644 --- a/components/engine/pkg/discovery/entry.go +++ b/components/engine/pkg/discovery/entry.go @@ -1,9 +1,6 @@ package discovery -import ( - "fmt" - "net" -) +import "net" // NewEntry creates a new entry. func NewEntry(url string) (*Entry, error) { @@ -27,7 +24,7 @@ func (e *Entry) Equals(cmp *Entry) bool { // String returns the string form of an entry. func (e *Entry) String() string { - return fmt.Sprintf("%s:%s", e.Host, e.Port) + return net.JoinHostPort(e.Host, e.Port) } // Entries is a list of *Entry with some helpers. diff --git a/components/engine/pkg/filenotify/poller.go b/components/engine/pkg/filenotify/poller.go index 0d92afd4cb..5261085346 100644 --- a/components/engine/pkg/filenotify/poller.go +++ b/components/engine/pkg/filenotify/poller.go @@ -118,8 +118,6 @@ func (w *filePoller) Close() error { w.remove(name) delete(w.watches, name) } - close(w.events) - close(w.errors) return nil } @@ -146,6 +144,7 @@ func (w *filePoller) sendErr(e error, chClose <-chan struct{}) error { // watch is responsible for polling the specified file for changes // upon finding changes to a file or errors, sendEvent/sendErr is called func (w *filePoller) watch(f *os.File, lastFi os.FileInfo, chClose chan struct{}) { + defer f.Close() for { time.Sleep(watchWaitTime) select { diff --git a/components/engine/pkg/filenotify/poller_test.go b/components/engine/pkg/filenotify/poller_test.go index 49e6e6486c..4f5026237c 100644 --- a/components/engine/pkg/filenotify/poller_test.go +++ b/components/engine/pkg/filenotify/poller_test.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "os" + "runtime" "testing" "time" @@ -36,6 +37,9 @@ func TestPollerAddRemove(t *testing.T) { } func TestPollerEvent(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("No chmod on Windows") + } w := NewPollingWatcher() f, err := ioutil.TempFile("", "test-poller") @@ -89,24 +93,6 @@ func TestPollerClose(t *testing.T) { t.Fatal(err) } - select { - case _, open := <-w.Events(): - if open { - t.Fatal("event chan should be closed") - } - default: - t.Fatal("event chan should be closed") - } - - select { - case _, open := <-w.Errors(): - if open { - t.Fatal("errors chan should be closed") - } - default: - t.Fatal("errors chan should be closed") - } - f, err := ioutil.TempFile("", "asdf") if err != nil { t.Fatal(err) diff --git a/components/engine/pkg/fileutils/fileutils.go b/components/engine/pkg/fileutils/fileutils.go index b5057ecd12..c1e309fee1 100644 --- a/components/engine/pkg/fileutils/fileutils.go +++ b/components/engine/pkg/fileutils/fileutils.go @@ -52,7 +52,7 @@ func CleanPatterns(patterns []string) ([]string, [][]string, bool, error) { if exclusion(pattern) { pattern = pattern[1:] } - patternDirs = append(patternDirs, strings.Split(pattern, "/")) + patternDirs = append(patternDirs, strings.Split(pattern, string(os.PathSeparator))) } return cleanedPatterns, patternDirs, exceptions, nil @@ -83,8 +83,9 @@ func Matches(file string, patterns []string) (bool, error) { // The more generic fileutils.Matches() can't make these assumptions. func OptimizedMatches(file string, patterns []string, patDirs [][]string) (bool, error) { matched := false + file = filepath.FromSlash(file) parentPath := filepath.Dir(file) - parentPathDirs := strings.Split(parentPath, "/") + parentPathDirs := strings.Split(parentPath, string(os.PathSeparator)) for i, pattern := range patterns { negative := false @@ -102,8 +103,8 @@ func OptimizedMatches(file string, patterns []string, patDirs [][]string) (bool, if !match && parentPath != "." { // Check to see if the pattern matches one of our parent dirs. if len(patDirs[i]) <= len(parentPathDirs) { - match, _ = regexpMatch(strings.Join(patDirs[i], "/"), - strings.Join(parentPathDirs[:len(patDirs[i])], "/")) + match, _ = regexpMatch(strings.Join(patDirs[i], string(os.PathSeparator)), + strings.Join(parentPathDirs[:len(patDirs[i])], string(os.PathSeparator))) } } @@ -125,6 +126,9 @@ func OptimizedMatches(file string, patterns []string, patDirs [][]string) (bool, // of directories. This means that we should be backwards compatible // with filepath.Match(). We'll end up supporting more stuff, due to // the fact that we're using regexp, but that's ok - it does no harm. +// +// As per the comment in golangs filepath.Match, on Windows, escaping +// is disabled. Instead, '\\' is treated as path separator. func regexpMatch(pattern, path string) (bool, error) { regStr := "^" diff --git a/components/engine/pkg/fileutils/fileutils_test.go b/components/engine/pkg/fileutils/fileutils_test.go index 2d584c6676..6df1be89bb 100644 --- a/components/engine/pkg/fileutils/fileutils_test.go +++ b/components/engine/pkg/fileutils/fileutils_test.go @@ -125,6 +125,10 @@ func TestCopyFile(t *testing.T) { // Reading a symlink to a directory must return the directory func TestReadSymlinkedDirectoryExistingDirectory(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } var err error if err = os.Mkdir("/tmp/testReadSymlinkToExistingDirectory", 0777); err != nil { t.Errorf("failed to create directory: %s", err) @@ -167,6 +171,10 @@ func TestReadSymlinkedDirectoryNonExistingSymlink(t *testing.T) { // Reading a symlink to a file must fail func TestReadSymlinkedDirectoryToFile(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } var err error var file *os.File @@ -301,6 +309,10 @@ func TestMatchesWithMalformedPatterns(t *testing.T) { // Test lots of variants of patterns & strings func TestMatches(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } tests := []struct { pattern string text string diff --git a/components/engine/pkg/gitutils/gitutils_test.go b/components/engine/pkg/gitutils/gitutils_test.go index 4ef37ff736..ec288f00fa 100644 --- a/components/engine/pkg/gitutils/gitutils_test.go +++ b/components/engine/pkg/gitutils/gitutils_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "testing" ) @@ -76,6 +77,11 @@ func TestCheckoutGit(t *testing.T) { } defer os.RemoveAll(root) + eol := "\n" + if runtime.GOOS == "windows" { + eol = "\r\n" + } + gitDir := filepath.Join(root, "repo") _, err = git("init", gitDir) if err != nil { @@ -103,12 +109,14 @@ func TestCheckoutGit(t *testing.T) { t.Fatal(err) } - if err = os.Symlink("../subdir", filepath.Join(gitDir, "parentlink")); err != nil { - t.Fatal(err) - } + if runtime.GOOS != "windows" { + if err = os.Symlink("../subdir", filepath.Join(gitDir, "parentlink")); err != nil { + t.Fatal(err) + } - if err = os.Symlink("/subdir", filepath.Join(gitDir, "absolutelink")); err != nil { - t.Fatal(err) + if err = os.Symlink("/subdir", filepath.Join(gitDir, "absolutelink")); err != nil { + t.Fatal(err) + } } if _, err = gitWithinDir(gitDir, "add", "-A"); err != nil { @@ -143,24 +151,34 @@ func TestCheckoutGit(t *testing.T) { t.Fatal(err) } - cases := []struct { + type singleCase struct { frag string exp string fail bool - }{ + } + + cases := []singleCase{ {"", "FROM scratch", false}, {"master", "FROM scratch", false}, - {":subdir", "FROM scratch\nEXPOSE 5000", false}, + {":subdir", "FROM scratch" + eol + "EXPOSE 5000", false}, {":nosubdir", "", true}, // missing directory error {":Dockerfile", "", true}, // not a directory error {"master:nosubdir", "", true}, - {"master:subdir", "FROM scratch\nEXPOSE 5000", false}, - {"master:parentlink", "FROM scratch\nEXPOSE 5000", false}, - {"master:absolutelink", "FROM scratch\nEXPOSE 5000", false}, + {"master:subdir", "FROM scratch" + eol + "EXPOSE 5000", false}, {"master:../subdir", "", true}, - {"test", "FROM scratch\nEXPOSE 3000", false}, - {"test:", "FROM scratch\nEXPOSE 3000", false}, - {"test:subdir", "FROM busybox\nEXPOSE 5000", false}, + {"test", "FROM scratch" + eol + "EXPOSE 3000", false}, + {"test:", "FROM scratch" + eol + "EXPOSE 3000", false}, + {"test:subdir", "FROM busybox" + eol + "EXPOSE 5000", false}, + } + + if runtime.GOOS != "windows" { + // Windows GIT (2.7.1 x64) does not support parentlink/absolutelink. Sample output below + // git --work-tree .\repo --git-dir .\repo\.git add -A + // error: readlink("absolutelink"): Function not implemented + // error: unable to index file absolutelink + // fatal: adding files failed + cases = append(cases, singleCase{frag: "master:absolutelink", exp: "FROM scratch" + eol + "EXPOSE 5000", fail: false}) + cases = append(cases, singleCase{frag: "master:parentlink", exp: "FROM scratch" + eol + "EXPOSE 5000", fail: false}) } for _, c := range cases { diff --git a/components/engine/pkg/graphdb/graphdb_test.go b/components/engine/pkg/graphdb/graphdb_test.go index 9a8930bcd8..f0fb074b4d 100644 --- a/components/engine/pkg/graphdb/graphdb_test.go +++ b/components/engine/pkg/graphdb/graphdb_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path" + "runtime" "strconv" "testing" @@ -131,6 +132,10 @@ func TestParents(t *testing.T) { } func TestChildren(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) @@ -173,6 +178,11 @@ func TestChildren(t *testing.T) { } func TestListAllRootChildren(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } + db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) @@ -189,6 +199,10 @@ func TestListAllRootChildren(t *testing.T) { } func TestListAllSubChildren(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) @@ -231,6 +245,10 @@ func TestListAllSubChildren(t *testing.T) { } func TestAddSelfAsChild(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) @@ -257,6 +275,10 @@ func TestAddChildToNonExistentRoot(t *testing.T) { } func TestWalkAll(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) _, err := db.Set("/webapp", "1") @@ -303,6 +325,10 @@ func TestWalkAll(t *testing.T) { } func TestGetEntityByPath(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) _, err := db.Set("/webapp", "1") @@ -350,6 +376,10 @@ func TestGetEntityByPath(t *testing.T) { } func TestEnitiesPaths(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) _, err := db.Set("/webapp", "1") @@ -403,6 +433,10 @@ func TestDeleteRootEntity(t *testing.T) { } func TestDeleteEntity(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) _, err := db.Set("/webapp", "1") @@ -450,6 +484,10 @@ func TestDeleteEntity(t *testing.T) { } func TestCountRefs(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) @@ -467,6 +505,11 @@ func TestCountRefs(t *testing.T) { } func TestPurgeId(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } + db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) @@ -490,6 +533,10 @@ func TestPurgeId(t *testing.T) { // Regression test https://github.com/docker/docker/issues/12334 func TestPurgeIdRefPaths(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) @@ -527,6 +574,10 @@ func TestPurgeIdRefPaths(t *testing.T) { } func TestRename(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) @@ -556,6 +607,11 @@ func TestRename(t *testing.T) { } func TestCreateMultipleNames(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } + db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) @@ -597,6 +653,10 @@ func TestExistsTrue(t *testing.T) { } func TestExistsFalse(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) @@ -621,6 +681,10 @@ func TestGetNameWithTrailingSlash(t *testing.T) { } func TestConcurrentWrites(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } db, dbpath := newTestDb(t) defer destroyTestDb(dbpath) diff --git a/components/engine/pkg/httputils/resumablerequestreader_test.go b/components/engine/pkg/httputils/resumablerequestreader_test.go index e9d0578306..7006f04967 100644 --- a/components/engine/pkg/httputils/resumablerequestreader_test.go +++ b/components/engine/pkg/httputils/resumablerequestreader_test.go @@ -96,7 +96,7 @@ type errorReaderCloser struct{} func (errorReaderCloser) Close() error { return nil } func (errorReaderCloser) Read(p []byte) (n int, err error) { - return 0, fmt.Errorf("A error occured") + return 0, fmt.Errorf("An error occurred") } // If a an unknown error is encountered, return 0, nil and log it diff --git a/components/engine/pkg/idtools/idtools.go b/components/engine/pkg/idtools/idtools.go index a1301ee976..7341640046 100644 --- a/components/engine/pkg/idtools/idtools.go +++ b/components/engine/pkg/idtools/idtools.go @@ -171,7 +171,7 @@ func parseSubidFile(path, username string) (ranges, error) { } text := strings.TrimSpace(s.Text()) - if text == "" { + if text == "" || strings.HasPrefix(text, "#") { continue } parts := strings.Split(text, ":") diff --git a/components/engine/pkg/idtools/idtools_unix_test.go b/components/engine/pkg/idtools/idtools_unix_test.go index 55b338c96e..540d3079ee 100644 --- a/components/engine/pkg/idtools/idtools_unix_test.go +++ b/components/engine/pkg/idtools/idtools_unix_test.go @@ -241,3 +241,31 @@ func compareTrees(left, right map[string]node) error { } return nil } + +func TestParseSubidFileWithNewlinesAndComments(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "parsesubid") + if err != nil { + t.Fatal(err) + } + fnamePath := filepath.Join(tmpDir, "testsubuid") + fcontent := `tss:100000:65536 +# empty default subuid/subgid file + +dockremap:231072:65536` + if err := ioutil.WriteFile(fnamePath, []byte(fcontent), 0644); err != nil { + t.Fatal(err) + } + ranges, err := parseSubidFile(fnamePath, "dockremap") + if err != nil { + t.Fatal(err) + } + if len(ranges) != 1 { + t.Fatalf("wanted 1 element in ranges, got %d instead", len(ranges)) + } + if ranges[0].Start != 231072 { + t.Fatalf("wanted 231072, got %d instead", ranges[0].Start) + } + if ranges[0].Length != 65536 { + t.Fatalf("wanted 65536, got %d instead", ranges[0].Length) + } +} diff --git a/components/engine/pkg/integration/dockerCmd_utils.go b/components/engine/pkg/integration/dockerCmd_utils.go index c6b51e699f..fab3e062dd 100644 --- a/components/engine/pkg/integration/dockerCmd_utils.go +++ b/components/engine/pkg/integration/dockerCmd_utils.go @@ -9,6 +9,13 @@ import ( "github.com/go-check/check" ) +// We use the elongated quote mechanism for quoting error returns as +// the use of strconv.Quote or %q in fmt.Errorf will escape characters. This +// has a big downside on Windows where the args include paths, so instead +// of something like c:\directory\file.txt, the output would be +// c:\\directory\\file.txt. This is highly misleading. +const quote = `"` + var execCommand = exec.Command // DockerCmdWithError executes a docker command that is supposed to fail and returns @@ -23,7 +30,7 @@ func DockerCmdWithError(dockerBinary string, args ...string) (string, int, error func DockerCmdWithStdoutStderr(dockerBinary string, c *check.C, args ...string) (string, string, int) { stdout, stderr, status, err := RunCommandWithStdoutStderr(execCommand(dockerBinary, args...)) if c != nil { - c.Assert(err, check.IsNil, check.Commentf("%q failed with errors: %s, %v", strings.Join(args, " "), stderr, err)) + c.Assert(err, check.IsNil, check.Commentf(quote+"%v"+quote+" failed with errors: %s, %v", strings.Join(args, " "), stderr, err)) } return stdout, stderr, status } @@ -32,7 +39,7 @@ func DockerCmdWithStdoutStderr(dockerBinary string, c *check.C, args ...string) // command returns an error, it will fail and stop the tests. func DockerCmd(dockerBinary string, c *check.C, args ...string) (string, int) { out, status, err := RunCommandWithOutput(execCommand(dockerBinary, args...)) - c.Assert(err, check.IsNil, check.Commentf("%q failed with errors: %s, %v", strings.Join(args, " "), out, err)) + c.Assert(err, check.IsNil, check.Commentf(quote+"%v"+quote+" failed with errors: %s, %v", strings.Join(args, " "), out, err)) return out, status } @@ -41,7 +48,7 @@ func DockerCmd(dockerBinary string, c *check.C, args ...string) (string, int) { func DockerCmdWithTimeout(dockerBinary string, timeout time.Duration, args ...string) (string, int, error) { out, status, err := RunCommandWithOutputAndTimeout(execCommand(dockerBinary, args...), timeout) if err != nil { - return out, status, fmt.Errorf("%q failed with errors: %v : %q", strings.Join(args, " "), err, out) + return out, status, fmt.Errorf(quote+"%v"+quote+" failed with errors: %v : %q", strings.Join(args, " "), err, out) } return out, status, err } @@ -53,7 +60,7 @@ func DockerCmdInDir(dockerBinary string, path string, args ...string) (string, i dockerCommand.Dir = path out, status, err := RunCommandWithOutput(dockerCommand) if err != nil { - return out, status, fmt.Errorf("%q failed with errors: %v : %q", strings.Join(args, " "), err, out) + return out, status, fmt.Errorf(quote+"%v"+quote+" failed with errors: %v : %q", strings.Join(args, " "), err, out) } return out, status, err } @@ -65,7 +72,7 @@ func DockerCmdInDirWithTimeout(dockerBinary string, timeout time.Duration, path dockerCommand.Dir = path out, status, err := RunCommandWithOutputAndTimeout(dockerCommand, timeout) if err != nil { - return out, status, fmt.Errorf("%q failed with errors: %v : %q", strings.Join(args, " "), err, out) + return out, status, fmt.Errorf(quote+"%v"+quote+" failed with errors: %v : %q", strings.Join(args, " "), err, out) } return out, status, err } diff --git a/components/engine/pkg/integration/utils_test.go b/components/engine/pkg/integration/utils_test.go index 892083444a..d166489e62 100644 --- a/components/engine/pkg/integration/utils_test.go +++ b/components/engine/pkg/integration/utils_test.go @@ -5,14 +5,20 @@ import ( "io/ioutil" "os" "os/exec" - "path" + "path/filepath" "runtime" + "strconv" "strings" "testing" "time" ) func TestIsKilledFalseWithNonKilledProcess(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } + lsCmd := exec.Command("ls") lsCmd.Start() // Wait for it to finish @@ -23,6 +29,12 @@ func TestIsKilledFalseWithNonKilledProcess(t *testing.T) { } func TestIsKilledTrueWithKilledProcess(t *testing.T) { + // TODO Windows: Using golang 1.5.3, this seems to hit + // a bug in go where Process.Kill() causes a panic. + // Needs further investigation @jhowardmsft + if runtime.GOOS == "windows" { + t.SkipNow() + } longCmd := exec.Command("top") // Start a command longCmd.Start() @@ -41,30 +53,56 @@ func TestIsKilledTrueWithKilledProcess(t *testing.T) { } func TestRunCommandWithOutput(t *testing.T) { - echoHelloWorldCmd := exec.Command("echo", "hello", "world") + var ( + echoHelloWorldCmd *exec.Cmd + expected string + ) + if runtime.GOOS != "windows" { + echoHelloWorldCmd = exec.Command("echo", "hello", "world") + expected = "hello world\n" + } else { + echoHelloWorldCmd = exec.Command("cmd", "/s", "/c", "echo", "hello", "world") + expected = "hello world\r\n" + } + out, exitCode, err := RunCommandWithOutput(echoHelloWorldCmd) - expected := "hello world\n" if out != expected || exitCode != 0 || err != nil { t.Fatalf("Expected command to output %s, got %s, %v with exitCode %v", expected, out, err, exitCode) } } func TestRunCommandWithOutputError(t *testing.T) { + var ( + p string + wrongCmd *exec.Cmd + expected string + expectedExitCode int + ) + + if runtime.GOOS != "windows" { + p = "$PATH" + wrongCmd = exec.Command("ls", "-z") + expected = `ls: invalid option -- 'z' +Try 'ls --help' for more information. +` + expectedExitCode = 2 + } else { + p = "%PATH%" + wrongCmd = exec.Command("cmd", "/s", "/c", "dir", "/Z") + expected = "Invalid switch - " + strconv.Quote("Z") + ".\r\n" + expectedExitCode = 1 + } cmd := exec.Command("doesnotexists") out, exitCode, err := RunCommandWithOutput(cmd) - expectedError := `exec: "doesnotexists": executable file not found in $PATH` + expectedError := `exec: "doesnotexists": executable file not found in ` + p if out != "" || exitCode != 127 || err == nil || err.Error() != expectedError { t.Fatalf("Expected command to output %s, got %s, %v with exitCode %v", expectedError, out, err, exitCode) } - wrongLsCmd := exec.Command("ls", "-z") - expected := `ls: invalid option -- 'z' -Try 'ls --help' for more information. -` - out, exitCode, err = RunCommandWithOutput(wrongLsCmd) + out, exitCode, err = RunCommandWithOutput(wrongCmd) - if out != expected || exitCode != 2 || err == nil || err.Error() != "exit status 2" { - t.Fatalf("Expected command to output %s, got out:%s, err:%v with exitCode %v", expected, out, err, exitCode) + if out != expected || exitCode != expectedExitCode || err == nil || !strings.Contains(err.Error(), "exit status "+strconv.Itoa(expectedExitCode)) { + t.Fatalf("Expected command to output %s, got out:xxx%sxxx, err:%v with exitCode %v", expected, out, err, exitCode) } } @@ -78,9 +116,13 @@ func TestRunCommandWithStdoutStderr(t *testing.T) { } func TestRunCommandWithStdoutStderrError(t *testing.T) { + p := "$PATH" + if runtime.GOOS == "windows" { + p = "%PATH%" + } cmd := exec.Command("doesnotexists") stdout, stderr, exitCode, err := RunCommandWithStdoutStderr(cmd) - expectedError := `exec: "doesnotexists": executable file not found in $PATH` + expectedError := `exec: "doesnotexists": executable file not found in ` + p if stdout != "" || stderr != "" || exitCode != 127 || err == nil || err.Error() != expectedError { t.Fatalf("Expected command to output out:%s, stderr:%s, got stdout:%s, stderr:%s, err:%v with exitCode %v", "", "", stdout, stderr, err, exitCode) } @@ -97,6 +139,11 @@ Try 'ls --help' for more information. } func TestRunCommandWithOutputForDurationFinished(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } + cmd := exec.Command("ls") out, exitCode, timedOut, err := RunCommandWithOutputForDuration(cmd, 50*time.Millisecond) if out == "" || exitCode != 0 || timedOut || err != nil { @@ -105,6 +152,10 @@ func TestRunCommandWithOutputForDurationFinished(t *testing.T) { } func TestRunCommandWithOutputForDurationKilled(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } cmd := exec.Command("sh", "-c", "while true ; do echo 1 ; sleep .1 ; done") out, exitCode, timedOut, err := RunCommandWithOutputForDuration(cmd, 500*time.Millisecond) ones := strings.Split(out, "\n") @@ -127,6 +178,11 @@ func TestRunCommandWithOutputForDurationErrors(t *testing.T) { } func TestRunCommandWithOutputAndTimeoutFinished(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } + cmd := exec.Command("ls") out, exitCode, err := RunCommandWithOutputAndTimeout(cmd, 50*time.Millisecond) if out == "" || exitCode != 0 || err != nil { @@ -135,6 +191,11 @@ func TestRunCommandWithOutputAndTimeoutFinished(t *testing.T) { } func TestRunCommandWithOutputAndTimeoutKilled(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } + cmd := exec.Command("sh", "-c", "while true ; do echo 1 ; sleep .1 ; done") out, exitCode, err := RunCommandWithOutputAndTimeout(cmd, 500*time.Millisecond) ones := strings.Split(out, "\n") @@ -157,6 +218,15 @@ func TestRunCommandWithOutputAndTimeoutErrors(t *testing.T) { } func TestRunCommand(t *testing.T) { + // TODO Windows: Port this test + if runtime.GOOS == "windows" { + t.Skip("Needs porting to Windows") + } + + p := "$PATH" + if runtime.GOOS == "windows" { + p = "%PATH%" + } lsCmd := exec.Command("ls") exitCode, err := RunCommand(lsCmd) if exitCode != 0 || err != nil { @@ -166,7 +236,7 @@ func TestRunCommand(t *testing.T) { var expectedError string exitCode, err = RunCommand(exec.Command("doesnotexists")) - expectedError = `exec: "doesnotexists": executable file not found in $PATH` + expectedError = `exec: "doesnotexists": executable file not found in ` + p if exitCode != 127 || err == nil || err.Error() != expectedError { t.Fatalf("Expected runCommand to run the command successfully, got: exitCode:%d, err:%v", exitCode, err) } @@ -188,6 +258,10 @@ func TestRunCommandPipelineWithOutputWithNotEnoughCmds(t *testing.T) { } func TestRunCommandPipelineWithOutputErrors(t *testing.T) { + p := "$PATH" + if runtime.GOOS == "windows" { + p = "%PATH%" + } cmd1 := exec.Command("ls") cmd1.Stdout = os.Stdout cmd2 := exec.Command("anything really") @@ -199,7 +273,7 @@ func TestRunCommandPipelineWithOutputErrors(t *testing.T) { cmdWithError := exec.Command("doesnotexists") cmdCat := exec.Command("cat") _, _, err = RunCommandPipelineWithOutput(cmdWithError, cmdCat) - if err == nil || err.Error() != `starting doesnotexists failed with error: exec: "doesnotexists": executable file not found in $PATH` { + if err == nil || err.Error() != `starting doesnotexists failed with error: exec: "doesnotexists": executable file not found in `+p { t.Fatalf("Expected an error, got %v", err) } } @@ -250,8 +324,8 @@ func TestCompareDirectoryEntries(t *testing.T) { } defer os.RemoveAll(tmpFolder) - file1 := path.Join(tmpFolder, "file1") - file2 := path.Join(tmpFolder, "file2") + file1 := filepath.Join(tmpFolder, "file1") + file2 := filepath.Join(tmpFolder, "file2") os.Create(file1) os.Create(file2) @@ -311,6 +385,10 @@ func TestCompareDirectoryEntries(t *testing.T) { // FIXME make an "unhappy path" test for ListTar without "panicking" :-) func TestListTar(t *testing.T) { + // TODO Windows: Figure out why this fails. Should be portable. + if runtime.GOOS == "windows" { + t.Skip("Failing on Windows - needs further investigation") + } tmpFolder, err := ioutil.TempDir("", "integration-cli-utils-list-tar") if err != nil { t.Fatal(err) @@ -318,10 +396,10 @@ func TestListTar(t *testing.T) { defer os.RemoveAll(tmpFolder) // Let's create a Tar file - srcFile := path.Join(tmpFolder, "src") - tarFile := path.Join(tmpFolder, "src.tar") + srcFile := filepath.Join(tmpFolder, "src") + tarFile := filepath.Join(tmpFolder, "src.tar") os.Create(srcFile) - cmd := exec.Command("/bin/sh", "-c", "tar cf "+tarFile+" "+srcFile) + cmd := exec.Command("sh", "-c", "tar cf "+tarFile+" "+srcFile) _, err = cmd.CombinedOutput() if err != nil { t.Fatal(err) diff --git a/components/engine/pkg/mount/mount_test.go b/components/engine/pkg/mount/mount_unix_test.go similarity index 99% rename from components/engine/pkg/mount/mount_test.go rename to components/engine/pkg/mount/mount_unix_test.go index 5c7f1b86a0..d45fbc1b89 100644 --- a/components/engine/pkg/mount/mount_test.go +++ b/components/engine/pkg/mount/mount_unix_test.go @@ -1,3 +1,5 @@ +// +build !windows + package mount import ( diff --git a/components/engine/pkg/plugins/client.go b/components/engine/pkg/plugins/client.go index 72c7331c78..b54d6f2273 100644 --- a/components/engine/pkg/plugins/client.go +++ b/components/engine/pkg/plugins/client.go @@ -6,17 +6,17 @@ import ( "io" "io/ioutil" "net/http" - "strings" + "net/url" "time" "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/plugins/transport" "github.com/docker/go-connections/sockets" "github.com/docker/go-connections/tlsconfig" ) const ( - versionMimetype = "application/vnd.docker.plugins.v1.2+json" - defaultTimeOut = 30 + defaultTimeOut = 30 ) // NewClient creates a new plugin client (http). @@ -29,21 +29,38 @@ func NewClient(addr string, tlsConfig tlsconfig.Options) (*Client, error) { } tr.TLSClientConfig = c - protoAndAddr := strings.Split(addr, "://") - sockets.ConfigureTransport(tr, protoAndAddr[0], protoAndAddr[1]) + u, err := url.Parse(addr) + if err != nil { + return nil, err + } + socket := u.Host + if socket == "" { + // valid local socket addresses have the host empty. + socket = u.Path + } + if err := sockets.ConfigureTransport(tr, u.Scheme, socket); err != nil { + return nil, err + } + scheme := httpScheme(u) - scheme := protoAndAddr[0] - if scheme != "https" { - scheme = "http" + clientTransport := transport.NewHTTPTransport(tr, scheme, socket) + return NewClientWithTransport(clientTransport), nil +} + +// NewClientWithTransport creates a new plugin client with a given transport. +func NewClientWithTransport(tr transport.Transport) *Client { + return &Client{ + http: &http.Client{ + Transport: tr, + }, + requestFactory: tr, } - return &Client{&http.Client{Transport: tr}, scheme, protoAndAddr[1]}, nil } // Client represents a plugin client. type Client struct { - http *http.Client // http client to use - scheme string // scheme protocol of the plugin - addr string // http address of the plugin + http *http.Client // http client to use + requestFactory transport.RequestFactory } // Call calls the specified method with the specified arguments for the plugin. @@ -92,13 +109,10 @@ func (c *Client) SendFile(serviceMethod string, data io.Reader, ret interface{}) } func (c *Client) callWithRetry(serviceMethod string, data io.Reader, retry bool) (io.ReadCloser, error) { - req, err := http.NewRequest("POST", "/"+serviceMethod, data) + req, err := c.requestFactory.NewRequest(serviceMethod, data) if err != nil { return nil, err } - req.Header.Add("Accept", versionMimetype) - req.URL.Scheme = c.scheme - req.URL.Host = c.addr var retries int start := time.Now() @@ -115,13 +129,14 @@ func (c *Client) callWithRetry(serviceMethod string, data io.Reader, retry bool) return nil, err } retries++ - logrus.Warnf("Unable to connect to plugin: %s, retrying in %v", c.addr, timeOff) + logrus.Warnf("Unable to connect to plugin: %s, retrying in %v", req.URL, timeOff) time.Sleep(timeOff) continue } if resp.StatusCode != http.StatusOK { b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() if err != nil { return nil, &statusError{resp.StatusCode, serviceMethod, err.Error()} } @@ -160,3 +175,11 @@ func backoff(retries int) time.Duration { func abort(start time.Time, timeOff time.Duration) bool { return timeOff+time.Since(start) >= time.Duration(defaultTimeOut)*time.Second } + +func httpScheme(u *url.URL) string { + scheme := u.Scheme + if scheme != "https" { + scheme = "http" + } + return scheme +} diff --git a/components/engine/pkg/plugins/client_test.go b/components/engine/pkg/plugins/client_test.go index d9e14e20a4..3fa2ff46ad 100644 --- a/components/engine/pkg/plugins/client_test.go +++ b/components/engine/pkg/plugins/client_test.go @@ -4,10 +4,12 @@ import ( "io" "net/http" "net/http/httptest" + "net/url" "reflect" "testing" "time" + "github.com/docker/docker/pkg/plugins/transport" "github.com/docker/go-connections/tlsconfig" ) @@ -48,7 +50,7 @@ func TestEchoInputOutput(t *testing.T) { } header := w.Header() - header.Set("Content-Type", versionMimetype) + header.Set("Content-Type", transport.VersionMimetype) io.Copy(w, r.Body) }) @@ -119,9 +121,14 @@ func TestClientScheme(t *testing.T) { } for addr, scheme := range cases { - c, _ := NewClient(addr, tlsconfig.Options{InsecureSkipVerify: true}) - if c.scheme != scheme { - t.Fatalf("URL scheme mismatch, expected %s, got %s", scheme, c.scheme) + u, err := url.Parse(addr) + if err != nil { + t.Fatal(err) + } + s := httpScheme(u) + + if s != scheme { + t.Fatalf("URL scheme mismatch, expected %s, got %s", scheme, s) } } } diff --git a/components/engine/pkg/plugins/discovery_test.go b/components/engine/pkg/plugins/discovery_test.go index 5610fe1e9f..2e8dc704eb 100644 --- a/components/engine/pkg/plugins/discovery_test.go +++ b/components/engine/pkg/plugins/discovery_test.go @@ -1,16 +1,13 @@ package plugins import ( - "fmt" "io/ioutil" - "net" "os" "path/filepath" - "reflect" "testing" ) -func setup(t *testing.T) (string, func()) { +func Setup(t *testing.T) (string, func()) { tmpdir, err := ioutil.TempDir("", "docker-test") if err != nil { t.Fatal(err) @@ -25,56 +22,8 @@ func setup(t *testing.T) (string, func()) { } } -func TestLocalSocket(t *testing.T) { - tmpdir, unregister := setup(t) - defer unregister() - - cases := []string{ - filepath.Join(tmpdir, "echo.sock"), - filepath.Join(tmpdir, "echo", "echo.sock"), - } - - for _, c := range cases { - if err := os.MkdirAll(filepath.Dir(c), 0755); err != nil { - t.Fatal(err) - } - - l, err := net.Listen("unix", c) - if err != nil { - t.Fatal(err) - } - - r := newLocalRegistry() - p, err := r.Plugin("echo") - if err != nil { - t.Fatal(err) - } - - pp, err := r.Plugin("echo") - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(p, pp) { - t.Fatalf("Expected %v, was %v\n", p, pp) - } - - if p.Name != "echo" { - t.Fatalf("Expected plugin `echo`, got %s\n", p.Name) - } - - addr := fmt.Sprintf("unix://%s", c) - if p.Addr != addr { - t.Fatalf("Expected plugin addr `%s`, got %s\n", addr, p.Addr) - } - if p.TLSConfig.InsecureSkipVerify != true { - t.Fatalf("Expected TLS verification to be skipped") - } - l.Close() - } -} - func TestFileSpecPlugin(t *testing.T) { - tmpdir, unregister := setup(t) + tmpdir, unregister := Setup(t) defer unregister() cases := []struct { @@ -83,6 +32,7 @@ func TestFileSpecPlugin(t *testing.T) { addr string fail bool }{ + // TODO Windows: Factor out the unix:// variants. {filepath.Join(tmpdir, "echo.spec"), "echo", "unix://var/lib/docker/plugins/echo.sock", false}, {filepath.Join(tmpdir, "echo", "echo.spec"), "echo", "unix://var/lib/docker/plugins/echo.sock", false}, {filepath.Join(tmpdir, "foo.spec"), "foo", "tcp://localhost:8080", false}, @@ -123,7 +73,7 @@ func TestFileSpecPlugin(t *testing.T) { } func TestFileJSONSpecPlugin(t *testing.T) { - tmpdir, unregister := setup(t) + tmpdir, unregister := Setup(t) defer unregister() p := filepath.Join(tmpdir, "example.json") diff --git a/components/engine/pkg/plugins/discovery_unix_test.go b/components/engine/pkg/plugins/discovery_unix_test.go new file mode 100644 index 0000000000..8166ae00b7 --- /dev/null +++ b/components/engine/pkg/plugins/discovery_unix_test.go @@ -0,0 +1,61 @@ +// +build !windows + +package plugins + +import ( + "fmt" + "net" + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestLocalSocket(t *testing.T) { + // TODO Windows: Enable a similar version for Windows named pipes + tmpdir, unregister := Setup(t) + defer unregister() + + cases := []string{ + filepath.Join(tmpdir, "echo.sock"), + filepath.Join(tmpdir, "echo", "echo.sock"), + } + + for _, c := range cases { + if err := os.MkdirAll(filepath.Dir(c), 0755); err != nil { + t.Fatal(err) + } + + l, err := net.Listen("unix", c) + if err != nil { + t.Fatal(err) + } + + r := newLocalRegistry() + p, err := r.Plugin("echo") + if err != nil { + t.Fatal(err) + } + + pp, err := r.Plugin("echo") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(p, pp) { + t.Fatalf("Expected %v, was %v\n", p, pp) + } + + if p.Name != "echo" { + t.Fatalf("Expected plugin `echo`, got %s\n", p.Name) + } + + addr := fmt.Sprintf("unix://%s", c) + if p.Addr != addr { + t.Fatalf("Expected plugin addr `%s`, got %s\n", addr, p.Addr) + } + if p.TLSConfig.InsecureSkipVerify != true { + t.Fatalf("Expected TLS verification to be skipped") + } + l.Close() + } +} diff --git a/components/engine/pkg/plugins/errors.go b/components/engine/pkg/plugins/errors.go index a1826c8906..7988471026 100644 --- a/components/engine/pkg/plugins/errors.go +++ b/components/engine/pkg/plugins/errors.go @@ -11,7 +11,7 @@ type statusError struct { err string } -// Error returns a formated string for this error type +// Error returns a formatted string for this error type func (e *statusError) Error() string { return fmt.Sprintf("%s: %v", e.method, e.err) } diff --git a/components/engine/pkg/plugins/plugins.go b/components/engine/pkg/plugins/plugins.go index 7157107ba3..738686e919 100644 --- a/components/engine/pkg/plugins/plugins.go +++ b/components/engine/pkg/plugins/plugins.go @@ -199,17 +199,27 @@ func GetAll(imp string) ([]*Plugin, error) { err error } - chPl := make(chan plLoad, len(pluginNames)) + chPl := make(chan *plLoad, len(pluginNames)) + var wg sync.WaitGroup for _, name := range pluginNames { + if pl, ok := storage.plugins[name]; ok { + chPl <- &plLoad{pl, nil} + continue + } + + wg.Add(1) go func(name string) { + defer wg.Done() pl, err := loadWithRetry(name, false) - chPl <- plLoad{pl, err} + chPl <- &plLoad{pl, err} }(name) } + wg.Wait() + close(chPl) + var out []*Plugin - for i := 0; i < len(pluginNames); i++ { - pl := <-chPl + for pl := range chPl { if pl.err != nil { logrus.Error(err) continue diff --git a/components/engine/pkg/plugins/transport/http.go b/components/engine/pkg/plugins/transport/http.go new file mode 100644 index 0000000000..5be146af65 --- /dev/null +++ b/components/engine/pkg/plugins/transport/http.go @@ -0,0 +1,36 @@ +package transport + +import ( + "io" + "net/http" +) + +// httpTransport holds an http.RoundTripper +// and information about the scheme and address the transport +// sends request to. +type httpTransport struct { + http.RoundTripper + scheme string + addr string +} + +// NewHTTPTransport creates a new httpTransport. +func NewHTTPTransport(r http.RoundTripper, scheme, addr string) Transport { + return httpTransport{ + RoundTripper: r, + scheme: scheme, + addr: addr, + } +} + +// NewRequest creates a new http.Request and sets the URL +// scheme and address with the transport's fields. +func (t httpTransport) NewRequest(path string, data io.Reader) (*http.Request, error) { + req, err := newHTTPRequest(path, data) + if err != nil { + return nil, err + } + req.URL.Scheme = t.scheme + req.URL.Host = t.addr + return req, nil +} diff --git a/components/engine/pkg/plugins/transport/transport.go b/components/engine/pkg/plugins/transport/transport.go new file mode 100644 index 0000000000..d7f1e2100c --- /dev/null +++ b/components/engine/pkg/plugins/transport/transport.go @@ -0,0 +1,36 @@ +package transport + +import ( + "io" + "net/http" + "strings" +) + +// VersionMimetype is the Content-Type the engine sends to plugins. +const VersionMimetype = "application/vnd.docker.plugins.v1.2+json" + +// RequestFactory defines an interface that +// transports can implement to create new requests. +type RequestFactory interface { + NewRequest(path string, data io.Reader) (*http.Request, error) +} + +// Transport defines an interface that plugin transports +// must implement. +type Transport interface { + http.RoundTripper + RequestFactory +} + +// newHTTPRequest creates a new request with a path and a body. +func newHTTPRequest(path string, data io.Reader) (*http.Request, error) { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + req, err := http.NewRequest("POST", path, data) + if err != nil { + return nil, err + } + req.Header.Add("Accept", VersionMimetype) + return req, nil +} diff --git a/components/engine/pkg/pubsub/publisher.go b/components/engine/pkg/pubsub/publisher.go index 8529ffa322..09364617e4 100644 --- a/components/engine/pkg/pubsub/publisher.go +++ b/components/engine/pkg/pubsub/publisher.go @@ -5,6 +5,8 @@ import ( "time" ) +var wgPool = sync.Pool{New: func() interface{} { return new(sync.WaitGroup) }} + // NewPublisher creates a new pub/sub publisher to broadcast messages. // The duration is used as the send timeout as to not block the publisher publishing // messages to other clients if one client is slow or unresponsive. @@ -62,13 +64,18 @@ func (p *Publisher) Evict(sub chan interface{}) { // Publish sends the data in v to all subscribers currently registered with the publisher. func (p *Publisher) Publish(v interface{}) { p.m.RLock() - wg := new(sync.WaitGroup) + if len(p.subscribers) == 0 { + p.m.RUnlock() + return + } + + wg := wgPool.Get().(*sync.WaitGroup) for sub, topic := range p.subscribers { wg.Add(1) - go p.sendTopic(sub, topic, v, wg) } wg.Wait() + wgPool.Put(wg) p.m.RUnlock() } diff --git a/components/engine/pkg/stdcopy/stdcopy.go b/components/engine/pkg/stdcopy/stdcopy.go index 9be01841a9..b37ae39f8e 100644 --- a/components/engine/pkg/stdcopy/stdcopy.go +++ b/components/engine/pkg/stdcopy/stdcopy.go @@ -3,12 +3,24 @@ package stdcopy import ( "encoding/binary" "errors" + "fmt" "io" "github.com/Sirupsen/logrus" ) +// StdType is the type of standard stream +// a writer can multiplex to. +type StdType byte + const ( + // Stdin represents standard input stream type. + Stdin StdType = iota + // Stdout represents standard output stream type. + Stdout + // Stderr represents standard error steam type. + Stderr + stdWriterPrefixLen = 8 stdWriterFdIndex = 0 stdWriterSizeIndex = 4 @@ -16,38 +28,32 @@ const ( startingBufLen = 32*1024 + stdWriterPrefixLen + 1 ) -// StdType prefixes type and length to standard stream. -type StdType [stdWriterPrefixLen]byte - -var ( - // Stdin represents standard input stream type. - Stdin = StdType{0: 0} - // Stdout represents standard output stream type. - Stdout = StdType{0: 1} - // Stderr represents standard error steam type. - Stderr = StdType{0: 2} -) - -// StdWriter is wrapper of io.Writer with extra customized info. -type StdWriter struct { +// stdWriter is wrapper of io.Writer with extra customized info. +type stdWriter struct { io.Writer - prefix StdType - sizeBuf []byte + prefix byte } -func (w *StdWriter) Write(buf []byte) (n int, err error) { - var n1, n2 int +// Write sends the buffer to the underneath writer. +// It insert the prefix header before the buffer, +// so stdcopy.StdCopy knows where to multiplex the output. +// It makes stdWriter to implement io.Writer. +func (w *stdWriter) Write(buf []byte) (n int, err error) { if w == nil || w.Writer == nil { return 0, errors.New("Writer not instantiated") } - binary.BigEndian.PutUint32(w.prefix[4:], uint32(len(buf))) - n1, err = w.Writer.Write(w.prefix[:]) - if err != nil { - n = n1 - stdWriterPrefixLen - } else { - n2, err = w.Writer.Write(buf) - n = n1 + n2 - stdWriterPrefixLen + if buf == nil { + return 0, nil } + + header := [stdWriterPrefixLen]byte{stdWriterFdIndex: w.prefix} + binary.BigEndian.PutUint32(header[stdWriterSizeIndex:], uint32(len(buf))) + + line := append(header[:], buf...) + + n, err = w.Writer.Write(line) + n -= stdWriterPrefixLen + if n < 0 { n = 0 } @@ -60,16 +66,13 @@ func (w *StdWriter) Write(buf []byte) (n int, err error) { // This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection. // `t` indicates the id of the stream to encapsulate. // It can be stdcopy.Stdin, stdcopy.Stdout, stdcopy.Stderr. -func NewStdWriter(w io.Writer, t StdType) *StdWriter { - return &StdWriter{ - Writer: w, - prefix: t, - sizeBuf: make([]byte, 4), +func NewStdWriter(w io.Writer, t StdType) io.Writer { + return &stdWriter{ + Writer: w, + prefix: byte(t), } } -var errInvalidStdHeader = errors.New("Unrecognized input header") - // StdCopy is a modified version of io.Copy. // // StdCopy will demultiplex `src`, assuming that it contains two streams, @@ -110,18 +113,18 @@ func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) } // Check the first byte to know where to write - switch buf[stdWriterFdIndex] { - case 0: + switch StdType(buf[stdWriterFdIndex]) { + case Stdin: fallthrough - case 1: + case Stdout: // Write on stdout out = dstout - case 2: + case Stderr: // Write on stderr out = dsterr default: logrus.Debugf("Error selecting output fd: (%d)", buf[stdWriterFdIndex]) - return 0, errInvalidStdHeader + return 0, fmt.Errorf("Unrecognized input header: %d", buf[stdWriterFdIndex]) } // Retrieve the size of the frame diff --git a/components/engine/pkg/stdcopy/stdcopy_test.go b/components/engine/pkg/stdcopy/stdcopy_test.go index 88d88d41e6..3137a75239 100644 --- a/components/engine/pkg/stdcopy/stdcopy_test.go +++ b/components/engine/pkg/stdcopy/stdcopy_test.go @@ -17,10 +17,9 @@ func TestNewStdWriter(t *testing.T) { } func TestWriteWithUnitializedStdWriter(t *testing.T) { - writer := StdWriter{ - Writer: nil, - prefix: Stdout, - sizeBuf: make([]byte, 4), + writer := stdWriter{ + Writer: nil, + prefix: byte(Stdout), } n, err := writer.Write([]byte("Something here")) if n != 0 || err == nil { @@ -72,7 +71,7 @@ func TestWriteWithWriterError(t *testing.T) { t.Fatalf("Didn't get expected error.") } if n != expectedReturnedBytes { - t.Fatalf("Didn't get expected writen bytes %d, got %d.", + t.Fatalf("Didn't get expected written bytes %d, got %d.", expectedReturnedBytes, n) } } @@ -180,7 +179,7 @@ func TestStdCopyDetectsCorruptedFrame(t *testing.T) { src: buffer} written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader) if written != startingBufLen { - t.Fatalf("Expected 0 bytes read, got %d", written) + t.Fatalf("Expected %d bytes read, got %d", startingBufLen, written) } if err != nil { t.Fatal("Didn't get nil error") diff --git a/components/engine/pkg/symlink/fs_test.go b/components/engine/pkg/symlink/fs_unix_test.go similarity index 98% rename from components/engine/pkg/symlink/fs_test.go rename to components/engine/pkg/symlink/fs_unix_test.go index 89209484a3..7085c0b666 100644 --- a/components/engine/pkg/symlink/fs_test.go +++ b/components/engine/pkg/symlink/fs_unix_test.go @@ -1,3 +1,5 @@ +// +build !windows + // Licensed under the Apache License, Version 2.0; See LICENSE.APACHE package symlink @@ -10,6 +12,9 @@ import ( "testing" ) +// TODO Windows: This needs some serious work to port to Windows. For now, +// turning off testing in this package. + type dirOrLink struct { path string target string diff --git a/components/engine/pkg/sysinfo/sysinfo.go b/components/engine/pkg/sysinfo/sysinfo.go index 285b3ba58f..cbd0099957 100644 --- a/components/engine/pkg/sysinfo/sysinfo.go +++ b/components/engine/pkg/sysinfo/sysinfo.go @@ -14,15 +14,16 @@ type SysInfo struct { cgroupCPUInfo cgroupBlkioInfo cgroupCpusetInfo + cgroupPids // Whether IPv4 forwarding is supported or not, if this was disabled, networking will not work IPv4ForwardingDisabled bool // Whether bridge-nf-call-iptables is supported or not - BridgeNfCallIptablesDisabled bool + BridgeNFCallIPTablesDisabled bool // Whether bridge-nf-call-ip6tables is supported or not - BridgeNfCallIP6tablesDisabled bool + BridgeNFCallIP6TablesDisabled bool // Whether the cgroup has the mountpoint of "devices" or not CgroupDevicesEnabled bool @@ -90,6 +91,11 @@ type cgroupCpusetInfo struct { Mems string } +type cgroupPids struct { + // Whether Pids Limit is supported or not + PidsLimit bool +} + // IsCpusetCpusAvailable returns `true` if the provided string set is contained // in cgroup's cpuset.cpus set, `false` otherwise. // If error is not nil a parsing error occurred. diff --git a/components/engine/pkg/sysinfo/sysinfo_linux.go b/components/engine/pkg/sysinfo/sysinfo_linux.go index 766d13d4d9..41fb0d2bb9 100644 --- a/components/engine/pkg/sysinfo/sysinfo_linux.go +++ b/components/engine/pkg/sysinfo/sysinfo_linux.go @@ -44,14 +44,15 @@ func New(quiet bool) *SysInfo { sysInfo.cgroupCPUInfo = checkCgroupCPU(cgMounts, quiet) sysInfo.cgroupBlkioInfo = checkCgroupBlkioInfo(cgMounts, quiet) sysInfo.cgroupCpusetInfo = checkCgroupCpusetInfo(cgMounts, quiet) + sysInfo.cgroupPids = checkCgroupPids(quiet) } _, ok := cgMounts["devices"] sysInfo.CgroupDevicesEnabled = ok sysInfo.IPv4ForwardingDisabled = !readProcBool("/proc/sys/net/ipv4/ip_forward") - sysInfo.BridgeNfCallIptablesDisabled = !readProcBool("/proc/sys/net/bridge/bridge-nf-call-iptables") - sysInfo.BridgeNfCallIP6tablesDisabled = !readProcBool("/proc/sys/net/bridge/bridge-nf-call-ip6tables") + sysInfo.BridgeNFCallIPTablesDisabled = !readProcBool("/proc/sys/net/bridge/bridge-nf-call-iptables") + sysInfo.BridgeNFCallIP6TablesDisabled = !readProcBool("/proc/sys/net/bridge/bridge-nf-call-ip6tables") // Check if AppArmor is supported. if _, err := os.Stat("/sys/kernel/security/apparmor"); !os.IsNotExist(err) { @@ -216,6 +217,21 @@ func checkCgroupCpusetInfo(cgMounts map[string]string, quiet bool) cgroupCpusetI } } +// checkCgroupPids reads the pids information from the pids cgroup mount point. +func checkCgroupPids(quiet bool) cgroupPids { + _, err := cgroups.FindCgroupMountpoint("pids") + if err != nil { + if !quiet { + logrus.Warn(err) + } + return cgroupPids{} + } + + return cgroupPids{ + PidsLimit: true, + } +} + func cgroupEnabled(mountPoint, name string) bool { _, err := os.Stat(path.Join(mountPoint, name)) return err == nil diff --git a/components/engine/pkg/tarsum/tarsum_spec.md b/components/engine/pkg/tarsum/tarsum_spec.md index 77927ee707..89b2e49f98 100644 --- a/components/engine/pkg/tarsum/tarsum_spec.md +++ b/components/engine/pkg/tarsum/tarsum_spec.md @@ -223,7 +223,7 @@ with matching paths, and orders the list of file sums accordingly [3]. * [2] Tar http://en.wikipedia.org/wiki/Tar_%28computing%29 * [3] Name collision https://github.com/docker/docker/commit/c5e6362c53cbbc09ddbabd5a7323e04438b57d31 -## Acknowledgements +## Acknowledgments Joffrey F (shin-) and Guillaume J. Charmes (creack) on the initial work of the TarSum calculation. diff --git a/components/engine/pkg/term/tc_linux_cgo.go b/components/engine/pkg/term/tc_linux_cgo.go index 1005084fd2..a22cd9d105 100644 --- a/components/engine/pkg/term/tc_linux_cgo.go +++ b/components/engine/pkg/term/tc_linux_cgo.go @@ -27,7 +27,6 @@ func MakeRaw(fd uintptr) (*State, error) { newState := oldState.termios C.cfmakeraw((*C.struct_termios)(unsafe.Pointer(&newState))) - newState.Oflag = newState.Oflag | C.OPOST if err := tcset(fd, &newState); err != 0 { return nil, err } diff --git a/components/engine/pkg/term/term.go b/components/engine/pkg/term/term.go index 316c399053..11ed20937b 100644 --- a/components/engine/pkg/term/term.go +++ b/components/engine/pkg/term/term.go @@ -127,6 +127,5 @@ func handleInterrupt(fd uintptr, state *State) { go func() { _ = <-sigchan RestoreTerminal(fd, state) - os.Exit(0) }() } diff --git a/components/engine/profiles/apparmor/apparmor.go b/components/engine/profiles/apparmor/apparmor.go index 46178886e6..e392b2c692 100644 --- a/components/engine/profiles/apparmor/apparmor.go +++ b/components/engine/profiles/apparmor/apparmor.go @@ -30,10 +30,8 @@ type profileData struct { Imports []string // InnerImports defines the apparmor functions to import in the profile. InnerImports []string - // MajorVersion is the apparmor_parser major version. - MajorVersion int - // MinorVersion is the apparmor_parser minor version. - MinorVersion int + // Version is the {major, minor, patch} version of apparmor_parser as a single number. + Version int } // generateDefault creates an apparmor profile from ProfileData. @@ -91,7 +89,7 @@ func InstallDefault(name string) error { return nil } -// IsLoaded checks if a passed profile as been loaded into the kernel. +// IsLoaded checks if a passed profile has been loaded into the kernel. func IsLoaded(name string) error { file, err := os.Open("/sys/kernel/security/apparmor/profiles") if err != nil { diff --git a/components/engine/profiles/apparmor/template.go b/components/engine/profiles/apparmor/template.go index d52748c2bf..db867b9def 100644 --- a/components/engine/profiles/apparmor/template.go +++ b/components/engine/profiles/apparmor/template.go @@ -38,13 +38,13 @@ profile {{.Name}} flags=(attach_disconnected,mediate_deleted) { deny /sys/firmware/efi/efivars/** rwklx, deny /sys/kernel/security/** rwklx, -{{if ge .MajorVersion 2}}{{if ge .MinorVersion 8}} +{{if ge .Version 208095}} # suppress ptrace denials when using 'docker ps' or using 'ps' inside a container ptrace (trace,read) peer=docker-default, -{{end}}{{end}} -{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}} +{{end}} +{{if ge .Version 209000}} # docker daemon confinement requires explict allow rule for signal signal (receive) set=(kill,term) peer={{.ExecPath}}, -{{end}}{{end}} +{{end}} } ` diff --git a/components/engine/profiles/seccomp/default.json b/components/engine/profiles/seccomp/default.json index 532a523872..91f04d6ec4 100755 --- a/components/engine/profiles/seccomp/default.json +++ b/components/engine/profiles/seccomp/default.json @@ -1,1566 +1,1607 @@ { - "default_action": 2, + "defaultAction": "SCMP_ACT_ERRNO", "architectures": [ - "amd64", - "x86", - "x32" + "SCMP_ARCH_X86_64", + "SCMP_ARCH_X86", + "SCMP_ARCH_X32" ], "syscalls": [ { "name": "accept", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "accept4", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "access", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "alarm", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "arch_prctl", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "bind", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "brk", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "capget", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "capset", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "chdir", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "chmod", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "chown", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "chown32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "chroot", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "clock_getres", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "clock_gettime", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "clock_nanosleep", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "clone", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [ { "index": 0, "value": 2080505856, - "value_two": 0, - "op": 7 + "valueTwo": 0, + "op": "SCMP_CMP_MASKED_EQ" } ] }, { "name": "close", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "connect", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "creat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "dup", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "dup2", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "dup3", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "epoll_create", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "epoll_create1", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "epoll_ctl", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "epoll_ctl_old", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "epoll_pwait", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "epoll_wait", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "epoll_wait_old", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "eventfd", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "eventfd2", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "execve", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "execveat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "exit", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "exit_group", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "faccessat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fadvise64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fadvise64_64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fallocate", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fanotify_init", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fanotify_mark", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fchdir", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fchmod", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fchmodat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fchown", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fchown32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fchownat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fcntl", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fcntl64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fdatasync", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fgetxattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "flistxattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "flock", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fork", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fremovexattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fsetxattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fstat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fstat64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fstatat64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fstatfs", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fstatfs64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "fsync", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "ftruncate", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "ftruncate64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "futex", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "futimesat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getcpu", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getcwd", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getdents", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getdents64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getegid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getegid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "geteuid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "geteuid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getgid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getgid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getgroups", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getgroups32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getitimer", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getpeername", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getpgid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getpgrp", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getpid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getppid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getpriority", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getrandom", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getresgid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getresgid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getresuid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getresuid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getrlimit", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "get_robust_list", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getrusage", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getsid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getsockname", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getsockopt", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "get_thread_area", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "gettid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "gettimeofday", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getuid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getuid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "getxattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "inotify_add_watch", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "inotify_init", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "inotify_init1", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "inotify_rm_watch", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "io_cancel", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "ioctl", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "io_destroy", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "io_getevents", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "ioprio_get", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "ioprio_set", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "io_setup", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "io_submit", - "action": 4, + "action": "SCMP_ACT_ALLOW", + "args": [] + }, + { + "name": "ipc", + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "kill", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "lchown", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "lchown32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "lgetxattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "link", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "linkat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "listen", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "listxattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "llistxattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "_llseek", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "lremovexattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "lseek", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "lsetxattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "lstat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "lstat64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "madvise", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "memfd_create", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mincore", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mkdir", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mkdirat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mknod", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mknodat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mlock", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mlockall", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mmap", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mmap2", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mprotect", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mq_getsetattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mq_notify", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mq_open", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mq_timedreceive", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mq_timedsend", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mq_unlink", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "mremap", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "msgctl", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "msgget", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "msgrcv", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "msgsnd", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "msync", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "munlock", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "munlockall", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "munmap", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "nanosleep", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "newfstatat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "_newselect", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "open", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "openat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "pause", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, + { + "name": "personality", + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 0, + "valueTwo": 0, + "op": "SCMP_CMP_EQ" + } + ] + }, + { + "name": "personality", + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 8, + "valueTwo": 0, + "op": "SCMP_CMP_EQ" + } + ] + }, + { + "name": "personality", + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 4294967295, + "valueTwo": 0, + "op": "SCMP_CMP_EQ" + } + ] + }, { "name": "pipe", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "pipe2", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "poll", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "ppoll", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "prctl", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "pread64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "preadv", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "prlimit64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "pselect6", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "pwrite64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "pwritev", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "read", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "readahead", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "readlink", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "readlinkat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "readv", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "recv", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "recvfrom", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "recvmmsg", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "recvmsg", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "remap_file_pages", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "removexattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "rename", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "renameat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "renameat2", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "rmdir", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "rt_sigaction", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "rt_sigpending", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "rt_sigprocmask", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "rt_sigqueueinfo", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "rt_sigreturn", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "rt_sigsuspend", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "rt_sigtimedwait", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "rt_tgsigqueueinfo", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sched_getaffinity", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sched_getattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sched_getparam", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sched_get_priority_max", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sched_get_priority_min", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sched_getscheduler", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sched_rr_get_interval", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sched_setaffinity", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sched_setattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sched_setparam", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sched_setscheduler", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sched_yield", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "seccomp", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "select", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "semctl", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "semget", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "semop", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "semtimedop", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "send", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sendfile", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sendfile64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sendmmsg", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sendmsg", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sendto", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setdomainname", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setfsgid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setfsgid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setfsuid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setfsuid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setgid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setgid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setgroups", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setgroups32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sethostname", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setitimer", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setpgid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setpriority", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setregid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setregid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setresgid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setresgid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setresuid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setresuid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setreuid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setreuid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setrlimit", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "set_robust_list", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setsid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setsockopt", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "set_thread_area", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "set_tid_address", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setuid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setuid32", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "setxattr", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "shmat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "shmctl", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "shmdt", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "shmget", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "shutdown", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sigaltstack", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "signalfd", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "signalfd4", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sigreturn", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "socket", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "socketpair", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "splice", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "stat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "stat64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "statfs", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "statfs64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "symlink", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "symlinkat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sync", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sync_file_range", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "syncfs", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "sysinfo", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "syslog", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "tee", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "tgkill", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "time", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "timer_create", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "timer_delete", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "timerfd_create", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "timerfd_gettime", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "timerfd_settime", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "timer_getoverrun", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "timer_gettime", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "timer_settime", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "times", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "tkill", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "truncate", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "truncate64", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "ugetrlimit", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "umask", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "uname", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "unlink", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "unlinkat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "utime", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "utimensat", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "utimes", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "vfork", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "vhangup", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "vmsplice", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "wait4", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "waitid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "waitpid", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "write", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "writev", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "modify_ldt", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "breakpoint", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "cacheflush", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] }, { "name": "set_tls", - "action": 4, + "action": "SCMP_ACT_ALLOW", "args": [] } ] diff --git a/components/engine/profiles/seccomp/generate.go b/components/engine/profiles/seccomp/generate.go index 8654ec028a..bf56594765 100644 --- a/components/engine/profiles/seccomp/generate.go +++ b/components/engine/profiles/seccomp/generate.go @@ -20,11 +20,8 @@ func main() { } f := filepath.Join(wd, "default.json") - // get the default profile - p := seccomp.GetDefaultProfile() - // write the default profile to the file - b, err := json.MarshalIndent(p, "", "\t") + b, err := json.MarshalIndent(seccomp.DefaultProfile, "", "\t") if err != nil { panic(err) } diff --git a/components/engine/profiles/seccomp/seccomp.go b/components/engine/profiles/seccomp/seccomp.go index 611b80b246..12b78e3296 100644 --- a/components/engine/profiles/seccomp/seccomp.go +++ b/components/engine/profiles/seccomp/seccomp.go @@ -14,11 +14,11 @@ import ( //go:generate go run -tags 'seccomp' generate.go // GetDefaultProfile returns the default seccomp profile. -func GetDefaultProfile() *configs.Seccomp { - return defaultProfile +func GetDefaultProfile() (*configs.Seccomp, error) { + return setupSeccomp(DefaultProfile) } -// LoadProfile takes a file path a decodes the seccomp profile. +// LoadProfile takes a file path and decodes the seccomp profile. func LoadProfile(body string) (*configs.Seccomp, error) { var config types.Seccomp if err := json.Unmarshal([]byte(body), &config); err != nil { diff --git a/components/engine/profiles/seccomp/seccomp_default.go b/components/engine/profiles/seccomp/seccomp_default.go index 49bd259df4..181e9f5002 100644 --- a/components/engine/profiles/seccomp/seccomp_default.go +++ b/components/engine/profiles/seccomp/seccomp_default.go @@ -5,1597 +5,1635 @@ package seccomp import ( "syscall" - "github.com/opencontainers/runc/libcontainer/configs" + "github.com/docker/engine-api/types" libseccomp "github.com/seccomp/libseccomp-golang" ) -func arches() []string { +func arches() []types.Arch { var native, err = libseccomp.GetNativeArch() if err != nil { - return []string{} + return []types.Arch{} } var a = native.String() switch a { case "amd64": - return []string{"amd64", "x86", "x32"} + return []types.Arch{types.ArchX86_64, types.ArchX86, types.ArchX32} case "arm64": - return []string{"arm64", "arm"} + return []types.Arch{types.ArchARM, types.ArchAARCH64} case "mips64": - return []string{"mips64", "mips64n32", "mips"} + return []types.Arch{types.ArchMIPS, types.ArchMIPS64, types.ArchMIPS64N32} case "mips64n32": - return []string{"mips64", "mips64n32", "mips"} + return []types.Arch{types.ArchMIPS, types.ArchMIPS64, types.ArchMIPS64N32} case "mipsel64": - return []string{"mipsel64", "mipsel64n32", "mipsel"} + return []types.Arch{types.ArchMIPSEL, types.ArchMIPSEL64, types.ArchMIPSEL64N32} case "mipsel64n32": - return []string{"mipsel64", "mipsel64n32", "mipsel"} + return []types.Arch{types.ArchMIPSEL, types.ArchMIPSEL64, types.ArchMIPSEL64N32} default: - return []string{a} + return []types.Arch{} } } -// defaultProfile defines the whitelist for the default seccomp profile. -var defaultProfile = &configs.Seccomp{ - DefaultAction: configs.Errno, +// DefaultProfile defines the whitelist for the default seccomp profile. +var DefaultProfile = &types.Seccomp{ + DefaultAction: types.ActErrno, Architectures: arches(), - Syscalls: []*configs.Syscall{ + Syscalls: []*types.Syscall{ { Name: "accept", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "accept4", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "access", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "alarm", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "arch_prctl", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "bind", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "brk", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "capget", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "capset", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "chdir", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "chmod", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "chown", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "chown32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "chroot", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "clock_getres", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "clock_gettime", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "clock_nanosleep", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "clone", - Action: configs.Allow, - Args: []*configs.Arg{ + Action: types.ActAllow, + Args: []*types.Arg{ { Index: 0, Value: syscall.CLONE_NEWNS | syscall.CLONE_NEWUTS | syscall.CLONE_NEWIPC | syscall.CLONE_NEWUSER | syscall.CLONE_NEWPID | syscall.CLONE_NEWNET, ValueTwo: 0, - Op: configs.MaskEqualTo, + Op: types.OpMaskedEqual, }, }, }, { Name: "close", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "connect", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "creat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "dup", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "dup2", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "dup3", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "epoll_create", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "epoll_create1", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "epoll_ctl", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "epoll_ctl_old", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "epoll_pwait", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "epoll_wait", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "epoll_wait_old", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "eventfd", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "eventfd2", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "execve", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "execveat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "exit", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "exit_group", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "faccessat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fadvise64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fadvise64_64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fallocate", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fanotify_init", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fanotify_mark", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fchdir", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fchmod", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fchmodat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fchown", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fchown32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fchownat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fcntl", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fcntl64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fdatasync", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fgetxattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "flistxattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "flock", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fork", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fremovexattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fsetxattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fstat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fstat64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fstatat64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fstatfs", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fstatfs64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "fsync", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "ftruncate", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "ftruncate64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "futex", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "futimesat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getcpu", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getcwd", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getdents", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getdents64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getegid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getegid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "geteuid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "geteuid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getgid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getgid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getgroups", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getgroups32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getitimer", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getpeername", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getpgid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getpgrp", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getpid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getppid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getpriority", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getrandom", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getresgid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getresgid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getresuid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getresuid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getrlimit", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "get_robust_list", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getrusage", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getsid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getsockname", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getsockopt", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "get_thread_area", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "gettid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "gettimeofday", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getuid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getuid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "getxattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "inotify_add_watch", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "inotify_init", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "inotify_init1", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "inotify_rm_watch", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "io_cancel", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "ioctl", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "io_destroy", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "io_getevents", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "ioprio_get", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "ioprio_set", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "io_setup", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "io_submit", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, + }, + { + Name: "ipc", + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "kill", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "lchown", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "lchown32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "lgetxattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "link", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "linkat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "listen", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "listxattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "llistxattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "_llseek", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "lremovexattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "lseek", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "lsetxattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "lstat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "lstat64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "madvise", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "memfd_create", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mincore", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mkdir", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mkdirat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mknod", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mknodat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mlock", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mlockall", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mmap", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mmap2", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mprotect", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mq_getsetattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mq_notify", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mq_open", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mq_timedreceive", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mq_timedsend", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mq_unlink", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "mremap", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "msgctl", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "msgget", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "msgrcv", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "msgsnd", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "msync", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "munlock", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "munlockall", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "munmap", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "nanosleep", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "newfstatat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "_newselect", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "open", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "openat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "pause", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, + }, + { + Name: "personality", + Action: types.ActAllow, + Args: []*types.Arg{ + { + Index: 0, + Value: 0x0, + Op: types.OpEqualTo, + }, + }, + }, + { + Name: "personality", + Action: types.ActAllow, + Args: []*types.Arg{ + { + Index: 0, + Value: 0x0008, + Op: types.OpEqualTo, + }, + }, + }, + { + Name: "personality", + Action: types.ActAllow, + Args: []*types.Arg{ + { + Index: 0, + Value: 0xffffffff, + Op: types.OpEqualTo, + }, + }, }, { Name: "pipe", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "pipe2", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "poll", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "ppoll", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "prctl", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "pread64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "preadv", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "prlimit64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "pselect6", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "pwrite64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "pwritev", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "read", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "readahead", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "readlink", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "readlinkat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "readv", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "recv", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "recvfrom", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "recvmmsg", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "recvmsg", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "remap_file_pages", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "removexattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "rename", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "renameat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "renameat2", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "rmdir", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "rt_sigaction", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "rt_sigpending", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "rt_sigprocmask", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "rt_sigqueueinfo", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "rt_sigreturn", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "rt_sigsuspend", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "rt_sigtimedwait", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "rt_tgsigqueueinfo", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sched_getaffinity", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sched_getattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sched_getparam", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sched_get_priority_max", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sched_get_priority_min", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sched_getscheduler", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sched_rr_get_interval", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sched_setaffinity", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sched_setattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sched_setparam", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sched_setscheduler", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sched_yield", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "seccomp", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "select", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "semctl", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "semget", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "semop", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "semtimedop", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "send", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sendfile", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sendfile64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sendmmsg", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sendmsg", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sendto", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setdomainname", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setfsgid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setfsgid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setfsuid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setfsuid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setgid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setgid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setgroups", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setgroups32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sethostname", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setitimer", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setpgid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setpriority", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setregid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setregid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setresgid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setresgid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setresuid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setresuid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setreuid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setreuid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setrlimit", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "set_robust_list", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setsid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setsockopt", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "set_thread_area", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "set_tid_address", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setuid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setuid32", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "setxattr", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "shmat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "shmctl", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "shmdt", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "shmget", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "shutdown", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sigaltstack", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "signalfd", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "signalfd4", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sigreturn", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "socket", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "socketpair", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "splice", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "stat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "stat64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "statfs", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "statfs64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "symlink", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "symlinkat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sync", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sync_file_range", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "syncfs", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "sysinfo", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "syslog", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "tee", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "tgkill", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "time", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "timer_create", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "timer_delete", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "timerfd_create", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "timerfd_gettime", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "timerfd_settime", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "timer_getoverrun", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "timer_gettime", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "timer_settime", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "times", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "tkill", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "truncate", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "truncate64", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "ugetrlimit", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "umask", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "uname", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "unlink", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "unlinkat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "utime", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "utimensat", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "utimes", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "vfork", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "vhangup", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "vmsplice", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "wait4", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "waitid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "waitpid", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "write", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "writev", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, // i386 specific syscalls { Name: "modify_ldt", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, // arm specific syscalls { Name: "breakpoint", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "cacheflush", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, { Name: "set_tls", - Action: configs.Allow, - Args: []*configs.Arg{}, + Action: types.ActAllow, + Args: []*types.Arg{}, }, }, } diff --git a/components/engine/profiles/seccomp/seccomp_test.go b/components/engine/profiles/seccomp/seccomp_test.go index 11df61e94d..2c9929e925 100644 --- a/components/engine/profiles/seccomp/seccomp_test.go +++ b/components/engine/profiles/seccomp/seccomp_test.go @@ -12,7 +12,16 @@ func TestLoadProfile(t *testing.T) { if err != nil { t.Fatal(err) } + if _, err := LoadProfile(string(f)); err != nil { + t.Fatal(err) + } +} +func TestLoadDefaultProfile(t *testing.T) { + f, err := ioutil.ReadFile("default.json") + if err != nil { + t.Fatal(err) + } if _, err := LoadProfile(string(f)); err != nil { t.Fatal(err) } diff --git a/components/engine/profiles/seccomp/seccomp_unsupported.go b/components/engine/profiles/seccomp/seccomp_unsupported.go index 780c7d051d..649632920a 100644 --- a/components/engine/profiles/seccomp/seccomp_unsupported.go +++ b/components/engine/profiles/seccomp/seccomp_unsupported.go @@ -2,9 +2,9 @@ package seccomp -import "github.com/opencontainers/runc/libcontainer/configs" +import "github.com/docker/engine-api/types" var ( - // defaultProfile is a nil pointer on unsupported systems. - defaultProfile *configs.Seccomp + // DefaultProfile is a nil pointer on unsupported systems. + DefaultProfile *types.Seccomp ) diff --git a/components/engine/project/PACKAGERS.md b/components/engine/project/PACKAGERS.md index b80749f95b..b3f60472fd 100644 --- a/components/engine/project/PACKAGERS.md +++ b/components/engine/project/PACKAGERS.md @@ -164,6 +164,12 @@ SELinux, you will need to use the `selinux` build tag: export DOCKER_BUILDTAGS='selinux' ``` +If you're building a binary that may need to be used on platforms that include +seccomp, you will need to use the `seccomp` build tag: +```bash +export DOCKER_BUILDTAGS='seccomp' +``` + There are build tags for disabling graphdrivers as well. By default, support for all graphdrivers are built in. diff --git a/components/engine/project/REVIEWING.md b/components/engine/project/REVIEWING.md index d610b571cb..95b9945cdf 100644 --- a/components/engine/project/REVIEWING.md +++ b/components/engine/project/REVIEWING.md @@ -26,6 +26,7 @@ exist on the repository should apply to issues. Special status labels: + * `status/failing-ci`: indicates that the PR in its current state fails the test suite * `status/needs-attention`: calls for a collective discussion during a review session ## Specialty group labels diff --git a/components/engine/project/TOOLS.md b/components/engine/project/TOOLS.md index f52147f65e..26303c3021 100644 --- a/components/engine/project/TOOLS.md +++ b/components/engine/project/TOOLS.md @@ -42,7 +42,7 @@ The gordon-bot repository is maintained at ### NSQ We use [NSQ](https://github.com/bitly/nsq) for various aspects of the project -infrastucture. +infrastructure. #### Hooks diff --git a/components/engine/reference/reference.go b/components/engine/reference/reference.go index 1e3482475f..e355596eab 100644 --- a/components/engine/reference/reference.go +++ b/components/engine/reference/reference.go @@ -1,6 +1,7 @@ package reference import ( + "errors" "fmt" "strings" @@ -72,7 +73,10 @@ func ParseNamed(s string) (Named, error) { // WithName returns a named object representing the given string. If the input // is invalid ErrReferenceInvalidFormat will be returned. func WithName(name string) (Named, error) { - name = normalize(name) + name, err := normalize(name) + if err != nil { + return nil, err + } if err := validateName(name); err != nil { return nil, err } @@ -172,15 +176,18 @@ func splitHostname(name string) (hostname, remoteName string) { // normalize returns a repository name in its normalized form, meaning it // will not contain default hostname nor library/ prefix for official images. -func normalize(name string) string { +func normalize(name string) (string, error) { host, remoteName := splitHostname(name) + if strings.ToLower(remoteName) != remoteName { + return "", errors.New("invalid reference format: repository name must be lowercase") + } if host == DefaultHostname { if strings.HasPrefix(remoteName, DefaultRepoPrefix) { - return strings.TrimPrefix(remoteName, DefaultRepoPrefix) + return strings.TrimPrefix(remoteName, DefaultRepoPrefix), nil } - return remoteName + return remoteName, nil } - return name + return name, nil } func validateName(name string) error { diff --git a/components/engine/registry/auth.go b/components/engine/registry/auth.go index 7175598c71..a8fdb675c1 100644 --- a/components/engine/registry/auth.go +++ b/components/engine/registry/auth.go @@ -1,35 +1,28 @@ package registry import ( - "encoding/json" "fmt" "io/ioutil" "net/http" + "net/url" "strings" + "time" "github.com/Sirupsen/logrus" + "github.com/docker/distribution/registry/client/auth" + "github.com/docker/distribution/registry/client/transport" "github.com/docker/engine-api/types" registrytypes "github.com/docker/engine-api/types/registry" ) -// Login tries to register/login to the registry server. -func Login(authConfig *types.AuthConfig, registryEndpoint *Endpoint) (string, error) { - // Separates the v2 registry login logic from the v1 logic. - if registryEndpoint.Version == APIVersion2 { - return loginV2(authConfig, registryEndpoint, "" /* scope */) - } - return loginV1(authConfig, registryEndpoint) -} - // loginV1 tries to register/login to the v1 registry server. -func loginV1(authConfig *types.AuthConfig, registryEndpoint *Endpoint) (string, error) { - var ( - status string - respBody []byte - err error - respStatusCode = 0 - serverAddress = authConfig.ServerAddress - ) +func loginV1(authConfig *types.AuthConfig, apiEndpoint APIEndpoint, userAgent string) (string, error) { + registryEndpoint, err := apiEndpoint.ToV1Endpoint(userAgent, nil) + if err != nil { + return "", err + } + + serverAddress := registryEndpoint.String() logrus.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) @@ -39,186 +32,121 @@ func loginV1(authConfig *types.AuthConfig, registryEndpoint *Endpoint) (string, loginAgainstOfficialIndex := serverAddress == IndexServer - // to avoid sending the server address to the server it should be removed before being marshaled - authCopy := *authConfig - authCopy.ServerAddress = "" - - jsonBody, err := json.Marshal(authCopy) + req, err := http.NewRequest("GET", serverAddress+"users/", nil) if err != nil { - return "", fmt.Errorf("Config Error: %s", err) + return "", err } - - // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. - b := strings.NewReader(string(jsonBody)) - resp1, err := registryEndpoint.client.Post(serverAddress+"users/", "application/json; charset=utf-8", b) - if err != nil { - return "", fmt.Errorf("Server Error: %s", err) - } - defer resp1.Body.Close() - respStatusCode = resp1.StatusCode - respBody, err = ioutil.ReadAll(resp1.Body) - if err != nil { - return "", fmt.Errorf("Server Error: [%#v] %s", respStatusCode, err) - } - - if respStatusCode == 201 { - if loginAgainstOfficialIndex { - status = "Account created. Please use the confirmation link we sent" + - " to your e-mail to activate it." - } else { - // *TODO: Use registry configuration to determine what this says, if anything? - status = "Account created. Please see the documentation of the registry " + serverAddress + " for instructions how to activate it." - } - } else if respStatusCode == 400 { - if string(respBody) == "\"Username or email already exists\"" { - req, err := http.NewRequest("GET", serverAddress+"users/", nil) - req.SetBasicAuth(authConfig.Username, authConfig.Password) - resp, err := registryEndpoint.client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - if resp.StatusCode == 200 { - return "Login Succeeded", nil - } else if resp.StatusCode == 401 { - return "", fmt.Errorf("Wrong login/password, please try again") - } else if resp.StatusCode == 403 { - if loginAgainstOfficialIndex { - return "", fmt.Errorf("Login: Account is not Active. Please check your e-mail for a confirmation link.") - } - // *TODO: Use registry configuration to determine what this says, if anything? - return "", fmt.Errorf("Login: Account is not Active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress) - } else if resp.StatusCode == 500 { // Issue #14326 - logrus.Errorf("%s returned status code %d. Response Body :\n%s", req.URL.String(), resp.StatusCode, body) - return "", fmt.Errorf("Internal Server Error") - } - return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, resp.StatusCode, resp.Header) - } - return "", fmt.Errorf("Registration: %s", respBody) - - } else if respStatusCode == 401 { - // This case would happen with private registries where /v1/users is - // protected, so people can use `docker login` as an auth check. - req, err := http.NewRequest("GET", serverAddress+"users/", nil) - req.SetBasicAuth(authConfig.Username, authConfig.Password) - resp, err := registryEndpoint.client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - if resp.StatusCode == 200 { - return "Login Succeeded", nil - } else if resp.StatusCode == 401 { - return "", fmt.Errorf("Wrong login/password, please try again") - } else { - return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, - resp.StatusCode, resp.Header) - } - } else { - return "", fmt.Errorf("Unexpected status code [%d] : %s", respStatusCode, respBody) - } - return status, nil -} - -// loginV2 tries to login to the v2 registry server. The given registry endpoint has been -// pinged or setup with a list of authorization challenges. Each of these challenges are -// tried until one of them succeeds. Currently supported challenge schemes are: -// HTTP Basic Authorization -// Token Authorization with a separate token issuing server -// NOTE: the v2 logic does not attempt to create a user account if one doesn't exist. For -// now, users should create their account through other means like directly from a web page -// served by the v2 registry service provider. Whether this will be supported in the future -// is to be determined. -func loginV2(authConfig *types.AuthConfig, registryEndpoint *Endpoint, scope string) (string, error) { - logrus.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) - var ( - err error - allErrors []error - ) - - for _, challenge := range registryEndpoint.AuthChallenges { - params := make(map[string]string, len(challenge.Parameters)+1) - for k, v := range challenge.Parameters { - params[k] = v - } - params["scope"] = scope - logrus.Debugf("trying %q auth challenge with params %v", challenge.Scheme, params) - - switch strings.ToLower(challenge.Scheme) { - case "basic": - err = tryV2BasicAuthLogin(authConfig, params, registryEndpoint) - case "bearer": - err = tryV2TokenAuthLogin(authConfig, params, registryEndpoint) - default: - // Unsupported challenge types are explicitly skipped. - err = fmt.Errorf("unsupported auth scheme: %q", challenge.Scheme) - } - - if err == nil { - return "Login Succeeded", nil - } - - logrus.Debugf("error trying auth challenge %q: %s", challenge.Scheme, err) - - allErrors = append(allErrors, err) - } - - return "", fmt.Errorf("no successful auth challenge for %s - errors: %s", registryEndpoint, allErrors) -} - -func tryV2BasicAuthLogin(authConfig *types.AuthConfig, params map[string]string, registryEndpoint *Endpoint) error { - req, err := http.NewRequest("GET", registryEndpoint.Path(""), nil) - if err != nil { - return err - } - req.SetBasicAuth(authConfig.Username, authConfig.Password) - resp, err := registryEndpoint.client.Do(req) if err != nil { - return err + // fallback when request could not be completed + return "", fallbackError{ + err: err, + } } defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("basic auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode == http.StatusOK { + return "Login Succeeded", nil + } else if resp.StatusCode == http.StatusUnauthorized { + if loginAgainstOfficialIndex { + return "", fmt.Errorf("Wrong login/password, please try again. Haven't got a Docker ID? Create one at https://hub.docker.com") + } + return "", fmt.Errorf("Wrong login/password, please try again") + } else if resp.StatusCode == http.StatusForbidden { + if loginAgainstOfficialIndex { + return "", fmt.Errorf("Login: Account is not active. Please check your e-mail for a confirmation link.") + } + // *TODO: Use registry configuration to determine what this says, if anything? + return "", fmt.Errorf("Login: Account is not active. Please see the documentation of the registry %s for instructions how to activate it.", serverAddress) + } else if resp.StatusCode == http.StatusInternalServerError { // Issue #14326 + logrus.Errorf("%s returned status code %d. Response Body :\n%s", req.URL.String(), resp.StatusCode, body) + return "", fmt.Errorf("Internal Server Error") + } else { + return "", fmt.Errorf("Login: %s (Code: %d; Headers: %s)", body, + resp.StatusCode, resp.Header) } - - return nil } -func tryV2TokenAuthLogin(authConfig *types.AuthConfig, params map[string]string, registryEndpoint *Endpoint) error { - token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint) +type loginCredentialStore struct { + authConfig *types.AuthConfig +} + +func (lcs loginCredentialStore) Basic(*url.URL) (string, string) { + return lcs.authConfig.Username, lcs.authConfig.Password +} + +type fallbackError struct { + err error +} + +func (err fallbackError) Error() string { + return err.err.Error() +} + +// loginV2 tries to login to the v2 registry server. The given registry +// endpoint will be pinged to get authorization challenges. These challenges +// will be used to authenticate against the registry to validate credentials. +func loginV2(authConfig *types.AuthConfig, endpoint APIEndpoint, userAgent string) (string, error) { + logrus.Debugf("attempting v2 login to registry endpoint %s", endpoint) + + modifiers := DockerHeaders(userAgent, nil) + authTransport := transport.NewTransport(NewTransport(endpoint.TLSConfig), modifiers...) + + challengeManager, foundV2, err := PingV2Registry(endpoint, authTransport) if err != nil { - return err + if !foundV2 { + err = fallbackError{err: err} + } + return "", err } - req, err := http.NewRequest("GET", registryEndpoint.Path(""), nil) - if err != nil { - return err + creds := loginCredentialStore{ + authConfig: authConfig, } - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + tokenHandler := auth.NewTokenHandler(authTransport, creds, "") + basicHandler := auth.NewBasicHandler(creds) + modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler)) + tr := transport.NewTransport(authTransport, modifiers...) - resp, err := registryEndpoint.client.Do(req) + loginClient := &http.Client{ + Transport: tr, + Timeout: 15 * time.Second, + } + + endpointStr := strings.TrimRight(endpoint.URL.String(), "/") + "/v2/" + req, err := http.NewRequest("GET", endpointStr, nil) if err != nil { - return err + if !foundV2 { + err = fallbackError{err: err} + } + return "", err + } + + resp, err := loginClient.Do(req) + if err != nil { + if !foundV2 { + err = fallbackError{err: err} + } + return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("token auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + // TODO(dmcgowan): Attempt to further interpret result, status code and error code string + err := fmt.Errorf("login attempt to %s failed with status: %d %s", endpointStr, resp.StatusCode, http.StatusText(resp.StatusCode)) + if !foundV2 { + err = fallbackError{err: err} + } + return "", err } - return nil + return "Login Succeeded", nil + } // ResolveAuthConfig matches an auth configuration to a server address or a URL @@ -253,3 +181,63 @@ func ResolveAuthConfig(authConfigs map[string]types.AuthConfig, index *registryt // When all else fails, return an empty auth config return types.AuthConfig{} } + +// PingResponseError is used when the response from a ping +// was received but invalid. +type PingResponseError struct { + Err error +} + +func (err PingResponseError) Error() string { + return err.Error() +} + +// PingV2Registry attempts to ping a v2 registry and on success return a +// challenge manager for the supported authentication types and +// whether v2 was confirmed by the response. If a response is received but +// cannot be interpreted a PingResponseError will be returned. +func PingV2Registry(endpoint APIEndpoint, transport http.RoundTripper) (auth.ChallengeManager, bool, error) { + var ( + foundV2 = false + v2Version = auth.APIVersion{ + Type: "registry", + Version: "2.0", + } + ) + + pingClient := &http.Client{ + Transport: transport, + Timeout: 15 * time.Second, + } + endpointStr := strings.TrimRight(endpoint.URL.String(), "/") + "/v2/" + req, err := http.NewRequest("GET", endpointStr, nil) + if err != nil { + return nil, false, err + } + resp, err := pingClient.Do(req) + if err != nil { + return nil, false, err + } + defer resp.Body.Close() + + versions := auth.APIVersions(resp, DefaultRegistryVersionHeader) + for _, pingVersion := range versions { + if pingVersion == v2Version { + // The version header indicates we're definitely + // talking to a v2 registry. So don't allow future + // fallbacks to the v1 protocol. + + foundV2 = true + break + } + } + + challengeManager := auth.NewSimpleChallengeManager() + if err := challengeManager.AddResponse(resp); err != nil { + return nil, foundV2, PingResponseError{ + Err: err, + } + } + + return challengeManager, foundV2, nil +} diff --git a/components/engine/registry/auth_test.go b/components/engine/registry/auth_test.go index caff8667d1..eedee44ef7 100644 --- a/components/engine/registry/auth_test.go +++ b/components/engine/registry/auth_test.go @@ -14,7 +14,6 @@ func buildAuthConfigs() map[string]types.AuthConfig { authConfigs[registry] = types.AuthConfig{ Username: "docker-user", Password: "docker-pass", - Email: "docker@docker.io", } } @@ -30,9 +29,6 @@ func TestSameAuthDataPostSave(t *testing.T) { if authConfig.Password != "docker-pass" { t.Fail() } - if authConfig.Email != "docker@docker.io" { - t.Fail() - } if authConfig.Auth != "" { t.Fail() } @@ -62,17 +58,14 @@ func TestResolveAuthConfigFullURL(t *testing.T) { registryAuth := types.AuthConfig{ Username: "foo-user", Password: "foo-pass", - Email: "foo@example.com", } localAuth := types.AuthConfig{ Username: "bar-user", Password: "bar-pass", - Email: "bar@example.com", } officialAuth := types.AuthConfig{ Username: "baz-user", Password: "baz-pass", - Email: "baz@example.com", } authConfigs[IndexServer] = officialAuth @@ -105,7 +98,7 @@ func TestResolveAuthConfigFullURL(t *testing.T) { for configKey, registries := range validRegistries { configured, ok := expectedAuths[configKey] - if !ok || configured.Email == "" { + if !ok { t.Fail() } index := ®istrytypes.IndexInfo{ @@ -114,13 +107,13 @@ func TestResolveAuthConfigFullURL(t *testing.T) { for _, registry := range registries { authConfigs[registry] = configured resolved := ResolveAuthConfig(authConfigs, index) - if resolved.Email != configured.Email { - t.Errorf("%s -> %q != %q\n", registry, resolved.Email, configured.Email) + if resolved.Username != configured.Username || resolved.Password != configured.Password { + t.Errorf("%s -> %v != %v\n", registry, resolved, configured) } delete(authConfigs, registry) resolved = ResolveAuthConfig(authConfigs, index) - if resolved.Email == configured.Email { - t.Errorf("%s -> %q == %q\n", registry, resolved.Email, configured.Email) + if resolved.Username == configured.Username || resolved.Password == configured.Password { + t.Errorf("%s -> %v == %v\n", registry, resolved, configured) } } } diff --git a/components/engine/registry/authchallenge.go b/components/engine/registry/authchallenge.go deleted file mode 100644 index e300d82a05..0000000000 --- a/components/engine/registry/authchallenge.go +++ /dev/null @@ -1,150 +0,0 @@ -package registry - -import ( - "net/http" - "strings" -) - -// Octet types from RFC 2616. -type octetType byte - -// AuthorizationChallenge carries information -// from a WWW-Authenticate response header. -type AuthorizationChallenge struct { - Scheme string - Parameters map[string]string -} - -var octetTypes [256]octetType - -const ( - isToken octetType = 1 << iota - isSpace -) - -func init() { - // OCTET = - // CHAR = - // CTL = - // CR = - // LF = - // SP = - // HT = - // <"> = - // CRLF = CR LF - // LWS = [CRLF] 1*( SP | HT ) - // TEXT = - // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> - // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT - // token = 1* - // qdtext = > - - for c := 0; c < 256; c++ { - var t octetType - isCtl := c <= 31 || c == 127 - isChar := 0 <= c && c <= 127 - isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 - if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { - t |= isSpace - } - if isChar && !isCtl && !isSeparator { - t |= isToken - } - octetTypes[c] = t - } -} - -func parseAuthHeader(header http.Header) []*AuthorizationChallenge { - var challenges []*AuthorizationChallenge - for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] { - v, p := parseValueAndParams(h) - if v != "" { - challenges = append(challenges, &AuthorizationChallenge{Scheme: v, Parameters: p}) - } - } - return challenges -} - -func parseValueAndParams(header string) (value string, params map[string]string) { - params = make(map[string]string) - value, s := expectToken(header) - if value == "" { - return - } - value = strings.ToLower(value) - s = "," + skipSpace(s) - for strings.HasPrefix(s, ",") { - var pkey string - pkey, s = expectToken(skipSpace(s[1:])) - if pkey == "" { - return - } - if !strings.HasPrefix(s, "=") { - return - } - var pvalue string - pvalue, s = expectTokenOrQuoted(s[1:]) - if pvalue == "" { - return - } - pkey = strings.ToLower(pkey) - params[pkey] = pvalue - s = skipSpace(s) - } - return -} - -func skipSpace(s string) (rest string) { - i := 0 - for ; i < len(s); i++ { - if octetTypes[s[i]]&isSpace == 0 { - break - } - } - return s[i:] -} - -func expectToken(s string) (token, rest string) { - i := 0 - for ; i < len(s); i++ { - if octetTypes[s[i]]&isToken == 0 { - break - } - } - return s[:i], s[i:] -} - -func expectTokenOrQuoted(s string) (value string, rest string) { - if !strings.HasPrefix(s, "\"") { - return expectToken(s) - } - s = s[1:] - for i := 0; i < len(s); i++ { - switch s[i] { - case '"': - return s[:i], s[i+1:] - case '\\': - p := make([]byte, len(s)-1) - j := copy(p, s[:i]) - escape := true - for i = i + i; i < len(s); i++ { - b := s[i] - switch { - case escape: - escape = false - p[j] = b - j++ - case b == '\\': - escape = true - case b == '"': - return string(p[:j]), s[i+1:] - default: - p[j] = b - j++ - } - } - return "", "" - } - } - return "", "" -} diff --git a/components/engine/registry/config.go b/components/engine/registry/config.go index ec8ec271c9..7d8b6301aa 100644 --- a/components/engine/registry/config.go +++ b/components/engine/registry/config.go @@ -19,7 +19,7 @@ type Options struct { InsecureRegistries opts.ListOpts } -const ( +var ( // DefaultNamespace is the default namespace DefaultNamespace = "docker.io" // DefaultRegistryVersionHeader is the name of the default HTTP header @@ -27,7 +27,7 @@ const ( DefaultRegistryVersionHeader = "Docker-Distribution-Api-Version" // IndexServer is the v1 registry server used for user auth + account creation - IndexServer = DefaultV1Registry + "/v1/" + IndexServer = DefaultV1Registry.String() + "/v1/" // IndexName is the name of the index IndexName = "docker.io" @@ -49,6 +49,9 @@ var ( V2Only = false ) +// for mocking in unit tests +var lookupIP = net.LookupIP + // InstallFlags adds command-line options to the top-level flag parser for // the current process. func (options *Options) InstallFlags(cmd *flag.FlagSet, usageFn func(string) string) { diff --git a/components/engine/registry/config_unix.go b/components/engine/registry/config_unix.go index df970181de..c3c19162f2 100644 --- a/components/engine/registry/config_unix.go +++ b/components/engine/registry/config_unix.go @@ -2,12 +2,22 @@ package registry -const ( +import ( + "net/url" +) + +var ( // DefaultV1Registry is the URI of the default v1 registry - DefaultV1Registry = "https://index.docker.io" + DefaultV1Registry = &url.URL{ + Scheme: "https", + Host: "index.docker.io", + } // DefaultV2Registry is the URI of the default v2 registry - DefaultV2Registry = "https://registry-1.docker.io" + DefaultV2Registry = &url.URL{ + Scheme: "https", + Host: "registry-1.docker.io", + } ) var ( diff --git a/components/engine/registry/config_windows.go b/components/engine/registry/config_windows.go index d01b2618af..f1ee488b1f 100644 --- a/components/engine/registry/config_windows.go +++ b/components/engine/registry/config_windows.go @@ -1,21 +1,28 @@ package registry import ( + "net/url" "os" "path/filepath" "strings" ) -const ( +var ( // DefaultV1Registry is the URI of the default v1 registry - DefaultV1Registry = "https://registry-win-tp3.docker.io" + DefaultV1Registry = &url.URL{ + Scheme: "https", + Host: "registry-win-tp3.docker.io", + } // DefaultV2Registry is the URI of the default (official) v2 registry. // This is the windows-specific endpoint. // // Currently it is a TEMPORARY link that allows Microsoft to continue // development of Docker Engine for Windows. - DefaultV2Registry = "https://registry-win-tp3.docker.io" + DefaultV2Registry = &url.URL{ + Scheme: "https", + Host: "registry-win-tp3.docker.io", + } ) // CertsDir is the directory where certificates are stored diff --git a/components/engine/registry/endpoint.go b/components/engine/registry/endpoint.go deleted file mode 100644 index ef00431f43..0000000000 --- a/components/engine/registry/endpoint.go +++ /dev/null @@ -1,277 +0,0 @@ -package registry - -import ( - "crypto/tls" - "encoding/json" - "fmt" - "io/ioutil" - "net" - "net/http" - "net/url" - "strings" - - "github.com/Sirupsen/logrus" - "github.com/docker/distribution/registry/api/v2" - "github.com/docker/distribution/registry/client/transport" - registrytypes "github.com/docker/engine-api/types/registry" -) - -// for mocking in unit tests -var lookupIP = net.LookupIP - -// scans string for api version in the URL path. returns the trimmed address, if version found, string and API version. -func scanForAPIVersion(address string) (string, APIVersion) { - var ( - chunks []string - apiVersionStr string - ) - - if strings.HasSuffix(address, "/") { - address = address[:len(address)-1] - } - - chunks = strings.Split(address, "/") - apiVersionStr = chunks[len(chunks)-1] - - for k, v := range apiVersions { - if apiVersionStr == v { - address = strings.Join(chunks[:len(chunks)-1], "/") - return address, k - } - } - - return address, APIVersionUnknown -} - -// NewEndpoint parses the given address to return a registry endpoint. v can be used to -// specify a specific endpoint version -func NewEndpoint(index *registrytypes.IndexInfo, userAgent string, metaHeaders http.Header, v APIVersion) (*Endpoint, error) { - tlsConfig, err := newTLSConfig(index.Name, index.Secure) - if err != nil { - return nil, err - } - endpoint, err := newEndpoint(GetAuthConfigKey(index), tlsConfig, userAgent, metaHeaders) - if err != nil { - return nil, err - } - if v != APIVersionUnknown { - endpoint.Version = v - } - if err := validateEndpoint(endpoint); err != nil { - return nil, err - } - - return endpoint, nil -} - -func validateEndpoint(endpoint *Endpoint) error { - logrus.Debugf("pinging registry endpoint %s", endpoint) - - // Try HTTPS ping to registry - endpoint.URL.Scheme = "https" - if _, err := endpoint.Ping(); err != nil { - if endpoint.IsSecure { - // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` - // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. - return fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) - } - - // If registry is insecure and HTTPS failed, fallback to HTTP. - logrus.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) - endpoint.URL.Scheme = "http" - - var err2 error - if _, err2 = endpoint.Ping(); err2 == nil { - return nil - } - - return fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) - } - - return nil -} - -func newEndpoint(address string, tlsConfig *tls.Config, userAgent string, metaHeaders http.Header) (*Endpoint, error) { - var ( - endpoint = new(Endpoint) - trimmedAddress string - err error - ) - - if !strings.HasPrefix(address, "http") { - address = "https://" + address - } - - endpoint.IsSecure = (tlsConfig == nil || !tlsConfig.InsecureSkipVerify) - - trimmedAddress, endpoint.Version = scanForAPIVersion(address) - - if endpoint.URL, err = url.Parse(trimmedAddress); err != nil { - return nil, err - } - - // TODO(tiborvass): make sure a ConnectTimeout transport is used - tr := NewTransport(tlsConfig) - endpoint.client = HTTPClient(transport.NewTransport(tr, DockerHeaders(userAgent, metaHeaders)...)) - return endpoint, nil -} - -// Endpoint stores basic information about a registry endpoint. -type Endpoint struct { - client *http.Client - URL *url.URL - Version APIVersion - IsSecure bool - AuthChallenges []*AuthorizationChallenge - URLBuilder *v2.URLBuilder -} - -// Get the formatted URL for the root of this registry Endpoint -func (e *Endpoint) String() string { - return fmt.Sprintf("%s/v%d/", e.URL, e.Version) -} - -// VersionString returns a formatted string of this -// endpoint address using the given API Version. -func (e *Endpoint) VersionString(version APIVersion) string { - return fmt.Sprintf("%s/v%d/", e.URL, version) -} - -// Path returns a formatted string for the URL -// of this endpoint with the given path appended. -func (e *Endpoint) Path(path string) string { - return fmt.Sprintf("%s/v%d/%s", e.URL, e.Version, path) -} - -// Ping pings the remote endpoint with v2 and v1 pings to determine the API -// version. It returns a PingResult containing the discovered version. The -// PingResult also indicates whether the registry is standalone or not. -func (e *Endpoint) Ping() (PingResult, error) { - // The ping logic to use is determined by the registry endpoint version. - switch e.Version { - case APIVersion1: - return e.pingV1() - case APIVersion2: - return e.pingV2() - } - - // APIVersionUnknown - // We should try v2 first... - e.Version = APIVersion2 - regInfo, errV2 := e.pingV2() - if errV2 == nil { - return regInfo, nil - } - - // ... then fallback to v1. - e.Version = APIVersion1 - regInfo, errV1 := e.pingV1() - if errV1 == nil { - return regInfo, nil - } - - e.Version = APIVersionUnknown - return PingResult{}, fmt.Errorf("unable to ping registry endpoint %s\nv2 ping attempt failed with error: %s\n v1 ping attempt failed with error: %s", e, errV2, errV1) -} - -func (e *Endpoint) pingV1() (PingResult, error) { - logrus.Debugf("attempting v1 ping for registry endpoint %s", e) - - if e.String() == IndexServer { - // Skip the check, we know this one is valid - // (and we never want to fallback to http in case of error) - return PingResult{Standalone: false}, nil - } - - req, err := http.NewRequest("GET", e.Path("_ping"), nil) - if err != nil { - return PingResult{Standalone: false}, err - } - - resp, err := e.client.Do(req) - if err != nil { - return PingResult{Standalone: false}, err - } - - defer resp.Body.Close() - - jsonString, err := ioutil.ReadAll(resp.Body) - if err != nil { - return PingResult{Standalone: false}, fmt.Errorf("error while reading the http response: %s", err) - } - - // If the header is absent, we assume true for compatibility with earlier - // versions of the registry. default to true - info := PingResult{ - Standalone: true, - } - if err := json.Unmarshal(jsonString, &info); err != nil { - logrus.Debugf("Error unmarshalling the _ping PingResult: %s", err) - // don't stop here. Just assume sane defaults - } - if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { - logrus.Debugf("Registry version header: '%s'", hdr) - info.Version = hdr - } - logrus.Debugf("PingResult.Version: %q", info.Version) - - standalone := resp.Header.Get("X-Docker-Registry-Standalone") - logrus.Debugf("Registry standalone header: '%s'", standalone) - // Accepted values are "true" (case-insensitive) and "1". - if strings.EqualFold(standalone, "true") || standalone == "1" { - info.Standalone = true - } else if len(standalone) > 0 { - // there is a header set, and it is not "true" or "1", so assume fails - info.Standalone = false - } - logrus.Debugf("PingResult.Standalone: %t", info.Standalone) - return info, nil -} - -func (e *Endpoint) pingV2() (PingResult, error) { - logrus.Debugf("attempting v2 ping for registry endpoint %s", e) - - req, err := http.NewRequest("GET", e.Path(""), nil) - if err != nil { - return PingResult{}, err - } - - resp, err := e.client.Do(req) - if err != nil { - return PingResult{}, err - } - defer resp.Body.Close() - - // The endpoint may have multiple supported versions. - // Ensure it supports the v2 Registry API. - var supportsV2 bool - -HeaderLoop: - for _, supportedVersions := range resp.Header[http.CanonicalHeaderKey("Docker-Distribution-API-Version")] { - for _, versionName := range strings.Fields(supportedVersions) { - if versionName == "registry/2.0" { - supportsV2 = true - break HeaderLoop - } - } - } - - if !supportsV2 { - return PingResult{}, fmt.Errorf("%s does not appear to be a v2 registry endpoint", e) - } - - if resp.StatusCode == http.StatusOK { - // It would seem that no authentication/authorization is required. - // So we don't need to parse/add any authorization schemes. - return PingResult{Standalone: true}, nil - } - - if resp.StatusCode == http.StatusUnauthorized { - // Parse the WWW-Authenticate Header and store the challenges - // on this endpoint object. - e.AuthChallenges = parseAuthHeader(resp.Header) - return PingResult{}, nil - } - - return PingResult{}, fmt.Errorf("v2 registry endpoint returned status %d: %q", resp.StatusCode, http.StatusText(resp.StatusCode)) -} diff --git a/components/engine/registry/endpoint_test.go b/components/engine/registry/endpoint_test.go index 4677e0c9e5..8451d3f678 100644 --- a/components/engine/registry/endpoint_test.go +++ b/components/engine/registry/endpoint_test.go @@ -14,12 +14,13 @@ func TestEndpointParse(t *testing.T) { }{ {IndexServer, IndexServer}, {"http://0.0.0.0:5000/v1/", "http://0.0.0.0:5000/v1/"}, - {"http://0.0.0.0:5000/v2/", "http://0.0.0.0:5000/v2/"}, - {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v0/"}, - {"0.0.0.0:5000", "https://0.0.0.0:5000/v0/"}, + {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v1/"}, + {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, + {"http://0.0.0.0:5000/nonversion/", "http://0.0.0.0:5000/nonversion/v1/"}, + {"http://0.0.0.0:5000/v0/", "http://0.0.0.0:5000/v0/v1/"}, } for _, td := range testData { - e, err := newEndpoint(td.str, nil, "", nil) + e, err := newV1EndpointFromStr(td.str, nil, "", nil) if err != nil { t.Errorf("%q: %s", td.str, err) } @@ -33,21 +34,26 @@ func TestEndpointParse(t *testing.T) { } } +func TestEndpointParseInvalid(t *testing.T) { + testData := []string{ + "http://0.0.0.0:5000/v2/", + } + for _, td := range testData { + e, err := newV1EndpointFromStr(td, nil, "", nil) + if err == nil { + t.Errorf("expected error parsing %q: parsed as %q", td, e) + } + } +} + // Ensure that a registry endpoint that responds with a 401 only is determined -// to be a v1 registry unless it includes a valid v2 API header. -func TestValidateEndpointAmbiguousAPIVersion(t *testing.T) { +// to be a valid v1 registry endpoint +func TestValidateEndpoint(t *testing.T) { requireBasicAuthHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Add("WWW-Authenticate", `Basic realm="localhost"`) w.WriteHeader(http.StatusUnauthorized) }) - requireBasicAuthHandlerV2 := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // This mock server supports v2.0, v2.1, v42.0, and v100.0 - w.Header().Add("Docker-Distribution-API-Version", "registry/100.0 registry/42.0") - w.Header().Add("Docker-Distribution-API-Version", "registry/2.0 registry/2.1") - requireBasicAuthHandler.ServeHTTP(w, r) - }) - // Make a test server which should validate as a v1 server. testServer := httptest.NewServer(requireBasicAuthHandler) defer testServer.Close() @@ -57,37 +63,16 @@ func TestValidateEndpointAmbiguousAPIVersion(t *testing.T) { t.Fatal(err) } - testEndpoint := Endpoint{ - URL: testServerURL, - Version: APIVersionUnknown, - client: HTTPClient(NewTransport(nil)), + testEndpoint := V1Endpoint{ + URL: testServerURL, + client: HTTPClient(NewTransport(nil)), } if err = validateEndpoint(&testEndpoint); err != nil { t.Fatal(err) } - if testEndpoint.Version != APIVersion1 { - t.Fatalf("expected endpoint to validate to %d, got %d", APIVersion1, testEndpoint.Version) - } - - // Make a test server which should validate as a v2 server. - testServer = httptest.NewServer(requireBasicAuthHandlerV2) - defer testServer.Close() - - testServerURL, err = url.Parse(testServer.URL) - if err != nil { - t.Fatal(err) - } - - testEndpoint.URL = testServerURL - testEndpoint.Version = APIVersionUnknown - - if err = validateEndpoint(&testEndpoint); err != nil { - t.Fatal(err) - } - - if testEndpoint.Version != APIVersion2 { - t.Fatalf("expected endpoint to validate to %d, got %d", APIVersion2, testEndpoint.Version) + if testEndpoint.URL.Scheme != "http" { + t.Fatalf("expecting to validate endpoint as http, got url %s", testEndpoint.String()) } } diff --git a/components/engine/registry/endpoint_v1.go b/components/engine/registry/endpoint_v1.go new file mode 100644 index 0000000000..58e2600ef9 --- /dev/null +++ b/components/engine/registry/endpoint_v1.go @@ -0,0 +1,199 @@ +package registry + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/registry/client/transport" + registrytypes "github.com/docker/engine-api/types/registry" +) + +// V1Endpoint stores basic information about a V1 registry endpoint. +type V1Endpoint struct { + client *http.Client + URL *url.URL + IsSecure bool +} + +// NewV1Endpoint parses the given address to return a registry endpoint. v can be used to +// specify a specific endpoint version +func NewV1Endpoint(index *registrytypes.IndexInfo, userAgent string, metaHeaders http.Header) (*V1Endpoint, error) { + tlsConfig, err := newTLSConfig(index.Name, index.Secure) + if err != nil { + return nil, err + } + + endpoint, err := newV1EndpointFromStr(GetAuthConfigKey(index), tlsConfig, userAgent, metaHeaders) + if err != nil { + return nil, err + } + + if err := validateEndpoint(endpoint); err != nil { + return nil, err + } + + return endpoint, nil +} + +func validateEndpoint(endpoint *V1Endpoint) error { + logrus.Debugf("pinging registry endpoint %s", endpoint) + + // Try HTTPS ping to registry + endpoint.URL.Scheme = "https" + if _, err := endpoint.Ping(); err != nil { + if endpoint.IsSecure { + // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` + // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. + return fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) + } + + // If registry is insecure and HTTPS failed, fallback to HTTP. + logrus.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) + endpoint.URL.Scheme = "http" + + var err2 error + if _, err2 = endpoint.Ping(); err2 == nil { + return nil + } + + return fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) + } + + return nil +} + +func newV1Endpoint(address url.URL, tlsConfig *tls.Config, userAgent string, metaHeaders http.Header) (*V1Endpoint, error) { + endpoint := &V1Endpoint{ + IsSecure: (tlsConfig == nil || !tlsConfig.InsecureSkipVerify), + URL: new(url.URL), + } + + *endpoint.URL = address + + // TODO(tiborvass): make sure a ConnectTimeout transport is used + tr := NewTransport(tlsConfig) + endpoint.client = HTTPClient(transport.NewTransport(tr, DockerHeaders(userAgent, metaHeaders)...)) + return endpoint, nil +} + +// trimV1Address trims the version off the address and returns the +// trimmed address or an error if there is a non-V1 version. +func trimV1Address(address string) (string, error) { + var ( + chunks []string + apiVersionStr string + ) + + if strings.HasSuffix(address, "/") { + address = address[:len(address)-1] + } + + chunks = strings.Split(address, "/") + apiVersionStr = chunks[len(chunks)-1] + if apiVersionStr == "v1" { + return strings.Join(chunks[:len(chunks)-1], "/"), nil + } + + for k, v := range apiVersions { + if k != APIVersion1 && apiVersionStr == v { + return "", fmt.Errorf("unsupported V1 version path %s", apiVersionStr) + } + } + + return address, nil +} + +func newV1EndpointFromStr(address string, tlsConfig *tls.Config, userAgent string, metaHeaders http.Header) (*V1Endpoint, error) { + if !strings.HasPrefix(address, "http://") && !strings.HasPrefix(address, "https://") { + address = "https://" + address + } + + address, err := trimV1Address(address) + if err != nil { + return nil, err + } + + uri, err := url.Parse(address) + if err != nil { + return nil, err + } + + endpoint, err := newV1Endpoint(*uri, tlsConfig, userAgent, metaHeaders) + if err != nil { + return nil, err + } + + return endpoint, nil +} + +// Get the formatted URL for the root of this registry Endpoint +func (e *V1Endpoint) String() string { + return e.URL.String() + "/v1/" +} + +// Path returns a formatted string for the URL +// of this endpoint with the given path appended. +func (e *V1Endpoint) Path(path string) string { + return e.URL.String() + "/v1/" + path +} + +// Ping returns a PingResult which indicates whether the registry is standalone or not. +func (e *V1Endpoint) Ping() (PingResult, error) { + logrus.Debugf("attempting v1 ping for registry endpoint %s", e) + + if e.String() == IndexServer { + // Skip the check, we know this one is valid + // (and we never want to fallback to http in case of error) + return PingResult{Standalone: false}, nil + } + + req, err := http.NewRequest("GET", e.Path("_ping"), nil) + if err != nil { + return PingResult{Standalone: false}, err + } + + resp, err := e.client.Do(req) + if err != nil { + return PingResult{Standalone: false}, err + } + + defer resp.Body.Close() + + jsonString, err := ioutil.ReadAll(resp.Body) + if err != nil { + return PingResult{Standalone: false}, fmt.Errorf("error while reading the http response: %s", err) + } + + // If the header is absent, we assume true for compatibility with earlier + // versions of the registry. default to true + info := PingResult{ + Standalone: true, + } + if err := json.Unmarshal(jsonString, &info); err != nil { + logrus.Debugf("Error unmarshalling the _ping PingResult: %s", err) + // don't stop here. Just assume sane defaults + } + if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { + logrus.Debugf("Registry version header: '%s'", hdr) + info.Version = hdr + } + logrus.Debugf("PingResult.Version: %q", info.Version) + + standalone := resp.Header.Get("X-Docker-Registry-Standalone") + logrus.Debugf("Registry standalone header: '%s'", standalone) + // Accepted values are "true" (case-insensitive) and "1". + if strings.EqualFold(standalone, "true") || standalone == "1" { + info.Standalone = true + } else if len(standalone) > 0 { + // there is a header set, and it is not "true" or "1", so assume fails + info.Standalone = false + } + logrus.Debugf("PingResult.Standalone: %t", info.Standalone) + return info, nil +} diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 6214d41af3..9071d9dc14 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -13,13 +13,9 @@ import ( "path/filepath" "runtime" "strings" - "syscall" "time" "github.com/Sirupsen/logrus" - "github.com/docker/distribution/registry/api/errcode" - "github.com/docker/distribution/registry/api/v2" - "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client/transport" "github.com/docker/go-connections/tlsconfig" ) @@ -169,51 +165,6 @@ func addRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque return nil } -// ShouldV2Fallback returns true if this error is a reason to fall back to v1. -func ShouldV2Fallback(err errcode.Error) bool { - switch err.Code { - case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown, v2.ErrorCodeNameUnknown: - return true - } - return false -} - -// ErrNoSupport is an error type used for errors indicating that an operation -// is not supported. It encapsulates a more specific error. -type ErrNoSupport struct{ Err error } - -func (e ErrNoSupport) Error() string { - if e.Err == nil { - return "not supported" - } - return e.Err.Error() -} - -// ContinueOnError returns true if we should fallback to the next endpoint -// as a result of this error. -func ContinueOnError(err error) bool { - switch v := err.(type) { - case errcode.Errors: - if len(v) == 0 { - return true - } - return ContinueOnError(v[0]) - case ErrNoSupport: - return ContinueOnError(v.Err) - case errcode.Error: - return ShouldV2Fallback(v) - case *client.UnexpectedHTTPResponseError: - return true - case error: - return !strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())) - } - // let's be nice and fallback if the error is a completely - // unexpected one. - // If new errors have to be handled in some way, please - // add them to the switch above. - return true -} - // NewTransport returns a new HTTP transport. If tlsConfig is nil, it uses the // default TLS configuration. func NewTransport(tlsConfig *tls.Config) *http.Transport { diff --git a/components/engine/registry/registry_test.go b/components/engine/registry/registry_test.go index 98a3aa1c8b..02eb683d05 100644 --- a/components/engine/registry/registry_test.go +++ b/components/engine/registry/registry_test.go @@ -25,7 +25,7 @@ const ( func spawnTestRegistrySession(t *testing.T) *Session { authConfig := &types.AuthConfig{} - endpoint, err := NewEndpoint(makeIndex("/v1/"), "", nil, APIVersionUnknown) + endpoint, err := NewV1Endpoint(makeIndex("/v1/"), "", nil) if err != nil { t.Fatal(err) } @@ -53,7 +53,7 @@ func spawnTestRegistrySession(t *testing.T) *Session { func TestPingRegistryEndpoint(t *testing.T) { testPing := func(index *registrytypes.IndexInfo, expectedStandalone bool, assertMessage string) { - ep, err := NewEndpoint(index, "", nil, APIVersionUnknown) + ep, err := NewV1Endpoint(index, "", nil) if err != nil { t.Fatal(err) } @@ -72,8 +72,8 @@ func TestPingRegistryEndpoint(t *testing.T) { func TestEndpoint(t *testing.T) { // Simple wrapper to fail test if err != nil - expandEndpoint := func(index *registrytypes.IndexInfo) *Endpoint { - endpoint, err := NewEndpoint(index, "", nil, APIVersionUnknown) + expandEndpoint := func(index *registrytypes.IndexInfo) *V1Endpoint { + endpoint, err := NewV1Endpoint(index, "", nil) if err != nil { t.Fatal(err) } @@ -82,7 +82,7 @@ func TestEndpoint(t *testing.T) { assertInsecureIndex := func(index *registrytypes.IndexInfo) { index.Secure = true - _, err := NewEndpoint(index, "", nil, APIVersionUnknown) + _, err := NewV1Endpoint(index, "", nil) assertNotEqual(t, err, nil, index.Name+": Expected error for insecure index") assertEqual(t, strings.Contains(err.Error(), "insecure-registry"), true, index.Name+": Expected insecure-registry error for insecure index") index.Secure = false @@ -90,7 +90,7 @@ func TestEndpoint(t *testing.T) { assertSecureIndex := func(index *registrytypes.IndexInfo) { index.Secure = true - _, err := NewEndpoint(index, "", nil, APIVersionUnknown) + _, err := NewV1Endpoint(index, "", nil) assertNotEqual(t, err, nil, index.Name+": Expected cert error for secure index") assertEqual(t, strings.Contains(err.Error(), "certificate signed by unknown authority"), true, index.Name+": Expected cert error for secure index") index.Secure = false @@ -100,51 +100,33 @@ func TestEndpoint(t *testing.T) { index.Name = makeURL("/v1/") endpoint := expandEndpoint(index) assertEqual(t, endpoint.String(), index.Name, "Expected endpoint to be "+index.Name) - if endpoint.Version != APIVersion1 { - t.Fatal("Expected endpoint to be v1") - } assertInsecureIndex(index) index.Name = makeURL("") endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), index.Name+"/v1/", index.Name+": Expected endpoint to be "+index.Name+"/v1/") - if endpoint.Version != APIVersion1 { - t.Fatal("Expected endpoint to be v1") - } assertInsecureIndex(index) httpURL := makeURL("") index.Name = strings.SplitN(httpURL, "://", 2)[1] endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), httpURL+"/v1/", index.Name+": Expected endpoint to be "+httpURL+"/v1/") - if endpoint.Version != APIVersion1 { - t.Fatal("Expected endpoint to be v1") - } assertInsecureIndex(index) index.Name = makeHTTPSURL("/v1/") endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), index.Name, "Expected endpoint to be "+index.Name) - if endpoint.Version != APIVersion1 { - t.Fatal("Expected endpoint to be v1") - } assertSecureIndex(index) index.Name = makeHTTPSURL("") endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), index.Name+"/v1/", index.Name+": Expected endpoint to be "+index.Name+"/v1/") - if endpoint.Version != APIVersion1 { - t.Fatal("Expected endpoint to be v1") - } assertSecureIndex(index) httpsURL := makeHTTPSURL("") index.Name = strings.SplitN(httpsURL, "://", 2)[1] endpoint = expandEndpoint(index) assertEqual(t, endpoint.String(), httpsURL+"/v1/", index.Name+": Expected endpoint to be "+httpsURL+"/v1/") - if endpoint.Version != APIVersion1 { - t.Fatal("Expected endpoint to be v1") - } assertSecureIndex(index) badEndpoints := []string{ @@ -156,7 +138,7 @@ func TestEndpoint(t *testing.T) { } for _, address := range badEndpoints { index.Name = address - _, err := NewEndpoint(index, "", nil, APIVersionUnknown) + _, err := NewV1Endpoint(index, "", nil) checkNotEqual(t, err, nil, "Expected error while expanding bad endpoint") } } @@ -673,7 +655,7 @@ func TestNewIndexInfo(t *testing.T) { func TestMirrorEndpointLookup(t *testing.T) { containsMirror := func(endpoints []APIEndpoint) bool { for _, pe := range endpoints { - if pe.URL == "my.mirror" { + if pe.URL.Host == "my.mirror" { return true } } @@ -685,7 +667,7 @@ func TestMirrorEndpointLookup(t *testing.T) { if err != nil { t.Error(err) } - pushAPIEndpoints, err := s.LookupPushEndpoints(imageName) + pushAPIEndpoints, err := s.LookupPushEndpoints(imageName.Hostname()) if err != nil { t.Fatal(err) } @@ -693,7 +675,7 @@ func TestMirrorEndpointLookup(t *testing.T) { t.Fatal("Push endpoint should not contain mirror") } - pullAPIEndpoints, err := s.LookupPullEndpoints(imageName) + pullAPIEndpoints, err := s.LookupPullEndpoints(imageName.Hostname()) if err != nil { t.Fatal(err) } diff --git a/components/engine/registry/service.go b/components/engine/registry/service.go index 861cdb4645..d9ea71b372 100644 --- a/components/engine/registry/service.go +++ b/components/engine/registry/service.go @@ -6,6 +6,7 @@ import ( "net/url" "strings" + "github.com/Sirupsen/logrus" "github.com/docker/docker/reference" "github.com/docker/engine-api/types" registrytypes "github.com/docker/engine-api/types/registry" @@ -28,29 +29,31 @@ func NewService(options *Options) *Service { // Auth contacts the public registry with the provided credentials, // and returns OK if authentication was successful. // It can be used to verify the validity of a client's credentials. -func (s *Service) Auth(authConfig *types.AuthConfig, userAgent string) (string, error) { - addr := authConfig.ServerAddress - if addr == "" { - // Use the official registry address if not specified. - addr = IndexServer - } - index, err := s.ResolveIndex(addr) +func (s *Service) Auth(authConfig *types.AuthConfig, userAgent string) (status string, err error) { + endpoints, err := s.LookupPushEndpoints(authConfig.ServerAddress) if err != nil { return "", err } - endpointVersion := APIVersion(APIVersionUnknown) - if V2Only { - // Override the endpoint to only attempt a v2 ping - endpointVersion = APIVersion2 - } + for _, endpoint := range endpoints { + login := loginV2 + if endpoint.Version == APIVersion1 { + login = loginV1 + } - endpoint, err := NewEndpoint(index, userAgent, nil, endpointVersion) - if err != nil { + status, err = login(authConfig, endpoint, userAgent) + if err == nil { + return + } + if fErr, ok := err.(fallbackError); ok { + err = fErr.err + logrus.Infof("Error logging in to %s endpoint, trying next endpoint: %v", endpoint.Version, err) + continue + } return "", err } - authConfig.ServerAddress = endpoint.String() - return Login(authConfig, endpoint) + + return "", err } // splitReposSearchTerm breaks a search term into an index name and remote name @@ -85,7 +88,7 @@ func (s *Service) Search(term string, authConfig *types.AuthConfig, userAgent st } // *TODO: Search multiple indexes. - endpoint, err := NewEndpoint(index, userAgent, http.Header(headers), APIVersionUnknown) + endpoint, err := NewV1Endpoint(index, userAgent, http.Header(headers)) if err != nil { return nil, err } @@ -121,7 +124,7 @@ func (s *Service) ResolveIndex(name string) (*registrytypes.IndexInfo, error) { // APIEndpoint represents a remote API endpoint type APIEndpoint struct { Mirror bool - URL string + URL *url.URL Version APIVersion Official bool TrimHostname bool @@ -129,8 +132,8 @@ type APIEndpoint struct { } // ToV1Endpoint returns a V1 API endpoint based on the APIEndpoint -func (e APIEndpoint) ToV1Endpoint(userAgent string, metaHeaders http.Header) (*Endpoint, error) { - return newEndpoint(e.URL, e.TLSConfig, userAgent, metaHeaders) +func (e APIEndpoint) ToV1Endpoint(userAgent string, metaHeaders http.Header) (*V1Endpoint, error) { + return newV1Endpoint(*e.URL, e.TLSConfig, userAgent, metaHeaders) } // TLSConfig constructs a client TLS configuration based on server defaults @@ -138,26 +141,22 @@ func (s *Service) TLSConfig(hostname string) (*tls.Config, error) { return newTLSConfig(hostname, isSecureIndex(s.Config, hostname)) } -func (s *Service) tlsConfigForMirror(mirror string) (*tls.Config, error) { - mirrorURL, err := url.Parse(mirror) - if err != nil { - return nil, err - } +func (s *Service) tlsConfigForMirror(mirrorURL *url.URL) (*tls.Config, error) { return s.TLSConfig(mirrorURL.Host) } -// LookupPullEndpoints creates an list of endpoints to try to pull from, in order of preference. +// LookupPullEndpoints creates a list of endpoints to try to pull from, in order of preference. // It gives preference to v2 endpoints over v1, mirrors over the actual // registry, and HTTPS over plain HTTP. -func (s *Service) LookupPullEndpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { - return s.lookupEndpoints(repoName) +func (s *Service) LookupPullEndpoints(hostname string) (endpoints []APIEndpoint, err error) { + return s.lookupEndpoints(hostname) } -// LookupPushEndpoints creates an list of endpoints to try to push to, in order of preference. +// LookupPushEndpoints creates a list of endpoints to try to push to, in order of preference. // It gives preference to v2 endpoints over v1, and HTTPS over plain HTTP. // Mirrors are not included. -func (s *Service) LookupPushEndpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { - allEndpoints, err := s.lookupEndpoints(repoName) +func (s *Service) LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error) { + allEndpoints, err := s.lookupEndpoints(hostname) if err == nil { for _, endpoint := range allEndpoints { if !endpoint.Mirror { @@ -168,8 +167,8 @@ func (s *Service) LookupPushEndpoints(repoName reference.Named) (endpoints []API return endpoints, err } -func (s *Service) lookupEndpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { - endpoints, err = s.lookupV2Endpoints(repoName) +func (s *Service) lookupEndpoints(hostname string) (endpoints []APIEndpoint, err error) { + endpoints, err = s.lookupV2Endpoints(hostname) if err != nil { return nil, err } @@ -178,7 +177,7 @@ func (s *Service) lookupEndpoints(repoName reference.Named) (endpoints []APIEndp return endpoints, nil } - legacyEndpoints, err := s.lookupV1Endpoints(repoName) + legacyEndpoints, err := s.lookupV1Endpoints(hostname) if err != nil { return nil, err } diff --git a/components/engine/registry/service_v1.go b/components/engine/registry/service_v1.go index 340ce9576a..56121eea4b 100644 --- a/components/engine/registry/service_v1.go +++ b/components/engine/registry/service_v1.go @@ -1,18 +1,15 @@ package registry import ( - "fmt" - "strings" + "net/url" - "github.com/docker/docker/reference" "github.com/docker/go-connections/tlsconfig" ) -func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { +func (s *Service) lookupV1Endpoints(hostname string) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg - nameString := repoName.FullName() - if strings.HasPrefix(nameString, DefaultNamespace+"/") { + if hostname == DefaultNamespace { endpoints = append(endpoints, APIEndpoint{ URL: DefaultV1Registry, Version: APIVersion1, @@ -23,12 +20,6 @@ func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEn return endpoints, nil } - slashIndex := strings.IndexRune(nameString, '/') - if slashIndex <= 0 { - return nil, fmt.Errorf("invalid repo name: missing '/': %s", nameString) - } - hostname := nameString[:slashIndex] - tlsConfig, err = s.TLSConfig(hostname) if err != nil { return nil, err @@ -36,7 +27,10 @@ func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEn endpoints = []APIEndpoint{ { - URL: "https://" + hostname, + URL: &url.URL{ + Scheme: "https", + Host: hostname, + }, Version: APIVersion1, TrimHostname: true, TLSConfig: tlsConfig, @@ -45,7 +39,10 @@ func (s *Service) lookupV1Endpoints(repoName reference.Named) (endpoints []APIEn if tlsConfig.InsecureSkipVerify { endpoints = append(endpoints, APIEndpoint{ // or this - URL: "http://" + hostname, + URL: &url.URL{ + Scheme: "http", + Host: hostname, + }, Version: APIVersion1, TrimHostname: true, // used to check if supposed to be secure via InsecureSkipVerify diff --git a/components/engine/registry/service_v2.go b/components/engine/registry/service_v2.go index f89326d515..9c909f186e 100644 --- a/components/engine/registry/service_v2.go +++ b/components/engine/registry/service_v2.go @@ -1,26 +1,31 @@ package registry import ( - "fmt" + "net/url" "strings" - "github.com/docker/docker/reference" "github.com/docker/go-connections/tlsconfig" ) -func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEndpoint, err error) { +func (s *Service) lookupV2Endpoints(hostname string) (endpoints []APIEndpoint, err error) { var cfg = tlsconfig.ServerDefault tlsConfig := &cfg - nameString := repoName.FullName() - if strings.HasPrefix(nameString, DefaultNamespace+"/") { + if hostname == DefaultNamespace { // v2 mirrors for _, mirror := range s.Config.Mirrors { - mirrorTLSConfig, err := s.tlsConfigForMirror(mirror) + if !strings.HasPrefix(mirror, "http://") && !strings.HasPrefix(mirror, "https://") { + mirror = "https://" + mirror + } + mirrorURL, err := url.Parse(mirror) + if err != nil { + return nil, err + } + mirrorTLSConfig, err := s.tlsConfigForMirror(mirrorURL) if err != nil { return nil, err } endpoints = append(endpoints, APIEndpoint{ - URL: mirror, + URL: mirrorURL, // guess mirrors are v2 Version: APIVersion2, Mirror: true, @@ -40,12 +45,6 @@ func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEn return endpoints, nil } - slashIndex := strings.IndexRune(nameString, '/') - if slashIndex <= 0 { - return nil, fmt.Errorf("invalid repo name: missing '/': %s", nameString) - } - hostname := nameString[:slashIndex] - tlsConfig, err = s.TLSConfig(hostname) if err != nil { return nil, err @@ -53,7 +52,10 @@ func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEn endpoints = []APIEndpoint{ { - URL: "https://" + hostname, + URL: &url.URL{ + Scheme: "https", + Host: hostname, + }, Version: APIVersion2, TrimHostname: true, TLSConfig: tlsConfig, @@ -62,7 +64,10 @@ func (s *Service) lookupV2Endpoints(repoName reference.Named) (endpoints []APIEn if tlsConfig.InsecureSkipVerify { endpoints = append(endpoints, APIEndpoint{ - URL: "http://" + hostname, + URL: &url.URL{ + Scheme: "http", + Host: hostname, + }, Version: APIVersion2, TrimHostname: true, // used to check if supposed to be secure via InsecureSkipVerify diff --git a/components/engine/registry/session.go b/components/engine/registry/session.go index 4b18d0d1a1..bd0dfb2cbd 100644 --- a/components/engine/registry/session.go +++ b/components/engine/registry/session.go @@ -37,7 +37,7 @@ var ( // A Session is used to communicate with a V1 registry type Session struct { - indexEndpoint *Endpoint + indexEndpoint *V1Endpoint client *http.Client // TODO(tiborvass): remove authConfig authConfig *types.AuthConfig @@ -163,7 +163,7 @@ func (tr *authTransport) CancelRequest(req *http.Request) { // NewSession creates a new session // TODO(tiborvass): remove authConfig param once registry client v2 is vendored -func NewSession(client *http.Client, authConfig *types.AuthConfig, endpoint *Endpoint) (r *Session, err error) { +func NewSession(client *http.Client, authConfig *types.AuthConfig, endpoint *V1Endpoint) (r *Session, err error) { r = &Session{ authConfig: authConfig, client: client, @@ -175,7 +175,7 @@ func NewSession(client *http.Client, authConfig *types.AuthConfig, endpoint *End // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside all our requests. - if endpoint.VersionString(1) != IndexServer && endpoint.URL.Scheme == "https" { + if endpoint.String() != IndexServer && endpoint.URL.Scheme == "https" { info, err := endpoint.Ping() if err != nil { return nil, err @@ -405,7 +405,7 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) { // GetRepositoryData returns lists of images and endpoints for the repository func (r *Session) GetRepositoryData(name reference.Named) (*RepositoryData, error) { - repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), name.RemoteName()) + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.String(), name.RemoteName()) logrus.Debugf("[registry] Calling GET %s", repositoryTarget) @@ -444,7 +444,7 @@ func (r *Session) GetRepositoryData(name reference.Named) (*RepositoryData, erro var endpoints []string if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String()) if err != nil { return nil, err } @@ -634,7 +634,7 @@ func (r *Session) PushImageJSONIndex(remote reference.Named, imgList []*ImgData, if validate { suffix = "images" } - u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote.RemoteName(), suffix) + u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.String(), remote.RemoteName(), suffix) logrus.Debugf("[registry] PUT %s", u) logrus.Debugf("Image list pushed to index:\n%s", imgListJSON) headers := map[string][]string{ @@ -680,7 +680,7 @@ func (r *Session) PushImageJSONIndex(remote reference.Named, imgList []*ImgData, if res.Header.Get("X-Docker-Endpoints") == "" { return nil, fmt.Errorf("Index response didn't contain any endpoints") } - endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1)) + endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String()) if err != nil { return nil, err } @@ -722,7 +722,7 @@ func shouldRedirect(response *http.Response) bool { // SearchRepositories performs a search against the remote repository func (r *Session) SearchRepositories(term string) (*registrytypes.SearchResults, error) { logrus.Debugf("Index server: %s", r.indexEndpoint) - u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term) + u := r.indexEndpoint.String() + "search?q=" + url.QueryEscape(term) req, err := http.NewRequest("GET", u, nil) if err != nil { @@ -752,7 +752,6 @@ func (r *Session) GetAuthConfig(withPasswd bool) *types.AuthConfig { return &types.AuthConfig{ Username: r.authConfig.Username, Password: password, - Email: r.authConfig.Email, } } diff --git a/components/engine/registry/token.go b/components/engine/registry/token.go deleted file mode 100644 index d91bd45505..0000000000 --- a/components/engine/registry/token.go +++ /dev/null @@ -1,81 +0,0 @@ -package registry - -import ( - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "strings" -) - -type tokenResponse struct { - Token string `json:"token"` -} - -func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint) (string, error) { - realm, ok := params["realm"] - if !ok { - return "", errors.New("no realm specified for token auth challenge") - } - - realmURL, err := url.Parse(realm) - if err != nil { - return "", fmt.Errorf("invalid token auth challenge realm: %s", err) - } - - if realmURL.Scheme == "" { - if registryEndpoint.IsSecure { - realmURL.Scheme = "https" - } else { - realmURL.Scheme = "http" - } - } - - req, err := http.NewRequest("GET", realmURL.String(), nil) - if err != nil { - return "", err - } - - reqParams := req.URL.Query() - service := params["service"] - scope := params["scope"] - - if service != "" { - reqParams.Add("service", service) - } - - for _, scopeField := range strings.Fields(scope) { - reqParams.Add("scope", scopeField) - } - - if username != "" { - reqParams.Add("account", username) - req.SetBasicAuth(username, password) - } - - req.URL.RawQuery = reqParams.Encode() - - resp, err := registryEndpoint.client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("token auth attempt for registry %s: %s request failed with status: %d %s", registryEndpoint, req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) - } - - decoder := json.NewDecoder(resp.Body) - - tr := new(tokenResponse) - if err = decoder.Decode(tr); err != nil { - return "", fmt.Errorf("unable to decode token response: %s", err) - } - - if tr.Token == "" { - return "", errors.New("authorization server did not include a token in the response") - } - - return tr.Token, nil -} diff --git a/components/engine/registry/types.go b/components/engine/registry/types.go index ee88276e4e..4247fed6fe 100644 --- a/components/engine/registry/types.go +++ b/components/engine/registry/types.go @@ -46,18 +46,18 @@ func (av APIVersion) String() string { return apiVersions[av] } -var apiVersions = map[APIVersion]string{ - 1: "v1", - 2: "v2", -} - // API Version identifiers. const ( - APIVersionUnknown = iota - APIVersion1 + _ = iota + APIVersion1 APIVersion = iota APIVersion2 ) +var apiVersions = map[APIVersion]string{ + APIVersion1: "v1", + APIVersion2: "v2", +} + // RepositoryInfo describes a repository type RepositoryInfo struct { reference.Named diff --git a/components/engine/runconfig/compare.go b/components/engine/runconfig/compare.go index e774ba2294..61346aabf4 100644 --- a/components/engine/runconfig/compare.go +++ b/components/engine/runconfig/compare.go @@ -17,19 +17,17 @@ func Compare(a, b *container.Config) bool { return false } - if a.Cmd.Len() != b.Cmd.Len() || + if len(a.Cmd) != len(b.Cmd) || len(a.Env) != len(b.Env) || len(a.Labels) != len(b.Labels) || len(a.ExposedPorts) != len(b.ExposedPorts) || - a.Entrypoint.Len() != b.Entrypoint.Len() || + len(a.Entrypoint) != len(b.Entrypoint) || len(a.Volumes) != len(b.Volumes) { return false } - aCmd := a.Cmd.Slice() - bCmd := b.Cmd.Slice() - for i := 0; i < len(aCmd); i++ { - if aCmd[i] != bCmd[i] { + for i := 0; i < len(a.Cmd); i++ { + if a.Cmd[i] != b.Cmd[i] { return false } } @@ -49,10 +47,8 @@ func Compare(a, b *container.Config) bool { } } - aEntrypoint := a.Entrypoint.Slice() - bEntrypoint := b.Entrypoint.Slice() - for i := 0; i < len(aEntrypoint); i++ { - if aEntrypoint[i] != bEntrypoint[i] { + for i := 0; i < len(a.Entrypoint); i++ { + if a.Entrypoint[i] != b.Entrypoint[i] { return false } } diff --git a/components/engine/runconfig/compare_test.go b/components/engine/runconfig/compare_test.go index e67f659ef9..9c17c553f3 100644 --- a/components/engine/runconfig/compare_test.go +++ b/components/engine/runconfig/compare_test.go @@ -34,12 +34,12 @@ func TestCompare(t *testing.T) { volumes3["/test3"] = struct{}{} envs1 := []string{"ENV1=value1", "ENV2=value2"} envs2 := []string{"ENV1=value1", "ENV3=value3"} - entrypoint1 := strslice.New("/bin/sh", "-c") - entrypoint2 := strslice.New("/bin/sh", "-d") - entrypoint3 := strslice.New("/bin/sh", "-c", "echo") - cmd1 := strslice.New("/bin/sh", "-c") - cmd2 := strslice.New("/bin/sh", "-d") - cmd3 := strslice.New("/bin/sh", "-c", "echo") + entrypoint1 := strslice.StrSlice{"/bin/sh", "-c"} + entrypoint2 := strslice.StrSlice{"/bin/sh", "-d"} + entrypoint3 := strslice.StrSlice{"/bin/sh", "-c", "echo"} + cmd1 := strslice.StrSlice{"/bin/sh", "-c"} + cmd2 := strslice.StrSlice{"/bin/sh", "-d"} + cmd3 := strslice.StrSlice{"/bin/sh", "-c", "echo"} labels1 := map[string]string{"LABEL1": "value1", "LABEL2": "value2"} labels2 := map[string]string{"LABEL1": "value1", "LABEL2": "value3"} labels3 := map[string]string{"LABEL1": "value1", "LABEL2": "value2", "LABEL3": "value3"} diff --git a/components/engine/runconfig/config.go b/components/engine/runconfig/config.go index f62b471bce..9f3f9c5e63 100644 --- a/components/engine/runconfig/config.go +++ b/components/engine/runconfig/config.go @@ -44,8 +44,8 @@ func DecodeContainerConfig(src io.Reader) (*container.Config, *container.HostCon return nil, nil, nil, err } - // Validate the isolation level - if err := ValidateIsolationLevel(hc); err != nil { + // Validate isolation + if err := ValidateIsolation(hc); err != nil { return nil, nil, nil, err } return w.Config, hc, w.NetworkingConfig, nil diff --git a/components/engine/runconfig/config_test.go b/components/engine/runconfig/config_test.go index b36d027bea..5804b12d0e 100644 --- a/components/engine/runconfig/config_test.go +++ b/components/engine/runconfig/config_test.go @@ -16,7 +16,7 @@ import ( type f struct { file string - entrypoint *strslice.StrSlice + entrypoint strslice.StrSlice } func TestDecodeContainerConfig(t *testing.T) { @@ -29,14 +29,14 @@ func TestDecodeContainerConfig(t *testing.T) { if runtime.GOOS != "windows" { image = "ubuntu" fixtures = []f{ - {"fixtures/unix/container_config_1_14.json", strslice.New()}, - {"fixtures/unix/container_config_1_17.json", strslice.New("bash")}, - {"fixtures/unix/container_config_1_19.json", strslice.New("bash")}, + {"fixtures/unix/container_config_1_14.json", strslice.StrSlice{}}, + {"fixtures/unix/container_config_1_17.json", strslice.StrSlice{"bash"}}, + {"fixtures/unix/container_config_1_19.json", strslice.StrSlice{"bash"}}, } } else { image = "windows" fixtures = []f{ - {"fixtures/windows/container_config_1_19.json", strslice.New("cmd")}, + {"fixtures/windows/container_config_1_19.json", strslice.StrSlice{"cmd"}}, } } @@ -55,7 +55,7 @@ func TestDecodeContainerConfig(t *testing.T) { t.Fatalf("Expected %s image, found %s\n", image, c.Image) } - if c.Entrypoint.Len() != f.entrypoint.Len() { + if len(c.Entrypoint) != len(f.entrypoint) { t.Fatalf("Expected %v, found %v\n", f.entrypoint, c.Entrypoint) } @@ -65,7 +65,7 @@ func TestDecodeContainerConfig(t *testing.T) { } } -// TestDecodeContainerConfigIsolation validates the isolation level passed +// TestDecodeContainerConfigIsolation validates isolation passed // to the daemon in the hostConfig structure. Note this is platform specific // as to what level of container isolation is supported. func TestDecodeContainerConfigIsolation(t *testing.T) { @@ -77,17 +77,30 @@ func TestDecodeContainerConfigIsolation(t *testing.T) { } } - // Blank isolation level (== default) + // Blank isolation (== default) if _, _, _, err := callDecodeContainerConfigIsolation(""); err != nil { t.Fatal("Blank isolation should have succeeded") } - // Default isolation level + // Default isolation if _, _, _, err := callDecodeContainerConfigIsolation("default"); err != nil { t.Fatal("default isolation should have succeeded") } - // Hyper-V Containers isolation level (Valid on Windows only) + // Process isolation (Valid on Windows only) + if runtime.GOOS == "windows" { + if _, _, _, err := callDecodeContainerConfigIsolation("process"); err != nil { + t.Fatal("process isolation should have succeeded") + } + } else { + if _, _, _, err := callDecodeContainerConfigIsolation("process"); err != nil { + if !strings.Contains(err.Error(), `invalid --isolation: "process"`) { + t.Fatal(err) + } + } + } + + // Hyper-V Containers isolation (Valid on Windows only) if runtime.GOOS == "windows" { if _, _, _, err := callDecodeContainerConfigIsolation("hyperv"); err != nil { t.Fatal("hyperv isolation should have succeeded") @@ -102,7 +115,7 @@ func TestDecodeContainerConfigIsolation(t *testing.T) { } // callDecodeContainerConfigIsolation is a utility function to call -// DecodeContainerConfig for validating isolation levels +// DecodeContainerConfig for validating isolation func callDecodeContainerConfigIsolation(isolation string) (*container.Config, *container.HostConfig, *networktypes.NetworkingConfig, error) { var ( b []byte @@ -112,7 +125,7 @@ func callDecodeContainerConfigIsolation(isolation string) (*container.Config, *c Config: &container.Config{}, HostConfig: &container.HostConfig{ NetworkMode: "none", - Isolation: container.IsolationLevel(isolation)}, + Isolation: container.Isolation(isolation)}, } if b, err = json.Marshal(w); err != nil { return nil, nil, nil, fmt.Errorf("Error on marshal %s", err.Error()) diff --git a/components/engine/runconfig/config_unix.go b/components/engine/runconfig/config_unix.go index 16a8d94d4a..e5902fb024 100644 --- a/components/engine/runconfig/config_unix.go +++ b/components/engine/runconfig/config_unix.go @@ -7,7 +7,7 @@ import ( networktypes "github.com/docker/engine-api/types/network" ) -// ContainerConfigWrapper is a Config wrapper that hold the container Config (portable) +// ContainerConfigWrapper is a Config wrapper that holds the container Config (portable) // and the corresponding HostConfig (non-portable). type ContainerConfigWrapper struct { *container.Config diff --git a/components/engine/runconfig/config_windows.go b/components/engine/runconfig/config_windows.go index 08d9b023ec..50a5238000 100644 --- a/components/engine/runconfig/config_windows.go +++ b/components/engine/runconfig/config_windows.go @@ -5,7 +5,7 @@ import ( networktypes "github.com/docker/engine-api/types/network" ) -// ContainerConfigWrapper is a Config wrapper that hold the container Config (portable) +// ContainerConfigWrapper is a Config wrapper that holds the container Config (portable) // and the corresponding HostConfig (non-portable). type ContainerConfigWrapper struct { *container.Config diff --git a/components/engine/runconfig/hostconfig_test.go b/components/engine/runconfig/hostconfig_test.go index ef82a7143f..f8d266cf0a 100644 --- a/components/engine/runconfig/hostconfig_test.go +++ b/components/engine/runconfig/hostconfig_test.go @@ -190,11 +190,11 @@ func TestDecodeHostConfig(t *testing.T) { t.Fatalf("Expected 1 bind, found %d\n", l) } - if c.CapAdd.Len() != 1 && c.CapAdd.Slice()[0] != "NET_ADMIN" { + if len(c.CapAdd) != 1 && c.CapAdd[0] != "NET_ADMIN" { t.Fatalf("Expected CapAdd NET_ADMIN, got %v", c.CapAdd) } - if c.CapDrop.Len() != 1 && c.CapDrop.Slice()[0] != "NET_ADMIN" { + if len(c.CapDrop) != 1 && c.CapDrop[0] != "NET_ADMIN" { t.Fatalf("Expected CapDrop MKNOD, got %v", c.CapDrop) } } diff --git a/components/engine/runconfig/hostconfig_unix.go b/components/engine/runconfig/hostconfig_unix.go index 28d209b694..1f01e486fa 100644 --- a/components/engine/runconfig/hostconfig_unix.go +++ b/components/engine/runconfig/hostconfig_unix.go @@ -70,10 +70,10 @@ func ValidateNetMode(c *container.Config, hc *container.HostConfig) error { return nil } -// ValidateIsolationLevel performs platform specific validation of the -// isolation level in the hostconfig structure. Linux only supports "default" +// ValidateIsolation performs platform specific validation of +// isolation in the hostconfig structure. Linux only supports "default" // which is LXC container isolation -func ValidateIsolationLevel(hc *container.HostConfig) error { +func ValidateIsolation(hc *container.HostConfig) error { // We may not be passed a host config, such as in the case of docker commit if hc == nil { return nil diff --git a/components/engine/runconfig/hostconfig_windows.go b/components/engine/runconfig/hostconfig_windows.go index 56aa171819..ea36666b89 100644 --- a/components/engine/runconfig/hostconfig_windows.go +++ b/components/engine/runconfig/hostconfig_windows.go @@ -34,10 +34,10 @@ func ValidateNetMode(c *container.Config, hc *container.HostConfig) error { return nil } -// ValidateIsolationLevel performs platform specific validation of the -// isolation level in the hostconfig structure. Windows supports 'default' (or +// ValidateIsolation performs platform specific validation of the +// isolation in the hostconfig structure. Windows supports 'default' (or // blank), 'process', or 'hyperv'. -func ValidateIsolationLevel(hc *container.HostConfig) error { +func ValidateIsolation(hc *container.HostConfig) error { // We may not be passed a host config, such as in the case of docker commit if hc == nil { return nil diff --git a/components/engine/runconfig/opts/parse.go b/components/engine/runconfig/opts/parse.go index c3683b5911..74d6ce993f 100644 --- a/components/engine/runconfig/opts/parse.go +++ b/components/engine/runconfig/opts/parse.go @@ -85,13 +85,14 @@ func Parse(cmd *flag.FlagSet, args []string) (*container.Config, *container.Host flIPv4Address = cmd.String([]string{"-ip"}, "", "Container IPv4 address (e.g. 172.30.100.104)") flIPv6Address = cmd.String([]string{"-ip6"}, "", "Container IPv6 address (e.g. 2001:db8::33)") flIpcMode = cmd.String([]string{"-ipc"}, "", "IPC namespace to use") + flPidsLimit = cmd.Int64([]string{"-pids-limit"}, 0, "Tune container pids limit (set -1 for unlimited)") flRestartPolicy = cmd.String([]string{"-restart"}, "no", "Restart policy to apply when a container exits") flReadonlyRootfs = cmd.Bool([]string{"-read-only"}, false, "Mount the container's root filesystem as read only") flLoggingDriver = cmd.String([]string{"-log-driver"}, "", "Logging driver for container") flCgroupParent = cmd.String([]string{"-cgroup-parent"}, "", "Optional parent cgroup for the container") flVolumeDriver = cmd.String([]string{"-volume-driver"}, "", "Optional volume driver for the container") flStopSignal = cmd.String([]string{"-stop-signal"}, signal.DefaultStopSignal, fmt.Sprintf("Signal to stop a container, %v by default", signal.DefaultStopSignal)) - flIsolation = cmd.String([]string{"-isolation"}, "", "Container isolation level") + flIsolation = cmd.String([]string{"-isolation"}, "", "Container isolation technology") flShmSize = cmd.String([]string{"-shm-size"}, "", "Size of /dev/shm, default value is 64MB") ) @@ -191,7 +192,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*container.Config, *container.Host swappiness := *flSwappiness if swappiness != -1 && (swappiness < 0 || swappiness > 100) { - return nil, nil, nil, cmd, fmt.Errorf("Invalid value: %d. Valid memory swappiness range is 0-100", swappiness) + return nil, nil, nil, cmd, fmt.Errorf("invalid value: %d. Valid memory swappiness range is 0-100", swappiness) } var shmSize int64 @@ -228,15 +229,15 @@ func Parse(cmd *flag.FlagSet, args []string) (*container.Config, *container.Host var ( parsedArgs = cmd.Args() - runCmd *strslice.StrSlice - entrypoint *strslice.StrSlice + runCmd strslice.StrSlice + entrypoint strslice.StrSlice image = cmd.Arg(0) ) if len(parsedArgs) > 1 { - runCmd = strslice.New(parsedArgs[1:]...) + runCmd = strslice.StrSlice(parsedArgs[1:]) } if *flEntrypoint != "" { - entrypoint = strslice.New(*flEntrypoint) + entrypoint = strslice.StrSlice{*flEntrypoint} } var ( @@ -257,7 +258,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*container.Config, *container.Host // Merge in exposed ports to the map of published ports for _, e := range flExpose.GetAll() { if strings.Contains(e, ":") { - return nil, nil, nil, cmd, fmt.Errorf("Invalid port format for --expose: %s", e) + return nil, nil, nil, cmd, fmt.Errorf("invalid port format for --expose: %s", e) } //support two formats for expose, original format /[] or /[] proto, port := nat.SplitProtoPort(e) @@ -265,7 +266,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*container.Config, *container.Host //if expose a port, the start and end port are the same start, end, err := nat.ParsePortRange(port) if err != nil { - return nil, nil, nil, cmd, fmt.Errorf("Invalid range format for --expose: %s, error: %s", e, err) + return nil, nil, nil, cmd, fmt.Errorf("invalid range format for --expose: %s, error: %s", e, err) } for i := start; i <= end; i++ { p, err := nat.NewPort(proto, strconv.FormatUint(i, 10)) @@ -343,6 +344,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*container.Config, *container.Host CpusetCpus: *flCpusetCpus, CpusetMems: *flCpusetMems, CPUQuota: *flCPUQuota, + PidsLimit: *flPidsLimit, BlkioWeight: *flBlkioWeight, BlkioWeightDevice: flBlkioWeightDevice.GetList(), BlkioDeviceReadBps: flDeviceReadBps.GetList(), @@ -375,7 +377,9 @@ func Parse(cmd *flag.FlagSet, args []string) (*container.Config, *container.Host Entrypoint: entrypoint, WorkingDir: *flWorkingDir, Labels: ConvertKVStringsToMap(labels), - StopSignal: *flStopSignal, + } + if cmd.IsSet("-stop-signal") { + config.StopSignal = *flStopSignal } hostConfig := &container.HostConfig{ @@ -400,15 +404,15 @@ func Parse(cmd *flag.FlagSet, args []string) (*container.Config, *container.Host IpcMode: ipcMode, PidMode: pidMode, UTSMode: utsMode, - CapAdd: strslice.New(flCapAdd.GetAll()...), - CapDrop: strslice.New(flCapDrop.GetAll()...), + CapAdd: strslice.StrSlice(flCapAdd.GetAll()), + CapDrop: strslice.StrSlice(flCapDrop.GetAll()), GroupAdd: flGroupAdd.GetAll(), RestartPolicy: restartPolicy, SecurityOpt: securityOpts, ReadonlyRootfs: *flReadonlyRootfs, LogConfig: container.LogConfig{Type: *flLoggingDriver, Config: loggingOpts}, VolumeDriver: *flVolumeDriver, - Isolation: container.IsolationLevel(*flIsolation), + Isolation: container.Isolation(*flIsolation), ShmSize: shmSize, Resources: resources, Tmpfs: tmpfs, @@ -489,7 +493,7 @@ func ConvertKVStringsToMap(values []string) map[string]string { func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]string, error) { loggingOptsMap := ConvertKVStringsToMap(loggingOpts) if loggingDriver == "none" && len(loggingOpts) > 0 { - return map[string]string{}, fmt.Errorf("Invalid logging opts for driver %s", loggingDriver) + return map[string]string{}, fmt.Errorf("invalid logging opts for driver %s", loggingDriver) } return loggingOptsMap, nil } @@ -498,17 +502,17 @@ func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]st func parseSecurityOpts(securityOpts []string) ([]string, error) { for key, opt := range securityOpts { con := strings.SplitN(opt, ":", 2) - if len(con) == 1 { + if len(con) == 1 && con[0] != "no-new-privileges" { return securityOpts, fmt.Errorf("Invalid --security-opt: %q", opt) } if con[0] == "seccomp" && con[1] != "unconfined" { f, err := ioutil.ReadFile(con[1]) if err != nil { - return securityOpts, fmt.Errorf("Opening seccomp profile (%s) failed: %v", con[1], err) + return securityOpts, fmt.Errorf("opening seccomp profile (%s) failed: %v", con[1], err) } b := bytes.NewBuffer(nil) if err := json.Compact(b, f); err != nil { - return securityOpts, fmt.Errorf("Compacting json for seccomp profile (%s) failed: %v", con[1], err) + return securityOpts, fmt.Errorf("compacting json for seccomp profile (%s) failed: %v", con[1], err) } securityOpts[key] = fmt.Sprintf("seccomp:%s", b.Bytes()) } @@ -577,7 +581,7 @@ func ParseDevice(device string) (container.DeviceMapping, error) { case 1: src = arr[0] default: - return container.DeviceMapping{}, fmt.Errorf("Invalid device specification: %s", device) + return container.DeviceMapping{}, fmt.Errorf("invalid device specification: %s", device) } if dst == "" { diff --git a/components/engine/runconfig/opts/parse_test.go b/components/engine/runconfig/opts/parse_test.go index 3ca9e32b32..834b51d142 100644 --- a/components/engine/runconfig/opts/parse_test.go +++ b/components/engine/runconfig/opts/parse_test.go @@ -401,14 +401,14 @@ func TestParseHostname(t *testing.T) { func TestParseWithExpose(t *testing.T) { invalids := map[string]string{ - ":": "Invalid port format for --expose: :", - "8080:9090": "Invalid port format for --expose: 8080:9090", - "/tcp": "Invalid range format for --expose: /tcp, error: Empty string specified for ports.", - "/udp": "Invalid range format for --expose: /udp, error: Empty string specified for ports.", - "NaN/tcp": `Invalid range format for --expose: NaN/tcp, error: strconv.ParseUint: parsing "NaN": invalid syntax`, - "NaN-NaN/tcp": `Invalid range format for --expose: NaN-NaN/tcp, error: strconv.ParseUint: parsing "NaN": invalid syntax`, - "8080-NaN/tcp": `Invalid range format for --expose: 8080-NaN/tcp, error: strconv.ParseUint: parsing "NaN": invalid syntax`, - "1234567890-8080/tcp": `Invalid range format for --expose: 1234567890-8080/tcp, error: strconv.ParseUint: parsing "1234567890": value out of range`, + ":": "invalid port format for --expose: :", + "8080:9090": "invalid port format for --expose: 8080:9090", + "/tcp": "invalid range format for --expose: /tcp, error: Empty string specified for ports.", + "/udp": "invalid range format for --expose: /udp, error: Empty string specified for ports.", + "NaN/tcp": `invalid range format for --expose: NaN/tcp, error: strconv.ParseUint: parsing "NaN": invalid syntax`, + "NaN-NaN/tcp": `invalid range format for --expose: NaN-NaN/tcp, error: strconv.ParseUint: parsing "NaN": invalid syntax`, + "8080-NaN/tcp": `invalid range format for --expose: 8080-NaN/tcp, error: strconv.ParseUint: parsing "NaN": invalid syntax`, + "1234567890-8080/tcp": `invalid range format for --expose: 1234567890-8080/tcp, error: strconv.ParseUint: parsing "1234567890": value out of range`, } valids := map[string][]nat.Port{ "8080/tcp": {"8080/tcp"}, @@ -578,8 +578,8 @@ func TestParseRestartPolicy(t *testing.T) { func TestParseLoggingOpts(t *testing.T) { // logging opts ko - if _, _, _, _, err := parseRun([]string{"--log-driver=none", "--log-opt=anything", "img", "cmd"}); err == nil || err.Error() != "Invalid logging opts for driver none" { - t.Fatalf("Expected an error with message 'Invalid logging opts for driver none', got %v", err) + if _, _, _, _, err := parseRun([]string{"--log-driver=none", "--log-opt=anything", "img", "cmd"}); err == nil || err.Error() != "invalid logging opts for driver none" { + t.Fatalf("Expected an error with message 'invalid logging opts for driver none', got %v", err) } // logging opts ok _, hostconfig, _, _, err := parseRun([]string{"--log-driver=syslog", "--log-opt=something", "img", "cmd"}) @@ -648,7 +648,7 @@ func TestParseEntryPoint(t *testing.T) { if err != nil { t.Fatal(err) } - if config.Entrypoint.Len() != 1 && config.Entrypoint.Slice()[0] != "anything" { + if len(config.Entrypoint) != 1 && config.Entrypoint[0] != "anything" { t.Fatalf("Expected entrypoint 'anything', got %v", config.Entrypoint) } } diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 49f50ddd43..d3dd00abf4 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -7,7 +7,6 @@ import ( "runtime" "strings" - "github.com/docker/distribution/registry/api/errcode" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/stringid" ) @@ -86,22 +85,3 @@ func ReplaceOrAppendEnvValues(defaults, overrides []string) []string { return defaults } - -// GetErrorMessage returns the human readable message associated with -// the passed-in error. In some cases the default Error() func returns -// something that is less than useful so based on its types this func -// will go and get a better piece of text. -func GetErrorMessage(err error) string { - switch err.(type) { - case errcode.Error: - e, _ := err.(errcode.Error) - return e.Message - - case errcode.ErrorCode: - ec, _ := err.(errcode.ErrorCode) - return ec.Message() - - default: - return err.Error() - } -} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/LICENSE b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/LICENSE new file mode 100644 index 0000000000..7448756763 --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/common.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/common.go new file mode 100644 index 0000000000..5141bf92d6 --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/common.go @@ -0,0 +1,342 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tar implements access to tar archives. +// It aims to cover most of the variations, including those produced +// by GNU and BSD tars. +// +// References: +// http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5 +// http://www.gnu.org/software/tar/manual/html_node/Standard.html +// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html +package tar + +import ( + "bytes" + "errors" + "fmt" + "os" + "path" + "time" +) + +const ( + blockSize = 512 + + // Types + TypeReg = '0' // regular file + TypeRegA = '\x00' // regular file + TypeLink = '1' // hard link + TypeSymlink = '2' // symbolic link + TypeChar = '3' // character device node + TypeBlock = '4' // block device node + TypeDir = '5' // directory + TypeFifo = '6' // fifo node + TypeCont = '7' // reserved + TypeXHeader = 'x' // extended header + TypeXGlobalHeader = 'g' // global extended header + TypeGNULongName = 'L' // Next file has a long name + TypeGNULongLink = 'K' // Next file symlinks to a file w/ a long name + TypeGNUSparse = 'S' // sparse file +) + +// A Header represents a single header in a tar archive. +// Some fields may not be populated. +type Header struct { + Name string // name of header file entry + Mode int64 // permission and mode bits + Uid int // user id of owner + Gid int // group id of owner + Size int64 // length in bytes + ModTime time.Time // modified time + Typeflag byte // type of header entry + Linkname string // target name of link + Uname string // user name of owner + Gname string // group name of owner + Devmajor int64 // major number of character or block device + Devminor int64 // minor number of character or block device + AccessTime time.Time // access time + ChangeTime time.Time // status change time + Xattrs map[string]string + Winheaders map[string]string +} + +// File name constants from the tar spec. +const ( + fileNameSize = 100 // Maximum number of bytes in a standard tar name. + fileNamePrefixSize = 155 // Maximum number of ustar extension bytes. +) + +// FileInfo returns an os.FileInfo for the Header. +func (h *Header) FileInfo() os.FileInfo { + return headerFileInfo{h} +} + +// headerFileInfo implements os.FileInfo. +type headerFileInfo struct { + h *Header +} + +func (fi headerFileInfo) Size() int64 { return fi.h.Size } +func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() } +func (fi headerFileInfo) ModTime() time.Time { return fi.h.ModTime } +func (fi headerFileInfo) Sys() interface{} { return fi.h } + +// Name returns the base name of the file. +func (fi headerFileInfo) Name() string { + if fi.IsDir() { + return path.Base(path.Clean(fi.h.Name)) + } + return path.Base(fi.h.Name) +} + +// Mode returns the permission and mode bits for the headerFileInfo. +func (fi headerFileInfo) Mode() (mode os.FileMode) { + // Set file permission bits. + mode = os.FileMode(fi.h.Mode).Perm() + + // Set setuid, setgid and sticky bits. + if fi.h.Mode&c_ISUID != 0 { + // setuid + mode |= os.ModeSetuid + } + if fi.h.Mode&c_ISGID != 0 { + // setgid + mode |= os.ModeSetgid + } + if fi.h.Mode&c_ISVTX != 0 { + // sticky + mode |= os.ModeSticky + } + + // Set file mode bits. + // clear perm, setuid, setgid and sticky bits. + m := os.FileMode(fi.h.Mode) &^ 07777 + if m == c_ISDIR { + // directory + mode |= os.ModeDir + } + if m == c_ISFIFO { + // named pipe (FIFO) + mode |= os.ModeNamedPipe + } + if m == c_ISLNK { + // symbolic link + mode |= os.ModeSymlink + } + if m == c_ISBLK { + // device file + mode |= os.ModeDevice + } + if m == c_ISCHR { + // Unix character device + mode |= os.ModeDevice + mode |= os.ModeCharDevice + } + if m == c_ISSOCK { + // Unix domain socket + mode |= os.ModeSocket + } + + switch fi.h.Typeflag { + case TypeSymlink: + // symbolic link + mode |= os.ModeSymlink + case TypeChar: + // character device node + mode |= os.ModeDevice + mode |= os.ModeCharDevice + case TypeBlock: + // block device node + mode |= os.ModeDevice + case TypeDir: + // directory + mode |= os.ModeDir + case TypeFifo: + // fifo node + mode |= os.ModeNamedPipe + } + + return mode +} + +// sysStat, if non-nil, populates h from system-dependent fields of fi. +var sysStat func(fi os.FileInfo, h *Header) error + +// Mode constants from the tar spec. +const ( + c_ISUID = 04000 // Set uid + c_ISGID = 02000 // Set gid + c_ISVTX = 01000 // Save text (sticky bit) + c_ISDIR = 040000 // Directory + c_ISFIFO = 010000 // FIFO + c_ISREG = 0100000 // Regular file + c_ISLNK = 0120000 // Symbolic link + c_ISBLK = 060000 // Block special file + c_ISCHR = 020000 // Character special file + c_ISSOCK = 0140000 // Socket +) + +// Keywords for the PAX Extended Header +const ( + paxAtime = "atime" + paxCharset = "charset" + paxComment = "comment" + paxCtime = "ctime" // please note that ctime is not a valid pax header. + paxGid = "gid" + paxGname = "gname" + paxLinkpath = "linkpath" + paxMtime = "mtime" + paxPath = "path" + paxSize = "size" + paxUid = "uid" + paxUname = "uname" + paxXattr = "SCHILY.xattr." + paxWindows = "MSWINDOWS." + paxNone = "" +) + +// FileInfoHeader creates a partially-populated Header from fi. +// If fi describes a symlink, FileInfoHeader records link as the link target. +// If fi describes a directory, a slash is appended to the name. +// Because os.FileInfo's Name method returns only the base name of +// the file it describes, it may be necessary to modify the Name field +// of the returned header to provide the full path name of the file. +func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) { + if fi == nil { + return nil, errors.New("tar: FileInfo is nil") + } + fm := fi.Mode() + h := &Header{ + Name: fi.Name(), + ModTime: fi.ModTime(), + Mode: int64(fm.Perm()), // or'd with c_IS* constants later + } + switch { + case fm.IsRegular(): + h.Mode |= c_ISREG + h.Typeflag = TypeReg + h.Size = fi.Size() + case fi.IsDir(): + h.Typeflag = TypeDir + h.Mode |= c_ISDIR + h.Name += "/" + case fm&os.ModeSymlink != 0: + h.Typeflag = TypeSymlink + h.Mode |= c_ISLNK + h.Linkname = link + case fm&os.ModeDevice != 0: + if fm&os.ModeCharDevice != 0 { + h.Mode |= c_ISCHR + h.Typeflag = TypeChar + } else { + h.Mode |= c_ISBLK + h.Typeflag = TypeBlock + } + case fm&os.ModeNamedPipe != 0: + h.Typeflag = TypeFifo + h.Mode |= c_ISFIFO + case fm&os.ModeSocket != 0: + h.Mode |= c_ISSOCK + default: + return nil, fmt.Errorf("archive/tar: unknown file mode %v", fm) + } + if fm&os.ModeSetuid != 0 { + h.Mode |= c_ISUID + } + if fm&os.ModeSetgid != 0 { + h.Mode |= c_ISGID + } + if fm&os.ModeSticky != 0 { + h.Mode |= c_ISVTX + } + // If possible, populate additional fields from OS-specific + // FileInfo fields. + if sys, ok := fi.Sys().(*Header); ok { + // This FileInfo came from a Header (not the OS). Use the + // original Header to populate all remaining fields. + h.Uid = sys.Uid + h.Gid = sys.Gid + h.Uname = sys.Uname + h.Gname = sys.Gname + h.AccessTime = sys.AccessTime + h.ChangeTime = sys.ChangeTime + if sys.Xattrs != nil { + h.Xattrs = make(map[string]string) + for k, v := range sys.Xattrs { + h.Xattrs[k] = v + } + } + if sys.Typeflag == TypeLink { + // hard link + h.Typeflag = TypeLink + h.Size = 0 + h.Linkname = sys.Linkname + } + } + if sysStat != nil { + return h, sysStat(fi, h) + } + return h, nil +} + +var zeroBlock = make([]byte, blockSize) + +// POSIX specifies a sum of the unsigned byte values, but the Sun tar uses signed byte values. +// We compute and return both. +func checksum(header []byte) (unsigned int64, signed int64) { + for i := 0; i < len(header); i++ { + if i == 148 { + // The chksum field (header[148:156]) is special: it should be treated as space bytes. + unsigned += ' ' * 8 + signed += ' ' * 8 + i += 7 + continue + } + unsigned += int64(header[i]) + signed += int64(int8(header[i])) + } + return +} + +type slicer []byte + +func (sp *slicer) next(n int) (b []byte) { + s := *sp + b, *sp = s[0:n], s[n:] + return +} + +func isASCII(s string) bool { + for _, c := range s { + if c >= 0x80 { + return false + } + } + return true +} + +func toASCII(s string) string { + if isASCII(s) { + return s + } + var buf bytes.Buffer + for _, c := range s { + if c < 0x80 { + buf.WriteByte(byte(c)) + } + } + return buf.String() +} + +// isHeaderOnlyType checks if the given type flag is of the type that has no +// data section even if a size is specified. +func isHeaderOnlyType(flag byte) bool { + switch flag { + case TypeLink, TypeSymlink, TypeChar, TypeBlock, TypeDir, TypeFifo: + return true + default: + return false + } +} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/reader.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/reader.go new file mode 100644 index 0000000000..6aee36c192 --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/reader.go @@ -0,0 +1,996 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +// TODO(dsymonds): +// - pax extensions + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "math" + "os" + "strconv" + "strings" + "time" +) + +var ( + ErrHeader = errors.New("archive/tar: invalid tar header") +) + +const maxNanoSecondIntSize = 9 + +// A Reader provides sequential access to the contents of a tar archive. +// A tar archive consists of a sequence of files. +// The Next method advances to the next file in the archive (including the first), +// and then it can be treated as an io.Reader to access the file's data. +type Reader struct { + r io.Reader + err error + pad int64 // amount of padding (ignored) after current file entry + curr numBytesReader // reader for current file entry + hdrBuff [blockSize]byte // buffer to use in readHeader +} + +type parser struct { + err error // Last error seen +} + +// A numBytesReader is an io.Reader with a numBytes method, returning the number +// of bytes remaining in the underlying encoded data. +type numBytesReader interface { + io.Reader + numBytes() int64 +} + +// A regFileReader is a numBytesReader for reading file data from a tar archive. +type regFileReader struct { + r io.Reader // underlying reader + nb int64 // number of unread bytes for current file entry +} + +// A sparseFileReader is a numBytesReader for reading sparse file data from a +// tar archive. +type sparseFileReader struct { + rfr numBytesReader // Reads the sparse-encoded file data + sp []sparseEntry // The sparse map for the file + pos int64 // Keeps track of file position + total int64 // Total size of the file +} + +// A sparseEntry holds a single entry in a sparse file's sparse map. +// +// Sparse files are represented using a series of sparseEntrys. +// Despite the name, a sparseEntry represents an actual data fragment that +// references data found in the underlying archive stream. All regions not +// covered by a sparseEntry are logically filled with zeros. +// +// For example, if the underlying raw file contains the 10-byte data: +// var compactData = "abcdefgh" +// +// And the sparse map has the following entries: +// var sp = []sparseEntry{ +// {offset: 2, numBytes: 5} // Data fragment for [2..7] +// {offset: 18, numBytes: 3} // Data fragment for [18..21] +// } +// +// Then the content of the resulting sparse file with a "real" size of 25 is: +// var sparseData = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4 +type sparseEntry struct { + offset int64 // Starting position of the fragment + numBytes int64 // Length of the fragment +} + +// Keywords for GNU sparse files in a PAX extended header +const ( + paxGNUSparseNumBlocks = "GNU.sparse.numblocks" + paxGNUSparseOffset = "GNU.sparse.offset" + paxGNUSparseNumBytes = "GNU.sparse.numbytes" + paxGNUSparseMap = "GNU.sparse.map" + paxGNUSparseName = "GNU.sparse.name" + paxGNUSparseMajor = "GNU.sparse.major" + paxGNUSparseMinor = "GNU.sparse.minor" + paxGNUSparseSize = "GNU.sparse.size" + paxGNUSparseRealSize = "GNU.sparse.realsize" +) + +// Keywords for old GNU sparse headers +const ( + oldGNUSparseMainHeaderOffset = 386 + oldGNUSparseMainHeaderIsExtendedOffset = 482 + oldGNUSparseMainHeaderNumEntries = 4 + oldGNUSparseExtendedHeaderIsExtendedOffset = 504 + oldGNUSparseExtendedHeaderNumEntries = 21 + oldGNUSparseOffsetSize = 12 + oldGNUSparseNumBytesSize = 12 +) + +// NewReader creates a new Reader reading from r. +func NewReader(r io.Reader) *Reader { return &Reader{r: r} } + +// Next advances to the next entry in the tar archive. +// +// io.EOF is returned at the end of the input. +func (tr *Reader) Next() (*Header, error) { + if tr.err != nil { + return nil, tr.err + } + + var hdr *Header + var extHdrs map[string]string + + // Externally, Next iterates through the tar archive as if it is a series of + // files. Internally, the tar format often uses fake "files" to add meta + // data that describes the next file. These meta data "files" should not + // normally be visible to the outside. As such, this loop iterates through + // one or more "header files" until it finds a "normal file". +loop: + for { + tr.err = tr.skipUnread() + if tr.err != nil { + return nil, tr.err + } + + hdr = tr.readHeader() + if tr.err != nil { + return nil, tr.err + } + + // Check for PAX/GNU special headers and files. + switch hdr.Typeflag { + case TypeXHeader: + extHdrs, tr.err = parsePAX(tr) + if tr.err != nil { + return nil, tr.err + } + continue loop // This is a meta header affecting the next header + case TypeGNULongName, TypeGNULongLink: + var realname []byte + realname, tr.err = ioutil.ReadAll(tr) + if tr.err != nil { + return nil, tr.err + } + + // Convert GNU extensions to use PAX headers. + if extHdrs == nil { + extHdrs = make(map[string]string) + } + var p parser + switch hdr.Typeflag { + case TypeGNULongName: + extHdrs[paxPath] = p.parseString(realname) + case TypeGNULongLink: + extHdrs[paxLinkpath] = p.parseString(realname) + } + if p.err != nil { + tr.err = p.err + return nil, tr.err + } + continue loop // This is a meta header affecting the next header + default: + mergePAX(hdr, extHdrs) + + // Check for a PAX format sparse file + sp, err := tr.checkForGNUSparsePAXHeaders(hdr, extHdrs) + if err != nil { + tr.err = err + return nil, err + } + if sp != nil { + // Current file is a PAX format GNU sparse file. + // Set the current file reader to a sparse file reader. + tr.curr, tr.err = newSparseFileReader(tr.curr, sp, hdr.Size) + if tr.err != nil { + return nil, tr.err + } + } + break loop // This is a file, so stop + } + } + return hdr, nil +} + +// checkForGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. If they are found, then +// this function reads the sparse map and returns it. Unknown sparse formats are ignored, causing the file to +// be treated as a regular file. +func (tr *Reader) checkForGNUSparsePAXHeaders(hdr *Header, headers map[string]string) ([]sparseEntry, error) { + var sparseFormat string + + // Check for sparse format indicators + major, majorOk := headers[paxGNUSparseMajor] + minor, minorOk := headers[paxGNUSparseMinor] + sparseName, sparseNameOk := headers[paxGNUSparseName] + _, sparseMapOk := headers[paxGNUSparseMap] + sparseSize, sparseSizeOk := headers[paxGNUSparseSize] + sparseRealSize, sparseRealSizeOk := headers[paxGNUSparseRealSize] + + // Identify which, if any, sparse format applies from which PAX headers are set + if majorOk && minorOk { + sparseFormat = major + "." + minor + } else if sparseNameOk && sparseMapOk { + sparseFormat = "0.1" + } else if sparseSizeOk { + sparseFormat = "0.0" + } else { + // Not a PAX format GNU sparse file. + return nil, nil + } + + // Check for unknown sparse format + if sparseFormat != "0.0" && sparseFormat != "0.1" && sparseFormat != "1.0" { + return nil, nil + } + + // Update hdr from GNU sparse PAX headers + if sparseNameOk { + hdr.Name = sparseName + } + if sparseSizeOk { + realSize, err := strconv.ParseInt(sparseSize, 10, 0) + if err != nil { + return nil, ErrHeader + } + hdr.Size = realSize + } else if sparseRealSizeOk { + realSize, err := strconv.ParseInt(sparseRealSize, 10, 0) + if err != nil { + return nil, ErrHeader + } + hdr.Size = realSize + } + + // Set up the sparse map, according to the particular sparse format in use + var sp []sparseEntry + var err error + switch sparseFormat { + case "0.0", "0.1": + sp, err = readGNUSparseMap0x1(headers) + case "1.0": + sp, err = readGNUSparseMap1x0(tr.curr) + } + return sp, err +} + +// mergePAX merges well known headers according to PAX standard. +// In general headers with the same name as those found +// in the header struct overwrite those found in the header +// struct with higher precision or longer values. Esp. useful +// for name and linkname fields. +func mergePAX(hdr *Header, headers map[string]string) error { + for k, v := range headers { + switch k { + case paxPath: + hdr.Name = v + case paxLinkpath: + hdr.Linkname = v + case paxGname: + hdr.Gname = v + case paxUname: + hdr.Uname = v + case paxUid: + uid, err := strconv.ParseInt(v, 10, 0) + if err != nil { + return err + } + hdr.Uid = int(uid) + case paxGid: + gid, err := strconv.ParseInt(v, 10, 0) + if err != nil { + return err + } + hdr.Gid = int(gid) + case paxAtime: + t, err := parsePAXTime(v) + if err != nil { + return err + } + hdr.AccessTime = t + case paxMtime: + t, err := parsePAXTime(v) + if err != nil { + return err + } + hdr.ModTime = t + case paxCtime: + t, err := parsePAXTime(v) + if err != nil { + return err + } + hdr.ChangeTime = t + case paxSize: + size, err := strconv.ParseInt(v, 10, 0) + if err != nil { + return err + } + hdr.Size = int64(size) + default: + if strings.HasPrefix(k, paxXattr) { + if hdr.Xattrs == nil { + hdr.Xattrs = make(map[string]string) + } + hdr.Xattrs[k[len(paxXattr):]] = v + } else if strings.HasPrefix(k, paxWindows) { + if hdr.Winheaders == nil { + hdr.Winheaders = make(map[string]string) + } + hdr.Winheaders[k[len(paxWindows):]] = v + } + } + } + return nil +} + +// parsePAXTime takes a string of the form %d.%d as described in +// the PAX specification. +func parsePAXTime(t string) (time.Time, error) { + buf := []byte(t) + pos := bytes.IndexByte(buf, '.') + var seconds, nanoseconds int64 + var err error + if pos == -1 { + seconds, err = strconv.ParseInt(t, 10, 0) + if err != nil { + return time.Time{}, err + } + } else { + seconds, err = strconv.ParseInt(string(buf[:pos]), 10, 0) + if err != nil { + return time.Time{}, err + } + nano_buf := string(buf[pos+1:]) + // Pad as needed before converting to a decimal. + // For example .030 -> .030000000 -> 30000000 nanoseconds + if len(nano_buf) < maxNanoSecondIntSize { + // Right pad + nano_buf += strings.Repeat("0", maxNanoSecondIntSize-len(nano_buf)) + } else if len(nano_buf) > maxNanoSecondIntSize { + // Right truncate + nano_buf = nano_buf[:maxNanoSecondIntSize] + } + nanoseconds, err = strconv.ParseInt(string(nano_buf), 10, 0) + if err != nil { + return time.Time{}, err + } + } + ts := time.Unix(seconds, nanoseconds) + return ts, nil +} + +// parsePAX parses PAX headers. +// If an extended header (type 'x') is invalid, ErrHeader is returned +func parsePAX(r io.Reader) (map[string]string, error) { + buf, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + sbuf := string(buf) + + // For GNU PAX sparse format 0.0 support. + // This function transforms the sparse format 0.0 headers into sparse format 0.1 headers. + var sparseMap bytes.Buffer + + headers := make(map[string]string) + // Each record is constructed as + // "%d %s=%s\n", length, keyword, value + for len(sbuf) > 0 { + key, value, residual, err := parsePAXRecord(sbuf) + if err != nil { + return nil, ErrHeader + } + sbuf = residual + + keyStr := string(key) + if keyStr == paxGNUSparseOffset || keyStr == paxGNUSparseNumBytes { + // GNU sparse format 0.0 special key. Write to sparseMap instead of using the headers map. + sparseMap.WriteString(value) + sparseMap.Write([]byte{','}) + } else { + // Normal key. Set the value in the headers map. + headers[keyStr] = string(value) + } + } + if sparseMap.Len() != 0 { + // Add sparse info to headers, chopping off the extra comma + sparseMap.Truncate(sparseMap.Len() - 1) + headers[paxGNUSparseMap] = sparseMap.String() + } + return headers, nil +} + +// parsePAXRecord parses the input PAX record string into a key-value pair. +// If parsing is successful, it will slice off the currently read record and +// return the remainder as r. +// +// A PAX record is of the following form: +// "%d %s=%s\n" % (size, key, value) +func parsePAXRecord(s string) (k, v, r string, err error) { + // The size field ends at the first space. + sp := strings.IndexByte(s, ' ') + if sp == -1 { + return "", "", s, ErrHeader + } + + // Parse the first token as a decimal integer. + n, perr := strconv.ParseInt(s[:sp], 10, 0) // Intentionally parse as native int + if perr != nil || n < 5 || int64(len(s)) < n { + return "", "", s, ErrHeader + } + + // Extract everything between the space and the final newline. + rec, nl, rem := s[sp+1:n-1], s[n-1:n], s[n:] + if nl != "\n" { + return "", "", s, ErrHeader + } + + // The first equals separates the key from the value. + eq := strings.IndexByte(rec, '=') + if eq == -1 { + return "", "", s, ErrHeader + } + return rec[:eq], rec[eq+1:], rem, nil +} + +// parseString parses bytes as a NUL-terminated C-style string. +// If a NUL byte is not found then the whole slice is returned as a string. +func (*parser) parseString(b []byte) string { + n := 0 + for n < len(b) && b[n] != 0 { + n++ + } + return string(b[0:n]) +} + +// parseNumeric parses the input as being encoded in either base-256 or octal. +// This function may return negative numbers. +// If parsing fails or an integer overflow occurs, err will be set. +func (p *parser) parseNumeric(b []byte) int64 { + // Check for base-256 (binary) format first. + // If the first bit is set, then all following bits constitute a two's + // complement encoded number in big-endian byte order. + if len(b) > 0 && b[0]&0x80 != 0 { + // Handling negative numbers relies on the following identity: + // -a-1 == ^a + // + // If the number is negative, we use an inversion mask to invert the + // data bytes and treat the value as an unsigned number. + var inv byte // 0x00 if positive or zero, 0xff if negative + if b[0]&0x40 != 0 { + inv = 0xff + } + + var x uint64 + for i, c := range b { + c ^= inv // Inverts c only if inv is 0xff, otherwise does nothing + if i == 0 { + c &= 0x7f // Ignore signal bit in first byte + } + if (x >> 56) > 0 { + p.err = ErrHeader // Integer overflow + return 0 + } + x = x<<8 | uint64(c) + } + if (x >> 63) > 0 { + p.err = ErrHeader // Integer overflow + return 0 + } + if inv == 0xff { + return ^int64(x) + } + return int64(x) + } + + // Normal case is base-8 (octal) format. + return p.parseOctal(b) +} + +func (p *parser) parseOctal(b []byte) int64 { + // Because unused fields are filled with NULs, we need + // to skip leading NULs. Fields may also be padded with + // spaces or NULs. + // So we remove leading and trailing NULs and spaces to + // be sure. + b = bytes.Trim(b, " \x00") + + if len(b) == 0 { + return 0 + } + x, perr := strconv.ParseUint(p.parseString(b), 8, 64) + if perr != nil { + p.err = ErrHeader + } + return int64(x) +} + +// skipUnread skips any unread bytes in the existing file entry, as well as any +// alignment padding. It returns io.ErrUnexpectedEOF if any io.EOF is +// encountered in the data portion; it is okay to hit io.EOF in the padding. +// +// Note that this function still works properly even when sparse files are being +// used since numBytes returns the bytes remaining in the underlying io.Reader. +func (tr *Reader) skipUnread() error { + dataSkip := tr.numBytes() // Number of data bytes to skip + totalSkip := dataSkip + tr.pad // Total number of bytes to skip + tr.curr, tr.pad = nil, 0 + + // If possible, Seek to the last byte before the end of the data section. + // Do this because Seek is often lazy about reporting errors; this will mask + // the fact that the tar stream may be truncated. We can rely on the + // io.CopyN done shortly afterwards to trigger any IO errors. + var seekSkipped int64 // Number of bytes skipped via Seek + if sr, ok := tr.r.(io.Seeker); ok && dataSkip > 1 { + // Not all io.Seeker can actually Seek. For example, os.Stdin implements + // io.Seeker, but calling Seek always returns an error and performs + // no action. Thus, we try an innocent seek to the current position + // to see if Seek is really supported. + pos1, err := sr.Seek(0, os.SEEK_CUR) + if err == nil { + // Seek seems supported, so perform the real Seek. + pos2, err := sr.Seek(dataSkip-1, os.SEEK_CUR) + if err != nil { + tr.err = err + return tr.err + } + seekSkipped = pos2 - pos1 + } + } + + var copySkipped int64 // Number of bytes skipped via CopyN + copySkipped, tr.err = io.CopyN(ioutil.Discard, tr.r, totalSkip-seekSkipped) + if tr.err == io.EOF && seekSkipped+copySkipped < dataSkip { + tr.err = io.ErrUnexpectedEOF + } + return tr.err +} + +func (tr *Reader) verifyChecksum(header []byte) bool { + if tr.err != nil { + return false + } + + var p parser + given := p.parseOctal(header[148:156]) + unsigned, signed := checksum(header) + return p.err == nil && (given == unsigned || given == signed) +} + +// readHeader reads the next block header and assumes that the underlying reader +// is already aligned to a block boundary. +// +// The err will be set to io.EOF only when one of the following occurs: +// * Exactly 0 bytes are read and EOF is hit. +// * Exactly 1 block of zeros is read and EOF is hit. +// * At least 2 blocks of zeros are read. +func (tr *Reader) readHeader() *Header { + header := tr.hdrBuff[:] + copy(header, zeroBlock) + + if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { + return nil // io.EOF is okay here + } + + // Two blocks of zero bytes marks the end of the archive. + if bytes.Equal(header, zeroBlock[0:blockSize]) { + if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { + return nil // io.EOF is okay here + } + if bytes.Equal(header, zeroBlock[0:blockSize]) { + tr.err = io.EOF + } else { + tr.err = ErrHeader // zero block and then non-zero block + } + return nil + } + + if !tr.verifyChecksum(header) { + tr.err = ErrHeader + return nil + } + + // Unpack + var p parser + hdr := new(Header) + s := slicer(header) + + hdr.Name = p.parseString(s.next(100)) + hdr.Mode = p.parseNumeric(s.next(8)) + hdr.Uid = int(p.parseNumeric(s.next(8))) + hdr.Gid = int(p.parseNumeric(s.next(8))) + hdr.Size = p.parseNumeric(s.next(12)) + hdr.ModTime = time.Unix(p.parseNumeric(s.next(12)), 0) + s.next(8) // chksum + hdr.Typeflag = s.next(1)[0] + hdr.Linkname = p.parseString(s.next(100)) + + // The remainder of the header depends on the value of magic. + // The original (v7) version of tar had no explicit magic field, + // so its magic bytes, like the rest of the block, are NULs. + magic := string(s.next(8)) // contains version field as well. + var format string + switch { + case magic[:6] == "ustar\x00": // POSIX tar (1003.1-1988) + if string(header[508:512]) == "tar\x00" { + format = "star" + } else { + format = "posix" + } + case magic == "ustar \x00": // old GNU tar + format = "gnu" + } + + switch format { + case "posix", "gnu", "star": + hdr.Uname = p.parseString(s.next(32)) + hdr.Gname = p.parseString(s.next(32)) + devmajor := s.next(8) + devminor := s.next(8) + if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock { + hdr.Devmajor = p.parseNumeric(devmajor) + hdr.Devminor = p.parseNumeric(devminor) + } + var prefix string + switch format { + case "posix", "gnu": + prefix = p.parseString(s.next(155)) + case "star": + prefix = p.parseString(s.next(131)) + hdr.AccessTime = time.Unix(p.parseNumeric(s.next(12)), 0) + hdr.ChangeTime = time.Unix(p.parseNumeric(s.next(12)), 0) + } + if len(prefix) > 0 { + hdr.Name = prefix + "/" + hdr.Name + } + } + + if p.err != nil { + tr.err = p.err + return nil + } + + nb := hdr.Size + if isHeaderOnlyType(hdr.Typeflag) { + nb = 0 + } + if nb < 0 { + tr.err = ErrHeader + return nil + } + + // Set the current file reader. + tr.pad = -nb & (blockSize - 1) // blockSize is a power of two + tr.curr = ®FileReader{r: tr.r, nb: nb} + + // Check for old GNU sparse format entry. + if hdr.Typeflag == TypeGNUSparse { + // Get the real size of the file. + hdr.Size = p.parseNumeric(header[483:495]) + if p.err != nil { + tr.err = p.err + return nil + } + + // Read the sparse map. + sp := tr.readOldGNUSparseMap(header) + if tr.err != nil { + return nil + } + + // Current file is a GNU sparse file. Update the current file reader. + tr.curr, tr.err = newSparseFileReader(tr.curr, sp, hdr.Size) + if tr.err != nil { + return nil + } + } + + return hdr +} + +// readOldGNUSparseMap reads the sparse map as stored in the old GNU sparse format. +// The sparse map is stored in the tar header if it's small enough. If it's larger than four entries, +// then one or more extension headers are used to store the rest of the sparse map. +func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry { + var p parser + isExtended := header[oldGNUSparseMainHeaderIsExtendedOffset] != 0 + spCap := oldGNUSparseMainHeaderNumEntries + if isExtended { + spCap += oldGNUSparseExtendedHeaderNumEntries + } + sp := make([]sparseEntry, 0, spCap) + s := slicer(header[oldGNUSparseMainHeaderOffset:]) + + // Read the four entries from the main tar header + for i := 0; i < oldGNUSparseMainHeaderNumEntries; i++ { + offset := p.parseNumeric(s.next(oldGNUSparseOffsetSize)) + numBytes := p.parseNumeric(s.next(oldGNUSparseNumBytesSize)) + if p.err != nil { + tr.err = p.err + return nil + } + if offset == 0 && numBytes == 0 { + break + } + sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) + } + + for isExtended { + // There are more entries. Read an extension header and parse its entries. + sparseHeader := make([]byte, blockSize) + if _, tr.err = io.ReadFull(tr.r, sparseHeader); tr.err != nil { + return nil + } + isExtended = sparseHeader[oldGNUSparseExtendedHeaderIsExtendedOffset] != 0 + s = slicer(sparseHeader) + for i := 0; i < oldGNUSparseExtendedHeaderNumEntries; i++ { + offset := p.parseNumeric(s.next(oldGNUSparseOffsetSize)) + numBytes := p.parseNumeric(s.next(oldGNUSparseNumBytesSize)) + if p.err != nil { + tr.err = p.err + return nil + } + if offset == 0 && numBytes == 0 { + break + } + sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) + } + } + return sp +} + +// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format +// version 1.0. The format of the sparse map consists of a series of +// newline-terminated numeric fields. The first field is the number of entries +// and is always present. Following this are the entries, consisting of two +// fields (offset, numBytes). This function must stop reading at the end +// boundary of the block containing the last newline. +// +// Note that the GNU manual says that numeric values should be encoded in octal +// format. However, the GNU tar utility itself outputs these values in decimal. +// As such, this library treats values as being encoded in decimal. +func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) { + var cntNewline int64 + var buf bytes.Buffer + var blk = make([]byte, blockSize) + + // feedTokens copies data in numBlock chunks from r into buf until there are + // at least cnt newlines in buf. It will not read more blocks than needed. + var feedTokens = func(cnt int64) error { + for cntNewline < cnt { + if _, err := io.ReadFull(r, blk); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + buf.Write(blk) + for _, c := range blk { + if c == '\n' { + cntNewline++ + } + } + } + return nil + } + + // nextToken gets the next token delimited by a newline. This assumes that + // at least one newline exists in the buffer. + var nextToken = func() string { + cntNewline-- + tok, _ := buf.ReadString('\n') + return tok[:len(tok)-1] // Cut off newline + } + + // Parse for the number of entries. + // Use integer overflow resistant math to check this. + if err := feedTokens(1); err != nil { + return nil, err + } + numEntries, err := strconv.ParseInt(nextToken(), 10, 0) // Intentionally parse as native int + if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { + return nil, ErrHeader + } + + // Parse for all member entries. + // numEntries is trusted after this since a potential attacker must have + // committed resources proportional to what this library used. + if err := feedTokens(2 * numEntries); err != nil { + return nil, err + } + sp := make([]sparseEntry, 0, numEntries) + for i := int64(0); i < numEntries; i++ { + offset, err := strconv.ParseInt(nextToken(), 10, 64) + if err != nil { + return nil, ErrHeader + } + numBytes, err := strconv.ParseInt(nextToken(), 10, 64) + if err != nil { + return nil, ErrHeader + } + sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) + } + return sp, nil +} + +// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format +// version 0.1. The sparse map is stored in the PAX headers. +func readGNUSparseMap0x1(extHdrs map[string]string) ([]sparseEntry, error) { + // Get number of entries. + // Use integer overflow resistant math to check this. + numEntriesStr := extHdrs[paxGNUSparseNumBlocks] + numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) // Intentionally parse as native int + if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { + return nil, ErrHeader + } + + // There should be two numbers in sparseMap for each entry. + sparseMap := strings.Split(extHdrs[paxGNUSparseMap], ",") + if int64(len(sparseMap)) != 2*numEntries { + return nil, ErrHeader + } + + // Loop through the entries in the sparse map. + // numEntries is trusted now. + sp := make([]sparseEntry, 0, numEntries) + for i := int64(0); i < numEntries; i++ { + offset, err := strconv.ParseInt(sparseMap[2*i], 10, 64) + if err != nil { + return nil, ErrHeader + } + numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 64) + if err != nil { + return nil, ErrHeader + } + sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) + } + return sp, nil +} + +// numBytes returns the number of bytes left to read in the current file's entry +// in the tar archive, or 0 if there is no current file. +func (tr *Reader) numBytes() int64 { + if tr.curr == nil { + // No current file, so no bytes + return 0 + } + return tr.curr.numBytes() +} + +// Read reads from the current entry in the tar archive. +// It returns 0, io.EOF when it reaches the end of that entry, +// until Next is called to advance to the next entry. +// +// Calling Read on special types like TypeLink, TypeSymLink, TypeChar, +// TypeBlock, TypeDir, and TypeFifo returns 0, io.EOF regardless of what +// the Header.Size claims. +func (tr *Reader) Read(b []byte) (n int, err error) { + if tr.err != nil { + return 0, tr.err + } + if tr.curr == nil { + return 0, io.EOF + } + + n, err = tr.curr.Read(b) + if err != nil && err != io.EOF { + tr.err = err + } + return +} + +func (rfr *regFileReader) Read(b []byte) (n int, err error) { + if rfr.nb == 0 { + // file consumed + return 0, io.EOF + } + if int64(len(b)) > rfr.nb { + b = b[0:rfr.nb] + } + n, err = rfr.r.Read(b) + rfr.nb -= int64(n) + + if err == io.EOF && rfr.nb > 0 { + err = io.ErrUnexpectedEOF + } + return +} + +// numBytes returns the number of bytes left to read in the file's data in the tar archive. +func (rfr *regFileReader) numBytes() int64 { + return rfr.nb +} + +// newSparseFileReader creates a new sparseFileReader, but validates all of the +// sparse entries before doing so. +func newSparseFileReader(rfr numBytesReader, sp []sparseEntry, total int64) (*sparseFileReader, error) { + if total < 0 { + return nil, ErrHeader // Total size cannot be negative + } + + // Validate all sparse entries. These are the same checks as performed by + // the BSD tar utility. + for i, s := range sp { + switch { + case s.offset < 0 || s.numBytes < 0: + return nil, ErrHeader // Negative values are never okay + case s.offset > math.MaxInt64-s.numBytes: + return nil, ErrHeader // Integer overflow with large length + case s.offset+s.numBytes > total: + return nil, ErrHeader // Region extends beyond the "real" size + case i > 0 && sp[i-1].offset+sp[i-1].numBytes > s.offset: + return nil, ErrHeader // Regions can't overlap and must be in order + } + } + return &sparseFileReader{rfr: rfr, sp: sp, total: total}, nil +} + +// readHole reads a sparse hole ending at endOffset. +func (sfr *sparseFileReader) readHole(b []byte, endOffset int64) int { + n64 := endOffset - sfr.pos + if n64 > int64(len(b)) { + n64 = int64(len(b)) + } + n := int(n64) + for i := 0; i < n; i++ { + b[i] = 0 + } + sfr.pos += n64 + return n +} + +// Read reads the sparse file data in expanded form. +func (sfr *sparseFileReader) Read(b []byte) (n int, err error) { + // Skip past all empty fragments. + for len(sfr.sp) > 0 && sfr.sp[0].numBytes == 0 { + sfr.sp = sfr.sp[1:] + } + + // If there are no more fragments, then it is possible that there + // is one last sparse hole. + if len(sfr.sp) == 0 { + // This behavior matches the BSD tar utility. + // However, GNU tar stops returning data even if sfr.total is unmet. + if sfr.pos < sfr.total { + return sfr.readHole(b, sfr.total), nil + } + return 0, io.EOF + } + + // In front of a data fragment, so read a hole. + if sfr.pos < sfr.sp[0].offset { + return sfr.readHole(b, sfr.sp[0].offset), nil + } + + // In a data fragment, so read from it. + // This math is overflow free since we verify that offset and numBytes can + // be safely added when creating the sparseFileReader. + endPos := sfr.sp[0].offset + sfr.sp[0].numBytes // End offset of fragment + bytesLeft := endPos - sfr.pos // Bytes left in fragment + if int64(len(b)) > bytesLeft { + b = b[:bytesLeft] + } + + n, err = sfr.rfr.Read(b) + sfr.pos += int64(n) + if err == io.EOF { + if sfr.pos < endPos { + err = io.ErrUnexpectedEOF // There was supposed to be more data + } else if sfr.pos < sfr.total { + err = nil // There is still an implicit sparse hole at the end + } + } + + if sfr.pos == endPos { + sfr.sp = sfr.sp[1:] // We are done with this fragment, so pop it + } + return n, err +} + +// numBytes returns the number of bytes left to read in the sparse file's +// sparse-encoded data in the tar archive. +func (sfr *sparseFileReader) numBytes() int64 { + return sfr.rfr.numBytes() +} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/stat_atim.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/stat_atim.go new file mode 100644 index 0000000000..cf9cc79c59 --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/stat_atim.go @@ -0,0 +1,20 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux dragonfly openbsd solaris + +package tar + +import ( + "syscall" + "time" +) + +func statAtime(st *syscall.Stat_t) time.Time { + return time.Unix(st.Atim.Unix()) +} + +func statCtime(st *syscall.Stat_t) time.Time { + return time.Unix(st.Ctim.Unix()) +} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/stat_atimespec.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/stat_atimespec.go new file mode 100644 index 0000000000..6f17dbe307 --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/stat_atimespec.go @@ -0,0 +1,20 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin freebsd netbsd + +package tar + +import ( + "syscall" + "time" +) + +func statAtime(st *syscall.Stat_t) time.Time { + return time.Unix(st.Atimespec.Unix()) +} + +func statCtime(st *syscall.Stat_t) time.Time { + return time.Unix(st.Ctimespec.Unix()) +} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/stat_unix.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/stat_unix.go new file mode 100644 index 0000000000..cb843db4cf --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/stat_unix.go @@ -0,0 +1,32 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux darwin dragonfly freebsd openbsd netbsd solaris + +package tar + +import ( + "os" + "syscall" +) + +func init() { + sysStat = statUnix +} + +func statUnix(fi os.FileInfo, h *Header) error { + sys, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return nil + } + h.Uid = int(sys.Uid) + h.Gid = int(sys.Gid) + // TODO(bradfitz): populate username & group. os/user + // doesn't cache LookupId lookups, and lacks group + // lookup functions. + h.AccessTime = statAtime(sys) + h.ChangeTime = statCtime(sys) + // TODO(bradfitz): major/minor device numbers? + return nil +} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/writer.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/writer.go new file mode 100644 index 0000000000..05027a35a4 --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/archive/tar/writer.go @@ -0,0 +1,419 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +// TODO(dsymonds): +// - catch more errors (no first header, etc.) + +import ( + "bytes" + "errors" + "fmt" + "io" + "path" + "sort" + "strconv" + "strings" + "time" +) + +var ( + ErrWriteTooLong = errors.New("archive/tar: write too long") + ErrFieldTooLong = errors.New("archive/tar: header field too long") + ErrWriteAfterClose = errors.New("archive/tar: write after close") + errInvalidHeader = errors.New("archive/tar: header field too long or contains invalid values") +) + +// A Writer provides sequential writing of a tar archive in POSIX.1 format. +// A tar archive consists of a sequence of files. +// Call WriteHeader to begin a new file, and then call Write to supply that file's data, +// writing at most hdr.Size bytes in total. +type Writer struct { + w io.Writer + err error + nb int64 // number of unwritten bytes for current file entry + pad int64 // amount of padding to write after current file entry + closed bool + usedBinary bool // whether the binary numeric field extension was used + preferPax bool // use pax header instead of binary numeric header + hdrBuff [blockSize]byte // buffer to use in writeHeader when writing a regular header + paxHdrBuff [blockSize]byte // buffer to use in writeHeader when writing a pax header +} + +type formatter struct { + err error // Last error seen +} + +// NewWriter creates a new Writer writing to w. +func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } + +// Flush finishes writing the current file (optional). +func (tw *Writer) Flush() error { + if tw.nb > 0 { + tw.err = fmt.Errorf("archive/tar: missed writing %d bytes", tw.nb) + return tw.err + } + + n := tw.nb + tw.pad + for n > 0 && tw.err == nil { + nr := n + if nr > blockSize { + nr = blockSize + } + var nw int + nw, tw.err = tw.w.Write(zeroBlock[0:nr]) + n -= int64(nw) + } + tw.nb = 0 + tw.pad = 0 + return tw.err +} + +// Write s into b, terminating it with a NUL if there is room. +func (f *formatter) formatString(b []byte, s string) { + if len(s) > len(b) { + f.err = ErrFieldTooLong + return + } + ascii := toASCII(s) + copy(b, ascii) + if len(ascii) < len(b) { + b[len(ascii)] = 0 + } +} + +// Encode x as an octal ASCII string and write it into b with leading zeros. +func (f *formatter) formatOctal(b []byte, x int64) { + s := strconv.FormatInt(x, 8) + // leading zeros, but leave room for a NUL. + for len(s)+1 < len(b) { + s = "0" + s + } + f.formatString(b, s) +} + +// fitsInBase256 reports whether x can be encoded into n bytes using base-256 +// encoding. Unlike octal encoding, base-256 encoding does not require that the +// string ends with a NUL character. Thus, all n bytes are available for output. +// +// If operating in binary mode, this assumes strict GNU binary mode; which means +// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is +// equivalent to the sign bit in two's complement form. +func fitsInBase256(n int, x int64) bool { + var binBits = uint(n-1) * 8 + return n >= 9 || (x >= -1<= 0; i-- { + b[i] = byte(x) + x >>= 8 + } + b[0] |= 0x80 // Highest bit indicates binary format + return + } + + f.formatOctal(b, 0) // Last resort, just write zero + f.err = ErrFieldTooLong +} + +var ( + minTime = time.Unix(0, 0) + // There is room for 11 octal digits (33 bits) of mtime. + maxTime = minTime.Add((1<<33 - 1) * time.Second) +) + +// WriteHeader writes hdr and prepares to accept the file's contents. +// WriteHeader calls Flush if it is not the first header. +// Calling after a Close will return ErrWriteAfterClose. +func (tw *Writer) WriteHeader(hdr *Header) error { + return tw.writeHeader(hdr, true) +} + +// WriteHeader writes hdr and prepares to accept the file's contents. +// WriteHeader calls Flush if it is not the first header. +// Calling after a Close will return ErrWriteAfterClose. +// As this method is called internally by writePax header to allow it to +// suppress writing the pax header. +func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { + if tw.closed { + return ErrWriteAfterClose + } + if tw.err == nil { + tw.Flush() + } + if tw.err != nil { + return tw.err + } + + // a map to hold pax header records, if any are needed + paxHeaders := make(map[string]string) + + // TODO(shanemhansen): we might want to use PAX headers for + // subsecond time resolution, but for now let's just capture + // too long fields or non ascii characters + + var f formatter + var header []byte + + // We need to select which scratch buffer to use carefully, + // since this method is called recursively to write PAX headers. + // If allowPax is true, this is the non-recursive call, and we will use hdrBuff. + // If allowPax is false, we are being called by writePAXHeader, and hdrBuff is + // already being used by the non-recursive call, so we must use paxHdrBuff. + header = tw.hdrBuff[:] + if !allowPax { + header = tw.paxHdrBuff[:] + } + copy(header, zeroBlock) + s := slicer(header) + + // Wrappers around formatter that automatically sets paxHeaders if the + // argument extends beyond the capacity of the input byte slice. + var formatString = func(b []byte, s string, paxKeyword string) { + needsPaxHeader := paxKeyword != paxNone && len(s) > len(b) || !isASCII(s) + if needsPaxHeader { + paxHeaders[paxKeyword] = s + return + } + f.formatString(b, s) + } + var formatNumeric = func(b []byte, x int64, paxKeyword string) { + // Try octal first. + s := strconv.FormatInt(x, 8) + if len(s) < len(b) { + f.formatOctal(b, x) + return + } + + // If it is too long for octal, and PAX is preferred, use a PAX header. + if paxKeyword != paxNone && tw.preferPax { + f.formatOctal(b, 0) + s := strconv.FormatInt(x, 10) + paxHeaders[paxKeyword] = s + return + } + + tw.usedBinary = true + f.formatNumeric(b, x) + } + + // keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax + pathHeaderBytes := s.next(fileNameSize) + + formatString(pathHeaderBytes, hdr.Name, paxPath) + + // Handle out of range ModTime carefully. + var modTime int64 + if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) { + modTime = hdr.ModTime.Unix() + } + + f.formatOctal(s.next(8), hdr.Mode) // 100:108 + formatNumeric(s.next(8), int64(hdr.Uid), paxUid) // 108:116 + formatNumeric(s.next(8), int64(hdr.Gid), paxGid) // 116:124 + formatNumeric(s.next(12), hdr.Size, paxSize) // 124:136 + formatNumeric(s.next(12), modTime, paxNone) // 136:148 --- consider using pax for finer granularity + s.next(8) // chksum (148:156) + s.next(1)[0] = hdr.Typeflag // 156:157 + + formatString(s.next(100), hdr.Linkname, paxLinkpath) + + copy(s.next(8), []byte("ustar\x0000")) // 257:265 + formatString(s.next(32), hdr.Uname, paxUname) // 265:297 + formatString(s.next(32), hdr.Gname, paxGname) // 297:329 + formatNumeric(s.next(8), hdr.Devmajor, paxNone) // 329:337 + formatNumeric(s.next(8), hdr.Devminor, paxNone) // 337:345 + + // keep a reference to the prefix to allow to overwrite it later if we detect that we can use ustar longnames instead of pax + prefixHeaderBytes := s.next(155) + formatString(prefixHeaderBytes, "", paxNone) // 345:500 prefix + + // Use the GNU magic instead of POSIX magic if we used any GNU extensions. + if tw.usedBinary { + copy(header[257:265], []byte("ustar \x00")) + } + + _, paxPathUsed := paxHeaders[paxPath] + // try to use a ustar header when only the name is too long + if !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed { + prefix, suffix, ok := splitUSTARPath(hdr.Name) + if ok { + // Since we can encode in USTAR format, disable PAX header. + delete(paxHeaders, paxPath) + + // Update the path fields + formatString(pathHeaderBytes, suffix, paxNone) + formatString(prefixHeaderBytes, prefix, paxNone) + } + } + + // The chksum field is terminated by a NUL and a space. + // This is different from the other octal fields. + chksum, _ := checksum(header) + f.formatOctal(header[148:155], chksum) // Never fails + header[155] = ' ' + + // Check if there were any formatting errors. + if f.err != nil { + tw.err = f.err + return tw.err + } + + if allowPax { + for k, v := range hdr.Xattrs { + paxHeaders[paxXattr+k] = v + } + for k, v := range hdr.Winheaders { + paxHeaders[paxWindows+k] = v + } + } + + if len(paxHeaders) > 0 { + if !allowPax { + return errInvalidHeader + } + if err := tw.writePAXHeader(hdr, paxHeaders); err != nil { + return err + } + } + tw.nb = int64(hdr.Size) + tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize + + _, tw.err = tw.w.Write(header) + return tw.err +} + +// splitUSTARPath splits a path according to USTAR prefix and suffix rules. +// If the path is not splittable, then it will return ("", "", false). +func splitUSTARPath(name string) (prefix, suffix string, ok bool) { + length := len(name) + if length <= fileNameSize || !isASCII(name) { + return "", "", false + } else if length > fileNamePrefixSize+1 { + length = fileNamePrefixSize + 1 + } else if name[length-1] == '/' { + length-- + } + + i := strings.LastIndex(name[:length], "/") + nlen := len(name) - i - 1 // nlen is length of suffix + plen := i // plen is length of prefix + if i <= 0 || nlen > fileNameSize || nlen == 0 || plen > fileNamePrefixSize { + return "", "", false + } + return name[:i], name[i+1:], true +} + +// writePaxHeader writes an extended pax header to the +// archive. +func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error { + // Prepare extended header + ext := new(Header) + ext.Typeflag = TypeXHeader + // Setting ModTime is required for reader parsing to + // succeed, and seems harmless enough. + ext.ModTime = hdr.ModTime + // The spec asks that we namespace our pseudo files + // with the current pid. However, this results in differing outputs + // for identical inputs. As such, the constant 0 is now used instead. + // golang.org/issue/12358 + dir, file := path.Split(hdr.Name) + fullName := path.Join(dir, "PaxHeaders.0", file) + + ascii := toASCII(fullName) + if len(ascii) > 100 { + ascii = ascii[:100] + } + ext.Name = ascii + // Construct the body + var buf bytes.Buffer + + // Keys are sorted before writing to body to allow deterministic output. + var keys []string + for k := range paxHeaders { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + fmt.Fprint(&buf, formatPAXRecord(k, paxHeaders[k])) + } + + ext.Size = int64(len(buf.Bytes())) + if err := tw.writeHeader(ext, false); err != nil { + return err + } + if _, err := tw.Write(buf.Bytes()); err != nil { + return err + } + if err := tw.Flush(); err != nil { + return err + } + return nil +} + +// formatPAXRecord formats a single PAX record, prefixing it with the +// appropriate length. +func formatPAXRecord(k, v string) string { + const padding = 3 // Extra padding for ' ', '=', and '\n' + size := len(k) + len(v) + padding + size += len(strconv.Itoa(size)) + record := fmt.Sprintf("%d %s=%s\n", size, k, v) + + // Final adjustment if adding size field increased the record size. + if len(record) != size { + size = len(record) + record = fmt.Sprintf("%d %s=%s\n", size, k, v) + } + return record +} + +// Write writes to the current entry in the tar archive. +// Write returns the error ErrWriteTooLong if more than +// hdr.Size bytes are written after WriteHeader. +func (tw *Writer) Write(b []byte) (n int, err error) { + if tw.closed { + err = ErrWriteAfterClose + return + } + overwrite := false + if int64(len(b)) > tw.nb { + b = b[0:tw.nb] + overwrite = true + } + n, err = tw.w.Write(b) + tw.nb -= int64(n) + if err == nil && overwrite { + err = ErrWriteTooLong + return + } + tw.err = err + return +} + +// Close closes the tar archive, flushing any unwritten +// data to the underlying writer. +func (tw *Writer) Close() error { + if tw.err != nil || tw.closed { + return tw.err + } + tw.Flush() + tw.closed = true + if tw.err != nil { + return tw.err + } + + // trailer: two zero blocks + for i := 0; i < 2; i++ { + _, tw.err = tw.w.Write(zeroBlock) + if tw.err != nil { + break + } + } + return tw.err +} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/backup.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/backup.go new file mode 100644 index 0000000000..bfefd42c4d --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/backup.go @@ -0,0 +1,241 @@ +package winio + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "runtime" + "syscall" + "unicode/utf16" +) + +//sys backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupRead +//sys backupWrite(h syscall.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupWrite + +const ( + BackupData = uint32(iota + 1) + BackupEaData + BackupSecurity + BackupAlternateData + BackupLink + BackupPropertyData + BackupObjectId + BackupReparseData + BackupSparseBlock + BackupTxfsData + + StreamSparseAttributes = uint32(8) +) + +// BackupHeader represents a backup stream of a file. +type BackupHeader struct { + Id uint32 // The backup stream ID + Attributes uint32 // Stream attributes + Size int64 // The size of the stream in bytes + Name string // The name of the stream (for BackupAlternateData only). + Offset int64 // The offset of the stream in the file (for BackupSparseBlock only). +} + +type win32StreamId struct { + StreamId uint32 + Attributes uint32 + Size uint64 + NameSize uint32 +} + +// BackupStreamReader reads from a stream produced by the BackupRead Win32 API and produces a series +// of BackupHeader values. +type BackupStreamReader struct { + r io.Reader + bytesLeft int64 +} + +// NewBackupStreamReader produces a BackupStreamReader from any io.Reader. +func NewBackupStreamReader(r io.Reader) *BackupStreamReader { + return &BackupStreamReader{r, 0} +} + +// Next returns the next backup stream and prepares for calls to Write(). It skips the remainder of the current stream if +// it was not completely read. +func (r *BackupStreamReader) Next() (*BackupHeader, error) { + if r.bytesLeft > 0 { + if _, err := io.Copy(ioutil.Discard, r); err != nil { + return nil, err + } + } + var wsi win32StreamId + if err := binary.Read(r.r, binary.LittleEndian, &wsi); err != nil { + return nil, err + } + hdr := &BackupHeader{ + Id: wsi.StreamId, + Attributes: wsi.Attributes, + Size: int64(wsi.Size), + } + if wsi.NameSize != 0 { + name := make([]uint16, int(wsi.NameSize/2)) + if err := binary.Read(r.r, binary.LittleEndian, name); err != nil { + return nil, err + } + hdr.Name = syscall.UTF16ToString(name) + } + if wsi.StreamId == BackupSparseBlock { + if err := binary.Read(r.r, binary.LittleEndian, &hdr.Offset); err != nil { + return nil, err + } + hdr.Size -= 8 + } + r.bytesLeft = hdr.Size + return hdr, nil +} + +// Read reads from the current backup stream. +func (r *BackupStreamReader) Read(b []byte) (int, error) { + if r.bytesLeft == 0 { + return 0, io.EOF + } + if int64(len(b)) > r.bytesLeft { + b = b[:r.bytesLeft] + } + n, err := r.r.Read(b) + r.bytesLeft -= int64(n) + if err == io.EOF { + err = io.ErrUnexpectedEOF + } else if r.bytesLeft == 0 && err == nil { + err = io.EOF + } + return n, err +} + +// BackupStreamWriter writes a stream compatible with the BackupWrite Win32 API. +type BackupStreamWriter struct { + w io.Writer + bytesLeft int64 +} + +// NewBackupStreamWriter produces a BackupStreamWriter on top of an io.Writer. +func NewBackupStreamWriter(w io.Writer) *BackupStreamWriter { + return &BackupStreamWriter{w, 0} +} + +// WriteHeader writes the next backup stream header and prepares for calls to Write(). +func (w *BackupStreamWriter) WriteHeader(hdr *BackupHeader) error { + if w.bytesLeft != 0 { + return fmt.Errorf("missing %d bytes", w.bytesLeft) + } + name := utf16.Encode([]rune(hdr.Name)) + wsi := win32StreamId{ + StreamId: hdr.Id, + Attributes: hdr.Attributes, + Size: uint64(hdr.Size), + NameSize: uint32(len(name) * 2), + } + if hdr.Id == BackupSparseBlock { + // Include space for the int64 block offset + wsi.Size += 8 + } + if err := binary.Write(w.w, binary.LittleEndian, &wsi); err != nil { + return err + } + if len(name) != 0 { + if err := binary.Write(w.w, binary.LittleEndian, name); err != nil { + return err + } + } + if hdr.Id == BackupSparseBlock { + if err := binary.Write(w.w, binary.LittleEndian, hdr.Offset); err != nil { + return err + } + } + w.bytesLeft = hdr.Size + return nil +} + +// Write writes to the current backup stream. +func (w *BackupStreamWriter) Write(b []byte) (int, error) { + if w.bytesLeft < int64(len(b)) { + return 0, fmt.Errorf("too many bytes by %d", int64(len(b))-w.bytesLeft) + } + n, err := w.w.Write(b) + w.bytesLeft -= int64(n) + return n, err +} + +// BackupFileReader provides an io.ReadCloser interface on top of the BackupRead Win32 API. +type BackupFileReader struct { + f *os.File + includeSecurity bool + ctx uintptr +} + +// NewBackupFileReader returns a new BackupFileReader from a file handle. If includeSecurity is true, +// Read will attempt to read the security descriptor of the file. +func NewBackupFileReader(f *os.File, includeSecurity bool) *BackupFileReader { + r := &BackupFileReader{f, includeSecurity, 0} + runtime.SetFinalizer(r, func(r *BackupFileReader) { r.Close() }) + return r +} + +// Read reads a backup stream from the file by calling the Win32 API BackupRead(). +func (r *BackupFileReader) Read(b []byte) (int, error) { + var bytesRead uint32 + err := backupRead(syscall.Handle(r.f.Fd()), b, &bytesRead, false, r.includeSecurity, &r.ctx) + if err != nil { + return 0, &os.PathError{"BackupRead", r.f.Name(), err} + } + if bytesRead == 0 { + return 0, io.EOF + } + return int(bytesRead), nil +} + +// Close frees Win32 resources associated with the BackupFileReader. It does not close +// the underlying file. +func (r *BackupFileReader) Close() error { + if r.ctx != 0 { + backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx) + r.ctx = 0 + } + return nil +} + +// BackupFileWriter provides an io.WriteCloser interface on top of the BackupWrite Win32 API. +type BackupFileWriter struct { + f *os.File + includeSecurity bool + ctx uintptr +} + +// NewBackupFileWrtier returns a new BackupFileWriter from a file handle. If includeSecurity is true, +// Write() will attempt to restore the security descriptor from the stream. +func NewBackupFileWriter(f *os.File, includeSecurity bool) *BackupFileWriter { + w := &BackupFileWriter{f, includeSecurity, 0} + runtime.SetFinalizer(w, func(w *BackupFileWriter) { w.Close() }) + return w +} + +// Write restores a portion of the file using the provided backup stream. +func (w *BackupFileWriter) Write(b []byte) (int, error) { + var bytesWritten uint32 + err := backupWrite(syscall.Handle(w.f.Fd()), b, &bytesWritten, false, w.includeSecurity, &w.ctx) + if err != nil { + return 0, &os.PathError{"BackupWrite", w.f.Name(), err} + } + if int(bytesWritten) != len(b) { + return int(bytesWritten), errors.New("not all bytes could be written") + } + return len(b), nil +} + +// Close frees Win32 resources associated with the BackupFileWriter. It does not +// close the underlying file. +func (w *BackupFileWriter) Close() error { + if w.ctx != 0 { + backupWrite(syscall.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx) + w.ctx = 0 + } + return nil +} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/backuptar/tar.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/backuptar/tar.go new file mode 100644 index 0000000000..c988574fdf --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/backuptar/tar.go @@ -0,0 +1,362 @@ +package backuptar + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/Microsoft/go-winio" + "github.com/Microsoft/go-winio/archive/tar" // until archive/tar supports pax extensions in its interface +) + +const ( + c_ISUID = 04000 // Set uid + c_ISGID = 02000 // Set gid + c_ISVTX = 01000 // Save text (sticky bit) + c_ISDIR = 040000 // Directory + c_ISFIFO = 010000 // FIFO + c_ISREG = 0100000 // Regular file + c_ISLNK = 0120000 // Symbolic link + c_ISBLK = 060000 // Block special file + c_ISCHR = 020000 // Character special file + c_ISSOCK = 0140000 // Socket +) + +const ( + hdrFileAttributes = "fileattr" + hdrAccessTime = "accesstime" + hdrChangeTime = "changetime" + hdrCreateTime = "createtime" + hdrWriteTime = "writetime" + hdrSecurityDescriptor = "sd" + hdrMountPoint = "mountpoint" +) + +func writeZeroes(w io.Writer, count int64) error { + buf := make([]byte, 8192) + c := len(buf) + for i := int64(0); i < count; i += int64(c) { + if int64(c) > count-i { + c = int(count - i) + } + _, err := w.Write(buf[:c]) + if err != nil { + return err + } + } + return nil +} + +func copySparse(t *tar.Writer, br *winio.BackupStreamReader) error { + curOffset := int64(0) + for { + bhdr, err := br.Next() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + return err + } + if bhdr.Id != winio.BackupSparseBlock { + return fmt.Errorf("unexpected stream %d", bhdr.Id) + } + + // archive/tar does not support writing sparse files + // so just write zeroes to catch up to the current offset. + err = writeZeroes(t, bhdr.Offset-curOffset) + if bhdr.Size == 0 { + break + } + n, err := io.Copy(t, br) + if err != nil { + return err + } + curOffset = bhdr.Offset + n + } + return nil +} + +func win32TimeFromTar(key string, hdrs map[string]string, unixTime time.Time) syscall.Filetime { + if s, ok := hdrs[key]; ok { + n, err := strconv.ParseUint(s, 10, 64) + if err == nil { + return syscall.Filetime{uint32(n & 0xffffffff), uint32(n >> 32)} + } + } + return syscall.NsecToFiletime(unixTime.UnixNano()) +} + +func win32TimeToTar(ft syscall.Filetime) (string, time.Time) { + return fmt.Sprintf("%d", uint64(ft.LowDateTime)+(uint64(ft.HighDateTime)<<32)), time.Unix(0, ft.Nanoseconds()) +} + +// Writes a file to a tar writer using data from a Win32 backup stream. +// +// This encodes Win32 metadata as tar pax vendor extensions starting with MSWINDOWS. +// +// The additional Win32 metadata is: +// +// MSWINDOWS.fileattr: The Win32 file attributes, as a decimal value +// +// MSWINDOWS.accesstime: The last access time, as a Filetime expressed as a 64-bit decimal value. +// +// MSWINDOWS.createtime: The creation time, as a Filetime expressed as a 64-bit decimal value. +// +// MSWINDOWS.changetime: The creation time, as a Filetime expressed as a 64-bit decimal value. +// +// MSWINDOWS.writetime: The creation time, as a Filetime expressed as a 64-bit decimal value. +// +// MSWINDOWS.sd: The Win32 security descriptor, in SDDL (string) format +// +// MSWINDOWS.mountpoint: If present, this is a mount point and not a symlink, even though the type is '2' (symlink) +func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size int64, fileInfo *winio.FileBasicInfo) error { + name = filepath.ToSlash(name) + hdr := &tar.Header{ + Name: name, + Size: size, + Typeflag: tar.TypeReg, + Winheaders: make(map[string]string), + } + hdr.Winheaders[hdrFileAttributes] = fmt.Sprintf("%d", fileInfo.FileAttributes) + hdr.Winheaders[hdrAccessTime], hdr.AccessTime = win32TimeToTar(fileInfo.LastAccessTime) + hdr.Winheaders[hdrChangeTime], hdr.ChangeTime = win32TimeToTar(fileInfo.ChangeTime) + hdr.Winheaders[hdrCreateTime], _ = win32TimeToTar(fileInfo.CreationTime) + hdr.Winheaders[hdrWriteTime], hdr.ModTime = win32TimeToTar(fileInfo.LastWriteTime) + + if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 { + hdr.Mode |= c_ISDIR + hdr.Size = 0 + hdr.Typeflag = tar.TypeDir + } + + br := winio.NewBackupStreamReader(r) + var dataHdr *winio.BackupHeader + for dataHdr == nil { + bhdr, err := br.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + switch bhdr.Id { + case winio.BackupData: + hdr.Mode |= c_ISREG + dataHdr = bhdr + case winio.BackupSecurity: + sd, err := ioutil.ReadAll(br) + if err != nil { + return err + } + sddl, err := winio.SecurityDescriptorToSddl(sd) + if err != nil { + return err + } + hdr.Winheaders[hdrSecurityDescriptor] = sddl + + case winio.BackupReparseData: + hdr.Mode |= c_ISLNK + hdr.Typeflag = tar.TypeSymlink + reparseBuffer, err := ioutil.ReadAll(br) + rp, err := winio.DecodeReparsePoint(reparseBuffer) + if err != nil { + return err + } + if rp.IsMountPoint { + hdr.Winheaders[hdrMountPoint] = "1" + } + hdr.Linkname = rp.Target + case winio.BackupEaData, winio.BackupLink, winio.BackupPropertyData, winio.BackupObjectId, winio.BackupTxfsData: + // ignore these streams + default: + return fmt.Errorf("%s: unknown stream ID %d", name, bhdr.Id) + } + } + + err := t.WriteHeader(hdr) + if err != nil { + return err + } + + if dataHdr != nil { + // A data stream was found. Copy the data. + if (dataHdr.Attributes & winio.StreamSparseAttributes) == 0 { + if size != dataHdr.Size { + return fmt.Errorf("%s: mismatch between file size %d and header size %d", name, size, dataHdr.Size) + } + _, err = io.Copy(t, br) + if err != nil { + return err + } + } else { + err = copySparse(t, br) + if err != nil { + return err + } + } + } + + // Look for streams after the data stream. The only ones we handle are alternate data streams. + // Other streams may have metadata that could be serialized, but the tar header has already + // been written. In practice, this means that we don't get EA or TXF metadata. + for { + bhdr, err := br.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + switch bhdr.Id { + case winio.BackupAlternateData: + altName := bhdr.Name + if strings.HasSuffix(altName, ":$DATA") { + altName = altName[:len(altName)-len(":$DATA")] + } + if (bhdr.Attributes & winio.StreamSparseAttributes) == 0 { + hdr = &tar.Header{ + Name: name + altName, + Mode: hdr.Mode, + Typeflag: tar.TypeReg, + Size: bhdr.Size, + ModTime: hdr.ModTime, + AccessTime: hdr.AccessTime, + ChangeTime: hdr.ChangeTime, + } + err = t.WriteHeader(hdr) + if err != nil { + return err + } + _, err = io.Copy(t, br) + if err != nil { + return err + } + + } else { + // Unsupported for now, since the size of the alternate stream is not present + // in the backup stream until after the data has been read. + return errors.New("tar of sparse alternate data streams is unsupported") + } + case winio.BackupEaData, winio.BackupLink, winio.BackupPropertyData, winio.BackupObjectId, winio.BackupTxfsData: + // ignore these streams + default: + return fmt.Errorf("%s: unknown stream ID %d after data", name, bhdr.Id) + } + } + return nil +} + +// Retrieves basic Win32 file information from a tar header, using the additional metadata written by +// WriteTarFileFromBackupStream. +func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *winio.FileBasicInfo, err error) { + name = hdr.Name + if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA { + size = hdr.Size + } + fileInfo = &winio.FileBasicInfo{ + LastAccessTime: win32TimeFromTar(hdrAccessTime, hdr.Winheaders, hdr.AccessTime), + LastWriteTime: win32TimeFromTar(hdrWriteTime, hdr.Winheaders, hdr.ModTime), + ChangeTime: win32TimeFromTar(hdrChangeTime, hdr.Winheaders, hdr.ChangeTime), + CreationTime: win32TimeFromTar(hdrCreateTime, hdr.Winheaders, hdr.ModTime), + } + if attrStr, ok := hdr.Winheaders[hdrFileAttributes]; ok { + attr, err := strconv.ParseUint(attrStr, 10, 32) + if err != nil { + return "", 0, nil, err + } + fileInfo.FileAttributes = uintptr(attr) + } else { + if hdr.Typeflag == tar.TypeDir { + fileInfo.FileAttributes |= syscall.FILE_ATTRIBUTE_DIRECTORY + } + } + return +} + +// Writes a Win32 backup stream from the current tar file. Since this function may process multiple +// tar file entries in order to collect all the alternate data streams for the file, it returns the next +// tar file that was not processed, or io.EOF is there are no more. +func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (*tar.Header, error) { + bw := winio.NewBackupStreamWriter(w) + if sddl, ok := hdr.Winheaders[hdrSecurityDescriptor]; ok { + sd, err := winio.SddlToSecurityDescriptor(sddl) + if err != nil { + return nil, err + } + bhdr := winio.BackupHeader{ + Id: winio.BackupSecurity, + Size: int64(len(sd)), + } + err = bw.WriteHeader(&bhdr) + if err != nil { + return nil, err + } + _, err = bw.Write(sd) + if err != nil { + return nil, err + } + } + if hdr.Typeflag == tar.TypeSymlink { + _, isMountPoint := hdr.Winheaders[hdrMountPoint] + rp := winio.ReparsePoint{ + Target: hdr.Linkname, + IsMountPoint: isMountPoint, + } + reparse := winio.EncodeReparsePoint(&rp) + bhdr := winio.BackupHeader{ + Id: winio.BackupReparseData, + Size: int64(len(reparse)), + } + err := bw.WriteHeader(&bhdr) + if err != nil { + return nil, err + } + _, err = bw.Write(reparse) + if err != nil { + return nil, err + } + } + if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA { + bhdr := winio.BackupHeader{ + Id: winio.BackupData, + Size: hdr.Size, + } + err := bw.WriteHeader(&bhdr) + if err != nil { + return nil, err + } + _, err = io.Copy(bw, t) + if err != nil { + return nil, err + } + } + // Copy all the alternate data streams and return the next non-ADS header. + for { + ahdr, err := t.Next() + if err != nil { + return nil, err + } + if ahdr.Typeflag != tar.TypeReg || !strings.HasPrefix(ahdr.Name, hdr.Name+":") { + return ahdr, nil + } + bhdr := winio.BackupHeader{ + Id: winio.BackupAlternateData, + Size: ahdr.Size, + Name: ahdr.Name[len(hdr.Name)+1:] + ":$DATA", + } + err = bw.WriteHeader(&bhdr) + if err != nil { + return nil, err + } + _, err = io.Copy(bw, t) + if err != nil { + return nil, err + } + } +} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/fileinfo.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/fileinfo.go new file mode 100644 index 0000000000..dc05a8b334 --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/fileinfo.go @@ -0,0 +1,30 @@ +package winio + +import ( + "os" + "syscall" + "unsafe" +) + +//sys getFileInformationByHandleEx(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = GetFileInformationByHandleEx +//sys setFileInformationByHandle(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = SetFileInformationByHandle + +type FileBasicInfo struct { + CreationTime, LastAccessTime, LastWriteTime, ChangeTime syscall.Filetime + FileAttributes uintptr // includes padding +} + +func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) { + bi := &FileBasicInfo{} + if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), 0, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil { + return nil, &os.PathError{"GetFileInformationByHandleEx", f.Name(), err} + } + return bi, nil +} + +func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error { + if err := setFileInformationByHandle(syscall.Handle(f.Fd()), 0, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil { + return &os.PathError{"SetFileInformationByHandle", f.Name(), err} + } + return nil +} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/mksyscall_windows.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/mksyscall_windows.go deleted file mode 100644 index 652074c7f1..0000000000 --- a/components/engine/vendor/src/github.com/Microsoft/go-winio/mksyscall_windows.go +++ /dev/null @@ -1,797 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -/* -mksyscall_windows generates windows system call bodies - -It parses all files specified on command line containing function -prototypes (like syscall_windows.go) and prints system call bodies -to standard output. - -The prototypes are marked by lines beginning with "//sys" and read -like func declarations if //sys is replaced by func, but: - -* The parameter lists must give a name for each argument. This - includes return parameters. - -* The parameter lists must give a type for each argument: - the (x, y, z int) shorthand is not allowed. - -* If the return parameter is an error number, it must be named err. - -* If go func name needs to be different from it's winapi dll name, - the winapi name could be specified at the end, after "=" sign, like - //sys LoadLibrary(libname string) (handle uint32, err error) = LoadLibraryA - -* Each function that returns err needs to supply a condition, that - return value of winapi will be tested against to detect failure. - This would set err to windows "last-error", otherwise it will be nil. - The value can be provided at end of //sys declaration, like - //sys LoadLibrary(libname string) (handle uint32, err error) [failretval==-1] = LoadLibraryA - and is [failretval==0] by default. - -Usage: - mksyscall_windows [flags] [path ...] - -The flags are: - -output - Specify output file name (outputs to console if blank). - -trace - Generate print statement after every syscall. -*/ -package main - -import ( - "bufio" - "bytes" - "errors" - "flag" - "fmt" - "go/format" - "go/parser" - "go/token" - "io" - "io/ioutil" - "log" - "os" - "strconv" - "strings" - "text/template" -) - -var ( - filename = flag.String("output", "", "output file name (standard output if omitted)") - printTraceFlag = flag.Bool("trace", false, "generate print statement after every syscall") -) - -func trim(s string) string { - return strings.Trim(s, " \t") -} - -var packageName string - -func packagename() string { - return packageName -} - -func syscalldot() string { - if packageName == "syscall" { - return "" - } - return "syscall." -} - -// Param is function parameter -type Param struct { - Name string - Type string - fn *Fn - tmpVarIdx int -} - -// tmpVar returns temp variable name that will be used to represent p during syscall. -func (p *Param) tmpVar() string { - if p.tmpVarIdx < 0 { - p.tmpVarIdx = p.fn.curTmpVarIdx - p.fn.curTmpVarIdx++ - } - return fmt.Sprintf("_p%d", p.tmpVarIdx) -} - -// BoolTmpVarCode returns source code for bool temp variable. -func (p *Param) BoolTmpVarCode() string { - const code = `var %s uint32 - if %s { - %s = 1 - } else { - %s = 0 - }` - tmp := p.tmpVar() - return fmt.Sprintf(code, tmp, p.Name, tmp, tmp) -} - -// SliceTmpVarCode returns source code for slice temp variable. -func (p *Param) SliceTmpVarCode() string { - const code = `var %s *%s - if len(%s) > 0 { - %s = &%s[0] - }` - tmp := p.tmpVar() - return fmt.Sprintf(code, tmp, p.Type[2:], p.Name, tmp, p.Name) -} - -// StringTmpVarCode returns source code for string temp variable. -func (p *Param) StringTmpVarCode() string { - errvar := p.fn.Rets.ErrorVarName() - if errvar == "" { - errvar = "_" - } - tmp := p.tmpVar() - const code = `var %s %s - %s, %s = %s(%s)` - s := fmt.Sprintf(code, tmp, p.fn.StrconvType(), tmp, errvar, p.fn.StrconvFunc(), p.Name) - if errvar == "-" { - return s - } - const morecode = ` - if %s != nil { - return - }` - return s + fmt.Sprintf(morecode, errvar) -} - -// TmpVarCode returns source code for temp variable. -func (p *Param) TmpVarCode() string { - switch { - case p.Type == "bool": - return p.BoolTmpVarCode() - case strings.HasPrefix(p.Type, "[]"): - return p.SliceTmpVarCode() - default: - return "" - } -} - -// TmpVarHelperCode returns source code for helper's temp variable. -func (p *Param) TmpVarHelperCode() string { - if p.Type != "string" { - return "" - } - return p.StringTmpVarCode() -} - -// SyscallArgList returns source code fragments representing p parameter -// in syscall. Slices are translated into 2 syscall parameters: pointer to -// the first element and length. -func (p *Param) SyscallArgList() []string { - t := p.HelperType() - var s string - switch { - case t[0] == '*': - s = fmt.Sprintf("unsafe.Pointer(%s)", p.Name) - case t == "bool": - s = p.tmpVar() - case strings.HasPrefix(t, "[]"): - return []string{ - fmt.Sprintf("uintptr(unsafe.Pointer(%s))", p.tmpVar()), - fmt.Sprintf("uintptr(len(%s))", p.Name), - } - default: - s = p.Name - } - return []string{fmt.Sprintf("uintptr(%s)", s)} -} - -// IsError determines if p parameter is used to return error. -func (p *Param) IsError() bool { - return p.Name == "err" && p.Type == "error" -} - -// HelperType returns type of parameter p used in helper function. -func (p *Param) HelperType() string { - if p.Type == "string" { - return p.fn.StrconvType() - } - return p.Type -} - -// join concatenates parameters ps into a string with sep separator. -// Each parameter is converted into string by applying fn to it -// before conversion. -func join(ps []*Param, fn func(*Param) string, sep string) string { - if len(ps) == 0 { - return "" - } - a := make([]string, 0) - for _, p := range ps { - a = append(a, fn(p)) - } - return strings.Join(a, sep) -} - -// Rets describes function return parameters. -type Rets struct { - Name string - Type string - ReturnsError bool - FailCond string -} - -// ErrorVarName returns error variable name for r. -func (r *Rets) ErrorVarName() string { - if r.ReturnsError { - return "err" - } - if r.Type == "error" { - return r.Name - } - return "" -} - -// ToParams converts r into slice of *Param. -func (r *Rets) ToParams() []*Param { - ps := make([]*Param, 0) - if len(r.Name) > 0 { - ps = append(ps, &Param{Name: r.Name, Type: r.Type}) - } - if r.ReturnsError { - ps = append(ps, &Param{Name: "err", Type: "error"}) - } - return ps -} - -// List returns source code of syscall return parameters. -func (r *Rets) List() string { - s := join(r.ToParams(), func(p *Param) string { return p.Name + " " + p.Type }, ", ") - if len(s) > 0 { - s = "(" + s + ")" - } - return s -} - -// PrintList returns source code of trace printing part correspondent -// to syscall return values. -func (r *Rets) PrintList() string { - return join(r.ToParams(), func(p *Param) string { return fmt.Sprintf(`"%s=", %s, `, p.Name, p.Name) }, `", ", `) -} - -// SetReturnValuesCode returns source code that accepts syscall return values. -func (r *Rets) SetReturnValuesCode() string { - if r.Name == "" && !r.ReturnsError { - return "" - } - retvar := "r0" - if r.Name == "" { - retvar = "r1" - } - errvar := "_" - if r.ReturnsError { - errvar = "e1" - } - return fmt.Sprintf("%s, _, %s := ", retvar, errvar) -} - -func (r *Rets) useLongHandleErrorCode(retvar string) string { - const code = `if %s { - if e1 != 0 { - err = error(e1) - } else { - err = %sEINVAL - } - }` - cond := retvar + " == 0" - if r.FailCond != "" { - cond = strings.Replace(r.FailCond, "failretval", retvar, 1) - } - return fmt.Sprintf(code, cond, syscalldot()) -} - -// SetErrorCode returns source code that sets return parameters. -func (r *Rets) SetErrorCode() string { - const code = `if r0 != 0 { - %s = %sErrno(r0) - }` - if r.Name == "" && !r.ReturnsError { - return "" - } - if r.Name == "" { - return r.useLongHandleErrorCode("r1") - } - if r.Type == "error" { - return fmt.Sprintf(code, r.Name, syscalldot()) - } - s := "" - switch { - case r.Type[0] == '*': - s = fmt.Sprintf("%s = (%s)(unsafe.Pointer(r0))", r.Name, r.Type) - case r.Type == "bool": - s = fmt.Sprintf("%s = r0 != 0", r.Name) - default: - s = fmt.Sprintf("%s = %s(r0)", r.Name, r.Type) - } - if !r.ReturnsError { - return s - } - return s + "\n\t" + r.useLongHandleErrorCode(r.Name) -} - -// Fn describes syscall function. -type Fn struct { - Name string - Params []*Param - Rets *Rets - PrintTrace bool - confirmproc bool - dllname string - dllfuncname string - src string - // TODO: get rid of this field and just use parameter index instead - curTmpVarIdx int // insure tmp variables have uniq names -} - -// extractParams parses s to extract function parameters. -func extractParams(s string, f *Fn) ([]*Param, error) { - s = trim(s) - if s == "" { - return nil, nil - } - a := strings.Split(s, ",") - ps := make([]*Param, len(a)) - for i := range ps { - s2 := trim(a[i]) - b := strings.Split(s2, " ") - if len(b) != 2 { - b = strings.Split(s2, "\t") - if len(b) != 2 { - return nil, errors.New("Could not extract function parameter from \"" + s2 + "\"") - } - } - ps[i] = &Param{ - Name: trim(b[0]), - Type: trim(b[1]), - fn: f, - tmpVarIdx: -1, - } - } - return ps, nil -} - -// extractSection extracts text out of string s starting after start -// and ending just before end. found return value will indicate success, -// and prefix, body and suffix will contain correspondent parts of string s. -func extractSection(s string, start, end rune) (prefix, body, suffix string, found bool) { - s = trim(s) - if strings.HasPrefix(s, string(start)) { - // no prefix - body = s[1:] - } else { - a := strings.SplitN(s, string(start), 2) - if len(a) != 2 { - return "", "", s, false - } - prefix = a[0] - body = a[1] - } - a := strings.SplitN(body, string(end), 2) - if len(a) != 2 { - return "", "", "", false - } - return prefix, a[0], a[1], true -} - -// newFn parses string s and return created function Fn. -func newFn(s string) (*Fn, error) { - s = trim(s) - f := &Fn{ - Rets: &Rets{}, - src: s, - PrintTrace: *printTraceFlag, - } - // function name and args - prefix, body, s, found := extractSection(s, '(', ')') - if !found || prefix == "" { - return nil, errors.New("Could not extract function name and parameters from \"" + f.src + "\"") - } - f.Name = prefix - var err error - f.Params, err = extractParams(body, f) - if err != nil { - return nil, err - } - // return values - _, body, s, found = extractSection(s, '(', ')') - if found { - r, err := extractParams(body, f) - if err != nil { - return nil, err - } - switch len(r) { - case 0: - case 1: - if r[0].IsError() { - f.Rets.ReturnsError = true - } else { - f.Rets.Name = r[0].Name - f.Rets.Type = r[0].Type - } - case 2: - if !r[1].IsError() { - return nil, errors.New("Only last windows error is allowed as second return value in \"" + f.src + "\"") - } - f.Rets.ReturnsError = true - f.Rets.Name = r[0].Name - f.Rets.Type = r[0].Type - default: - return nil, errors.New("Too many return values in \"" + f.src + "\"") - } - } - // fail condition - _, body, s, found = extractSection(s, '[', ']') - if found { - f.Rets.FailCond = body - } - // dll and dll function names - s = trim(s) - if s == "" { - return f, nil - } - if !strings.HasPrefix(s, "=") { - return nil, errors.New("Could not extract dll name from \"" + f.src + "\"") - } - s = trim(s[1:]) - a := strings.Split(s, ".") - switch len(a) { - case 1: - f.dllfuncname = a[0] - case 2: - f.dllname = a[0] - f.dllfuncname = a[1] - default: - return nil, errors.New("Could not extract dll name from \"" + f.src + "\"") - } - if f.dllfuncname[len(f.dllfuncname)-1] == '?' { - f.confirmproc = true - f.dllfuncname = f.dllfuncname[0 : len(f.dllfuncname)-1] - } - return f, nil -} - -// DLLName returns DLL name for function f. -func (f *Fn) DLLName() string { - if f.dllname == "" { - return "kernel32" - } - return f.dllname -} - -// DLLName returns DLL function name for function f. -func (f *Fn) DLLFuncName() string { - if f.dllfuncname == "" { - return f.Name - } - return f.dllfuncname -} - -func (f *Fn) ConfirmProc() bool { - return f.confirmproc -} - -// ParamList returns source code for function f parameters. -func (f *Fn) ParamList() string { - return join(f.Params, func(p *Param) string { return p.Name + " " + p.Type }, ", ") -} - -// HelperParamList returns source code for helper function f parameters. -func (f *Fn) HelperParamList() string { - return join(f.Params, func(p *Param) string { return p.Name + " " + p.HelperType() }, ", ") -} - -// ParamPrintList returns source code of trace printing part correspondent -// to syscall input parameters. -func (f *Fn) ParamPrintList() string { - return join(f.Params, func(p *Param) string { return fmt.Sprintf(`"%s=", %s, `, p.Name, p.Name) }, `", ", `) -} - -// ParamCount return number of syscall parameters for function f. -func (f *Fn) ParamCount() int { - n := 0 - for _, p := range f.Params { - n += len(p.SyscallArgList()) - } - return n -} - -// SyscallParamCount determines which version of Syscall/Syscall6/Syscall9/... -// to use. It returns parameter count for correspondent SyscallX function. -func (f *Fn) SyscallParamCount() int { - n := f.ParamCount() - switch { - case n <= 3: - return 3 - case n <= 6: - return 6 - case n <= 9: - return 9 - case n <= 12: - return 12 - case n <= 15: - return 15 - default: - panic("too many arguments to system call") - } -} - -// Syscall determines which SyscallX function to use for function f. -func (f *Fn) Syscall() string { - c := f.SyscallParamCount() - if c == 3 { - return syscalldot() + "Syscall" - } - return syscalldot() + "Syscall" + strconv.Itoa(c) -} - -// SyscallParamList returns source code for SyscallX parameters for function f. -func (f *Fn) SyscallParamList() string { - a := make([]string, 0) - for _, p := range f.Params { - a = append(a, p.SyscallArgList()...) - } - for len(a) < f.SyscallParamCount() { - a = append(a, "0") - } - return strings.Join(a, ", ") -} - -// HelperCallParamList returns source code of call into function f helper. -func (f *Fn) HelperCallParamList() string { - a := make([]string, 0, len(f.Params)) - for _, p := range f.Params { - s := p.Name - if p.Type == "string" { - s = p.tmpVar() - } - a = append(a, s) - } - return strings.Join(a, ", ") -} - -// IsUTF16 is true, if f is W (utf16) function. It is false -// for all A (ascii) functions. -func (_ *Fn) IsUTF16() bool { - return true -} - -// StrconvFunc returns name of Go string to OS string function for f. -func (f *Fn) StrconvFunc() string { - if f.IsUTF16() { - return syscalldot() + "UTF16PtrFromString" - } - return syscalldot() + "BytePtrFromString" -} - -// StrconvType returns Go type name used for OS string for f. -func (f *Fn) StrconvType() string { - if f.IsUTF16() { - return "*uint16" - } - return "*byte" -} - -// HasStringParam is true, if f has at least one string parameter. -// Otherwise it is false. -func (f *Fn) HasStringParam() bool { - for _, p := range f.Params { - if p.Type == "string" { - return true - } - } - return false -} - -// HelperName returns name of function f helper. -func (f *Fn) HelperName() string { - if !f.HasStringParam() { - return f.Name - } - return "_" + f.Name -} - -// Source files and functions. -type Source struct { - Funcs []*Fn - Files []string -} - -// ParseFiles parses files listed in fs and extracts all syscall -// functions listed in sys comments. It returns source files -// and functions collection *Source if successful. -func ParseFiles(fs []string) (*Source, error) { - src := &Source{ - Funcs: make([]*Fn, 0), - Files: make([]string, 0), - } - for _, file := range fs { - if err := src.ParseFile(file); err != nil { - return nil, err - } - } - return src, nil -} - -// DLLs return dll names for a source set src. -func (src *Source) DLLs() []string { - uniq := make(map[string]bool) - r := make([]string, 0) - for _, f := range src.Funcs { - name := f.DLLName() - if _, found := uniq[name]; !found { - uniq[name] = true - r = append(r, name) - } - } - return r -} - -// ParseFile adds additional file path to a source set src. -func (src *Source) ParseFile(path string) error { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - s := bufio.NewScanner(file) - for s.Scan() { - t := trim(s.Text()) - if len(t) < 7 { - continue - } - if !strings.HasPrefix(t, "//sys") { - continue - } - t = t[5:] - if !(t[0] == ' ' || t[0] == '\t') { - continue - } - f, err := newFn(t[1:]) - if err != nil { - return err - } - src.Funcs = append(src.Funcs, f) - } - if err := s.Err(); err != nil { - return err - } - src.Files = append(src.Files, path) - - // get package name - fset := token.NewFileSet() - _, err = file.Seek(0, 0) - if err != nil { - return err - } - pkg, err := parser.ParseFile(fset, "", file, parser.PackageClauseOnly) - if err != nil { - return err - } - packageName = pkg.Name.Name - - return nil -} - -// Generate output source file from a source set src. -func (src *Source) Generate(w io.Writer) error { - funcMap := template.FuncMap{ - "packagename": packagename, - "syscalldot": syscalldot, - } - t := template.Must(template.New("main").Funcs(funcMap).Parse(srcTemplate)) - err := t.Execute(w, src) - if err != nil { - return errors.New("Failed to execute template: " + err.Error()) - } - return nil -} - -func usage() { - fmt.Fprintf(os.Stderr, "usage: mksyscall_windows [flags] [path ...]\n") - flag.PrintDefaults() - os.Exit(1) -} - -func main() { - flag.Usage = usage - flag.Parse() - if len(flag.Args()) <= 0 { - fmt.Fprintf(os.Stderr, "no files to parse provided\n") - usage() - } - - src, err := ParseFiles(flag.Args()) - if err != nil { - log.Fatal(err) - } - - var buf bytes.Buffer - if err := src.Generate(&buf); err != nil { - log.Fatal(err) - } - - data, err := format.Source(buf.Bytes()) - if err != nil { - log.Fatal(err) - } - if *filename == "" { - _, err = os.Stdout.Write(data) - } else { - err = ioutil.WriteFile(*filename, data, 0644) - } - if err != nil { - log.Fatal(err) - } -} - -// TODO: use println instead to print in the following template -const srcTemplate = ` - -{{define "main"}}// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT - -package {{packagename}} - -import "unsafe"{{if syscalldot}} -import "syscall"{{end}} - -var _ unsafe.Pointer - -var ( -{{template "dlls" .}} -{{template "funcnames" .}}) -{{range .Funcs}}{{if .HasStringParam}}{{template "helperbody" .}}{{end}}{{template "funcbody" .}}{{end}} -{{end}} - -{{/* help functions */}} - -{{define "dlls"}}{{range .DLLs}} mod{{.}} = {{syscalldot}}NewLazyDLL("{{.}}.dll") -{{end}}{{end}} - -{{define "funcnames"}}{{range .Funcs}} proc{{.DLLFuncName}} = mod{{.DLLName}}.NewProc("{{.DLLFuncName}}") -{{end}}{{end}} - -{{define "helperbody"}} -func {{.Name}}({{.ParamList}}) {{template "results" .}}{ -{{template "helpertmpvars" .}} return {{.HelperName}}({{.HelperCallParamList}}) -} -{{end}} - -{{define "funcbody"}} -func {{.HelperName}}({{.HelperParamList}}) {{template "results" .}}{ -{{template "tmpvars" .}} {{template "syscallcheck" .}}{{template "syscall" .}} -{{template "seterror" .}}{{template "printtrace" .}} return -} -{{end}} - -{{define "helpertmpvars"}}{{range .Params}}{{if .TmpVarHelperCode}} {{.TmpVarHelperCode}} -{{end}}{{end}}{{end}} - -{{define "tmpvars"}}{{range .Params}}{{if .TmpVarCode}} {{.TmpVarCode}} -{{end}}{{end}}{{end}} - -{{define "results"}}{{if .Rets.List}}{{.Rets.List}} {{end}}{{end}} - -{{define "syscallcheck"}}{{if .ConfirmProc}}if {{.Rets.ErrorVarName}} = proc{{.DLLFuncName}}.Find(); {{.Rets.ErrorVarName}} != nil { - return -} -{{end}}{{end}} - -{{define "syscall"}}{{.Rets.SetReturnValuesCode}}{{.Syscall}}(proc{{.DLLFuncName}}.Addr(), {{.ParamCount}}, {{.SyscallParamList}}){{end}} - -{{define "seterror"}}{{if .Rets.SetErrorCode}} {{.Rets.SetErrorCode}} -{{end}}{{end}} - -{{define "printtrace"}}{{if .PrintTrace}} print("SYSCALL: {{.Name}}(", {{.ParamPrintList}}") (", {{.Rets.PrintList}}")\n") -{{end}}{{end}} - -` diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/pipe.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/pipe.go index b281b5e23c..0e398fffcb 100644 --- a/components/engine/vendor/src/github.com/Microsoft/go-winio/pipe.go +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/pipe.go @@ -213,7 +213,7 @@ func ListenPipe(path, sddl string) (net.Listener, error) { err error ) if sddl != "" { - sd, err = sddlToSecurityDescriptor(sddl) + sd, err = SddlToSecurityDescriptor(sddl) if err != nil { return nil, err } diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/privilege.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/privilege.go new file mode 100644 index 0000000000..e87c5737d4 --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/privilege.go @@ -0,0 +1,147 @@ +package winio + +import ( + "bytes" + "encoding/binary" + "fmt" + "runtime" + "syscall" + "unicode/utf16" +) + +//sys adjustTokenPrivileges(token syscall.Handle, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (err error) = advapi32.AdjustTokenPrivileges +//sys impersonateSelf(level uint32) (err error) = advapi32.ImpersonateSelf +//sys revertToSelf() (err error) = advapi32.RevertToSelf +//sys openThreadToken(thread syscall.Handle, accessMask uint32, openAsSelf bool, token *syscall.Handle) (err error) = advapi32.OpenThreadToken +//sys getCurrentThread() (h syscall.Handle) = GetCurrentThread +//sys lookupPrivilegeValue(systemName string, name string, luid *uint64) (err error) = advapi32.LookupPrivilegeValueW +//sys lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size *uint32) (err error) = advapi32.LookupPrivilegeNameW +//sys lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) = advapi32.LookupPrivilegeDisplayNameW + +const ( + SE_PRIVILEGE_ENABLED = 2 + + SeBackupPrivilege = "SeBackupPrivilege" + SeRestorePrivilege = "SeRestorePrivilege" +) + +const ( + securityAnonymous = iota + securityIdentification + securityImpersonation + securityDelegation +) + +type PrivilegeError struct { + privileges []uint64 +} + +func (e *PrivilegeError) Error() string { + s := "" + if len(e.privileges) > 1 { + s = "Could not enable privileges " + } else { + s = "Could not enable privilege " + } + for i, p := range e.privileges { + if i != 0 { + s += ", " + } + s += `"` + s += getPrivilegeName(p) + s += `"` + } + return s +} + +func RunWithPrivilege(name string, fn func() error) error { + return RunWithPrivileges([]string{name}, fn) +} + +func RunWithPrivileges(names []string, fn func() error) error { + var privileges []uint64 + for _, name := range names { + p := uint64(0) + err := lookupPrivilegeValue("", name, &p) + if err != nil { + return err + } + privileges = append(privileges, p) + } + runtime.LockOSThread() + defer runtime.UnlockOSThread() + token, err := newThreadToken() + if err != nil { + return err + } + defer releaseThreadToken(token) + err = adjustPrivileges(token, privileges) + if err != nil { + return err + } + return fn() +} + +func adjustPrivileges(token syscall.Handle, privileges []uint64) error { + var b bytes.Buffer + binary.Write(&b, binary.LittleEndian, uint32(len(privileges))) + for _, p := range privileges { + binary.Write(&b, binary.LittleEndian, p) + binary.Write(&b, binary.LittleEndian, uint32(SE_PRIVILEGE_ENABLED)) + } + prevState := make([]byte, b.Len()) + reqSize := uint32(0) + if err := adjustTokenPrivileges(token, false, &b.Bytes()[0], uint32(len(prevState)), &prevState[0], &reqSize); err != nil { + return err + } + if int(binary.LittleEndian.Uint32(prevState[0:4])) < len(privileges) { + return &PrivilegeError{privileges} + } + return nil +} + +func getPrivilegeName(luid uint64) string { + var nameBuffer [256]uint16 + bufSize := uint32(len(nameBuffer)) + err := lookupPrivilegeName("", &luid, &nameBuffer[0], &bufSize) + if err != nil { + return fmt.Sprintf("", luid) + } + + var displayNameBuffer [256]uint16 + displayBufSize := uint32(len(displayNameBuffer)) + var langId uint32 + err = lookupPrivilegeDisplayName("", &nameBuffer[0], &displayNameBuffer[0], &displayBufSize, &langId) + if err != nil { + return fmt.Sprintf("", utf16.Decode(nameBuffer[:bufSize])) + } + + return string(utf16.Decode(displayNameBuffer[:displayBufSize])) +} + +func newThreadToken() (syscall.Handle, error) { + err := impersonateSelf(securityImpersonation) + if err != nil { + panic(err) + return 0, err + } + + var token syscall.Handle + err = openThreadToken(getCurrentThread(), syscall.TOKEN_ADJUST_PRIVILEGES|syscall.TOKEN_QUERY, false, &token) + if err != nil { + rerr := revertToSelf() + if rerr != nil { + panic(rerr) + } + return 0, err + } + return token, nil +} + +func releaseThreadToken(h syscall.Handle) { + err := revertToSelf() + if err != nil { + panic(err) + } + syscall.Close(h) +} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/reparse.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/reparse.go new file mode 100644 index 0000000000..96d7b9a877 --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/reparse.go @@ -0,0 +1,124 @@ +package winio + +import ( + "bytes" + "encoding/binary" + "fmt" + "strings" + "unicode/utf16" + "unsafe" +) + +const ( + reparseTagMountPoint = 0xA0000003 + reparseTagSymlink = 0xA000000C +) + +type reparseDataBuffer struct { + ReparseTag uint32 + ReparseDataLength uint16 + Reserved uint16 + SubstituteNameOffset uint16 + SubstituteNameLength uint16 + PrintNameOffset uint16 + PrintNameLength uint16 +} + +// ReparsePoint describes a Win32 symlink or mount point. +type ReparsePoint struct { + Target string + IsMountPoint bool +} + +// UnsupportedReparsePointError is returned when trying to decode a non-symlink or +// mount point reparse point. +type UnsupportedReparsePointError struct { + Tag uint32 +} + +func (e *UnsupportedReparsePointError) Error() string { + return fmt.Sprintf("unsupported reparse point %x", e.Tag) +} + +// DecodeReparsePoint decodes a Win32 REPARSE_DATA_BUFFER structure containing either a symlink +// or a mount point. +func DecodeReparsePoint(b []byte) (*ReparsePoint, error) { + isMountPoint := false + tag := binary.LittleEndian.Uint32(b[0:4]) + switch tag { + case reparseTagMountPoint: + isMountPoint = true + case reparseTagSymlink: + default: + return nil, &UnsupportedReparsePointError{tag} + } + nameOffset := 16 + binary.LittleEndian.Uint16(b[12:14]) + if !isMountPoint { + nameOffset += 4 + } + nameLength := binary.LittleEndian.Uint16(b[14:16]) + name := make([]uint16, nameLength/2) + err := binary.Read(bytes.NewReader(b[nameOffset:nameOffset+nameLength]), binary.LittleEndian, &name) + if err != nil { + return nil, err + } + return &ReparsePoint{string(utf16.Decode(name)), isMountPoint}, nil +} + +func isDriveLetter(c byte) bool { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') +} + +// EncodeReparsePoint encodes a Win32 REPARSE_DATA_BUFFER structure describing a symlink or +// mount point. +func EncodeReparsePoint(rp *ReparsePoint) []byte { + // Generate an NT path and determine if this is a relative path. + var ntTarget string + relative := false + if strings.HasPrefix(rp.Target, `\\?\`) { + ntTarget = rp.Target + } else if strings.HasPrefix(rp.Target, `\\`) { + ntTarget = `\??\UNC\` + rp.Target[2:] + } else if len(rp.Target) >= 2 && isDriveLetter(rp.Target[0]) && rp.Target[1] == ':' { + ntTarget = `\??\` + rp.Target + } else { + ntTarget = rp.Target + relative = true + } + + // The paths must be NUL-terminated even though they are counted strings. + target16 := utf16.Encode([]rune(rp.Target + "\x00")) + ntTarget16 := utf16.Encode([]rune(ntTarget + "\x00")) + + size := int(unsafe.Sizeof(reparseDataBuffer{})) - 8 + size += len(ntTarget16)*2 + len(target16)*2 + + tag := uint32(reparseTagMountPoint) + if !rp.IsMountPoint { + tag = reparseTagSymlink + size += 4 // Add room for symlink flags + } + + data := reparseDataBuffer{ + ReparseTag: tag, + ReparseDataLength: uint16(size), + SubstituteNameOffset: 0, + SubstituteNameLength: uint16((len(ntTarget16) - 1) * 2), + PrintNameOffset: uint16(len(ntTarget16) * 2), + PrintNameLength: uint16((len(target16) - 1) * 2), + } + + var b bytes.Buffer + binary.Write(&b, binary.LittleEndian, &data) + if !rp.IsMountPoint { + flags := uint32(0) + if relative { + flags |= 1 + } + binary.Write(&b, binary.LittleEndian, flags) + } + + binary.Write(&b, binary.LittleEndian, ntTarget16) + binary.Write(&b, binary.LittleEndian, target16) + return b.Bytes() +} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/sd.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/sd.go index 2df5a7c52b..60ab56ce7a 100644 --- a/components/engine/vendor/src/github.com/Microsoft/go-winio/sd.go +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/sd.go @@ -8,6 +8,7 @@ import ( //sys lookupAccountName(systemName *uint16, accountName string, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) = advapi32.LookupAccountNameW //sys convertSidToStringSid(sid *byte, str **uint16) (err error) = advapi32.ConvertSidToStringSidW //sys convertStringSecurityDescriptorToSecurityDescriptor(str string, revision uint32, sd *uintptr, size *uint32) (err error) = advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW +//sys convertSecurityDescriptorToStringSecurityDescriptor(sd *byte, revision uint32, secInfo uint32, sddl **uint16, sddlSize *uint32) (err error) = advapi32.ConvertSecurityDescriptorToStringSecurityDescriptorW //sys localFree(mem uintptr) = LocalFree //sys getSecurityDescriptorLength(sd uintptr) (len uint32) = advapi32.GetSecurityDescriptorLength @@ -70,7 +71,7 @@ func LookupSidByName(name string) (sid string, err error) { return sid, nil } -func sddlToSecurityDescriptor(sddl string) ([]byte, error) { +func SddlToSecurityDescriptor(sddl string) ([]byte, error) { var sdBuffer uintptr err := convertStringSecurityDescriptorToSecurityDescriptor(sddl, 1, &sdBuffer, nil) if err != nil { @@ -78,6 +79,18 @@ func sddlToSecurityDescriptor(sddl string) ([]byte, error) { } defer localFree(sdBuffer) sd := make([]byte, getSecurityDescriptorLength(sdBuffer)) - copy(sd, (*[1 << 30]byte)(unsafe.Pointer(sdBuffer))[:len(sd)]) + copy(sd, (*[0xffff]byte)(unsafe.Pointer(sdBuffer))[:len(sd)]) return sd, nil } + +func SecurityDescriptorToSddl(sd []byte) (string, error) { + var sddl *uint16 + // The returned string length seems to including an aribtrary number of terminating NULs. + // Don't use it. + err := convertSecurityDescriptorToStringSecurityDescriptor(&sd[0], 1, 0xff, &sddl, nil) + if err != nil { + return "", err + } + defer localFree(uintptr(unsafe.Pointer(sddl))) + return syscall.UTF16ToString((*[0xffff]uint16)(unsafe.Pointer(sddl))[:]), nil +} diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/syscall.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/syscall.go index 20767dcb22..96fdff7b49 100644 --- a/components/engine/vendor/src/github.com/Microsoft/go-winio/syscall.go +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/syscall.go @@ -1,3 +1,3 @@ package winio -//go:generate go run mksyscall_windows.go -output zsyscall.go file.go pipe.go sd.go +//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsyscall.go file.go pipe.go sd.go fileinfo.go privilege.go backup.go diff --git a/components/engine/vendor/src/github.com/Microsoft/go-winio/zsyscall.go b/components/engine/vendor/src/github.com/Microsoft/go-winio/zsyscall.go index bfe4ac3470..19d53dc305 100644 --- a/components/engine/vendor/src/github.com/Microsoft/go-winio/zsyscall.go +++ b/components/engine/vendor/src/github.com/Microsoft/go-winio/zsyscall.go @@ -22,8 +22,21 @@ var ( procLookupAccountNameW = modadvapi32.NewProc("LookupAccountNameW") procConvertSidToStringSidW = modadvapi32.NewProc("ConvertSidToStringSidW") procConvertStringSecurityDescriptorToSecurityDescriptorW = modadvapi32.NewProc("ConvertStringSecurityDescriptorToSecurityDescriptorW") + procConvertSecurityDescriptorToStringSecurityDescriptorW = modadvapi32.NewProc("ConvertSecurityDescriptorToStringSecurityDescriptorW") procLocalFree = modkernel32.NewProc("LocalFree") procGetSecurityDescriptorLength = modadvapi32.NewProc("GetSecurityDescriptorLength") + procGetFileInformationByHandleEx = modkernel32.NewProc("GetFileInformationByHandleEx") + procSetFileInformationByHandle = modkernel32.NewProc("SetFileInformationByHandle") + procAdjustTokenPrivileges = modadvapi32.NewProc("AdjustTokenPrivileges") + procImpersonateSelf = modadvapi32.NewProc("ImpersonateSelf") + procRevertToSelf = modadvapi32.NewProc("RevertToSelf") + procOpenThreadToken = modadvapi32.NewProc("OpenThreadToken") + procGetCurrentThread = modkernel32.NewProc("GetCurrentThread") + procLookupPrivilegeValueW = modadvapi32.NewProc("LookupPrivilegeValueW") + procLookupPrivilegeNameW = modadvapi32.NewProc("LookupPrivilegeNameW") + procLookupPrivilegeDisplayNameW = modadvapi32.NewProc("LookupPrivilegeDisplayNameW") + procBackupRead = modkernel32.NewProc("BackupRead") + procBackupWrite = modkernel32.NewProc("BackupWrite") ) func cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) { @@ -206,6 +219,18 @@ func _convertStringSecurityDescriptorToSecurityDescriptor(str *uint16, revision return } +func convertSecurityDescriptorToStringSecurityDescriptor(sd *byte, revision uint32, secInfo uint32, sddl **uint16, sddlSize *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procConvertSecurityDescriptorToStringSecurityDescriptorW.Addr(), 5, uintptr(unsafe.Pointer(sd)), uintptr(revision), uintptr(secInfo), uintptr(unsafe.Pointer(sddl)), uintptr(unsafe.Pointer(sddlSize)), 0) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + func localFree(mem uintptr) { syscall.Syscall(procLocalFree.Addr(), 1, uintptr(mem), 0, 0) return @@ -216,3 +241,217 @@ func getSecurityDescriptorLength(sd uintptr) (len uint32) { len = uint32(r0) return } + +func getFileInformationByHandleEx(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procGetFileInformationByHandleEx.Addr(), 4, uintptr(h), uintptr(class), uintptr(unsafe.Pointer(buffer)), uintptr(size), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func setFileInformationByHandle(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procSetFileInformationByHandle.Addr(), 4, uintptr(h), uintptr(class), uintptr(unsafe.Pointer(buffer)), uintptr(size), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func adjustTokenPrivileges(token syscall.Handle, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (err error) { + var _p0 uint32 + if releaseAll { + _p0 = 1 + } else { + _p0 = 0 + } + r1, _, e1 := syscall.Syscall6(procAdjustTokenPrivileges.Addr(), 6, uintptr(token), uintptr(_p0), uintptr(unsafe.Pointer(input)), uintptr(outputSize), uintptr(unsafe.Pointer(output)), uintptr(unsafe.Pointer(requiredSize))) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func impersonateSelf(level uint32) (err error) { + r1, _, e1 := syscall.Syscall(procImpersonateSelf.Addr(), 1, uintptr(level), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func revertToSelf() (err error) { + r1, _, e1 := syscall.Syscall(procRevertToSelf.Addr(), 0, 0, 0, 0) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func openThreadToken(thread syscall.Handle, accessMask uint32, openAsSelf bool, token *syscall.Handle) (err error) { + var _p0 uint32 + if openAsSelf { + _p0 = 1 + } else { + _p0 = 0 + } + r1, _, e1 := syscall.Syscall6(procOpenThreadToken.Addr(), 4, uintptr(thread), uintptr(accessMask), uintptr(_p0), uintptr(unsafe.Pointer(token)), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func getCurrentThread() (h syscall.Handle) { + r0, _, _ := syscall.Syscall(procGetCurrentThread.Addr(), 0, 0, 0, 0) + h = syscall.Handle(r0) + return +} + +func lookupPrivilegeValue(systemName string, name string, luid *uint64) (err error) { + var _p0 *uint16 + _p0, err = syscall.UTF16PtrFromString(systemName) + if err != nil { + return + } + var _p1 *uint16 + _p1, err = syscall.UTF16PtrFromString(name) + if err != nil { + return + } + return _lookupPrivilegeValue(_p0, _p1, luid) +} + +func _lookupPrivilegeValue(systemName *uint16, name *uint16, luid *uint64) (err error) { + r1, _, e1 := syscall.Syscall(procLookupPrivilegeValueW.Addr(), 3, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(luid))) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size *uint32) (err error) { + var _p0 *uint16 + _p0, err = syscall.UTF16PtrFromString(systemName) + if err != nil { + return + } + return _lookupPrivilegeName(_p0, luid, buffer, size) +} + +func _lookupPrivilegeName(systemName *uint16, luid *uint64, buffer *uint16, size *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procLookupPrivilegeNameW.Addr(), 4, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(luid)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) { + var _p0 *uint16 + _p0, err = syscall.UTF16PtrFromString(systemName) + if err != nil { + return + } + return _lookupPrivilegeDisplayName(_p0, name, buffer, size, languageId) +} + +func _lookupPrivilegeDisplayName(systemName *uint16, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procLookupPrivilegeDisplayNameW.Addr(), 5, uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), uintptr(unsafe.Pointer(languageId)), 0) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) { + var _p0 *byte + if len(b) > 0 { + _p0 = &b[0] + } + var _p1 uint32 + if abort { + _p1 = 1 + } else { + _p1 = 0 + } + var _p2 uint32 + if processSecurity { + _p2 = 1 + } else { + _p2 = 0 + } + r1, _, e1 := syscall.Syscall9(procBackupRead.Addr(), 7, uintptr(h), uintptr(unsafe.Pointer(_p0)), uintptr(len(b)), uintptr(unsafe.Pointer(bytesRead)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(context)), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func backupWrite(h syscall.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) { + var _p0 *byte + if len(b) > 0 { + _p0 = &b[0] + } + var _p1 uint32 + if abort { + _p1 = 1 + } else { + _p1 = 0 + } + var _p2 uint32 + if processSecurity { + _p2 = 1 + } else { + _p2 = 0 + } + r1, _, e1 := syscall.Syscall9(procBackupWrite.Addr(), 7, uintptr(h), uintptr(unsafe.Pointer(_p0)), uintptr(len(b)), uintptr(unsafe.Pointer(bytesWritten)), uintptr(_p1), uintptr(_p2), uintptr(unsafe.Pointer(context)), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = error(e1) + } else { + err = syscall.EINVAL + } + } + return +} diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/copylayer.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/copylayer.go deleted file mode 100644 index abbe134e20..0000000000 --- a/components/engine/vendor/src/github.com/Microsoft/hcsshim/copylayer.go +++ /dev/null @@ -1,34 +0,0 @@ -package hcsshim - -import "github.com/Sirupsen/logrus" - -// CopyLayer performs a commit of the srcId (which is expected to be a read-write -// layer) into a new read-only layer at dstId. This requires the full list of -// on-disk paths to parent layers, provided in parentLayerPaths, in order to -// complete the commit. -func CopyLayer(info DriverInfo, srcId, dstId string, parentLayerPaths []string) error { - title := "hcsshim::CopyLayer " - logrus.Debugf(title+"srcId %s dstId", srcId, dstId) - - // Generate layer descriptors - layers, err := layerPathsToDescriptors(parentLayerPaths) - if err != nil { - return err - } - - // Convert info to API calling convention - infop, err := convertDriverInfo(info) - if err != nil { - return err - } - - err = copyLayer(&infop, srcId, dstId, layers) - if err != nil { - err = makeErrorf(err, title, "srcId=%s dstId=%d", srcId, dstId) - logrus.Error(err) - return err - } - - logrus.Debugf(title+" - succeeded srcId=%s dstId=%s", srcId, dstId) - return nil -} diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/createprocess.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/createprocess.go index a170a9a1e3..a2b6298546 100644 --- a/components/engine/vendor/src/github.com/Microsoft/hcsshim/createprocess.go +++ b/components/engine/vendor/src/github.com/Microsoft/hcsshim/createprocess.go @@ -48,10 +48,9 @@ func makeOpenFiles(hs []syscall.Handle) (_ []io.ReadWriteCloser, err error) { // CreateProcessInComputeSystem starts a process in a container. This is invoked, for example, // as a result of docker run, docker exec, or RUN in Dockerfile. If successful, // it returns the PID of the process. -func CreateProcessInComputeSystem(id string, useStdin bool, useStdout bool, useStderr bool, params CreateProcessParams) (_ uint32, _ io.WriteCloser, _ io.ReadCloser, _ io.ReadCloser, hr uint32, err error) { +func CreateProcessInComputeSystem(id string, useStdin bool, useStdout bool, useStderr bool, params CreateProcessParams) (_ uint32, _ io.WriteCloser, _ io.ReadCloser, _ io.ReadCloser, err error) { title := "HCSShim::CreateProcessInComputeSystem" logrus.Debugf(title+" id=%s", id) - hr = 0xFFFFFFFF // If we are not emulating a console, ignore any console size passed to us if !params.EmulateConsole { @@ -82,14 +81,13 @@ func CreateProcessInComputeSystem(id string, useStdin bool, useStdout bool, useS err = createProcessWithStdHandlesInComputeSystem(id, string(paramsJson), &pid, stdinParam, stdoutParam, stderrParam) if err != nil { - winerr := makeErrorf(err, title, "id=%s params=%v", id, params) - hr = winerr.HResult() + herr := makeErrorf(err, title, "id=%s params=%v", id, params) // Windows TP4: Hyper-V Containers may return this error with more than one // concurrent exec. Do not log it as an error - if hr != Win32InvalidArgument { - logrus.Error(winerr) + if err != WSAEINVAL { + logrus.Error(herr) } - err = winerr + err = herr return } @@ -99,5 +97,5 @@ func CreateProcessInComputeSystem(id string, useStdin bool, useStdout bool, useS } logrus.Debugf(title+" - succeeded id=%s params=%s pid=%d", id, paramsJson, pid) - return pid, pipes[0], pipes[1], pipes[2], 0, nil + return pid, pipes[0], pipes[1], pipes[2], nil } diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/exportlayer.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/exportlayer.go index 629dc04d5a..e197d575e5 100644 --- a/components/engine/vendor/src/github.com/Microsoft/hcsshim/exportlayer.go +++ b/components/engine/vendor/src/github.com/Microsoft/hcsshim/exportlayer.go @@ -1,6 +1,15 @@ package hcsshim -import "github.com/Sirupsen/logrus" +import ( + "io" + "io/ioutil" + "os" + "runtime" + "syscall" + + "github.com/Microsoft/go-winio" + "github.com/Sirupsen/logrus" +) // ExportLayer will create a folder at exportFolderPath and fill that folder with // the transport format version of the layer identified by layerId. This transport @@ -34,3 +43,114 @@ func ExportLayer(info DriverInfo, layerId string, exportFolderPath string, paren logrus.Debugf(title+"succeeded flavour=%d layerId=%s folder=%s", info.Flavour, layerId, exportFolderPath) return nil } + +type LayerReader interface { + Next() (string, int64, *winio.FileBasicInfo, error) + Read(b []byte) (int, error) + Close() error +} + +// FilterLayerReader provides an interface for extracting the contents of an on-disk layer. +type FilterLayerReader struct { + context uintptr +} + +// Next reads the next available file from a layer, ensuring that parent directories are always read +// before child files and directories. +// +// Next returns the file's relative path, size, and basic file metadata. Read() should be used to +// extract a Win32 backup stream with the remainder of the metadata and the data. +func (r *FilterLayerReader) Next() (string, int64, *winio.FileBasicInfo, error) { + var fileNamep *uint16 + fileInfo := &winio.FileBasicInfo{} + var deleted uint32 + var fileSize int64 + err := exportLayerNext(r.context, &fileNamep, fileInfo, &fileSize, &deleted) + if err != nil { + if err == syscall.ERROR_NO_MORE_FILES { + err = io.EOF + } else { + err = makeError(err, "ExportLayerNext", "") + } + return "", 0, nil, err + } + fileName := convertAndFreeCoTaskMemString(fileNamep) + if deleted != 0 { + fileInfo = nil + } + if fileName[0] == '\\' { + fileName = fileName[1:] + } + return fileName, fileSize, fileInfo, nil +} + +// Read reads from the current file's Win32 backup stream. +func (r *FilterLayerReader) Read(b []byte) (int, error) { + var bytesRead uint32 + err := exportLayerRead(r.context, b, &bytesRead) + if err != nil { + return 0, makeError(err, "ExportLayerRead", "") + } + if bytesRead == 0 { + return 0, io.EOF + } + return int(bytesRead), nil +} + +// Close frees resources associated with the layer reader. It will return an +// error if there was an error while reading the layer or of the layer was not +// completely read. +func (r *FilterLayerReader) Close() (err error) { + if r.context != 0 { + err = exportLayerEnd(r.context) + if err != nil { + err = makeError(err, "ExportLayerEnd", "") + } + r.context = 0 + } + return +} + +// NewLayerReader returns a new layer reader for reading the contents of an on-disk layer. +func NewLayerReader(info DriverInfo, layerId string, parentLayerPaths []string) (LayerReader, error) { + if procExportLayerBegin.Find() != nil { + // The new layer reader is not available on this Windows build. Fall back to the + // legacy export code path. + path, err := ioutil.TempDir("", "hcs") + if err != nil { + return nil, err + } + err = ExportLayer(info, layerId, path, parentLayerPaths) + if err != nil { + os.RemoveAll(path) + return nil, err + } + return &legacyLayerReaderWrapper{NewLegacyLayerReader(path)}, nil + } + + layers, err := layerPathsToDescriptors(parentLayerPaths) + if err != nil { + return nil, err + } + infop, err := convertDriverInfo(info) + if err != nil { + return nil, err + } + r := &FilterLayerReader{} + err = exportLayerBegin(&infop, layerId, layers, &r.context) + if err != nil { + return nil, makeError(err, "ExportLayerBegin", "") + } + runtime.SetFinalizer(r, func(r *FilterLayerReader) { r.Close() }) + return r, err +} + +type legacyLayerReaderWrapper struct { + *LegacyLayerReader +} + +func (r *legacyLayerReaderWrapper) Close() error { + err := r.LegacyLayerReader.Close() + os.RemoveAll(r.root) + return err +} diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/hcsshim.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/hcsshim.go index fc894ca935..88650b8ac9 100644 --- a/components/engine/vendor/src/github.com/Microsoft/hcsshim/hcsshim.go +++ b/components/engine/vendor/src/github.com/Microsoft/hcsshim/hcsshim.go @@ -27,6 +27,18 @@ import ( //sys nameToGuid(name string, guid *GUID) (hr error) = vmcompute.NameToGuid? //sys prepareLayer(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.PrepareLayer? //sys unprepareLayer(info *driverInfo, id string) (hr error) = vmcompute.UnprepareLayer? +//sys processBaseImage(path string) (hr error) = vmcompute.ProcessBaseImage? +//sys processUtilityImage(path string) (hr error) = vmcompute.ProcessUtilityImage? + +//sys importLayerBegin(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) = vmcompute.ImportLayerBegin? +//sys importLayerNext(context uintptr, fileName string, fileInfo *winio.FileBasicInfo) (hr error) = vmcompute.ImportLayerNext? +//sys importLayerWrite(context uintptr, buffer []byte) (hr error) = vmcompute.ImportLayerWrite? +//sys importLayerEnd(context uintptr) (hr error) = vmcompute.ImportLayerEnd? + +//sys exportLayerBegin(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) = vmcompute.ExportLayerBegin? +//sys exportLayerNext(context uintptr, fileName **uint16, fileInfo *winio.FileBasicInfo, fileSize *int64, deleted *uint32) (hr error) = vmcompute.ExportLayerNext? +//sys exportLayerRead(context uintptr, buffer []byte, bytesRead *uint32) (hr error) = vmcompute.ExportLayerRead? +//sys exportLayerEnd(context uintptr) (hr error) = vmcompute.ExportLayerEnd? //sys createComputeSystem(id string, configuration string) (hr error) = vmcompute.CreateComputeSystem? //sys createProcessWithStdHandlesInComputeSystem(id string, paramsJson string, pid *uint32, stdin *syscall.Handle, stdout *syscall.Handle, stderr *syscall.Handle) (hr error) = vmcompute.CreateProcessWithStdHandlesInComputeSystem? @@ -43,47 +55,62 @@ const ( // Specific user-visible exit codes WaitErrExecFailed = 32767 - // Known Win32 RC values which should be trapped - Win32PipeHasBeenEnded = 0x8007006d // WaitForProcessInComputeSystem: The pipe has been ended - Win32SystemShutdownIsInProgress = 0x8007045B // ShutdownComputeSystem: A system shutdown is in progress - Win32SpecifiedPathInvalid = 0x800700A1 // ShutdownComputeSystem: The specified path is invalid - Win32SystemCannotFindThePathSpecified = 0x80070003 // ShutdownComputeSystem: The system cannot find the path specified - Win32InvalidArgument = 0x80072726 // CreateProcessInComputeSystem: An invalid argument was supplied - EFail = 0x80004005 + ERROR_GEN_FAILURE = syscall.Errno(31) + ERROR_SHUTDOWN_IN_PROGRESS = syscall.Errno(1115) + WSAEINVAL = syscall.Errno(10022) // Timeout on wait calls TimeoutInfinite = 0xFFFFFFFF ) -type hcsError struct { +type HcsError struct { title string rest string - err error + Err error } -type Win32Error interface { - error - HResult() uint32 +func makeError(err error, title, rest string) error { + // Pass through DLL errors directly since they do not originate from HCS. + if _, ok := err.(*syscall.DLLError); ok { + return err + } + return &HcsError{title, rest, err} } -func makeError(err error, title, rest string) Win32Error { - return &hcsError{title, rest, err} -} - -func makeErrorf(err error, title, format string, a ...interface{}) Win32Error { +func makeErrorf(err error, title, format string, a ...interface{}) error { return makeError(err, title, fmt.Sprintf(format, a...)) } -func (e *hcsError) HResult() uint32 { - if hr, ok := e.err.(syscall.Errno); ok { - return uint32(hr) - } else { - return EFail +func win32FromError(err error) uint32 { + if herr, ok := err.(*HcsError); ok { + return win32FromError(herr.Err) } + if code, ok := err.(syscall.Errno); ok { + return uint32(code) + } + return uint32(ERROR_GEN_FAILURE) } -func (e *hcsError) Error() string { - return fmt.Sprintf("%s- Win32 API call returned error r1=0x%x err=%s%s", e.title, e.HResult(), e.err, e.rest) +func win32FromHresult(hr uintptr) uintptr { + if hr&0x1fff0000 == 0x00070000 { + return hr & 0xffff + } + return hr +} + +func (e *HcsError) Error() string { + s := e.title + if len(s) > 0 && s[len(s)-1] != ' ' { + s += " " + } + s += fmt.Sprintf("failed in Win32: %s (0x%x)", e.Err, win32FromError(e.Err)) + if e.rest != "" { + if e.rest[0] != ' ' { + s += " " + } + s += e.rest + } + return s } func convertAndFreeCoTaskMemString(buffer *uint16) string { diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/hnsfuncs.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/hnsfuncs.go index affff7f86f..590d6e381f 100644 --- a/components/engine/vendor/src/github.com/Microsoft/hcsshim/hnsfuncs.go +++ b/components/engine/vendor/src/github.com/Microsoft/hcsshim/hnsfuncs.go @@ -36,12 +36,16 @@ type MacPool struct { // HNSNetwork represents a network in HNS type HNSNetwork struct { - Id string `json:",omitempty"` - Name string `json:",omitempty"` - Type string `json:",omitempty"` - Policies []json.RawMessage `json:",omitempty"` - MacPools []MacPool `json:",omitempty"` - Subnets []Subnet `json:",omitempty"` + Id string `json:",omitempty"` + Name string `json:",omitempty"` + Type string `json:",omitempty"` + NetworkAdapterName string `json:",omitempty"` + SourceMac string `json:",omitempty"` + Policies []json.RawMessage `json:",omitempty"` + MacPools []MacPool `json:",omitempty"` + Subnets []Subnet `json:",omitempty"` + DNSSuffix string `json:",omitempty"` + DNSServerList string `json:",omitempty"` } // HNSEndpoint represents a network endpoint in HNS @@ -53,6 +57,10 @@ type HNSEndpoint struct { Policies []json.RawMessage `json:",omitempty"` MacAddress string `json:",omitempty"` IPAddress net.IP `json:",omitempty"` + DNSSuffix string `json:",omitempty"` + DNSServerList string `json:",omitempty"` + GatewayAddress string `json:",omitempty"` + PrefixLength uint8 `json:",omitempty"` } type hnsNetworkResponse struct { diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/importlayer.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/importlayer.go index 3ab1124e9c..800b300b3a 100644 --- a/components/engine/vendor/src/github.com/Microsoft/hcsshim/importlayer.go +++ b/components/engine/vendor/src/github.com/Microsoft/hcsshim/importlayer.go @@ -1,6 +1,13 @@ package hcsshim -import "github.com/Sirupsen/logrus" +import ( + "io/ioutil" + "os" + "runtime" + + "github.com/Microsoft/go-winio" + "github.com/Sirupsen/logrus" +) // ImportLayer will take the contents of the folder at importFolderPath and import // that into a layer with the id layerId. Note that in order to correctly populate @@ -33,3 +40,114 @@ func ImportLayer(info DriverInfo, layerId string, importFolderPath string, paren logrus.Debugf(title+"succeeded flavour=%d layerId=%s folder=%s", info.Flavour, layerId, importFolderPath) return nil } + +type LayerWriter interface { + Add(name string, fileInfo *winio.FileBasicInfo) error + Remove(name string) error + Write(b []byte) (int, error) + Close() error +} + +// FilterLayerWriter provides an interface to write the contents of a layer to the file system. +type FilterLayerWriter struct { + context uintptr +} + +// Add adds a file or directory to the layer. The file's parent directory must have already been added. +// +// name contains the file's relative path. fileInfo contains file times and file attributes; the rest +// of the file metadata and the file data must be written as a Win32 backup stream to the Write() method. +// winio.BackupStreamWriter can be used to facilitate this. +func (w *FilterLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) error { + if name[0] != '\\' { + name = `\` + name + } + err := importLayerNext(w.context, name, fileInfo) + if err != nil { + return makeError(err, "ImportLayerNext", "") + } + return nil +} + +// Remove removes a file from the layer. The file must have been present in the parent layer. +// +// name contains the file's relative path. +func (w *FilterLayerWriter) Remove(name string) error { + if name[0] != '\\' { + name = `\` + name + } + err := importLayerNext(w.context, name, nil) + if err != nil { + return makeError(err, "ImportLayerNext", "") + } + return nil +} + +// Write writes more backup stream data to the current file. +func (w *FilterLayerWriter) Write(b []byte) (int, error) { + err := importLayerWrite(w.context, b) + if err != nil { + err = makeError(err, "ImportLayerWrite", "") + return 0, err + } + return len(b), err +} + +// Close completes the layer write operation. The error must be checked to ensure that the +// operation was successful. +func (w *FilterLayerWriter) Close() (err error) { + if w.context != 0 { + err = importLayerEnd(w.context) + if err != nil { + err = makeError(err, "ImportLayerEnd", "") + } + w.context = 0 + } + return +} + +type legacyLayerWriterWrapper struct { + *LegacyLayerWriter + info DriverInfo + layerId string + parentLayerPaths []string +} + +func (r *legacyLayerWriterWrapper) Close() error { + err := r.LegacyLayerWriter.Close() + if err == nil { + err = ImportLayer(r.info, r.layerId, r.root, r.parentLayerPaths) + } + os.RemoveAll(r.root) + return err +} + +// NewLayerWriter returns a new layer writer for creating a layer on disk. +func NewLayerWriter(info DriverInfo, layerId string, parentLayerPaths []string) (LayerWriter, error) { + if procImportLayerBegin.Find() != nil { + // The new layer reader is not available on this Windows build. Fall back to the + // legacy export code path. + path, err := ioutil.TempDir("", "hcs") + if err != nil { + return nil, err + } + return &legacyLayerWriterWrapper{NewLegacyLayerWriter(path), info, layerId, parentLayerPaths}, nil + } + layers, err := layerPathsToDescriptors(parentLayerPaths) + if err != nil { + return nil, err + } + + infop, err := convertDriverInfo(info) + if err != nil { + return nil, err + } + + w := &FilterLayerWriter{} + err = importLayerBegin(&infop, layerId, layers, &w.context) + if err != nil { + return nil, makeError(err, "ImportLayerStart", "") + } + runtime.SetFinalizer(w, func(w *FilterLayerWriter) { w.Close() }) + return w, nil +} diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/legacy.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/legacy.go new file mode 100644 index 0000000000..bc31f23656 --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/hcsshim/legacy.go @@ -0,0 +1,393 @@ +package hcsshim + +import ( + "bufio" + "encoding/binary" + "errors" + "io" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/Microsoft/go-winio" +) + +var errorIterationCanceled = errors.New("") + +func openFileOrDir(path string, mode uint32, createDisposition uint32) (file *os.File, err error) { + winPath, err := syscall.UTF16FromString(path) + if err != nil { + return + } + h, err := syscall.CreateFile(&winPath[0], mode, syscall.FILE_SHARE_READ, nil, createDisposition, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + if err != nil { + err = &os.PathError{"open", path, err} + return + } + file = os.NewFile(uintptr(h), path) + return +} + +type fileEntry struct { + path string + fi os.FileInfo + err error +} + +type LegacyLayerReader struct { + root string + result chan *fileEntry + proceed chan bool + currentFile *os.File + backupReader *winio.BackupFileReader + isTP4Format bool +} + +// NewLegacyLayerReader returns a new LayerReader that can read the Windows +// TP4 transport format from disk. +func NewLegacyLayerReader(root string) *LegacyLayerReader { + r := &LegacyLayerReader{ + root: root, + result: make(chan *fileEntry), + proceed: make(chan bool), + isTP4Format: IsTP4(), + } + go r.walk() + return r +} + +func readTombstones(path string) (map[string]([]string), error) { + tf, err := os.Open(filepath.Join(path, "tombstones.txt")) + if err != nil { + return nil, err + } + defer tf.Close() + s := bufio.NewScanner(tf) + if !s.Scan() || s.Text() != "\xef\xbb\xbfVersion 1.0" { + return nil, errors.New("Invalid tombstones file") + } + + ts := make(map[string]([]string)) + for s.Scan() { + t := s.Text()[1:] // skip leading `\` + dir := filepath.Dir(t) + ts[dir] = append(ts[dir], t) + } + if err = s.Err(); err != nil { + return nil, err + } + + return ts, nil +} + +func (r *LegacyLayerReader) walk() { + defer close(r.result) + if !<-r.proceed { + return + } + + ts, err := readTombstones(r.root) + if err != nil { + goto ErrorLoop + } + + err = filepath.Walk(r.root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if path == r.root || path == filepath.Join(r.root, "tombstones.txt") || strings.HasSuffix(path, ".$wcidirs$") { + return nil + } + r.result <- &fileEntry{path, info, nil} + if !<-r.proceed { + return errorIterationCanceled + } + + // List all the tombstones. + if info.IsDir() { + relPath, err := filepath.Rel(r.root, path) + if err != nil { + return err + } + if dts, ok := ts[relPath]; ok { + for _, t := range dts { + r.result <- &fileEntry{t, nil, nil} + if !<-r.proceed { + return errorIterationCanceled + } + } + } + } + return nil + }) + if err == errorIterationCanceled { + return + } + if err == nil { + err = io.EOF + } + +ErrorLoop: + for { + r.result <- &fileEntry{err: err} + if !<-r.proceed { + break + } + } +} + +func (r *LegacyLayerReader) reset() { + if r.backupReader != nil { + r.backupReader.Close() + r.backupReader = nil + } + if r.currentFile != nil { + r.currentFile.Close() + r.currentFile = nil + } +} + +func findBackupStreamSize(r io.Reader) (int64, error) { + br := winio.NewBackupStreamReader(r) + for { + hdr, err := br.Next() + if err != nil { + if err == io.EOF { + err = nil + } + return 0, err + } + if hdr.Id == winio.BackupData { + return hdr.Size, nil + } + } +} + +func (r *LegacyLayerReader) Next() (path string, size int64, fileInfo *winio.FileBasicInfo, err error) { + r.reset() + r.proceed <- true + fe := <-r.result + if fe == nil { + err = errors.New("LegacyLayerReader closed") + return + } + if fe.err != nil { + err = fe.err + return + } + + path, err = filepath.Rel(r.root, fe.path) + if err != nil { + return + } + + if fe.fi == nil { + // This is a tombstone. Return a nil fileInfo. + return + } + + if fe.fi.IsDir() && strings.HasPrefix(path, `Files\`) { + fe.path += ".$wcidirs$" + } + + f, err := openFileOrDir(fe.path, syscall.GENERIC_READ, syscall.OPEN_EXISTING) + if err != nil { + return + } + defer func() { + if f != nil { + f.Close() + } + }() + + fileInfo, err = winio.GetFileBasicInfo(f) + if err != nil { + return + } + + if !strings.HasPrefix(path, `Files\`) { + size = fe.fi.Size() + r.backupReader = winio.NewBackupFileReader(f, false) + if path == "Hives" || path == "Files" { + // The Hives directory has a non-deterministic file time because of the + // nature of the import process. Use the times from System_Delta. + var g *os.File + g, err = os.Open(filepath.Join(r.root, `Hives\System_Delta`)) + if err != nil { + return + } + attr := fileInfo.FileAttributes + fileInfo, err = winio.GetFileBasicInfo(g) + g.Close() + if err != nil { + return + } + fileInfo.FileAttributes = attr + } + + // The creation time and access time get reset for files outside of the Files path. + fileInfo.CreationTime = fileInfo.LastWriteTime + fileInfo.LastAccessTime = fileInfo.LastWriteTime + + } else { + beginning := int64(0) + if !r.isTP4Format { + // In TP5, the file attributes were added before the backup stream + var attr uint32 + err = binary.Read(f, binary.LittleEndian, &attr) + if err != nil { + return + } + fileInfo.FileAttributes = uintptr(attr) + beginning = 4 + } + + // Find the accurate file size. + if !fe.fi.IsDir() { + size, err = findBackupStreamSize(f) + if err != nil { + err = &os.PathError{"findBackupStreamSize", fe.path, err} + return + } + } + + // Return back to the beginning of the backup stream. + _, err = f.Seek(beginning, 0) + if err != nil { + return + } + } + + r.currentFile = f + f = nil + return +} + +func (r *LegacyLayerReader) Read(b []byte) (int, error) { + if r.backupReader == nil { + if r.currentFile == nil { + return 0, io.EOF + } + return r.currentFile.Read(b) + } + return r.backupReader.Read(b) +} + +func (r *LegacyLayerReader) Close() error { + r.proceed <- false + <-r.result + r.reset() + return nil +} + +type LegacyLayerWriter struct { + root string + currentFile *os.File + backupWriter *winio.BackupFileWriter + tombstones []string + isTP4Format bool +} + +// NewLegacyLayerWriter returns a LayerWriter that can write the TP4 transport format +// to disk. +func NewLegacyLayerWriter(root string) *LegacyLayerWriter { + return &LegacyLayerWriter{ + root: root, + isTP4Format: IsTP4(), + } +} + +func (w *LegacyLayerWriter) reset() { + if w.backupWriter != nil { + w.backupWriter.Close() + w.backupWriter = nil + } + if w.currentFile != nil { + w.currentFile.Close() + w.currentFile = nil + } +} + +func (w *LegacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) error { + w.reset() + path := filepath.Join(w.root, name) + + createDisposition := uint32(syscall.CREATE_NEW) + if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 { + err := os.Mkdir(path, 0) + if err != nil { + return err + } + path += ".$wcidirs$" + } + + f, err := openFileOrDir(path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, createDisposition) + if err != nil { + return err + } + defer func() { + if f != nil { + f.Close() + os.Remove(path) + } + }() + + strippedFi := *fileInfo + strippedFi.FileAttributes = 0 + err = winio.SetFileBasicInfo(f, &strippedFi) + if err != nil { + return err + } + + if strings.HasPrefix(name, `Hives\`) { + w.backupWriter = winio.NewBackupFileWriter(f, false) + } else { + if !w.isTP4Format { + // In TP5, the file attributes were added to the header + err = binary.Write(f, binary.LittleEndian, uint32(fileInfo.FileAttributes)) + if err != nil { + return err + } + } + } + + w.currentFile = f + f = nil + return nil +} + +func (w *LegacyLayerWriter) Remove(name string) error { + w.tombstones = append(w.tombstones, name) + return nil +} + +func (w *LegacyLayerWriter) Write(b []byte) (int, error) { + if w.backupWriter == nil { + if w.currentFile == nil { + return 0, errors.New("closed") + } + return w.currentFile.Write(b) + } + return w.backupWriter.Write(b) +} + +func (w *LegacyLayerWriter) Close() error { + w.reset() + tf, err := os.Create(filepath.Join(w.root, "tombstones.txt")) + if err != nil { + return err + } + defer tf.Close() + _, err = tf.Write([]byte("\xef\xbb\xbfVersion 1.0\n")) + if err != nil { + return err + } + for _, t := range w.tombstones { + _, err = tf.Write([]byte(filepath.Join(`\`, t) + "\n")) + if err != nil { + return err + } + } + return nil +} diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/mksyscall_windows.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/mksyscall_windows.go index 652074c7f1..7c9a00110a 100644 --- a/components/engine/vendor/src/github.com/Microsoft/hcsshim/mksyscall_windows.go +++ b/components/engine/vendor/src/github.com/Microsoft/hcsshim/mksyscall_windows.go @@ -294,6 +294,9 @@ func (r *Rets) SetErrorCode() string { const code = `if r0 != 0 { %s = %sErrno(r0) }` + const hrCode = `if int32(r0) < 0 { + %s = %sErrno(win32FromHresult(r0)) + }` if r.Name == "" && !r.ReturnsError { return "" } @@ -301,7 +304,11 @@ func (r *Rets) SetErrorCode() string { return r.useLongHandleErrorCode("r1") } if r.Type == "error" { - return fmt.Sprintf(code, r.Name, syscalldot()) + if r.Name == "hr" { + return fmt.Sprintf(hrCode, r.Name, syscalldot()) + } else { + return fmt.Sprintf(code, r.Name, syscalldot()) + } } s := "" switch { diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/processimage.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/processimage.go new file mode 100644 index 0000000000..fadb1b92c5 --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/hcsshim/processimage.go @@ -0,0 +1,23 @@ +package hcsshim + +import "os" + +// ProcessBaseLayer post-processes a base layer that has had its files extracted. +// The files should have been extracted to \Files. +func ProcessBaseLayer(path string) error { + err := processBaseImage(path) + if err != nil { + return &os.PathError{Op: "ProcessBaseLayer", Path: path, Err: err} + } + return nil +} + +// ProcessUtilityVMImage post-processes a utility VM image that has had its files extracted. +// The files should have been extracted to \Files. +func ProcessUtilityVMImage(path string) error { + err := processUtilityImage(path) + if err != nil { + return &os.PathError{Op: "ProcessUtilityVMImage", Path: path, Err: err} + } + return nil +} diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/shutdownterminatecomputesystem.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/shutdownterminatecomputesystem.go index d006edf77c..27ac734bd8 100644 --- a/components/engine/vendor/src/github.com/Microsoft/hcsshim/shutdownterminatecomputesystem.go +++ b/components/engine/vendor/src/github.com/Microsoft/hcsshim/shutdownterminatecomputesystem.go @@ -3,19 +3,19 @@ package hcsshim import "github.com/Sirupsen/logrus" // TerminateComputeSystem force terminates a container. -func TerminateComputeSystem(id string, timeout uint32, context string) (uint32, error) { +func TerminateComputeSystem(id string, timeout uint32, context string) error { return shutdownTerminate(false, id, timeout, context) } // ShutdownComputeSystem shuts down a container by requesting a shutdown within // the container operating system. -func ShutdownComputeSystem(id string, timeout uint32, context string) (uint32, error) { +func ShutdownComputeSystem(id string, timeout uint32, context string) error { return shutdownTerminate(true, id, timeout, context) } // shutdownTerminate is a wrapper for ShutdownComputeSystem and TerminateComputeSystem // which have very similar calling semantics -func shutdownTerminate(shutdown bool, id string, timeout uint32, context string) (uint32, error) { +func shutdownTerminate(shutdown bool, id string, timeout uint32, context string) error { var ( title = "HCSShim::" @@ -35,10 +35,9 @@ func shutdownTerminate(shutdown bool, id string, timeout uint32, context string) } if err != nil { - err := makeErrorf(err, title, "id=%s context=%s", id, context) - return err.HResult(), err + return makeErrorf(err, title, "id=%s context=%s", id, context) } logrus.Debugf(title+" succeeded id=%s context=%s", id, context) - return 0, nil + return nil } diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/version.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/version.go new file mode 100644 index 0000000000..ae10c23d42 --- /dev/null +++ b/components/engine/vendor/src/github.com/Microsoft/hcsshim/version.go @@ -0,0 +1,7 @@ +package hcsshim + +// IsTP4 returns whether the currently running Windows build is at least TP4. +func IsTP4() bool { + // HNSCall was not present in TP4 + return procHNSCall.Find() != nil +} diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/waitprocess.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/waitprocess.go index c9f4617711..e916140399 100644 --- a/components/engine/vendor/src/github.com/Microsoft/hcsshim/waitprocess.go +++ b/components/engine/vendor/src/github.com/Microsoft/hcsshim/waitprocess.go @@ -3,8 +3,8 @@ package hcsshim import "github.com/Sirupsen/logrus" // WaitForProcessInComputeSystem waits for a process ID to terminate and returns -// the exit code. Returns exitcode, errno, error -func WaitForProcessInComputeSystem(id string, processid uint32, timeout uint32) (int32, uint32, error) { +// the exit code. Returns exitcode, error +func WaitForProcessInComputeSystem(id string, processid uint32, timeout uint32) (int32, error) { title := "HCSShim::WaitForProcessInComputeSystem" logrus.Debugf(title+" id=%s processid=%d", id, processid) @@ -12,10 +12,9 @@ func WaitForProcessInComputeSystem(id string, processid uint32, timeout uint32) var exitCode uint32 err := waitForProcessInComputeSystem(id, processid, timeout, &exitCode) if err != nil { - err := makeErrorf(err, title, "id=%s", id) - return 0, err.HResult(), err + return 0, makeErrorf(err, title, "id=%s", id) } logrus.Debugf(title+" succeeded id=%s processid=%d exitcode=%d", id, processid, exitCode) - return int32(exitCode), 0, nil + return int32(exitCode), nil } diff --git a/components/engine/vendor/src/github.com/Microsoft/hcsshim/zhcsshim.go b/components/engine/vendor/src/github.com/Microsoft/hcsshim/zhcsshim.go index 15528aaa23..03924b6fcc 100644 --- a/components/engine/vendor/src/github.com/Microsoft/hcsshim/zhcsshim.go +++ b/components/engine/vendor/src/github.com/Microsoft/hcsshim/zhcsshim.go @@ -2,7 +2,11 @@ package hcsshim -import "unsafe" +import ( + "unsafe" + + "github.com/Microsoft/go-winio" +) import "syscall" var _ unsafe.Pointer @@ -26,6 +30,16 @@ var ( procNameToGuid = modvmcompute.NewProc("NameToGuid") procPrepareLayer = modvmcompute.NewProc("PrepareLayer") procUnprepareLayer = modvmcompute.NewProc("UnprepareLayer") + procProcessBaseImage = modvmcompute.NewProc("ProcessBaseImage") + procProcessUtilityImage = modvmcompute.NewProc("ProcessUtilityImage") + procImportLayerBegin = modvmcompute.NewProc("ImportLayerBegin") + procImportLayerNext = modvmcompute.NewProc("ImportLayerNext") + procImportLayerWrite = modvmcompute.NewProc("ImportLayerWrite") + procImportLayerEnd = modvmcompute.NewProc("ImportLayerEnd") + procExportLayerBegin = modvmcompute.NewProc("ExportLayerBegin") + procExportLayerNext = modvmcompute.NewProc("ExportLayerNext") + procExportLayerRead = modvmcompute.NewProc("ExportLayerRead") + procExportLayerEnd = modvmcompute.NewProc("ExportLayerEnd") procCreateComputeSystem = modvmcompute.NewProc("CreateComputeSystem") procCreateProcessWithStdHandlesInComputeSystem = modvmcompute.NewProc("CreateProcessWithStdHandlesInComputeSystem") procResizeConsoleInComputeSystem = modvmcompute.NewProc("ResizeConsoleInComputeSystem") @@ -56,8 +70,8 @@ func _activateLayer(info *driverInfo, id *uint16) (hr error) { return } r0, _, _ := syscall.Syscall(procActivateLayer.Addr(), 2, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -85,8 +99,8 @@ func _copyLayer(info *driverInfo, srcId *uint16, dstId *uint16, descriptors []WC return } r0, _, _ := syscall.Syscall6(procCopyLayer.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(srcId)), uintptr(unsafe.Pointer(dstId)), uintptr(unsafe.Pointer(_p2)), uintptr(len(descriptors)), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -110,8 +124,8 @@ func _createLayer(info *driverInfo, id *uint16, parent *uint16) (hr error) { return } r0, _, _ := syscall.Syscall(procCreateLayer.Addr(), 3, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(parent))) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -139,8 +153,8 @@ func _createSandboxLayer(info *driverInfo, id *uint16, parent *uint16, descripto return } r0, _, _ := syscall.Syscall6(procCreateSandboxLayer.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(parent)), uintptr(unsafe.Pointer(_p2)), uintptr(len(descriptors)), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -159,8 +173,8 @@ func _deactivateLayer(info *driverInfo, id *uint16) (hr error) { return } r0, _, _ := syscall.Syscall(procDeactivateLayer.Addr(), 2, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -179,8 +193,8 @@ func _destroyLayer(info *driverInfo, id *uint16) (hr error) { return } r0, _, _ := syscall.Syscall(procDestroyLayer.Addr(), 2, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -208,8 +222,8 @@ func _exportLayer(info *driverInfo, id *uint16, path *uint16, descriptors []WC_L return } r0, _, _ := syscall.Syscall6(procExportLayer.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(_p2)), uintptr(len(descriptors)), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -228,8 +242,8 @@ func _getLayerMountPath(info *driverInfo, id *uint16, length *uintptr, buffer *u return } r0, _, _ := syscall.Syscall6(procGetLayerMountPath.Addr(), 4, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(length)), uintptr(unsafe.Pointer(buffer)), 0, 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -239,8 +253,8 @@ func getBaseImages(buffer **uint16) (hr error) { return } r0, _, _ := syscall.Syscall(procGetBaseImages.Addr(), 1, uintptr(unsafe.Pointer(buffer)), 0, 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -268,8 +282,8 @@ func _importLayer(info *driverInfo, id *uint16, path *uint16, descriptors []WC_L return } r0, _, _ := syscall.Syscall6(procImportLayer.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(_p2)), uintptr(len(descriptors)), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -288,8 +302,8 @@ func _layerExists(info *driverInfo, id *uint16, exists *uint32) (hr error) { return } r0, _, _ := syscall.Syscall(procLayerExists.Addr(), 3, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(exists))) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -308,8 +322,8 @@ func _nameToGuid(name *uint16, guid *GUID) (hr error) { return } r0, _, _ := syscall.Syscall(procNameToGuid.Addr(), 2, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(guid)), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -332,8 +346,8 @@ func _prepareLayer(info *driverInfo, id *uint16, descriptors []WC_LAYER_DESCRIPT return } r0, _, _ := syscall.Syscall6(procPrepareLayer.Addr(), 4, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(_p1)), uintptr(len(descriptors)), 0, 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -352,8 +366,179 @@ func _unprepareLayer(info *driverInfo, id *uint16) (hr error) { return } r0, _, _ := syscall.Syscall(procUnprepareLayer.Addr(), 2, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func processBaseImage(path string) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(path) + if hr != nil { + return + } + return _processBaseImage(_p0) +} + +func _processBaseImage(path *uint16) (hr error) { + if hr = procProcessBaseImage.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procProcessBaseImage.Addr(), 1, uintptr(unsafe.Pointer(path)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func processUtilityImage(path string) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(path) + if hr != nil { + return + } + return _processUtilityImage(_p0) +} + +func _processUtilityImage(path *uint16) (hr error) { + if hr = procProcessUtilityImage.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procProcessUtilityImage.Addr(), 1, uintptr(unsafe.Pointer(path)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func importLayerBegin(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _importLayerBegin(info, _p0, descriptors, context) +} + +func _importLayerBegin(info *driverInfo, id *uint16, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) { + var _p1 *WC_LAYER_DESCRIPTOR + if len(descriptors) > 0 { + _p1 = &descriptors[0] + } + if hr = procImportLayerBegin.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procImportLayerBegin.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(_p1)), uintptr(len(descriptors)), uintptr(unsafe.Pointer(context)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func importLayerNext(context uintptr, fileName string, fileInfo *winio.FileBasicInfo) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(fileName) + if hr != nil { + return + } + return _importLayerNext(context, _p0, fileInfo) +} + +func _importLayerNext(context uintptr, fileName *uint16, fileInfo *winio.FileBasicInfo) (hr error) { + if hr = procImportLayerNext.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procImportLayerNext.Addr(), 3, uintptr(context), uintptr(unsafe.Pointer(fileName)), uintptr(unsafe.Pointer(fileInfo))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func importLayerWrite(context uintptr, buffer []byte) (hr error) { + var _p0 *byte + if len(buffer) > 0 { + _p0 = &buffer[0] + } + if hr = procImportLayerWrite.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procImportLayerWrite.Addr(), 3, uintptr(context), uintptr(unsafe.Pointer(_p0)), uintptr(len(buffer))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func importLayerEnd(context uintptr) (hr error) { + if hr = procImportLayerEnd.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procImportLayerEnd.Addr(), 1, uintptr(context), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func exportLayerBegin(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _exportLayerBegin(info, _p0, descriptors, context) +} + +func _exportLayerBegin(info *driverInfo, id *uint16, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) { + var _p1 *WC_LAYER_DESCRIPTOR + if len(descriptors) > 0 { + _p1 = &descriptors[0] + } + if hr = procExportLayerBegin.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procExportLayerBegin.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(_p1)), uintptr(len(descriptors)), uintptr(unsafe.Pointer(context)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func exportLayerNext(context uintptr, fileName **uint16, fileInfo *winio.FileBasicInfo, fileSize *int64, deleted *uint32) (hr error) { + if hr = procExportLayerNext.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procExportLayerNext.Addr(), 5, uintptr(context), uintptr(unsafe.Pointer(fileName)), uintptr(unsafe.Pointer(fileInfo)), uintptr(unsafe.Pointer(fileSize)), uintptr(unsafe.Pointer(deleted)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func exportLayerRead(context uintptr, buffer []byte, bytesRead *uint32) (hr error) { + var _p0 *byte + if len(buffer) > 0 { + _p0 = &buffer[0] + } + if hr = procExportLayerRead.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procExportLayerRead.Addr(), 4, uintptr(context), uintptr(unsafe.Pointer(_p0)), uintptr(len(buffer)), uintptr(unsafe.Pointer(bytesRead)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func exportLayerEnd(context uintptr) (hr error) { + if hr = procExportLayerEnd.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procExportLayerEnd.Addr(), 1, uintptr(context), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -377,8 +562,8 @@ func _createComputeSystem(id *uint16, configuration *uint16) (hr error) { return } r0, _, _ := syscall.Syscall(procCreateComputeSystem.Addr(), 2, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(configuration)), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -402,8 +587,8 @@ func _createProcessWithStdHandlesInComputeSystem(id *uint16, paramsJson *uint16, return } r0, _, _ := syscall.Syscall6(procCreateProcessWithStdHandlesInComputeSystem.Addr(), 6, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(paramsJson)), uintptr(unsafe.Pointer(pid)), uintptr(unsafe.Pointer(stdin)), uintptr(unsafe.Pointer(stdout)), uintptr(unsafe.Pointer(stderr))) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -422,8 +607,8 @@ func _resizeConsoleInComputeSystem(id *uint16, pid uint32, height uint16, width return } r0, _, _ := syscall.Syscall6(procResizeConsoleInComputeSystem.Addr(), 5, uintptr(unsafe.Pointer(id)), uintptr(pid), uintptr(height), uintptr(width), uintptr(flags), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -442,8 +627,8 @@ func _shutdownComputeSystem(id *uint16, timeout uint32) (hr error) { return } r0, _, _ := syscall.Syscall(procShutdownComputeSystem.Addr(), 2, uintptr(unsafe.Pointer(id)), uintptr(timeout), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -462,8 +647,8 @@ func _startComputeSystem(id *uint16) (hr error) { return } r0, _, _ := syscall.Syscall(procStartComputeSystem.Addr(), 1, uintptr(unsafe.Pointer(id)), 0, 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -482,8 +667,8 @@ func _terminateComputeSystem(id *uint16) (hr error) { return } r0, _, _ := syscall.Syscall(procTerminateComputeSystem.Addr(), 1, uintptr(unsafe.Pointer(id)), 0, 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -502,8 +687,8 @@ func _terminateProcessInComputeSystem(id *uint16, pid uint32) (hr error) { return } r0, _, _ := syscall.Syscall(procTerminateProcessInComputeSystem.Addr(), 2, uintptr(unsafe.Pointer(id)), uintptr(pid), 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -522,8 +707,8 @@ func _waitForProcessInComputeSystem(id *uint16, pid uint32, timeout uint32, exit return } r0, _, _ := syscall.Syscall6(procWaitForProcessInComputeSystem.Addr(), 4, uintptr(unsafe.Pointer(id)), uintptr(pid), uintptr(timeout), uintptr(unsafe.Pointer(exitCode)), 0, 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } @@ -552,8 +737,8 @@ func __hnsCall(method *uint16, path *uint16, object *uint16, response **uint16) return } r0, _, _ := syscall.Syscall6(procHNSCall.Addr(), 4, uintptr(unsafe.Pointer(method)), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(object)), uintptr(unsafe.Pointer(response)), 0, 0) - if r0 != 0 { - hr = syscall.Errno(r0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) } return } diff --git a/components/engine/vendor/src/github.com/docker/distribution/context/doc.go b/components/engine/vendor/src/github.com/docker/distribution/context/doc.go index 6fe1f817d8..3b4ab8882f 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/context/doc.go +++ b/components/engine/vendor/src/github.com/docker/distribution/context/doc.go @@ -1,6 +1,6 @@ // Package context provides several utilities for working with // golang.org/x/net/context in http requests. Primarily, the focus is on -// logging relevent request information but this package is not limited to +// logging relevant request information but this package is not limited to // that purpose. // // The easiest way to get started is to get the background context: diff --git a/components/engine/vendor/src/github.com/docker/distribution/context/trace.go b/components/engine/vendor/src/github.com/docker/distribution/context/trace.go index af4f1351e9..721964a848 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/context/trace.go +++ b/components/engine/vendor/src/github.com/docker/distribution/context/trace.go @@ -10,7 +10,7 @@ import ( // WithTrace allocates a traced timing span in a new context. This allows a // caller to track the time between calling WithTrace and the returned done // function. When the done function is called, a log message is emitted with a -// "trace.duration" field, corresponding to the elapased time and a +// "trace.duration" field, corresponding to the elapsed time and a // "trace.func" field, corresponding to the function that called WithTrace. // // The logging keys "trace.id" and "trace.parent.id" are provided to implement diff --git a/components/engine/vendor/src/github.com/docker/distribution/digest/set.go b/components/engine/vendor/src/github.com/docker/distribution/digest/set.go index 3fac41b40f..4b9313c1ae 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/digest/set.go +++ b/components/engine/vendor/src/github.com/docker/distribution/digest/set.go @@ -22,7 +22,7 @@ var ( // may be easily referenced by easily referenced by a string // representation of the digest as well as short representation. // The uniqueness of the short representation is based on other -// digests in the set. If digests are ommited from this set, +// digests in the set. If digests are omitted from this set, // collisions in a larger set may not be detected, therefore it // is important to always do short representation lookups on // the complete set of digests. To mitigate collisions, an diff --git a/components/engine/vendor/src/github.com/docker/distribution/manifest/schema1/manifest.go b/components/engine/vendor/src/github.com/docker/distribution/manifest/schema1/manifest.go index 160f9cd996..bff47bde05 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/manifest/schema1/manifest.go +++ b/components/engine/vendor/src/github.com/docker/distribution/manifest/schema1/manifest.go @@ -102,7 +102,7 @@ type SignedManifest struct { Canonical []byte `json:"-"` // all contains the byte representation of the Manifest including signatures - // and is retuend by Payload() + // and is returned by Payload() all []byte } diff --git a/components/engine/vendor/src/github.com/docker/distribution/reference/reference.go b/components/engine/vendor/src/github.com/docker/distribution/reference/reference.go index c188472a40..6f079cbb1a 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/reference/reference.go +++ b/components/engine/vendor/src/github.com/docker/distribution/reference/reference.go @@ -6,7 +6,7 @@ // reference := repository [ ":" tag ] [ "@" digest ] // name := [hostname '/'] component ['/' component]* // hostname := hostcomponent ['.' hostcomponent]* [':' port-number] -// hostcomponent := /([a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])/ +// hostcomponent := /([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])/ // port-number := /[0-9]+/ // component := alpha-numeric [separator alpha-numeric]* // alpha-numeric := /[a-z0-9]+/ diff --git a/components/engine/vendor/src/github.com/docker/distribution/reference/regexp.go b/components/engine/vendor/src/github.com/docker/distribution/reference/regexp.go index a4ffe5b642..9a7d366bc8 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/reference/regexp.go +++ b/components/engine/vendor/src/github.com/docker/distribution/reference/regexp.go @@ -22,7 +22,7 @@ var ( // hostnameComponentRegexp restricts the registry hostname component of a // repository name to start with a component as defined by hostnameRegexp // and followed by an optional port. - hostnameComponentRegexp = match(`(?:[a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])`) + hostnameComponentRegexp = match(`(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])`) // hostnameRegexp defines the structure of potential hostname components // that may be part of image names. This is purposely a subset of what is @@ -49,7 +49,7 @@ var ( // NameRegexp is the format for the name component of references. The // regexp has capturing groups for the hostname and name part omitting - // the seperating forward slash from either. + // the separating forward slash from either. NameRegexp = expression( optional(hostnameRegexp, literal(`/`)), nameComponentRegexp, diff --git a/components/engine/vendor/src/github.com/docker/distribution/registry/api/v2/descriptors.go b/components/engine/vendor/src/github.com/docker/distribution/registry/api/v2/descriptors.go index ad3da3efb9..7549ccc322 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/registry/api/v2/descriptors.go +++ b/components/engine/vendor/src/github.com/docker/distribution/registry/api/v2/descriptors.go @@ -271,7 +271,7 @@ type MethodDescriptor struct { // RequestDescriptor per API use case. type RequestDescriptor struct { // Name provides a short identifier for the request, usable as a title or - // to provide quick context for the particalar request. + // to provide quick context for the particular request. Name string // Description should cover the requests purpose, covering any details for @@ -303,14 +303,14 @@ type RequestDescriptor struct { // ResponseDescriptor describes the components of an API response. type ResponseDescriptor struct { // Name provides a short identifier for the response, usable as a title or - // to provide quick context for the particalar response. + // to provide quick context for the particular response. Name string // Description should provide a brief overview of the role of the // response. Description string - // StatusCode specifies the status recieved by this particular response. + // StatusCode specifies the status received by this particular response. StatusCode int // Headers covers any headers that may be returned from the response. diff --git a/components/engine/vendor/src/github.com/docker/distribution/registry/api/v2/errors.go b/components/engine/vendor/src/github.com/docker/distribution/registry/api/v2/errors.go index ece52a2cd0..97d6923aa0 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/registry/api/v2/errors.go +++ b/components/engine/vendor/src/github.com/docker/distribution/registry/api/v2/errors.go @@ -84,7 +84,7 @@ var ( }) // ErrorCodeManifestUnverified is returned when the manifest fails - // signature verfication. + // signature verification. ErrorCodeManifestUnverified = errcode.Register(errGroup, errcode.ErrorDescriptor{ Value: "MANIFEST_UNVERIFIED", Message: "manifest failed signature verification", diff --git a/components/engine/vendor/src/github.com/docker/distribution/registry/client/auth/session.go b/components/engine/vendor/src/github.com/docker/distribution/registry/client/auth/session.go index 50a94a3da3..f4c7ade41f 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/registry/client/auth/session.go +++ b/components/engine/vendor/src/github.com/docker/distribution/registry/client/auth/session.go @@ -15,6 +15,10 @@ import ( "github.com/docker/distribution/registry/client/transport" ) +// ErrNoBasicAuthCredentials is returned if a request can't be authorized with +// basic auth due to lack of credentials. +var ErrNoBasicAuthCredentials = errors.New("no basic auth credentials") + // AuthenticationHandler is an interface for authorizing a request from // params from a "WWW-Authenicate" header for a single scheme. type AuthenticationHandler interface { @@ -322,5 +326,5 @@ func (bh *basicHandler) AuthorizeRequest(req *http.Request, params map[string]st return nil } } - return errors.New("no basic auth credentials") + return ErrNoBasicAuthCredentials } diff --git a/components/engine/vendor/src/github.com/docker/distribution/registry/client/repository.go b/components/engine/vendor/src/github.com/docker/distribution/registry/client/repository.go index 1e8c4fa98e..ebf44d4733 100644 --- a/components/engine/vendor/src/github.com/docker/distribution/registry/client/repository.go +++ b/components/engine/vendor/src/github.com/docker/distribution/registry/client/repository.go @@ -36,8 +36,21 @@ func checkHTTPRedirect(req *http.Request, via []*http.Request) error { if len(via) > 0 { for headerName, headerVals := range via[0].Header { - if headerName == "Accept" || headerName == "Range" { - for _, val := range headerVals { + if headerName != "Accept" && headerName != "Range" { + continue + } + for _, val := range headerVals { + // Don't add to redirected request if redirected + // request already has a header with the same + // name and value. + hasValue := false + for _, existingVal := range req.Header[headerName] { + if existingVal == val { + hasValue = true + break + } + } + if !hasValue { req.Header.Add(headerName, val) } } diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/client_nounix.go b/components/engine/vendor/src/github.com/docker/engine-api/client/client_darwin.go similarity index 84% rename from components/engine/vendor/src/github.com/docker/engine-api/client/client_nounix.go rename to components/engine/vendor/src/github.com/docker/engine-api/client/client_darwin.go index d07ab84dcd..4b47a178c4 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/client_nounix.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/client_darwin.go @@ -1,5 +1,3 @@ -// +build windows darwin - package client // DefaultDockerHost defines os specific default if DOCKER_HOST is unset diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/client_windows.go b/components/engine/vendor/src/github.com/docker/engine-api/client/client_windows.go new file mode 100644 index 0000000000..07c0c7a774 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/client_windows.go @@ -0,0 +1,4 @@ +package client + +// DefaultDockerHost defines os specific default if DOCKER_HOST is unset +const DefaultDockerHost = "npipe:////./pipe/docker_engine" diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/copy.go b/components/engine/vendor/src/github.com/docker/engine-api/client/container_copy.go similarity index 100% rename from components/engine/vendor/src/github.com/docker/engine-api/client/copy.go rename to components/engine/vendor/src/github.com/docker/engine-api/client/container_copy.go diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/diff.go b/components/engine/vendor/src/github.com/docker/engine-api/client/container_diff.go similarity index 100% rename from components/engine/vendor/src/github.com/docker/engine-api/client/diff.go rename to components/engine/vendor/src/github.com/docker/engine-api/client/container_diff.go diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/exec.go b/components/engine/vendor/src/github.com/docker/engine-api/client/container_exec.go similarity index 100% rename from components/engine/vendor/src/github.com/docker/engine-api/client/exec.go rename to components/engine/vendor/src/github.com/docker/engine-api/client/container_exec.go diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/export.go b/components/engine/vendor/src/github.com/docker/engine-api/client/container_export.go similarity index 100% rename from components/engine/vendor/src/github.com/docker/engine-api/client/export.go rename to components/engine/vendor/src/github.com/docker/engine-api/client/container_export.go diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/kill.go b/components/engine/vendor/src/github.com/docker/engine-api/client/container_kill.go similarity index 100% rename from components/engine/vendor/src/github.com/docker/engine-api/client/kill.go rename to components/engine/vendor/src/github.com/docker/engine-api/client/container_kill.go diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/logs.go b/components/engine/vendor/src/github.com/docker/engine-api/client/container_logs.go similarity index 100% rename from components/engine/vendor/src/github.com/docker/engine-api/client/logs.go rename to components/engine/vendor/src/github.com/docker/engine-api/client/container_logs.go diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/pause.go b/components/engine/vendor/src/github.com/docker/engine-api/client/container_pause.go similarity index 100% rename from components/engine/vendor/src/github.com/docker/engine-api/client/pause.go rename to components/engine/vendor/src/github.com/docker/engine-api/client/container_pause.go diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/resize.go b/components/engine/vendor/src/github.com/docker/engine-api/client/container_resize.go similarity index 100% rename from components/engine/vendor/src/github.com/docker/engine-api/client/resize.go rename to components/engine/vendor/src/github.com/docker/engine-api/client/container_resize.go diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/wait.go b/components/engine/vendor/src/github.com/docker/engine-api/client/container_wait.go similarity index 100% rename from components/engine/vendor/src/github.com/docker/engine-api/client/wait.go rename to components/engine/vendor/src/github.com/docker/engine-api/client/container_wait.go diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/hijack.go b/components/engine/vendor/src/github.com/docker/engine-api/client/hijack.go index d40a136fdb..d9c8513883 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/hijack.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/hijack.go @@ -105,7 +105,12 @@ func tlsDialWithDialer(dialer *net.Dialer, network, addr string, config *tls.Con }) } - rawConn, err := dialer.Dial(network, addr) + proxyDialer, err := sockets.DialerFromEnvironment(dialer) + if err != nil { + return nil, err + } + + rawConn, err := proxyDialer.Dial(network, addr) if err != nil { return nil, err } diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/image_build.go b/components/engine/vendor/src/github.com/docker/engine-api/client/image_build.go index 175d654fa9..d5f96cbd54 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/image_build.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/image_build.go @@ -74,8 +74,8 @@ func imageBuildOptionsToQuery(options types.ImageBuildOptions) (url.Values, erro query.Set("pull", "1") } - if !container.IsolationLevel.IsDefault(options.IsolationLevel) { - query.Set("isolation", string(options.IsolationLevel)) + if !container.Isolation.IsDefault(options.Isolation) { + query.Set("isolation", string(options.Isolation)) } query.Set("cpusetcpus", options.CPUSetCPUs) diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/history.go b/components/engine/vendor/src/github.com/docker/engine-api/client/image_history.go similarity index 100% rename from components/engine/vendor/src/github.com/docker/engine-api/client/history.go rename to components/engine/vendor/src/github.com/docker/engine-api/client/image_history.go diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/image_load.go b/components/engine/vendor/src/github.com/docker/engine-api/client/image_load.go index 0c8880cab0..84ee19c309 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/image_load.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/image_load.go @@ -18,7 +18,8 @@ func (cli *Client) ImageLoad(ctx context.Context, input io.Reader, quiet bool) ( if quiet { v.Set("quiet", "1") } - resp, err := cli.postRaw(ctx, "/images/load", v, input, nil) + headers := map[string][]string{"Content-Type": {"application/x-tar"}} + resp, err := cli.postRaw(ctx, "/images/load", v, input, headers) if err != nil { return types.ImageLoadResponse{}, err } diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/network.go b/components/engine/vendor/src/github.com/docker/engine-api/client/network.go deleted file mode 100644 index 90b9adb29e..0000000000 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/network.go +++ /dev/null @@ -1,86 +0,0 @@ -package client - -import ( - "encoding/json" - "net/http" - "net/url" - - "github.com/docker/engine-api/types" - "github.com/docker/engine-api/types/filters" - "github.com/docker/engine-api/types/network" -) - -// NetworkCreate creates a new network in the docker host. -func (cli *Client) NetworkCreate(options types.NetworkCreate) (types.NetworkCreateResponse, error) { - var response types.NetworkCreateResponse - serverResp, err := cli.post("/networks/create", nil, options, nil) - if err != nil { - return response, err - } - - json.NewDecoder(serverResp.body).Decode(&response) - ensureReaderClosed(serverResp) - return response, err -} - -// NetworkRemove removes an existent network from the docker host. -func (cli *Client) NetworkRemove(networkID string) error { - resp, err := cli.delete("/networks/"+networkID, nil, nil) - ensureReaderClosed(resp) - return err -} - -// NetworkConnect connects a container to an existent network in the docker host. -func (cli *Client) NetworkConnect(networkID, containerID string, config *network.EndpointSettings) error { - nc := types.NetworkConnect{ - Container: containerID, - EndpointConfig: config, - } - resp, err := cli.post("/networks/"+networkID+"/connect", nil, nc, nil) - ensureReaderClosed(resp) - return err -} - -// NetworkDisconnect disconnects a container from an existent network in the docker host. -func (cli *Client) NetworkDisconnect(networkID, containerID string, force bool) error { - nd := types.NetworkDisconnect{Container: containerID, Force: force} - resp, err := cli.post("/networks/"+networkID+"/disconnect", nil, nd, nil) - ensureReaderClosed(resp) - return err -} - -// NetworkList returns the list of networks configured in the docker host. -func (cli *Client) NetworkList(options types.NetworkListOptions) ([]types.NetworkResource, error) { - query := url.Values{} - if options.Filters.Len() > 0 { - filterJSON, err := filters.ToParam(options.Filters) - if err != nil { - return nil, err - } - - query.Set("filters", filterJSON) - } - var networkResources []types.NetworkResource - resp, err := cli.get("/networks", query, nil) - if err != nil { - return networkResources, err - } - err = json.NewDecoder(resp.body).Decode(&networkResources) - ensureReaderClosed(resp) - return networkResources, err -} - -// NetworkInspect returns the information for a specific network configured in the docker host. -func (cli *Client) NetworkInspect(networkID string) (types.NetworkResource, error) { - var networkResource types.NetworkResource - resp, err := cli.get("/networks/"+networkID, nil, nil) - if err != nil { - if resp.statusCode == http.StatusNotFound { - return networkResource, networkNotFoundError{networkID} - } - return networkResource, err - } - err = json.NewDecoder(resp.body).Decode(&networkResource) - ensureReaderClosed(resp) - return networkResource, err -} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/network_connect.go b/components/engine/vendor/src/github.com/docker/engine-api/client/network_connect.go new file mode 100644 index 0000000000..103ab2b3f4 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/network_connect.go @@ -0,0 +1,17 @@ +package client + +import ( + "github.com/docker/engine-api/types" + "github.com/docker/engine-api/types/network" +) + +// NetworkConnect connects a container to an existent network in the docker host. +func (cli *Client) NetworkConnect(networkID, containerID string, config *network.EndpointSettings) error { + nc := types.NetworkConnect{ + Container: containerID, + EndpointConfig: config, + } + resp, err := cli.post("/networks/"+networkID+"/connect", nil, nc, nil) + ensureReaderClosed(resp) + return err +} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/network_create.go b/components/engine/vendor/src/github.com/docker/engine-api/client/network_create.go new file mode 100644 index 0000000000..39b249d8c6 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/network_create.go @@ -0,0 +1,20 @@ +package client + +import ( + "encoding/json" + + "github.com/docker/engine-api/types" +) + +// NetworkCreate creates a new network in the docker host. +func (cli *Client) NetworkCreate(options types.NetworkCreate) (types.NetworkCreateResponse, error) { + var response types.NetworkCreateResponse + serverResp, err := cli.post("/networks/create", nil, options, nil) + if err != nil { + return response, err + } + + json.NewDecoder(serverResp.body).Decode(&response) + ensureReaderClosed(serverResp) + return response, err +} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/network_disconnect.go b/components/engine/vendor/src/github.com/docker/engine-api/client/network_disconnect.go new file mode 100644 index 0000000000..3426a87bda --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/network_disconnect.go @@ -0,0 +1,13 @@ +package client + +import ( + "github.com/docker/engine-api/types" +) + +// NetworkDisconnect disconnects a container from an existent network in the docker host. +func (cli *Client) NetworkDisconnect(networkID, containerID string, force bool) error { + nd := types.NetworkDisconnect{Container: containerID, Force: force} + resp, err := cli.post("/networks/"+networkID+"/disconnect", nil, nd, nil) + ensureReaderClosed(resp) + return err +} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/network_inspect.go b/components/engine/vendor/src/github.com/docker/engine-api/client/network_inspect.go new file mode 100644 index 0000000000..e79f2c2436 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/network_inspect.go @@ -0,0 +1,23 @@ +package client + +import ( + "encoding/json" + "net/http" + + "github.com/docker/engine-api/types" +) + +// NetworkInspect returns the information for a specific network configured in the docker host. +func (cli *Client) NetworkInspect(networkID string) (types.NetworkResource, error) { + var networkResource types.NetworkResource + resp, err := cli.get("/networks/"+networkID, nil, nil) + if err != nil { + if resp.statusCode == http.StatusNotFound { + return networkResource, networkNotFoundError{networkID} + } + return networkResource, err + } + err = json.NewDecoder(resp.body).Decode(&networkResource) + ensureReaderClosed(resp) + return networkResource, err +} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/network_list.go b/components/engine/vendor/src/github.com/docker/engine-api/client/network_list.go new file mode 100644 index 0000000000..df6d2e44c0 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/network_list.go @@ -0,0 +1,30 @@ +package client + +import ( + "encoding/json" + "net/url" + + "github.com/docker/engine-api/types" + "github.com/docker/engine-api/types/filters" +) + +// NetworkList returns the list of networks configured in the docker host. +func (cli *Client) NetworkList(options types.NetworkListOptions) ([]types.NetworkResource, error) { + query := url.Values{} + if options.Filters.Len() > 0 { + filterJSON, err := filters.ToParam(options.Filters) + if err != nil { + return nil, err + } + + query.Set("filters", filterJSON) + } + var networkResources []types.NetworkResource + resp, err := cli.get("/networks", query, nil) + if err != nil { + return networkResources, err + } + err = json.NewDecoder(resp.body).Decode(&networkResources) + ensureReaderClosed(resp) + return networkResources, err +} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/network_remove.go b/components/engine/vendor/src/github.com/docker/engine-api/client/network_remove.go new file mode 100644 index 0000000000..728fdc2211 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/network_remove.go @@ -0,0 +1,8 @@ +package client + +// NetworkRemove removes an existent network from the docker host. +func (cli *Client) NetworkRemove(networkID string) error { + resp, err := cli.delete("/networks/"+networkID, nil, nil) + ensureReaderClosed(resp) + return err +} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/transport/client_mock.go b/components/engine/vendor/src/github.com/docker/engine-api/client/transport/client_mock.go deleted file mode 100644 index 8cc0cca5b1..0000000000 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/transport/client_mock.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build test - -package transport - -import ( - "crypto/tls" - "net/http" -) - -type mockClient struct { - *tlsInfo - do func(*http.Request) (*http.Response, error) -} - -// NewMockClient returns a mocked client that runs the function supplied as `client.Do` call -func NewMockClient(tlsConfig *tls.Config, doer func(*http.Request) (*http.Response, error)) Client { - return mockClient{ - tlsInfo: &tlsInfo{tlsConfig}, - do: doer, - } -} - -// Do executes the supplied function for the mock. -func (m mockClient) Do(req *http.Request) (*http.Response, error) { - return m.do(req) -} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/volume.go b/components/engine/vendor/src/github.com/docker/engine-api/client/volume.go deleted file mode 100644 index 597e31803d..0000000000 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/volume.go +++ /dev/null @@ -1,66 +0,0 @@ -package client - -import ( - "encoding/json" - "net/http" - "net/url" - - "github.com/docker/engine-api/types" - "github.com/docker/engine-api/types/filters" -) - -// VolumeList returns the volumes configured in the docker host. -func (cli *Client) VolumeList(filter filters.Args) (types.VolumesListResponse, error) { - var volumes types.VolumesListResponse - query := url.Values{} - - if filter.Len() > 0 { - filterJSON, err := filters.ToParam(filter) - if err != nil { - return volumes, err - } - query.Set("filters", filterJSON) - } - resp, err := cli.get("/volumes", query, nil) - if err != nil { - return volumes, err - } - - err = json.NewDecoder(resp.body).Decode(&volumes) - ensureReaderClosed(resp) - return volumes, err -} - -// VolumeInspect returns the information about a specific volume in the docker host. -func (cli *Client) VolumeInspect(volumeID string) (types.Volume, error) { - var volume types.Volume - resp, err := cli.get("/volumes/"+volumeID, nil, nil) - if err != nil { - if resp.statusCode == http.StatusNotFound { - return volume, volumeNotFoundError{volumeID} - } - return volume, err - } - err = json.NewDecoder(resp.body).Decode(&volume) - ensureReaderClosed(resp) - return volume, err -} - -// VolumeCreate creates a volume in the docker host. -func (cli *Client) VolumeCreate(options types.VolumeCreateRequest) (types.Volume, error) { - var volume types.Volume - resp, err := cli.post("/volumes/create", nil, options, nil) - if err != nil { - return volume, err - } - err = json.NewDecoder(resp.body).Decode(&volume) - ensureReaderClosed(resp) - return volume, err -} - -// VolumeRemove removes a volume from the docker host. -func (cli *Client) VolumeRemove(volumeID string) error { - resp, err := cli.delete("/volumes/"+volumeID, nil, nil) - ensureReaderClosed(resp) - return err -} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/volume_create.go b/components/engine/vendor/src/github.com/docker/engine-api/client/volume_create.go new file mode 100644 index 0000000000..98e8f796a3 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/volume_create.go @@ -0,0 +1,19 @@ +package client + +import ( + "encoding/json" + + "github.com/docker/engine-api/types" +) + +// VolumeCreate creates a volume in the docker host. +func (cli *Client) VolumeCreate(options types.VolumeCreateRequest) (types.Volume, error) { + var volume types.Volume + resp, err := cli.post("/volumes/create", nil, options, nil) + if err != nil { + return volume, err + } + err = json.NewDecoder(resp.body).Decode(&volume) + ensureReaderClosed(resp) + return volume, err +} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/volume_inspect.go b/components/engine/vendor/src/github.com/docker/engine-api/client/volume_inspect.go new file mode 100644 index 0000000000..dbd8444800 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/volume_inspect.go @@ -0,0 +1,23 @@ +package client + +import ( + "encoding/json" + "net/http" + + "github.com/docker/engine-api/types" +) + +// VolumeInspect returns the information about a specific volume in the docker host. +func (cli *Client) VolumeInspect(volumeID string) (types.Volume, error) { + var volume types.Volume + resp, err := cli.get("/volumes/"+volumeID, nil, nil) + if err != nil { + if resp.statusCode == http.StatusNotFound { + return volume, volumeNotFoundError{volumeID} + } + return volume, err + } + err = json.NewDecoder(resp.body).Decode(&volume) + ensureReaderClosed(resp) + return volume, err +} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/volume_list.go b/components/engine/vendor/src/github.com/docker/engine-api/client/volume_list.go new file mode 100644 index 0000000000..6659bc39e2 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/volume_list.go @@ -0,0 +1,31 @@ +package client + +import ( + "encoding/json" + "net/url" + + "github.com/docker/engine-api/types" + "github.com/docker/engine-api/types/filters" +) + +// VolumeList returns the volumes configured in the docker host. +func (cli *Client) VolumeList(filter filters.Args) (types.VolumesListResponse, error) { + var volumes types.VolumesListResponse + query := url.Values{} + + if filter.Len() > 0 { + filterJSON, err := filters.ToParam(filter) + if err != nil { + return volumes, err + } + query.Set("filters", filterJSON) + } + resp, err := cli.get("/volumes", query, nil) + if err != nil { + return volumes, err + } + + err = json.NewDecoder(resp.body).Decode(&volumes) + ensureReaderClosed(resp) + return volumes, err +} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/volume_remove.go b/components/engine/vendor/src/github.com/docker/engine-api/client/volume_remove.go new file mode 100644 index 0000000000..a8bd612de0 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/volume_remove.go @@ -0,0 +1,8 @@ +package client + +// VolumeRemove removes a volume from the docker host. +func (cli *Client) VolumeRemove(volumeID string) error { + resp, err := cli.delete("/volumes/"+volumeID, nil, nil) + ensureReaderClosed(resp) + return err +} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/auth.go b/components/engine/vendor/src/github.com/docker/engine-api/types/auth.go index 6cd4c36a83..056af6b842 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/auth.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/auth.go @@ -2,10 +2,21 @@ package types // AuthConfig contains authorization information for connecting to a Registry type AuthConfig struct { - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - Auth string `json:"auth"` - Email string `json:"email"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Auth string `json:"auth,omitempty"` + + // Email is an optional value associated with the username. + // This field is deprecated and will be removed in a later + // version of docker. + Email string `json:"email,omitempty"` + ServerAddress string `json:"serveraddress,omitempty"` + + // IdentityToken is used to authenticate the user and get + // an access token for the registry. + IdentityToken string `json:"identitytoken,omitempty"` + + // RegistryToken is a bearer token to be sent to a registry RegistryToken string `json:"registrytoken,omitempty"` } diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/client.go b/components/engine/vendor/src/github.com/docker/engine-api/types/client.go index 16c1cb101b..4880140367 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/client.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/client.go @@ -127,7 +127,7 @@ type ImageBuildOptions struct { Remove bool ForceRemove bool PullParent bool - IsolationLevel container.IsolationLevel + Isolation container.Isolation CPUSetCPUs string CPUSetMems string CPUShares int64 diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/container/config.go b/components/engine/vendor/src/github.com/docker/engine-api/types/container/config.go index b4e6205d21..b8747a5087 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/container/config.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/container/config.go @@ -24,12 +24,12 @@ type Config struct { OpenStdin bool // Open stdin StdinOnce bool // If true, close stdin after the 1 attached client disconnects. Env []string // List of environment variable to set in the container - Cmd *strslice.StrSlice // Command to run when starting the container + Cmd strslice.StrSlice // Command to run when starting the container ArgsEscaped bool `json:",omitempty"` // True if command is already escaped (Windows specific) Image string // Name of the image as it was passed by the operator (eg. could be symbolic) Volumes map[string]struct{} // List of volumes (mounts) used for the container WorkingDir string // Current directory (PWD) in the command will be launched - Entrypoint *strslice.StrSlice // Entrypoint to run when starting the container + Entrypoint strslice.StrSlice // Entrypoint to run when starting the container NetworkDisabled bool `json:",omitempty"` // Is network disabled MacAddress string `json:",omitempty"` // Mac Address of the container OnBuild []string // ONBUILD metadata that were defined on the image Dockerfile diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/container/host_config.go b/components/engine/vendor/src/github.com/docker/engine-api/types/container/host_config.go index b7c459ea8a..f8b9842295 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/container/host_config.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/container/host_config.go @@ -12,13 +12,13 @@ import ( // NetworkMode represents the container network stack. type NetworkMode string -// IsolationLevel represents the isolation level of a container. The supported +// Isolation represents the isolation technology of a container. The supported // values are platform specific -type IsolationLevel string +type Isolation string -// IsDefault indicates the default isolation level of a container. On Linux this +// IsDefault indicates the default isolation technology of a container. On Linux this // is the native driver. On Windows, this is a Windows Server Container. -func (i IsolationLevel) IsDefault() bool { +func (i Isolation) IsDefault() bool { return strings.ToLower(string(i)) == "default" || string(i) == "" } @@ -65,6 +65,30 @@ func (n IpcMode) Container() string { return "" } +// UsernsMode represents userns mode in the container. +type UsernsMode string + +// IsHost indicates whether the container uses the host's userns. +func (n UsernsMode) IsHost() bool { + return n == "host" +} + +// IsPrivate indicates whether the container uses the a private userns. +func (n UsernsMode) IsPrivate() bool { + return !(n.IsHost()) +} + +// Valid indicates whether the userns is valid. +func (n UsernsMode) Valid() bool { + parts := strings.Split(string(n), ":") + switch mode := parts[0]; mode { + case "", "host": + default: + return false + } + return true +} + // UTSMode represents the UTS namespace of the container. type UTSMode string @@ -166,6 +190,7 @@ type LogConfig struct { type Resources struct { // Applicable to all platforms CPUShares int64 `json:"CpuShares"` // CPU shares (relative weight vs. other containers) + Memory int64 // Memory limit (in bytes) // Applicable to UNIX platforms CgroupParent string // Parent cgroup. @@ -180,14 +205,19 @@ type Resources struct { CpusetCpus string // CpusetCpus 0-2, 0,1 CpusetMems string // CpusetMems 0-2, 0,1 Devices []DeviceMapping // List of devices to map inside the container + DiskQuota int64 // Disk limit (in bytes) KernelMemory int64 // Kernel memory limit (in bytes) - Memory int64 // Memory limit (in bytes) MemoryReservation int64 // Memory soft limit (in bytes) MemorySwap int64 // Total memory usage (memory + swap); set `-1` to enable unlimited swap MemorySwappiness *int64 // Tuning container memory swappiness behaviour OomKillDisable *bool // Whether to disable OOM Killer or not PidsLimit int64 // Setting pids limit for a container Ulimits []*units.Ulimit // List of ulimits to be set in the container + + // Applicable to Windows + BlkioIOps uint64 // Maximum IOps for the container system drive + BlkioBps uint64 // Maximum Bytes per second for the container system drive + SandboxSize uint64 // System drive will be expanded to at least this size (in bytes) } // UpdateConfig holds the mutable attributes of a Container. @@ -209,32 +239,36 @@ type HostConfig struct { NetworkMode NetworkMode // Network mode to use for the container PortBindings nat.PortMap // Port mapping between the exposed port (container) and the host RestartPolicy RestartPolicy // Restart policy to be used for the container + AutoRemove bool // Automatically remove container when it exits VolumeDriver string // Name of the volume driver used to mount volumes VolumesFrom []string // List of volumes to take from other container // Applicable to UNIX platforms - CapAdd *strslice.StrSlice // List of kernel capabilities to add to the container - CapDrop *strslice.StrSlice // List of kernel capabilities to remove from the container - DNS []string `json:"Dns"` // List of DNS server to lookup - DNSOptions []string `json:"DnsOptions"` // List of DNSOption to look for - DNSSearch []string `json:"DnsSearch"` // List of DNSSearch to look for - ExtraHosts []string // List of extra hosts - GroupAdd []string // List of additional groups that the container process will run as - IpcMode IpcMode // IPC namespace to use for the container - Links []string // List of links (in the name:alias form) - OomScoreAdj int // Container preference for OOM-killing - PidMode PidMode // PID namespace to use for the container - Privileged bool // Is the container in privileged mode - PublishAllPorts bool // Should docker publish all exposed port for the container - ReadonlyRootfs bool // Is the container root filesystem in read-only - SecurityOpt []string // List of string values to customize labels for MLS systems, such as SELinux. - Tmpfs map[string]string `json:",omitempty"` // List of tmpfs (mounts) used for the container - UTSMode UTSMode // UTS namespace to use for the container - ShmSize int64 // Total shm memory usage + CapAdd strslice.StrSlice // List of kernel capabilities to add to the container + CapDrop strslice.StrSlice // List of kernel capabilities to remove from the container + DNS []string `json:"Dns"` // List of DNS server to lookup + DNSOptions []string `json:"DnsOptions"` // List of DNSOption to look for + DNSSearch []string `json:"DnsSearch"` // List of DNSSearch to look for + ExtraHosts []string // List of extra hosts + GroupAdd []string // List of additional groups that the container process will run as + IpcMode IpcMode // IPC namespace to use for the container + Links []string // List of links (in the name:alias form) + OomScoreAdj int // Container preference for OOM-killing + PidMode PidMode // PID namespace to use for the container + Privileged bool // Is the container in privileged mode + PublishAllPorts bool // Should docker publish all exposed port for the container + ReadonlyRootfs bool // Is the container root filesystem in read-only + SecurityOpt []string // List of string values to customize labels for MLS systems, such as SELinux. + StorageOpt []string // Storage driver options per container. + Tmpfs map[string]string `json:",omitempty"` // List of tmpfs (mounts) used for the container + UTSMode UTSMode // UTS namespace to use for the container + UsernsMode UsernsMode // The user namespace to use for the container + ShmSize int64 // Total shm memory usage + Sysctls map[string]string `json:",omitempty"` // List of Namespaced sysctls used for the container // Applicable to Windows - ConsoleSize [2]int // Initial console size - Isolation IsolationLevel // Isolation level of the container (eg default, hyperv) + ConsoleSize [2]int // Initial console size + Isolation Isolation // Isolation technology of the container (eg default, hyperv) // Contains container's resources (cgroups, ulimits) Resources diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/container/hostconfig_unix.go b/components/engine/vendor/src/github.com/docker/engine-api/types/container/hostconfig_unix.go index c12534aec4..4171059a47 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/container/hostconfig_unix.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/container/hostconfig_unix.go @@ -4,8 +4,8 @@ package container import "strings" -// IsValid indicates is an isolation level is valid -func (i IsolationLevel) IsValid() bool { +// IsValid indicates if an isolation technology is valid +func (i Isolation) IsValid() bool { return i.IsDefault() } diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/container/hostconfig_windows.go b/components/engine/vendor/src/github.com/docker/engine-api/types/container/hostconfig_windows.go index 64ba756d2b..5726a77e0d 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/container/hostconfig_windows.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/container/hostconfig_windows.go @@ -1,7 +1,6 @@ package container import ( - "fmt" "strings" ) @@ -15,66 +14,74 @@ func (n NetworkMode) IsNone() bool { return n == "none" } +// IsContainer indicates whether container uses a container network stack. +// Returns false as windows doesn't support this mode +func (n NetworkMode) IsContainer() bool { + return false +} + +// IsBridge indicates whether container uses the bridge network stack +// in windows it is given the name NAT +func (n NetworkMode) IsBridge() bool { + return n == "nat" +} + +// IsHost indicates whether container uses the host network stack. +// returns false as this is not supported by windows +func (n NetworkMode) IsHost() bool { + return false +} + +// IsPrivate indicates whether container uses it's private network stack. +func (n NetworkMode) IsPrivate() bool { + return !(n.IsHost() || n.IsContainer()) +} + +// ConnectedContainer is the id of the container which network this container is connected to. +// Returns blank string on windows +func (n NetworkMode) ConnectedContainer() string { + return "" +} + // IsUserDefined indicates user-created network func (n NetworkMode) IsUserDefined() bool { - return !n.IsDefault() && !n.IsNone() + return !n.IsDefault() && !n.IsNone() && !n.IsBridge() } // IsHyperV indicates the use of a Hyper-V partition for isolation -func (i IsolationLevel) IsHyperV() bool { +func (i Isolation) IsHyperV() bool { return strings.ToLower(string(i)) == "hyperv" } // IsProcess indicates the use of process isolation -func (i IsolationLevel) IsProcess() bool { +func (i Isolation) IsProcess() bool { return strings.ToLower(string(i)) == "process" } -// IsValid indicates is an isolation level is valid -func (i IsolationLevel) IsValid() bool { +// IsValid indicates if an isolation technology is valid +func (i Isolation) IsValid() bool { return i.IsDefault() || i.IsHyperV() || i.IsProcess() } -// DefaultDaemonNetworkMode returns the default network stack the daemon should -// use. -func DefaultDaemonNetworkMode() NetworkMode { - return NetworkMode("default") -} - // NetworkName returns the name of the network stack. func (n NetworkMode) NetworkName() string { if n.IsDefault() { return "default" + } else if n.IsBridge() { + return "nat" + } else if n.IsNone() { + return "none" + } else if n.IsUserDefined() { + return n.UserDefined() } + return "" } -// ValidateNetMode ensures that the various combinations of requested -// network settings are valid. -func ValidateNetMode(c *Config, hc *HostConfig) error { - // We may not be passed a host config, such as in the case of docker commit - if hc == nil { - return nil +//UserDefined indicates user-created network +func (n NetworkMode) UserDefined() string { + if n.IsUserDefined() { + return string(n) } - parts := strings.Split(string(hc.NetworkMode), ":") - switch mode := parts[0]; mode { - case "default", "none": - default: - return fmt.Errorf("invalid --net: %s", hc.NetworkMode) - } - return nil -} - -// ValidateIsolationLevel performs platform specific validation of the -// isolation level in the hostconfig structure. Windows supports 'default' (or -// blank), 'process', or 'hyperv'. -func ValidateIsolationLevel(hc *HostConfig) error { - // We may not be passed a host config, such as in the case of docker commit - if hc == nil { - return nil - } - if !hc.Isolation.IsValid() { - return fmt.Errorf("invalid --isolation: %q. Windows supports 'default', 'process', or 'hyperv'", hc.Isolation) - } - return nil + return "" } diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/network/network.go b/components/engine/vendor/src/github.com/docker/engine-api/types/network/network.go index 48b2199622..bce60f5eec 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/network/network.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/network/network.go @@ -46,7 +46,7 @@ type EndpointSettings struct { } // NetworkingConfig represents the container's networking configuration for each of its interfaces -// Carries the networink configs specified in the `docker run` and `docker network connect` commands +// Carries the networking configs specified in the `docker run` and `docker network connect` commands type NetworkingConfig struct { - EndpointsConfig map[string]*EndpointSettings // Endpoint configs for each conencting network + EndpointsConfig map[string]*EndpointSettings // Endpoint configs for each connecting network } diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/strslice/strslice.go b/components/engine/vendor/src/github.com/docker/engine-api/types/strslice/strslice.go index 9f3ee620a3..bad493fb89 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/strslice/strslice.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/strslice/strslice.go @@ -1,29 +1,18 @@ package strslice -import ( - "encoding/json" - "strings" -) +import "encoding/json" // StrSlice represents a string or an array of strings. // We need to override the json decoder to accept both options. -type StrSlice struct { - parts []string -} +type StrSlice []string -// MarshalJSON Marshals (or serializes) the StrSlice into the json format. -// This method is needed to implement json.Marshaller. -func (e *StrSlice) MarshalJSON() ([]byte, error) { - if e == nil { - return []byte{}, nil - } - return json.Marshal(e.Slice()) -} - -// UnmarshalJSON decodes the byte slice whether it's a string or an array of strings. -// This method is needed to implement json.Unmarshaler. +// UnmarshalJSON decodes the byte slice whether it's a string or an array of +// strings. This method is needed to implement json.Unmarshaler. func (e *StrSlice) UnmarshalJSON(b []byte) error { if len(b) == 0 { + // With no input, we preserve the existing value by returning nil and + // leaving the target alone. This allows defining default values for + // the type. return nil } @@ -36,36 +25,6 @@ func (e *StrSlice) UnmarshalJSON(b []byte) error { p = append(p, s) } - e.parts = p + *e = p return nil } - -// Len returns the number of parts of the StrSlice. -func (e *StrSlice) Len() int { - if e == nil { - return 0 - } - return len(e.parts) -} - -// Slice gets the parts of the StrSlice as a Slice of string. -func (e *StrSlice) Slice() []string { - if e == nil { - return nil - } - return e.parts -} - -// ToString gets space separated string of all the parts. -func (e *StrSlice) ToString() string { - s := e.Slice() - if s == nil { - return "" - } - return strings.Join(s, " ") -} - -// New creates an StrSlice based on the specified parts (as strings). -func New(parts ...string) *StrSlice { - return &StrSlice{parts} -} diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/types.go b/components/engine/vendor/src/github.com/docker/engine-api/types/types.go index efb5a606c8..0b6494aa50 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/types.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/types.go @@ -39,6 +39,10 @@ type ContainerUpdateResponse struct { type AuthResponse struct { // Status is the authentication status Status string `json:"Status"` + + // IdentityToken is an opaque token used for authenticating + // a user after a successful login. + IdentityToken string `json:"IdentityToken,omitempty"` } // ContainerWaitResponse contains response of Remote API: @@ -148,6 +152,7 @@ type Container struct { NetworkMode string `json:",omitempty"` } NetworkSettings *SummaryNetworkSettings + Mounts []MountPoint } // CopyConfig contains request body of Remote API: @@ -203,6 +208,7 @@ type Info struct { Plugins PluginsInfo MemoryLimit bool SwapLimit bool + KernelMemory bool CPUCfsPeriod bool `json:"CpuCfsPeriod"` CPUCfsQuota bool `json:"CpuCfsQuota"` CPUShares bool @@ -217,6 +223,7 @@ type Info struct { SystemTime string ExecutionDriver string LoggingDriver string + CgroupDriver string NEventsListener int KernelVersion string OperatingSystem string @@ -238,8 +245,8 @@ type Info struct { ClusterAdvertise string } -// PluginsInfo is temp struct holds Plugins name -// registered with docker daemon. It used by Info struct +// PluginsInfo is a temp struct holding Plugins name +// registered with docker daemon. It is used by Info struct type PluginsInfo struct { // List of Volume plugins registered Volume []string diff --git a/components/engine/vendor/src/github.com/docker/go-connections/sockets/proxy.go b/components/engine/vendor/src/github.com/docker/go-connections/sockets/proxy.go new file mode 100644 index 0000000000..98e9a1dc61 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/go-connections/sockets/proxy.go @@ -0,0 +1,51 @@ +package sockets + +import ( + "net" + "net/url" + "os" + "strings" + + "golang.org/x/net/proxy" +) + +// GetProxyEnv allows access to the uppercase and the lowercase forms of +// proxy-related variables. See the Go specification for details on these +// variables. https://golang.org/pkg/net/http/ +func GetProxyEnv(key string) string { + proxyValue := os.Getenv(strings.ToUpper(key)) + if proxyValue == "" { + return os.Getenv(strings.ToLower(key)) + } + return proxyValue +} + +// DialerFromEnvironment takes in a "direct" *net.Dialer and returns a +// proxy.Dialer which will route the connections through the proxy using the +// given dialer. +func DialerFromEnvironment(direct *net.Dialer) (proxy.Dialer, error) { + allProxy := GetProxyEnv("all_proxy") + if len(allProxy) == 0 { + return direct, nil + } + + proxyURL, err := url.Parse(allProxy) + if err != nil { + return direct, err + } + + proxyFromURL, err := proxy.FromURL(proxyURL, direct) + if err != nil { + return direct, err + } + + noProxy := GetProxyEnv("no_proxy") + if len(noProxy) == 0 { + return proxyFromURL, nil + } + + perHost := proxy.NewPerHost(proxyFromURL, direct) + perHost.AddFromString(noProxy) + + return perHost, nil +} diff --git a/components/engine/vendor/src/github.com/docker/go-connections/sockets/sockets.go b/components/engine/vendor/src/github.com/docker/go-connections/sockets/sockets.go index 4bf5e84fc1..1739cecf2a 100644 --- a/components/engine/vendor/src/github.com/docker/go-connections/sockets/sockets.go +++ b/components/engine/vendor/src/github.com/docker/go-connections/sockets/sockets.go @@ -8,28 +8,35 @@ import ( ) // Why 32? See https://github.com/docker/docker/pull/8035. -const defaulTimeout = 32 * time.Second +const defaultTimeout = 32 * time.Second // ConfigureTransport configures the specified Transport according to the // specified proto and addr. -// If the proto is unix (using a unix socket to communicate) the compression -// is disabled. -func ConfigureTransport(tr *http.Transport, proto, addr string) { +// If the proto is unix (using a unix socket to communicate) or npipe the +// compression is disabled. +func ConfigureTransport(tr *http.Transport, proto, addr string) error { switch proto { case "unix": // No need for compression in local communications. tr.DisableCompression = true tr.Dial = func(_, _ string) (net.Conn, error) { - return net.DialTimeout(proto, addr, defaulTimeout) + return net.DialTimeout(proto, addr, defaultTimeout) } case "npipe": // No need for compression in local communications. tr.DisableCompression = true tr.Dial = func(_, _ string) (net.Conn, error) { - return DialPipe(addr, defaulTimeout) + return DialPipe(addr, defaultTimeout) } default: tr.Proxy = http.ProxyFromEnvironment - tr.Dial = (&net.Dialer{Timeout: defaulTimeout}).Dial + dialer, err := DialerFromEnvironment(&net.Dialer{ + Timeout: defaultTimeout, + }) + if err != nil { + return err + } + tr.Dial = dialer.Dial } + return nil } diff --git a/components/engine/vendor/src/github.com/docker/go-connections/sockets/unix_socket.go b/components/engine/vendor/src/github.com/docker/go-connections/sockets/unix_socket.go index c10acedca2..d1627349f8 100644 --- a/components/engine/vendor/src/github.com/docker/go-connections/sockets/unix_socket.go +++ b/components/engine/vendor/src/github.com/docker/go-connections/sockets/unix_socket.go @@ -1,4 +1,4 @@ -// +build linux freebsd +// +build linux freebsd solaris package sockets diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/CHANGELOG.md b/components/engine/vendor/src/github.com/docker/libnetwork/CHANGELOG.md index 1735a29938..c2ebca7730 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/CHANGELOG.md +++ b/components/engine/vendor/src/github.com/docker/libnetwork/CHANGELOG.md @@ -1,5 +1,46 @@ # Changelog +## 0.7.0-dev.5 (2016-03-08) +- Fixes https://github.com/docker/docker/issues/20847 +- Fixes https://github.com/docker/docker/issues/20997 +- Fixes issues unveiled by docker integ test over 0.7.0-dev.4 + +## 0.7.0-dev.4 (2016-03-07) +- Changed ownership of exposed ports and port-mapping options from Endpoint to Sandbox +- Implement DNS RR in the Docker embedded DNS server +- Fixes https://github.com/docker/libnetwork/issues/984 (multi container overlay veth leak) +- Libnetwork to program container's interface MAC address +- Fixed bug in iptables.Exists() logic +- Fixes https://github.com/docker/docker/issues/20694 +- Source external DNS queries from container namespace +- Added inbuilt nil IPAM driver +- Windows drivers integration fixes +- Extract hostname from (hostname.domainname). Related to https://github.com/docker/docker/issues/14282 +- Fixed race in sandbox statistics read +- Fixes https://github.com/docker/libnetwork/issues/892 (docker start fails when ipv6.disable=1) +- Fixed error message on bridge network creation conflict + +## 0.7.0-dev.3 (2016-02-17) +- Fixes https://github.com/docker/docker/issues/20350 +- Fixes https://github.com/docker/docker/issues/20145 +- Initial Windows HNS integration +- Allow passing global datastore config to libnetwork after boot +- Set Recursion Available bit in DNS query responses +- Make sure iptables chains are recreated on firewalld reload + +## 0.7.0-dev.2 (2016-02-11) +- Fixes https://github.com/docker/docker/issues/20140 + +## 0.7.0-dev.1 (2016-02-10) +- Expose EnableIPV6 option +- discoverapi refactoring +- Fixed a few typos & docs update + +## 0.6.1-rc2 (2016-02-09) +- Fixes https://github.com/docker/docker/issues/20132 +- Fixes https://github.com/docker/docker/issues/20140 +- Fixes https://github.com/docker/docker/issues/20019 + ## 0.6.1-rc1 (2016-02-05) - Fixes https://github.com/docker/docker/issues/20026 @@ -69,7 +110,7 @@ - DEPRECATE service discovery from default bridge network - Introduced new network UX - Support for multiple networks in bridge driver -- Local persistance with boltdb +- Local persistence with boltdb ## 0.4.0 (2015-07-24) @@ -82,6 +123,6 @@ - Fixed a bunch of issues with osl namespace mgmt ## 0.3.0 (2015-05-27) - + - Introduce CNM (Container Networking Model) - Replace docker networking with CNM & Bridge driver diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/Dockerfile.build b/components/engine/vendor/src/github.com/docker/libnetwork/Dockerfile.build index 2b767c2a57..8dcfd83255 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/Dockerfile.build +++ b/components/engine/vendor/src/github.com/docker/libnetwork/Dockerfile.build @@ -8,6 +8,5 @@ RUN cd /go/src && mkdir -p golang.org/x && \ RUN go get github.com/tools/godep \ github.com/golang/lint/golint \ golang.org/x/tools/cmd/vet \ - golang.org/x/tools/cmd/goimports \ golang.org/x/tools/cmd/cover\ github.com/mattn/goveralls diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/MAINTAINERS b/components/engine/vendor/src/github.com/docker/libnetwork/MAINTAINERS index 33f2dd2e18..da991614c3 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/MAINTAINERS +++ b/components/engine/vendor/src/github.com/docker/libnetwork/MAINTAINERS @@ -16,6 +16,8 @@ "icecrime", "mrjana", "mavenugo", + "sanimej", + "chenchun", ] [people] @@ -36,6 +38,11 @@ Email = "lk4d4@docker.com" GitHub = "LK4D4" + [people.chenchun] + Name = "Chun Chen" + Email = "ramichen@tencent.com" + GitHub = "chenchun" + [people.icecrime] Name = "Arnaud Porterie" Email = "arnaud@docker.com" @@ -50,3 +57,8 @@ Name = "Madhu Venugopal" Email = "madhu@docker.com" GitHub = "mavenugo" + + [people.sanimej] + Name = "Santhosh Manohar" + Email = "santhosh@docker.com" + GitHub = "sanimej" diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/Makefile b/components/engine/vendor/src/github.com/docker/libnetwork/Makefile index 2031826c7d..096d7fcc21 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/Makefile +++ b/components/engine/vendor/src/github.com/docker/libnetwork/Makefile @@ -53,7 +53,7 @@ check-code: check-format: @echo "Checking format... " - test -z "$$(goimports -l . | grep -v Godeps/_workspace/src/ | tee /dev/stderr)" + test -z "$$(gofmt -s -l . | grep -v Godeps/_workspace/src/ | tee /dev/stderr)" @echo "Done checking format" run-tests: diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/bitseq/sequence.go b/components/engine/vendor/src/github.com/docker/libnetwork/bitseq/sequence.go index 270a36aa63..0dc1bc4ad0 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/bitseq/sequence.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/bitseq/sequence.go @@ -163,7 +163,7 @@ func (s *sequence) toByteArray() ([]byte, error) { func (s *sequence) fromByteArray(data []byte) error { l := len(data) if l%12 != 0 { - return fmt.Errorf("cannot deserialize byte sequence of lenght %d (%v)", l, data) + return fmt.Errorf("cannot deserialize byte sequence of length %d (%v)", l, data) } p := s diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/config/config.go b/components/engine/vendor/src/github.com/docker/libnetwork/config/config.go index 80d2fc3af7..8da92f7a0e 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/config/config.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/config/config.go @@ -61,7 +61,23 @@ func ParseConfig(tomlCfgFile string) (*Config, error) { return cfg, nil } -// Option is a option setter function type used to pass varios configurations +// ParseConfigOptions parses the configuration options and returns +// a reference to the corresponding Config structure +func ParseConfigOptions(cfgOptions ...Option) *Config { + cfg := &Config{ + Daemon: DaemonCfg{ + DriverCfg: make(map[string]interface{}), + }, + Scopes: make(map[string]*datastore.ScopeCfg), + } + + cfg.ProcessOptions(cfgOptions...) + cfg.LoadDefaultScopes(cfg.Daemon.DataDir) + + return cfg +} + +// Option is an option setter function type used to pass various configurations // to the controller type Option func(c *Config) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/controller.go b/components/engine/vendor/src/github.com/docker/libnetwork/controller.go index ef214fd2ce..6abc4d1ddf 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/controller.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/controller.go @@ -56,6 +56,7 @@ import ( "github.com/docker/docker/pkg/stringid" "github.com/docker/libnetwork/config" "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/hostdiscovery" "github.com/docker/libnetwork/ipamapi" @@ -105,6 +106,9 @@ type NetworkController interface { // Stop network controller Stop() + + // ReloadCondfiguration updates the controller configuration + ReloadConfiguration(cfgOptions ...config.Option) error } // NetworkWalker is a client provided function which will be used to walk the Networks. @@ -128,7 +132,6 @@ type ipamData struct { } type driverTable map[string]*driverData - type ipamTable map[string]*ipamData type sandboxTable map[string]*sandbox @@ -152,22 +155,9 @@ type controller struct { // New creates a new instance of network controller. func New(cfgOptions ...config.Option) (NetworkController, error) { - var cfg *config.Config - cfg = &config.Config{ - Daemon: config.DaemonCfg{ - DriverCfg: make(map[string]interface{}), - }, - Scopes: make(map[string]*datastore.ScopeCfg), - } - - if len(cfgOptions) > 0 { - cfg.ProcessOptions(cfgOptions...) - } - cfg.LoadDefaultScopes(cfg.Daemon.DataDir) - c := &controller{ id: stringid.GenerateRandomID(), - cfg: cfg, + cfg: config.ParseConfigOptions(cfgOptions...), sandboxes: sandboxTable{}, drivers: driverTable{}, ipamDrivers: ipamTable{}, @@ -178,9 +168,9 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { return nil, err } - if cfg != nil && cfg.Cluster.Watcher != nil { - if err := c.initDiscovery(cfg.Cluster.Watcher); err != nil { - // Failing to initalize discovery is a bad situation to be in. + if c.cfg != nil && c.cfg.Cluster.Watcher != nil { + if err := c.initDiscovery(c.cfg.Cluster.Watcher); err != nil { + // Failing to initialize discovery is a bad situation to be in. // But it cannot fail creating the Controller log.Errorf("Failed to Initialize Discovery : %v", err) } @@ -205,6 +195,83 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { return c, nil } +var procReloadConfig = make(chan (bool), 1) + +func (c *controller) ReloadConfiguration(cfgOptions ...config.Option) error { + procReloadConfig <- true + defer func() { <-procReloadConfig }() + + // For now we accept the configuration reload only as a mean to provide a global store config after boot. + // Refuse the configuration if it alters an existing datastore client configuration. + update := false + cfg := config.ParseConfigOptions(cfgOptions...) + for s := range c.cfg.Scopes { + if _, ok := cfg.Scopes[s]; !ok { + return types.ForbiddenErrorf("cannot accept new configuration because it removes an existing datastore client") + } + } + for s, nSCfg := range cfg.Scopes { + if eSCfg, ok := c.cfg.Scopes[s]; ok { + if eSCfg.Client.Provider != nSCfg.Client.Provider || + eSCfg.Client.Address != nSCfg.Client.Address { + return types.ForbiddenErrorf("cannot accept new configuration because it modifies an existing datastore client") + } + } else { + update = true + } + } + if !update { + return nil + } + + c.Lock() + c.cfg = cfg + c.Unlock() + + if err := c.initStores(); err != nil { + return err + } + + if c.discovery == nil && c.cfg.Cluster.Watcher != nil { + if err := c.initDiscovery(c.cfg.Cluster.Watcher); err != nil { + log.Errorf("Failed to Initialize Discovery after configuration update: %v", err) + } + } + + var dsConfig *discoverapi.DatastoreConfigData + for scope, sCfg := range cfg.Scopes { + if scope == datastore.LocalScope || !sCfg.IsValid() { + continue + } + dsConfig = &discoverapi.DatastoreConfigData{ + Scope: scope, + Provider: sCfg.Client.Provider, + Address: sCfg.Client.Address, + Config: sCfg.Client.Config, + } + break + } + if dsConfig == nil { + return nil + } + + for nm, id := range c.getIpamDrivers() { + err := id.driver.DiscoverNew(discoverapi.DatastoreConfig, *dsConfig) + if err != nil { + log.Errorf("Failed to set datastore in driver %s: %v", nm, err) + } + } + + for nm, id := range c.getNetDrivers() { + err := id.driver.DiscoverNew(discoverapi.DatastoreConfig, *dsConfig) + if err != nil { + log.Errorf("Failed to set datastore in driver %s: %v", nm, err) + } + } + + return nil +} + func (c *controller) ID() string { return c.id } @@ -288,12 +355,12 @@ func (c *controller) pushNodeDiscovery(d *driverData, nodes []net.IP, add bool) return } for _, node := range nodes { - nodeData := driverapi.NodeDiscoveryData{Address: node.String(), Self: node.Equal(self)} + nodeData := discoverapi.NodeDiscoveryData{Address: node.String(), Self: node.Equal(self)} var err error if add { - err = d.driver.DiscoverNew(driverapi.NodeDiscovery, nodeData) + err = d.driver.DiscoverNew(discoverapi.NodeDiscovery, nodeData) } else { - err = d.driver.DiscoverDelete(driverapi.NodeDiscovery, nodeData) + err = d.driver.DiscoverDelete(discoverapi.NodeDiscovery, nodeData) } if err != nil { log.Debugf("discovery notification error : %v", err) @@ -725,6 +792,26 @@ func (c *controller) getIpamDriver(name string) (ipamapi.Ipam, error) { return id.driver, nil } +func (c *controller) getIpamDrivers() ipamTable { + c.Lock() + defer c.Unlock() + table := ipamTable{} + for i, d := range c.ipamDrivers { + table[i] = d + } + return table +} + +func (c *controller) getNetDrivers() driverTable { + c.Lock() + defer c.Unlock() + table := driverTable{} + for i, d := range c.drivers { + table[i] = d + } + return table +} + func (c *controller) Stop() { c.closeStores() c.stopExternalKeyListener() diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/datastore/datastore.go b/components/engine/vendor/src/github.com/docker/libnetwork/datastore/datastore.go index 0ba7b6ef2c..c15cd620c1 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/datastore/datastore.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/datastore/datastore.go @@ -13,6 +13,7 @@ import ( "github.com/docker/libkv/store/consul" "github.com/docker/libkv/store/etcd" "github.com/docker/libkv/store/zookeeper" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/types" ) @@ -30,7 +31,7 @@ type DataStore interface { DeleteObjectAtomic(kvObject KVObject) error // DeleteTree deletes a record DeleteTree(kvObject KVObject) error - // Watchable returns whether the store is watchable are not + // Watchable returns whether the store is watchable or not Watchable() bool // Watch for changes on a KVObject Watch(kvObject KVObject, stopCh <-chan struct{}) (<-chan KVObject, error) @@ -253,6 +254,34 @@ func NewDataStore(scope string, cfg *ScopeCfg) (DataStore, error) { return newClient(scope, cfg.Client.Provider, cfg.Client.Address, cfg.Client.Config, cached) } +// NewDataStoreFromConfig creates a new instance of LibKV data store starting from the datastore config data +func NewDataStoreFromConfig(dsc discoverapi.DatastoreConfigData) (DataStore, error) { + var ( + ok bool + sCfgP *store.Config + ) + + sCfgP, ok = dsc.Config.(*store.Config) + if !ok && dsc.Config != nil { + return nil, fmt.Errorf("cannot parse store configuration: %v", dsc.Config) + } + + scopeCfg := &ScopeCfg{ + Client: ScopeClientCfg{ + Address: dsc.Address, + Provider: dsc.Provider, + Config: sCfgP, + }, + } + + ds, err := NewDataStore(dsc.Scope, scopeCfg) + if err != nil { + return nil, fmt.Errorf("failed to construct datastore client from datastore configuration %v: %v", dsc, err) + } + + return ds, err +} + func (ds *datastore) Close() { ds.store.Close() } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/default_gateway.go b/components/engine/vendor/src/github.com/docker/libnetwork/default_gateway.go index 2df047a348..d8eb732701 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/default_gateway.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/default_gateway.go @@ -3,7 +3,6 @@ package libnetwork import ( "fmt" - "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" ) @@ -28,15 +27,15 @@ var procGwNetwork = make(chan (bool), 1) - its deleted when an endpoint with GW joins the container */ -func (sb *sandbox) setupDefaultGW(srcEp *endpoint) error { - var createOptions []EndpointOption - c := srcEp.getNetwork().getController() +func (sb *sandbox) setupDefaultGW() error { // check if the conitainer already has a GW endpoint if ep := sb.getEndpointInGWNetwork(); ep != nil { return nil } + c := sb.controller + // Look for default gw network. In case of error (includes not found), // retry and create it if needed in a serialized execution. n, err := c.NetworkByName(libnGWNetwork) @@ -46,19 +45,7 @@ func (sb *sandbox) setupDefaultGW(srcEp *endpoint) error { } } - if opt, ok := srcEp.generic[netlabel.PortMap]; ok { - if pb, ok := opt.([]types.PortBinding); ok { - createOptions = append(createOptions, CreateOptionPortMapping(pb)) - } - } - - if opt, ok := srcEp.generic[netlabel.ExposedPorts]; ok { - if exp, ok := opt.([]types.TransportPort); ok { - createOptions = append(createOptions, CreateOptionExposedPorts(exp)) - } - } - - createOptions = append(createOptions, CreateOptionAnonymous()) + createOptions := []EndpointOption{CreateOptionAnonymous()} eplen := gwEPlen if len(sb.containerID) < gwEPlen { @@ -74,9 +61,13 @@ func (sb *sandbox) setupDefaultGW(srcEp *endpoint) error { if err := epLocal.sbJoin(sb); err != nil { return fmt.Errorf("container %s: endpoint join on GW Network failed: %v", sb.containerID, err) } + return nil } +// If present, removes the endpoint connecting the sandbox to the default gw network. +// Unless it is the endpoint designated to provide the external connectivity. +// If the sandbox is being deleted, removes the endpoint unconditionally. func (sb *sandbox) clearDefaultGW() error { var ep *endpoint @@ -84,6 +75,10 @@ func (sb *sandbox) clearDefaultGW() error { return nil } + if ep == sb.getGatewayEndpoint() && !sb.inDelete { + return nil + } + if err := ep.sbLeave(sb, false); err != nil { return fmt.Errorf("container %s: endpoint leaving GW Network failed: %v", sb.containerID, err) } @@ -98,7 +93,7 @@ func (sb *sandbox) needDefaultGW() bool { for _, ep := range sb.getConnectedEndpoints() { if ep.endpointInGWNetwork() { - continue + return false } if ep.getNetwork().Type() == "null" || ep.getNetwork().Type() == "host" { continue @@ -165,3 +160,16 @@ func (c *controller) defaultGwNetwork() (Network, error) { } return n, err } + +// Returns the endpoint which is providing external connectivity to the sandbox +func (sb *sandbox) getGatewayEndpoint() *endpoint { + for _, ep := range sb.getConnectedEndpoints() { + if ep.getNetwork().Type() == "null" || ep.getNetwork().Type() == "host" { + continue + } + if len(ep.Gateway()) != 0 { + return ep + } + } + return nil +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/default_gateway_linux.go b/components/engine/vendor/src/github.com/docker/libnetwork/default_gateway_linux.go index c7f812778f..9376922a21 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/default_gateway_linux.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/default_gateway_linux.go @@ -5,8 +5,6 @@ import ( "strconv" "github.com/docker/libnetwork/drivers/bridge" - "github.com/docker/libnetwork/netlabel" - "github.com/docker/libnetwork/options" ) func (c *controller) createGWNetwork() (Network, error) { @@ -17,10 +15,9 @@ func (c *controller) createGWNetwork() (Network, error) { } n, err := c.NewNetwork("bridge", libnGWNetwork, - NetworkOptionGeneric(options.Generic{ - netlabel.GenericData: netOption, - netlabel.EnableIPv6: false, - })) + NetworkOptionDriverOpts(netOption), + NetworkOptionEnableIPv6(false), + ) if err != nil { return nil, fmt.Errorf("error creating external connectivity network: %v", err) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/discoverapi/discoverapi.go b/components/engine/vendor/src/github.com/docker/libnetwork/discoverapi/discoverapi.go new file mode 100644 index 0000000000..eeacc3204e --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/libnetwork/discoverapi/discoverapi.go @@ -0,0 +1,35 @@ +package discoverapi + +// Discover is an interface to be implemented by the componenet interested in receiving discover events +// like new node joining the cluster or datastore updates +type Discover interface { + // DiscoverNew is a notification for a new discovery event, Example:a new node joining a cluster + DiscoverNew(dType DiscoveryType, data interface{}) error + + // DiscoverDelete is a notification for a discovery delete event, Example:a node leaving a cluster + DiscoverDelete(dType DiscoveryType, data interface{}) error +} + +// DiscoveryType represents the type of discovery element the DiscoverNew function is invoked on +type DiscoveryType int + +const ( + // NodeDiscovery represents Node join/leave events provided by discovery + NodeDiscovery = iota + 1 + // DatastoreConfig represents an add/remove datastore event + DatastoreConfig +) + +// NodeDiscoveryData represents the structure backing the node discovery data json string +type NodeDiscoveryData struct { + Address string + Self bool +} + +// DatastoreConfigData is the data for the datastore update event message +type DatastoreConfigData struct { + Scope string + Provider string + Address string + Config interface{} +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/driverapi/driverapi.go b/components/engine/vendor/src/github.com/docker/libnetwork/driverapi/driverapi.go index 44a937fb73..4ea5e11278 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/driverapi/driverapi.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/driverapi/driverapi.go @@ -1,12 +1,18 @@ package driverapi -import "net" +import ( + "net" + + "github.com/docker/libnetwork/discoverapi" +) // NetworkPluginEndpointType represents the Endpoint Type used by Plugin system const NetworkPluginEndpointType = "NetworkDriver" // Driver is an interface that every plugin driver needs to implement. type Driver interface { + discoverapi.Discover + // CreateNetwork invokes the driver method to create a network passing // the network id and network specific config. The config mechanism will // eventually be replaced with labels which are yet to be introduced. @@ -36,11 +42,13 @@ type Driver interface { // Leave method is invoked when a Sandbox detaches from an endpoint. Leave(nid, eid string) error - // DiscoverNew is a notification for a new discovery event, Example:a new node joining a cluster - DiscoverNew(dType DiscoveryType, data interface{}) error + // ProgramExternalConnectivity invokes the driver method which does the necessary + // programming to allow the external connectivity dictated by the passed options + ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error - // DiscoverDelete is a notification for a discovery delete event, Example:a node leaving a cluster - DiscoverDelete(dType DiscoveryType, data interface{}) error + // RevokeExternalConnectivity aks the driver to remove any external connectivity + // programming that was done so far + RevokeExternalConnectivity(nid, eid string) error // Type returns the the type of this driver, the network type this driver manages Type() string @@ -88,8 +96,8 @@ type JoinInfo interface { // SetGatewayIPv6 sets the default IPv6 gateway when a container joins the endpoint. SetGatewayIPv6(net.IP) error - // AddStaticRoute adds a routes to the sandbox. - // It may be used in addtion to or instead of a default gateway (as above). + // AddStaticRoute adds a route to the sandbox. + // It may be used in addition to or instead of a default gateway (as above). AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP) error // DisableGatewayService tells libnetwork not to provide Default GW for the container @@ -107,20 +115,6 @@ type Capability struct { DataScope string } -// DiscoveryType represents the type of discovery element the DiscoverNew function is invoked on -type DiscoveryType int - -const ( - // NodeDiscovery represents Node join/leave events provided by discovery - NodeDiscovery = iota + 1 -) - -// NodeDiscoveryData represents the structure backing the node discovery data json string -type NodeDiscoveryData struct { - Address string - Self bool -} - // IPAMData represents the per-network ip related // operational information libnetwork will send // to the network driver during CreateNetwork() diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/driverapi/ipamdata.go b/components/engine/vendor/src/github.com/docker/libnetwork/driverapi/ipamdata.go index 9a2375bf8a..fc1c2af441 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/driverapi/ipamdata.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/driverapi/ipamdata.go @@ -64,7 +64,7 @@ func (i *IPAMData) UnmarshalJSON(data []byte) error { return nil } -// Validate checks wheter the IPAMData structure contains congruent data +// Validate checks whether the IPAMData structure contains congruent data func (i *IPAMData) Validate() error { var isV6 bool if i.Pool == nil { @@ -93,7 +93,7 @@ func (i *IPAMData) Validate() error { return nil } -// IsV6 returns wheter this is an IPv6 IPAMData structure +// IsV6 returns whether this is an IPv6 IPAMData structure func (i *IPAMData) IsV6() bool { return nil == i.Pool.IP.To4() } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers.go index 1b11f37324..566d330ff4 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers.go @@ -3,11 +3,14 @@ package libnetwork import ( "strings" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/ipamapi" - builtinIpam "github.com/docker/libnetwork/ipams/builtin" - remoteIpam "github.com/docker/libnetwork/ipams/remote" "github.com/docker/libnetwork/netlabel" + + builtinIpam "github.com/docker/libnetwork/ipams/builtin" + nullIpam "github.com/docker/libnetwork/ipams/null" + remoteIpam "github.com/docker/libnetwork/ipams/remote" ) type initializer struct { @@ -56,10 +59,12 @@ func makeDriverConfig(c *controller, ntype string) map[string]interface{} { if !v.IsValid() { continue } - - config[netlabel.MakeKVProvider(k)] = v.Client.Provider - config[netlabel.MakeKVProviderURL(k)] = v.Client.Address - config[netlabel.MakeKVProviderConfig(k)] = v.Client.Config + config[netlabel.MakeKVClient(k)] = discoverapi.DatastoreConfigData{ + Scope: k, + Provider: v.Client.Provider, + Address: v.Client.Address, + Config: v.Client.Config, + } } return config @@ -69,6 +74,7 @@ func initIpams(ic ipamapi.Callback, lDs, gDs interface{}) error { for _, fn := range [](func(ipamapi.Callback, interface{}, interface{}) error){ builtinIpam.Init, remoteIpam.Init, + nullIpam.Init, } { if err := fn(ic, lDs, gDs); err != nil { return err diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go index 2bb4350e9d..00e16e1e5b 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go @@ -15,6 +15,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/iptables" "github.com/docker/libnetwork/netlabel" @@ -73,9 +74,7 @@ type networkConfiguration struct { // endpointConfiguration represents the user specified configuration for the sandbox endpoint type endpointConfiguration struct { - MacAddress net.HardwareAddr - PortBindings []types.PortBinding - ExposedPorts []types.TransportPort + MacAddress net.HardwareAddr } // containerConfiguration represents the user specified configuration for a container @@ -84,6 +83,12 @@ type containerConfiguration struct { ChildEndpoints []string } +// cnnectivityConfiguration represents the user specified configuration regarding the external connectivity +type connectivityConfiguration struct { + PortBindings []types.PortBinding + ExposedPorts []types.TransportPort +} + type bridgeEndpoint struct { id string srcName string @@ -92,6 +97,7 @@ type bridgeEndpoint struct { macAddress net.HardwareAddr config *endpointConfiguration // User specified parameters containerConfig *containerConfiguration + extConnConfig *connectivityConfiguration portMapping []types.PortBinding // Operation port bindings } @@ -132,6 +138,9 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { if out, err := exec.Command("modprobe", "-va", "nf_nat").CombinedOutput(); err != nil { logrus.Warnf("Running modprobe nf_nat failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err) } + if out, err := exec.Command("modprobe", "-va", "xt_conntrack").CombinedOutput(); err != nil { + logrus.Warnf("Running modprobe xt_conntrack failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err) + } if err := iptables.FirewalldInit(); err != nil { logrus.Debugf("Fail to initialize firewalld: %v, using raw iptables instead", err) } @@ -179,7 +188,7 @@ func (c *networkConfiguration) Conflicts(o *networkConfiguration) error { return fmt.Errorf("same configuration") } - // Also empty, becasue only one network with empty name is allowed + // Also empty, because only one network with empty name is allowed if c.BridgeName == o.BridgeName { return fmt.Errorf("networks have same bridge name") } @@ -383,6 +392,8 @@ func (d *driver) configure(option map[string]interface{}) error { if err != nil { return err } + // Make sure on firewall reload, first thing being re-played is chains creation + iptables.OnReloaded(func() { logrus.Debugf("Recreating iptables chains on firewall reload"); setupIPChains(config) }) } d.Lock() @@ -444,7 +455,7 @@ func parseNetworkGenericOptions(data interface{}) (*networkConfiguration, error) func (c *networkConfiguration) processIPAM(id string, ipamV4Data, ipamV6Data []driverapi.IPAMData) error { if len(ipamV4Data) > 1 || len(ipamV6Data) > 1 { - return types.ForbiddenErrorf("bridge driver doesnt support multiple subnets") + return types.ForbiddenErrorf("bridge driver doesn't support multiple subnets") } if len(ipamV4Data) == 0 { @@ -537,6 +548,9 @@ func (d *driver) getNetworks() []*bridgeNetwork { // Create a new network using bridge plugin func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) error { + if len(ipV4Data) == 0 || ipV4Data[0].Pool.String() == "0.0.0.0/0" { + return types.BadRequestErrorf("ipv4 pool is empty") + } // Sanity checks d.Lock() if _, ok := d.networks[id]; ok { @@ -575,7 +589,7 @@ func (d *driver) createNetwork(config *networkConfiguration) error { nw.Unlock() if err := nwConfig.Conflicts(config); err != nil { return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s", - nwConfig.BridgeName, config.ID, nw.id, nw.config.BridgeName, err.Error()) + config.ID, config.BridgeName, nwConfig.ID, nwConfig.BridgeName, err.Error()) } } @@ -942,28 +956,19 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, } } - // Create the sandbox side pipe interface + // Store the sandbox side pipe interface parameters endpoint.srcName = containerIfName endpoint.macAddress = ifInfo.MacAddress() endpoint.addr = ifInfo.Address() endpoint.addrv6 = ifInfo.AddressIPv6() - // Down the interface before configuring mac address. - if err = netlink.LinkSetDown(sbox); err != nil { - return fmt.Errorf("could not set link down for container interface %s: %v", containerIfName, err) - } - - // Set the sbox's MAC. If specified, use the one configured by user, otherwise generate one based on IP. + // Set the sbox's MAC if not provided. If specified, use the one configured by user, otherwise generate one based on IP. if endpoint.macAddress == nil { endpoint.macAddress = electMacAddress(epConfig, endpoint.addr.IP) - if err := ifInfo.SetMacAddress(endpoint.macAddress); err != nil { + if err = ifInfo.SetMacAddress(endpoint.macAddress); err != nil { return err } } - err = netlink.LinkSetHardwareAddr(sbox, endpoint.macAddress) - if err != nil { - return fmt.Errorf("could not set mac address for container interface %s: %v", containerIfName, err) - } // Up the host interface after finishing all netlink configuration if err = netlink.LinkSetUp(host); err != nil { @@ -990,17 +995,11 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, } endpoint.addrv6 = &net.IPNet{IP: ip6, Mask: network.Mask} - if err := ifInfo.SetIPAddress(endpoint.addrv6); err != nil { + if err = ifInfo.SetIPAddress(endpoint.addrv6); err != nil { return err } } - // Program any required port mapping and store them in the endpoint - endpoint.portMapping, err = n.allocatePorts(epConfig, endpoint, config.DefaultBindingIP, d.config.EnableUserlandProxy) - if err != nil { - return err - } - return nil } @@ -1055,9 +1054,6 @@ func (d *driver) DeleteEndpoint(nid, eid string) error { } }() - // Remove port mappings. Do not stop endpoint delete on unmap failure - n.releasePorts(ep) - // Try removal of link. Discard error: it is a best effort. // Also make sure defer does not see this error either. if link, err := netlink.LinkByName(ep.srcName); err == nil { @@ -1098,10 +1094,10 @@ func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, erro m := make(map[string]interface{}) - if ep.config.ExposedPorts != nil { + if ep.extConnConfig != nil && ep.extConnConfig.ExposedPorts != nil { // Return a copy of the config data - epc := make([]types.TransportPort, 0, len(ep.config.ExposedPorts)) - for _, tp := range ep.config.ExposedPorts { + epc := make([]types.TransportPort, 0, len(ep.extConnConfig.ExposedPorts)) + for _, tp := range ep.extConnConfig.ExposedPorts { epc = append(epc, tp.GetCopy()) } m[netlabel.ExposedPorts] = epc @@ -1141,6 +1137,11 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, return EndpointNotFoundError(eid) } + endpoint.containerConfig, err = parseContainerOptions(options) + if err != nil { + return err + } + iNames := jinfo.InterfaceName() err = iNames.SetNames(endpoint.srcName, containerVethPrefix) if err != nil { @@ -1157,10 +1158,6 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, return err } - if !network.config.EnableICC { - return d.link(network, endpoint, options, true) - } - return nil } @@ -1183,32 +1180,87 @@ func (d *driver) Leave(nid, eid string) error { } if !network.config.EnableICC { - return d.link(network, endpoint, nil, false) + if err = d.link(network, endpoint, false); err != nil { + return err + } } return nil } -func (d *driver) link(network *bridgeNetwork, endpoint *bridgeEndpoint, options map[string]interface{}, enable bool) error { - var ( - cc *containerConfiguration - err error - ) +func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error { + defer osl.InitOSContext()() - if enable { - cc, err = parseContainerOptions(options) - if err != nil { - return err - } - } else { - cc = endpoint.containerConfig + network, err := d.getNetwork(nid) + if err != nil { + return err } + endpoint, err := network.getEndpoint(eid) + if err != nil { + return err + } + + if endpoint == nil { + return EndpointNotFoundError(eid) + } + + endpoint.extConnConfig, err = parseConnectivityOptions(options) + if err != nil { + return err + } + + // Program any required port mapping and store them in the endpoint + endpoint.portMapping, err = network.allocatePorts(endpoint, network.config.DefaultBindingIP, d.config.EnableUserlandProxy) + if err != nil { + return err + } + + if !network.config.EnableICC { + return d.link(network, endpoint, true) + } + + return nil +} + +func (d *driver) RevokeExternalConnectivity(nid, eid string) error { + defer osl.InitOSContext()() + + network, err := d.getNetwork(nid) + if err != nil { + return err + } + + endpoint, err := network.getEndpoint(eid) + if err != nil { + return err + } + + if endpoint == nil { + return EndpointNotFoundError(eid) + } + + err = network.releasePorts(endpoint) + if err != nil { + logrus.Warn(err) + } + + return nil +} + +func (d *driver) link(network *bridgeNetwork, endpoint *bridgeEndpoint, enable bool) error { + var err error + + cc := endpoint.containerConfig if cc == nil { return nil } + ec := endpoint.extConnConfig + if ec == nil { + return nil + } - if endpoint.config != nil && endpoint.config.ExposedPorts != nil { + if ec.ExposedPorts != nil { for _, p := range cc.ParentEndpoints { var parentEndpoint *bridgeEndpoint parentEndpoint, err = network.getEndpoint(p) @@ -1222,7 +1274,7 @@ func (d *driver) link(network *bridgeNetwork, endpoint *bridgeEndpoint, options l := newLink(parentEndpoint.addr.IP.String(), endpoint.addr.IP.String(), - endpoint.config.ExposedPorts, network.config.BridgeName) + ec.ExposedPorts, network.config.BridgeName) if enable { err = l.Enable() if err != nil { @@ -1249,13 +1301,13 @@ func (d *driver) link(network *bridgeNetwork, endpoint *bridgeEndpoint, options err = InvalidEndpointIDError(c) return err } - if childEndpoint.config == nil || childEndpoint.config.ExposedPorts == nil { + if childEndpoint.extConnConfig == nil || childEndpoint.extConnConfig.ExposedPorts == nil { continue } l := newLink(endpoint.addr.IP.String(), childEndpoint.addr.IP.String(), - childEndpoint.config.ExposedPorts, network.config.BridgeName) + childEndpoint.extConnConfig.ExposedPorts, network.config.BridgeName) if enable { err = l.Enable() if err != nil { @@ -1271,10 +1323,6 @@ func (d *driver) link(network *bridgeNetwork, endpoint *bridgeEndpoint, options } } - if enable { - endpoint.containerConfig = cc - } - return nil } @@ -1283,12 +1331,12 @@ func (d *driver) Type() string { } // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster -func (d *driver) DiscoverNew(dType driverapi.DiscoveryType, data interface{}) error { +func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { return nil } // DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster -func (d *driver) DiscoverDelete(dType driverapi.DiscoveryType, data interface{}) error { +func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { return nil } @@ -1307,22 +1355,6 @@ func parseEndpointOptions(epOptions map[string]interface{}) (*endpointConfigurat } } - if opt, ok := epOptions[netlabel.PortMap]; ok { - if bs, ok := opt.([]types.PortBinding); ok { - ec.PortBindings = bs - } else { - return nil, &ErrInvalidEndpointConfig{} - } - } - - if opt, ok := epOptions[netlabel.ExposedPorts]; ok { - if ports, ok := opt.([]types.TransportPort); ok { - ec.ExposedPorts = ports - } else { - return nil, &ErrInvalidEndpointConfig{} - } - } - return ec, nil } @@ -1348,6 +1380,32 @@ func parseContainerOptions(cOptions map[string]interface{}) (*containerConfigura } } +func parseConnectivityOptions(cOptions map[string]interface{}) (*connectivityConfiguration, error) { + if cOptions == nil { + return nil, nil + } + + cc := &connectivityConfiguration{} + + if opt, ok := cOptions[netlabel.PortMap]; ok { + if pb, ok := opt.([]types.PortBinding); ok { + cc.PortBindings = pb + } else { + return nil, types.BadRequestErrorf("Invalid port mapping data in connectivity configuration: %v", opt) + } + } + + if opt, ok := cOptions[netlabel.ExposedPorts]; ok { + if ports, ok := opt.([]types.TransportPort); ok { + cc.ExposedPorts = ports + } else { + return nil, types.BadRequestErrorf("Invalid exposed ports data in connectivity configuration: %v", opt) + } + } + + return cc, nil +} + func electMacAddress(epConfig *endpointConfiguration, ip net.IP) net.HardwareAddr { if epConfig != nil && epConfig.MacAddress != nil { return epConfig.MacAddress diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge_store.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge_store.go index 96c31d87e8..066469adc9 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge_store.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge_store.go @@ -6,9 +6,9 @@ import ( "net" "github.com/Sirupsen/logrus" - "github.com/docker/libkv/store" "github.com/docker/libkv/store/boltdb" "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" ) @@ -16,27 +16,15 @@ import ( const bridgePrefix = "bridge" func (d *driver) initStore(option map[string]interface{}) error { - var err error - - provider, provOk := option[netlabel.LocalKVProvider] - provURL, urlOk := option[netlabel.LocalKVProviderURL] - - if provOk && urlOk { - cfg := &datastore.ScopeCfg{ - Client: datastore.ScopeClientCfg{ - Provider: provider.(string), - Address: provURL.(string), - }, + if data, ok := option[netlabel.LocalKVClient]; ok { + var err error + dsc, ok := data.(discoverapi.DatastoreConfigData) + if !ok { + return types.InternalErrorf("incorrect data in datastore configuration: %v", data) } - - provConfig, confOk := option[netlabel.LocalKVProviderConfig] - if confOk { - cfg.Client.Config = provConfig.(*store.Config) - } - - d.store, err = datastore.NewDataStore(datastore.LocalScope, cfg) + d.store, err = datastore.NewDataStoreFromConfig(dsc) if err != nil { - return fmt.Errorf("bridge driver failed to initialize data store: %v", err) + return types.InternalErrorf("bridge driver failed to initialize data store: %v", err) } return d.populateNetworks() diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/port_mapping.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/port_mapping.go index 4dab8a0c89..965cc9a039 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/port_mapping.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/port_mapping.go @@ -14,8 +14,8 @@ var ( defaultBindingIP = net.IPv4(0, 0, 0, 0) ) -func (n *bridgeNetwork) allocatePorts(epConfig *endpointConfiguration, ep *bridgeEndpoint, reqDefBindIP net.IP, ulPxyEnabled bool) ([]types.PortBinding, error) { - if epConfig == nil || epConfig.PortBindings == nil { +func (n *bridgeNetwork) allocatePorts(ep *bridgeEndpoint, reqDefBindIP net.IP, ulPxyEnabled bool) ([]types.PortBinding, error) { + if ep.extConnConfig == nil || ep.extConnConfig.PortBindings == nil { return nil, nil } @@ -24,7 +24,7 @@ func (n *bridgeNetwork) allocatePorts(epConfig *endpointConfiguration, ep *bridg defHostIP = reqDefBindIP } - return n.allocatePortsInternal(epConfig.PortBindings, ep.addr.IP, defHostIP, ulPxyEnabled) + return n.allocatePortsInternal(ep.extConnConfig.PortBindings, ep.addr.IP, defHostIP, ulPxyEnabled) } func (n *bridgeNetwork) allocatePortsInternal(bindings []types.PortBinding, containerIP, defHostIP net.IP, ulPxyEnabled bool) ([]types.PortBinding, error) { diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go index f5ceed2130..16d61588f3 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go @@ -115,7 +115,7 @@ func (n *bridgeNetwork) setupIPTables(config *networkConfiguration, i *bridgeInt return iptables.ProgramChain(filterChain, config.BridgeName, hairpinMode, false) }) - n.portMapper.SetIptablesChain(filterChain, n.getNetworkBridgeName()) + n.portMapper.SetIptablesChain(natChain, n.getNetworkBridgeName()) } if err := ensureJumpRule("FORWARD", IsolationChain); err != nil { @@ -148,6 +148,9 @@ func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, hairp if err := programChainRule(natRule, "NAT", enable); err != nil { return err } + } + + if ipmasq && !hairpin { if err := programChainRule(skipDNAT, "SKIP DNAT", enable); err != nil { return err } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/host/host.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/host/host.go index 340cb2e6f0..bbf59c204c 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/host/host.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/host/host.go @@ -4,6 +4,7 @@ import ( "sync" "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/types" ) @@ -62,16 +63,24 @@ func (d *driver) Leave(nid, eid string) error { return nil } +func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error { + return nil +} + +func (d *driver) RevokeExternalConnectivity(nid, eid string) error { + return nil +} + func (d *driver) Type() string { return networkType } // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster -func (d *driver) DiscoverNew(dType driverapi.DiscoveryType, data interface{}) error { +func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { return nil } // DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster -func (d *driver) DiscoverDelete(dType driverapi.DiscoveryType, data interface{}) error { +func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { return nil } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/null/null.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/null/null.go index 5fbdd12956..ecc64d2db3 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/null/null.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/null/null.go @@ -4,6 +4,7 @@ import ( "sync" "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/types" ) @@ -62,16 +63,24 @@ func (d *driver) Leave(nid, eid string) error { return nil } +func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error { + return nil +} + +func (d *driver) RevokeExternalConnectivity(nid, eid string) error { + return nil +} + func (d *driver) Type() string { return networkType } // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster -func (d *driver) DiscoverNew(dType driverapi.DiscoveryType, data interface{}) error { +func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { return nil } // DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster -func (d *driver) DiscoverDelete(dType driverapi.DiscoveryType, data interface{}) error { +func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { return nil } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/filter.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/filter.go index 87b0c48aa0..0a69c6715b 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/filter.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/filter.go @@ -78,7 +78,7 @@ func setFilters(cname, brName string, remove bool) error { opt = "-D" } - // Everytime we set filters for a new subnet make sure to move the global overlay hook to the top of the both the OUTPUT and forward chains + // Every time we set filters for a new subnet make sure to move the global overlay hook to the top of the both the OUTPUT and forward chains if !remove { for _, chain := range []string{"OUTPUT", "FORWARD"} { exists := iptables.Exists(iptables.Filter, chain, "-j", globalChain) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/joinleave.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/joinleave.go index d87c032dfc..5b6792da6b 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/joinleave.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/joinleave.go @@ -54,7 +54,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, return err } - ep.ifName = overlayIfName + ep.ifName = containerIfName // Set the container interface and its peer MTU to 1450 to allow // for 50 bytes vxlan encap (inner eth header(14) + outer IP(20) + diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/ov_network.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/ov_network.go index f0b9b2b1f5..1bf91e3f21 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/ov_network.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/ov_network.go @@ -63,6 +63,9 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat if id == "" { return fmt.Errorf("invalid network id") } + if len(ipV4Data) == 0 || ipV4Data[0].Pool.String() == "0.0.0.0/0" { + return types.BadRequestErrorf("ipv4 pool is empty") + } // Since we perform lazy configuration make sure we try // configuring the driver when we enter CreateNetwork @@ -111,6 +114,14 @@ func (d *driver) DeleteNetwork(nid string) error { return n.releaseVxlanID() } +func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error { + return nil +} + +func (d *driver) RevokeExternalConnectivity(nid, eid string) error { + return nil +} + func (n *network) incEndpointCount() { n.Lock() defer n.Unlock() diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/overlay.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/overlay.go index 9e5eba4013..284e806030 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/overlay.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/overlay.go @@ -6,11 +6,12 @@ import ( "sync" "github.com/Sirupsen/logrus" - "github.com/docker/libkv/store" "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/idm" "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/types" "github.com/hashicorp/serf/serf" ) @@ -24,6 +25,8 @@ const ( vxlanVethMTU = 1450 ) +var initVxlanIdm = make(chan (bool), 1) + type driver struct { eventCh chan serf.Event notifyCh chan ovNotify @@ -35,7 +38,6 @@ type driver struct { serfInstance *serf.Serf networks networkTable store datastore.DataStore - ipAllocator *idm.Idm vxlanIdm *idm.Idm once sync.Once joinOnce sync.Once @@ -56,6 +58,18 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { config: config, } + if data, ok := config[netlabel.GlobalKVClient]; ok { + var err error + dsc, ok := data.(discoverapi.DatastoreConfigData) + if !ok { + return types.InternalErrorf("incorrect data in datastore configuration: %v", data) + } + d.store, err = datastore.NewDataStoreFromConfig(dsc) + if err != nil { + return types.InternalErrorf("failed to initialize data store: %v", err) + } + } + return dc.RegisterDriver(networkType, d, c) } @@ -73,48 +87,33 @@ func Fini(drv driverapi.Driver) { } func (d *driver) configure() error { + if d.store == nil { + return types.NoServiceErrorf("datastore is not available") + } + + if d.vxlanIdm == nil { + return d.initializeVxlanIdm() + } + + return nil +} + +func (d *driver) initializeVxlanIdm() error { var err error - if len(d.config) == 0 { + initVxlanIdm <- true + defer func() { <-initVxlanIdm }() + + if d.vxlanIdm != nil { return nil } - d.once.Do(func() { - provider, provOk := d.config[netlabel.GlobalKVProvider] - provURL, urlOk := d.config[netlabel.GlobalKVProviderURL] + d.vxlanIdm, err = idm.New(d.store, "vxlan-id", vxlanIDStart, vxlanIDEnd) + if err != nil { + return fmt.Errorf("failed to initialize vxlan id manager: %v", err) + } - if provOk && urlOk { - cfg := &datastore.ScopeCfg{ - Client: datastore.ScopeClientCfg{ - Provider: provider.(string), - Address: provURL.(string), - }, - } - provConfig, confOk := d.config[netlabel.GlobalKVProviderConfig] - if confOk { - cfg.Client.Config = provConfig.(*store.Config) - } - d.store, err = datastore.NewDataStore(datastore.GlobalScope, cfg) - if err != nil { - err = fmt.Errorf("failed to initialize data store: %v", err) - return - } - } - - d.vxlanIdm, err = idm.New(d.store, "vxlan-id", vxlanIDStart, vxlanIDEnd) - if err != nil { - err = fmt.Errorf("failed to initialize vxlan id manager: %v", err) - return - } - - d.ipAllocator, err = idm.New(d.store, "ipam-id", 1, 0xFFFF-2) - if err != nil { - err = fmt.Errorf("failed to initalize ipam id manager: %v", err) - return - } - }) - - return err + return nil } func (d *driver) Type() string { @@ -192,18 +191,33 @@ func (d *driver) pushLocalEndpointEvent(action, nid, eid string) { } // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster -func (d *driver) DiscoverNew(dType driverapi.DiscoveryType, data interface{}) error { - if dType == driverapi.NodeDiscovery { - nodeData, ok := data.(driverapi.NodeDiscoveryData) +func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { + switch dType { + case discoverapi.NodeDiscovery: + nodeData, ok := data.(discoverapi.NodeDiscoveryData) if !ok || nodeData.Address == "" { return fmt.Errorf("invalid discovery data") } d.nodeJoin(nodeData.Address, nodeData.Self) + case discoverapi.DatastoreConfig: + var err error + if d.store != nil { + return types.ForbiddenErrorf("cannot accept datastore configuration: Overlay driver has a datastore configured already") + } + dsc, ok := data.(discoverapi.DatastoreConfigData) + if !ok { + return types.InternalErrorf("incorrect data in datastore configuration: %v", data) + } + d.store, err = datastore.NewDataStoreFromConfig(dsc) + if err != nil { + return types.InternalErrorf("failed to initialize data store: %v", err) + } + default: } return nil } // DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster -func (d *driver) DiscoverDelete(dType driverapi.DiscoveryType, data interface{}) error { +func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { return nil } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/remote/api/api.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/remote/api/api.go index 6c9fb09521..c40a80bb87 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/remote/api/api.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/remote/api/api.go @@ -7,6 +7,7 @@ package api import ( "net" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" ) @@ -152,9 +153,32 @@ type LeaveResponse struct { Response } +// ProgramExternalConnectivityRequest describes the API for programming the external connectivity for the given endpoint. +type ProgramExternalConnectivityRequest struct { + NetworkID string + EndpointID string + Options map[string]interface{} +} + +// ProgramExternalConnectivityResponse is the answer to ProgramExternalConnectivityRequest. +type ProgramExternalConnectivityResponse struct { + Response +} + +// RevokeExternalConnectivityRequest describes the API for revoking the external connectivity for the given endpoint. +type RevokeExternalConnectivityRequest struct { + NetworkID string + EndpointID string +} + +// RevokeExternalConnectivityResponse is the answer to RevokeExternalConnectivityRequest. +type RevokeExternalConnectivityResponse struct { + Response +} + // DiscoveryNotification represents a discovery notification type DiscoveryNotification struct { - DiscoveryType driverapi.DiscoveryType + DiscoveryType discoverapi.DiscoveryType DiscoveryData interface{} } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/remote/driver.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/remote/driver.go index 0a7ab1865c..32533533dd 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/remote/driver.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/remote/driver.go @@ -3,15 +3,21 @@ package remote import ( "fmt" "net" + "strings" log "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/plugins" "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/drivers/remote/api" "github.com/docker/libnetwork/types" ) +const ( + missingMethod = "404 page not found" +) + type driver struct { endpoint *plugins.Client networkType string @@ -246,13 +252,42 @@ func (d *driver) Leave(nid, eid string) error { return d.call("Leave", leave, &api.LeaveResponse{}) } +// ProgramExternalConnectivity is invoked to program the rules to allow external connectivity for the endpoint. +func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error { + data := &api.ProgramExternalConnectivityRequest{ + NetworkID: nid, + EndpointID: eid, + Options: options, + } + err := d.call("ProgramExternalConnectivity", data, &api.ProgramExternalConnectivityResponse{}) + if err != nil && strings.Contains(err.Error(), missingMethod) { + // It is not mandatory yet to support this method + return nil + } + return err +} + +// RevokeExternalConnectivity method is invoked to remove any external connectivity programming related to the endpoint. +func (d *driver) RevokeExternalConnectivity(nid, eid string) error { + data := &api.RevokeExternalConnectivityRequest{ + NetworkID: nid, + EndpointID: eid, + } + err := d.call("RevokeExternalConnectivity", data, &api.RevokeExternalConnectivityResponse{}) + if err != nil && strings.Contains(err.Error(), missingMethod) { + // It is not mandatory yet to support this method + return nil + } + return err +} + func (d *driver) Type() string { return d.networkType } // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster -func (d *driver) DiscoverNew(dType driverapi.DiscoveryType, data interface{}) error { - if dType != driverapi.NodeDiscovery { +func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { + if dType != discoverapi.NodeDiscovery { return fmt.Errorf("Unknown discovery type : %v", dType) } notif := &api.DiscoveryNotification{ @@ -263,8 +298,8 @@ func (d *driver) DiscoverNew(dType driverapi.DiscoveryType, data interface{}) er } // DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster -func (d *driver) DiscoverDelete(dType driverapi.DiscoveryType, data interface{}) error { - if dType != driverapi.NodeDiscovery { +func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { + if dType != discoverapi.NodeDiscovery { return fmt.Errorf("Unknown discovery type : %v", dType) } notif := &api.DiscoveryNotification{ diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/windows/labels.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/windows/labels.go new file mode 100644 index 0000000000..7cccf2011c --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/windows/labels.go @@ -0,0 +1,12 @@ +package windows + +const ( + // NetworkName label for bridge driver + NetworkName = "com.docker.network.windowsshim.networkname" + + // HNSID of the discovered network + HNSID = "com.docker.network.windowsshim.hnsid" + + // RoutingDomain of the network + RoutingDomain = "com.docker.network.windowsshim.routingdomain" +) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/windows/windows.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/windows/windows.go index 5464a5f070..231e6a2d68 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/windows/windows.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/windows/windows.go @@ -1,64 +1,535 @@ +// +build windows + +// Shim for the Host Network Service (HNS) to manage networking for +// Windows Server containers and Hyper-V containers. This module +// is a basic libnetwork driver that passes all the calls to HNS +// It implements the 4 networking modes supported by HNS L2Bridge, +// L2Tunnel, NAT and Transparent(DHCP) +// +// The network are stored in memory and docker daemon ensures discovering +// and loading these networks on startup + package windows import ( + "encoding/json" + "fmt" + "net" + "strings" + "sync" + + "github.com/Microsoft/hcsshim" + log "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/types" ) -const networkType = "windows" - -// TODO Windows. This is a placeholder for now - -type driver struct{} - -// Init registers a new instance of null driver -func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { - c := driverapi.Capability{ - DataScope: datastore.LocalScope, - } - return dc.RegisterDriver(networkType, &driver{}, c) +// networkConfiguration for network specific configuration +type networkConfiguration struct { + ID string + Type string + Name string + HnsID string + RDID string } +// endpointConfiguration represents the user specified configuration for the sandbox endpoint +type endpointConfiguration struct { + MacAddress net.HardwareAddr + PortBindings []types.PortBinding + ExposedPorts []types.TransportPort +} + +type hnsEndpoint struct { + id string + profileID string + macAddress net.HardwareAddr + config *endpointConfiguration // User specified parameters + portMapping []types.PortBinding // Operation port bindings + addr *net.IPNet +} + +type hnsNetwork struct { + id string + config *networkConfiguration + endpoints map[string]*hnsEndpoint // key: endpoint id + driver *driver // The network's driver + sync.Mutex +} + +type driver struct { + name string + networks map[string]*hnsNetwork + sync.Mutex +} + +func isValidNetworkType(networkType string) bool { + if "l2bridge" == networkType || "l2tunnel" == networkType || "nat" == networkType || "transparent" == networkType { + return true + } + + return false +} + +// New constructs a new bridge driver +func newDriver(networkType string) *driver { + return &driver{name: networkType, networks: map[string]*hnsNetwork{}} +} + +// GetInit returns an initializer for the given network type +func GetInit(networkType string) func(dc driverapi.DriverCallback, config map[string]interface{}) error { + return func(dc driverapi.DriverCallback, config map[string]interface{}) error { + if !isValidNetworkType(networkType) { + return types.BadRequestErrorf("Network type not supported: %s", networkType) + } + + return dc.RegisterDriver(networkType, newDriver(networkType), driverapi.Capability{ + DataScope: datastore.LocalScope, + }) + } +} + +func (d *driver) getNetwork(id string) (*hnsNetwork, error) { + d.Lock() + defer d.Unlock() + + if nw, ok := d.networks[id]; ok { + return nw, nil + } + + return nil, types.NotFoundErrorf("network not found: %s", id) +} + +func (n *hnsNetwork) getEndpoint(eid string) (*hnsEndpoint, error) { + n.Lock() + defer n.Unlock() + + if ep, ok := n.endpoints[eid]; ok { + return ep, nil + } + + return nil, types.NotFoundErrorf("Endpoint not found: %s", eid) +} + +func (d *driver) parseNetworkOptions(id string, genericOptions map[string]string) (*networkConfiguration, error) { + config := &networkConfiguration{} + + for label, value := range genericOptions { + switch label { + case NetworkName: + config.Name = value + case HNSID: + config.HnsID = value + case RoutingDomain: + config.RDID = value + } + } + + config.ID = id + config.Type = d.name + return config, nil +} + +func (c *networkConfiguration) processIPAM(id string, ipamV4Data, ipamV6Data []driverapi.IPAMData) error { + if len(ipamV6Data) > 0 { + return types.ForbiddenErrorf("windowsshim driver doesn't support v6 subnets") + } + + if len(ipamV4Data) == 0 { + return types.BadRequestErrorf("network %s requires ipv4 configuration", id) + } + + return nil +} + +// Create a new network func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) error { + if _, err := d.getNetwork(id); err == nil { + return types.ForbiddenErrorf("network %s exists", id) + } + + genData, ok := option[netlabel.GenericData].(map[string]string) + if !ok { + return fmt.Errorf("Unknown generic data option") + } + + // Parse and validate the config. It should not conflict with existing networks' config + config, err := d.parseNetworkOptions(id, genData) + if err != nil { + return err + } + + err = config.processIPAM(id, ipV4Data, ipV6Data) + if err != nil { + return err + } + + network := &hnsNetwork{ + id: config.ID, + endpoints: make(map[string]*hnsEndpoint), + config: config, + driver: d, + } + + d.Lock() + d.networks[config.ID] = network + d.Unlock() + + // A non blank hnsid indicates that the network was discovered + // from HNS. No need to call HNS if this network was discovered + // from HNS + if config.HnsID == "" { + subnets := []hcsshim.Subnet{} + + for _, ipData := range ipV4Data { + subnet := hcsshim.Subnet{ + AddressPrefix: ipData.Pool.String(), + } + + if ipData.Gateway != nil { + subnet.GatewayAddress = ipData.Gateway.IP.String() + } + + subnets = append(subnets, subnet) + } + + network := &hcsshim.HNSNetwork{ + Name: config.Name, + Type: d.name, + Subnets: subnets, + } + + if network.Name == "" { + network.Name = id + } + + configurationb, err := json.Marshal(network) + if err != nil { + return err + } + + configuration := string(configurationb) + log.Debugf("HNSNetwork Request =%v Address Space=%v", configuration, subnets) + + hnsresponse, err := hcsshim.HNSNetworkRequest("POST", "", configuration) + if err != nil { + return err + } + + config.HnsID = hnsresponse.Id + genData[HNSID] = config.HnsID + } + return nil } func (d *driver) DeleteNetwork(nid string) error { + n, err := d.getNetwork(nid) + if err != nil { + return types.InternalMaskableErrorf("%s", err) + } + + n.Lock() + config := n.config + n.Unlock() + + // Cannot remove network if endpoints are still present + if len(n.endpoints) != 0 { + return fmt.Errorf("network %s has active endpoint", n.id) + } + + _, err = hcsshim.HNSNetworkRequest("DELETE", config.HnsID, "") + if err != nil { + return err + } + + d.Lock() + delete(d.networks, nid) + d.Unlock() + return nil } +func convertPortBindings(portBindings []types.PortBinding) ([]json.RawMessage, error) { + var pbs []json.RawMessage + + // Enumerate through the port bindings specified by the user and convert + // them into the internal structure matching the JSON blob that can be + // understood by the HCS. + for _, elem := range portBindings { + proto := strings.ToUpper(elem.Proto.String()) + if proto != "TCP" && proto != "UDP" { + return nil, fmt.Errorf("invalid protocol %s", elem.Proto.String()) + } + + if elem.HostPort != elem.HostPortEnd { + return nil, fmt.Errorf("Windows does not support more than one host port in NAT settings") + } + + if len(elem.HostIP) != 0 { + return nil, fmt.Errorf("Windows does not support host IP addresses in NAT settings") + } + + encodedPolicy, err := json.Marshal(hcsshim.NatPolicy{ + Type: "NAT", + ExternalPort: elem.HostPort, + InternalPort: elem.Port, + Protocol: elem.Proto.String(), + }) + + if err != nil { + return nil, err + } + pbs = append(pbs, encodedPolicy) + } + return pbs, nil +} + +func parsePortBindingPolicies(policies []json.RawMessage) ([]types.PortBinding, error) { + var bindings []types.PortBinding + hcsPolicy := &hcsshim.NatPolicy{} + + for _, elem := range policies { + + if err := json.Unmarshal([]byte(elem), &hcsPolicy); err != nil || hcsPolicy.Type != "NAT" { + continue + } + + binding := types.PortBinding{ + HostPort: hcsPolicy.ExternalPort, + HostPortEnd: hcsPolicy.ExternalPort, + Port: hcsPolicy.InternalPort, + Proto: types.ParseProtocol(hcsPolicy.Protocol), + HostIP: net.IPv4(0, 0, 0, 0), + } + + bindings = append(bindings, binding) + } + + return bindings, nil +} + +func parseEndpointOptions(epOptions map[string]interface{}) (*endpointConfiguration, error) { + if epOptions == nil { + return nil, nil + } + + ec := &endpointConfiguration{} + + if opt, ok := epOptions[netlabel.MacAddress]; ok { + if mac, ok := opt.(net.HardwareAddr); ok { + ec.MacAddress = mac + } else { + return nil, fmt.Errorf("Invalid endpoint configuration") + } + } + + if opt, ok := epOptions[netlabel.PortMap]; ok { + if bs, ok := opt.([]types.PortBinding); ok { + ec.PortBindings = bs + } else { + return nil, fmt.Errorf("Invalid endpoint configuration") + } + } + + if opt, ok := epOptions[netlabel.ExposedPorts]; ok { + if ports, ok := opt.([]types.TransportPort); ok { + ec.ExposedPorts = ports + } else { + return nil, fmt.Errorf("Invalid endpoint configuration") + } + } + + return ec, nil +} + func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, epOptions map[string]interface{}) error { + n, err := d.getNetwork(nid) + if err != nil { + return err + } + + // Check if endpoint id is good and retrieve corresponding endpoint + ep, err := n.getEndpoint(eid) + if err == nil && ep != nil { + return driverapi.ErrEndpointExists(eid) + } + + endpointStruct := &hcsshim.HNSEndpoint{ + VirtualNetwork: n.config.HnsID, + } + + ec, err := parseEndpointOptions(epOptions) + + if err != nil { + return err + } + + endpointStruct.Policies, err = convertPortBindings(ec.PortBindings) + + if err != nil { + return err + } + + configurationb, err := json.Marshal(endpointStruct) + if err != nil { + return err + } + + hnsresponse, err := hcsshim.HNSEndpointRequest("POST", "", string(configurationb)) + if err != nil { + return err + } + + mac, err := net.ParseMAC(hnsresponse.MacAddress) + if err != nil { + return err + } + + // TODO For now the ip mask is not in the info generated by HNS + endpoint := &hnsEndpoint{ + id: eid, + addr: &net.IPNet{IP: hnsresponse.IPAddress, Mask: hnsresponse.IPAddress.DefaultMask()}, + macAddress: mac, + } + + endpoint.profileID = hnsresponse.Id + endpoint.config = ec + endpoint.portMapping, err = parsePortBindingPolicies(hnsresponse.Policies) + + if err != nil { + hcsshim.HNSEndpointRequest("DELETE", hnsresponse.Id, "") + return err + } + + n.Lock() + n.endpoints[eid] = endpoint + n.Unlock() + + ifInfo.SetIPAddress(endpoint.addr) + ifInfo.SetMacAddress(endpoint.macAddress) + return nil } func (d *driver) DeleteEndpoint(nid, eid string) error { + n, err := d.getNetwork(nid) + if err != nil { + return types.InternalMaskableErrorf("%s", err) + } + + ep, err := n.getEndpoint(eid) + if err != nil { + return err + } + + n.Lock() + delete(n.endpoints, eid) + n.Unlock() + + _, err = hcsshim.HNSEndpointRequest("DELETE", ep.profileID, "") + if err != nil { + return err + } + return nil } func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, error) { - return make(map[string]interface{}, 0), nil + network, err := d.getNetwork(nid) + if err != nil { + return nil, err + } + + ep, err := network.getEndpoint(eid) + if err != nil { + return nil, err + } + + data := make(map[string]interface{}, 1) + data["hnsid"] = ep.profileID + if ep.config.ExposedPorts != nil { + // Return a copy of the config data + epc := make([]types.TransportPort, 0, len(ep.config.ExposedPorts)) + for _, tp := range ep.config.ExposedPorts { + epc = append(epc, tp.GetCopy()) + } + data[netlabel.ExposedPorts] = epc + } + + if ep.portMapping != nil { + // Return a copy of the operational data + pmc := make([]types.PortBinding, 0, len(ep.portMapping)) + for _, pm := range ep.portMapping { + pmc = append(pmc, pm.GetCopy()) + } + data[netlabel.PortMap] = pmc + } + + if len(ep.macAddress) != 0 { + data[netlabel.MacAddress] = ep.macAddress + } + return data, nil } // Join method is invoked when a Sandbox is attached to an endpoint. func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error { + network, err := d.getNetwork(nid) + if err != nil { + return err + } + + // Ensure that the endpoint exists + _, err = network.getEndpoint(eid) + if err != nil { + return err + } + + // This is just a stub for now + + jinfo.DisableGatewayService() return nil } // Leave method is invoked when a Sandbox detaches from an endpoint. func (d *driver) Leave(nid, eid string) error { + network, err := d.getNetwork(nid) + if err != nil { + return types.InternalMaskableErrorf("%s", err) + } + + // Ensure that the endpoint exists + _, err = network.getEndpoint(eid) + if err != nil { + return err + } + + // This is just a stub for now + + return nil +} + +func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error { + return nil +} + +func (d *driver) RevokeExternalConnectivity(nid, eid string) error { return nil } func (d *driver) Type() string { - return networkType + return d.name } // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster -func (d *driver) DiscoverNew(dType driverapi.DiscoveryType, data interface{}) error { +func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { return nil } // DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster -func (d *driver) DiscoverDelete(dType driverapi.DiscoveryType, data interface{}) error { +func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { return nil } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers_windows.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers_windows.go index 314ff8d58d..1e44b626d4 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers_windows.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers_windows.go @@ -1,9 +1,16 @@ package libnetwork -import "github.com/docker/libnetwork/drivers/windows" +import ( + "github.com/docker/libnetwork/drivers/null" + "github.com/docker/libnetwork/drivers/windows" +) func getInitializers() []initializer { return []initializer{ - {windows.Init, "windows"}, + {null.Init, "null"}, + {windows.GetInit("transparent"), "transparent"}, + {windows.GetInit("l2bridge"), "l2bridge"}, + {windows.GetInit("l2tunnel"), "l2tunnel"}, + {windows.GetInit("nat"), "nat"}, } } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/endpoint.go b/components/engine/vendor/src/github.com/docker/libnetwork/endpoint.go index 6455686a32..b0aacb9f44 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/endpoint.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/endpoint.go @@ -44,7 +44,7 @@ type Endpoint interface { Delete(force bool) error } -// EndpointOption is a option setter function type used to pass varios options to Network +// EndpointOption is an option setter function type used to pass various options to Network // and Endpoint interfaces methods. The various setter functions of type EndpointOption are // provided by libnetwork, they look like Option[...](...) type EndpointOption func(ep *endpoint) @@ -343,7 +343,7 @@ func (ep *endpoint) getNetworkFromStore() (*network, error) { return nil, fmt.Errorf("invalid network object in endpoint %s", ep.Name()) } - return ep.network.ctrlr.getNetworkFromStore(ep.network.id) + return ep.network.getController().getNetworkFromStore(ep.network.id) } func (ep *endpoint) Join(sbox Sandbox, options ...EndpointOption) error { @@ -359,22 +359,16 @@ func (ep *endpoint) Join(sbox Sandbox, options ...EndpointOption) error { sb.joinLeaveStart() defer sb.joinLeaveEnd() - return ep.sbJoin(sbox, options...) + return ep.sbJoin(sb, options...) } -func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error { - var err error - sb, ok := sbox.(*sandbox) - if !ok { - return types.BadRequestErrorf("not a valid Sandbox interface") - } - - network, err := ep.getNetworkFromStore() +func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error { + n, err := ep.getNetworkFromStore() if err != nil { return fmt.Errorf("failed to get network from store during join: %v", err) } - ep, err = network.getEndpointFromStore(ep.ID()) + ep, err = n.getEndpointFromStore(ep.ID()) if err != nil { return fmt.Errorf("failed to get endpoint from store during join: %v", err) } @@ -384,11 +378,8 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error { ep.Unlock() return types.ForbiddenErrorf("another container is attached to the same network endpoint") } - ep.Unlock() - - ep.Lock() - ep.network = network - ep.sandboxID = sbox.ID() + ep.network = n + ep.sandboxID = sb.ID() ep.joinInfo = &endpointJoinInfo{} epid := ep.id ep.Unlock() @@ -400,32 +391,29 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error { } }() - network.Lock() - nid := network.id - network.Unlock() + nid := n.ID() ep.processOptions(options...) - driver, err := network.driver(true) + d, err := n.driver(true) if err != nil { return fmt.Errorf("failed to join endpoint: %v", err) } - err = driver.Join(nid, epid, sbox.Key(), ep, sbox.Labels()) + err = d.Join(nid, epid, sb.Key(), ep, sb.Labels()) if err != nil { return err } defer func() { if err != nil { - // Do not alter global err variable, it's needed by the previous defer - if err := driver.Leave(nid, epid); err != nil { + if err := d.Leave(nid, epid); err != nil { log.Warnf("driver leave failed while rolling back join: %v", err) } } }() // Watch for service records - network.getController().watchSvcRecord(ep) + n.getController().watchSvcRecord(ep) address := "" if ip := ep.getFirstInterfaceAddress(); ip != nil { @@ -434,27 +422,23 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error { if err = sb.updateHostsFile(address); err != nil { return err } - if err = sb.updateDNS(network.enableIPv6); err != nil { + if err = sb.updateDNS(n.enableIPv6); err != nil { return err } - if err = network.getController().updateToStore(ep); err != nil { + if err = n.getController().updateToStore(ep); err != nil { return err } + // Current endpoint providing external connectivity for the sandbox + extEp := sb.getGatewayEndpoint() + sb.Lock() heap.Push(&sb.endpoints, ep) sb.Unlock() defer func() { if err != nil { - for i, e := range sb.getConnectedEndpoints() { - if e == ep { - sb.Lock() - heap.Remove(&sb.endpoints, i) - sb.Unlock() - return - } - } + sb.removeEndpoint(ep) } }() @@ -463,9 +447,39 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error { } if sb.needDefaultGW() { - return sb.setupDefaultGW(ep) + return sb.setupDefaultGW() } - return nil + + moveExtConn := sb.getGatewayEndpoint() != extEp + + if moveExtConn { + if extEp != nil { + log.Debugf("Revoking external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID()) + if err = d.RevokeExternalConnectivity(extEp.network.ID(), extEp.ID()); err != nil { + return types.InternalErrorf( + "driver failed revoking external connectivity on endpoint %s (%s): %v", + extEp.Name(), extEp.ID(), err) + } + defer func() { + if err != nil { + if e := d.ProgramExternalConnectivity(extEp.network.ID(), extEp.ID(), sb.Labels()); e != nil { + log.Warnf("Failed to roll-back external connectivity on endpoint %s (%s): %v", + extEp.Name(), extEp.ID(), e) + } + } + }() + } + if !n.internal { + log.Debugf("Programming external connectivity on endpoint %s (%s)", ep.Name(), ep.ID()) + if err = d.ProgramExternalConnectivity(n.ID(), ep.ID(), sb.Labels()); err != nil { + return types.InternalErrorf( + "driver failed programming external connectivity on endpoint %s (%s): %v", + ep.Name(), ep.ID(), err) + } + } + } + + return sb.clearDefaultGW() } func (ep *endpoint) rename(name string) error { @@ -533,15 +547,10 @@ func (ep *endpoint) Leave(sbox Sandbox, options ...EndpointOption) error { sb.joinLeaveStart() defer sb.joinLeaveEnd() - return ep.sbLeave(sbox, false, options...) + return ep.sbLeave(sb, false, options...) } -func (ep *endpoint) sbLeave(sbox Sandbox, force bool, options ...EndpointOption) error { - sb, ok := sbox.(*sandbox) - if !ok { - return types.BadRequestErrorf("not a valid Sandbox interface") - } - +func (ep *endpoint) sbLeave(sb *sandbox, force bool, options ...EndpointOption) error { n, err := ep.getNetworkFromStore() if err != nil { return fmt.Errorf("failed to get network from store during leave: %v", err) @@ -559,8 +568,8 @@ func (ep *endpoint) sbLeave(sbox Sandbox, force bool, options ...EndpointOption) if sid == "" { return types.ForbiddenErrorf("cannot leave endpoint with no attached sandbox") } - if sid != sbox.ID() { - return types.ForbiddenErrorf("unexpected sandbox ID in leave request. Expected %s. Got %s", ep.sandboxID, sbox.ID()) + if sid != sb.ID() { + return types.ForbiddenErrorf("unexpected sandbox ID in leave request. Expected %s. Got %s", ep.sandboxID, sb.ID()) } ep.processOptions(options...) @@ -575,7 +584,19 @@ func (ep *endpoint) sbLeave(sbox Sandbox, force bool, options ...EndpointOption) ep.network = n ep.Unlock() + // Current endpoint providing external connectivity to the sandbox + extEp := sb.getGatewayEndpoint() + moveExtConn := extEp != nil && (extEp.ID() == ep.ID()) + if d != nil { + if moveExtConn { + log.Debugf("Revoking external connectivity on endpoint %s (%s)", ep.Name(), ep.ID()) + if err := d.RevokeExternalConnectivity(n.id, ep.id); err != nil { + log.Warnf("driver failed revoking external connectivity on endpoint %s (%s): %v", + ep.Name(), ep.ID(), err) + } + } + if err := d.Leave(n.id, ep.id); err != nil { if _, ok := err.(types.MaskableError); !ok { log.Warnf("driver error disconnecting container %s : %v", ep.name, err) @@ -597,7 +618,24 @@ func (ep *endpoint) sbLeave(sbox Sandbox, force bool, options ...EndpointOption) } sb.deleteHostsEntries(n.getSvcRecords(ep)) - return nil + if !sb.inDelete && sb.needDefaultGW() { + if sb.getEPwithoutGateway() == nil { + return fmt.Errorf("endpoint without GW expected, but not found") + } + return sb.setupDefaultGW() + } + + // New endpoint providing external connectivity for the sandbox + extEp = sb.getGatewayEndpoint() + if moveExtConn && extEp != nil { + log.Debugf("Programming external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID()) + if err := d.ProgramExternalConnectivity(extEp.network.ID(), extEp.ID(), sb.Labels()); err != nil { + log.Warnf("driver failed programming external connectivity on endpoint %s: (%s) %v", + extEp.Name(), extEp.ID(), err) + } + } + + return sb.clearDefaultGW() } func (n *network) validateForceDelete(locator string) error { @@ -643,7 +681,7 @@ func (ep *endpoint) Delete(force bool) error { } if sb != nil { - if e := ep.sbLeave(sb, force); e != nil { + if e := ep.sbLeave(sb.(*sandbox), force); e != nil { log.Warnf("failed to leave sandbox for endpoint %s : %v", name, e) } } @@ -911,7 +949,7 @@ func (ep *endpoint) assignAddressVersion(ipVer int, ipam ipamapi.Ipam) error { } } if progAdd != nil { - return types.BadRequestErrorf("Invalid preferred address %s: It does not belong to any of this network's subnets", prefAdd) + return types.BadRequestErrorf("Invalid address %s: It does not belong to any of this network's subnets", prefAdd) } return fmt.Errorf("no available IPv%d addresses on this network's address pools: %s (%s)", ipVer, n.Name(), n.ID()) } @@ -929,9 +967,13 @@ func (ep *endpoint) releaseAddress() { log.Warnf("Failed to retrieve ipam driver to release interface address on delete of endpoint %s (%s): %v", ep.Name(), ep.ID(), err) return } - if err := ipam.ReleaseAddress(ep.iface.v4PoolID, ep.iface.addr.IP); err != nil { - log.Warnf("Failed to release ip address %s on delete of endpoint %s (%s): %v", ep.iface.addr.IP, ep.Name(), ep.ID(), err) + + if ep.iface.addr != nil { + if err := ipam.ReleaseAddress(ep.iface.v4PoolID, ep.iface.addr.IP); err != nil { + log.Warnf("Failed to release ip address %s on delete of endpoint %s (%s): %v", ep.iface.addr.IP, ep.Name(), ep.ID(), err) + } } + if ep.iface.addrv6 != nil && ep.iface.addrv6.IP.IsGlobalUnicast() { if err := ipam.ReleaseAddress(ep.iface.v6PoolID, ep.iface.addrv6.IP); err != nil { log.Warnf("Failed to release ip address %s on delete of endpoint %s (%s): %v", ep.iface.addrv6.IP, ep.Name(), ep.ID(), err) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/etchosts/etchosts.go b/components/engine/vendor/src/github.com/docker/libnetwork/etchosts/etchosts.go index 5f68372b90..4526532593 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/etchosts/etchosts.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/etchosts/etchosts.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "regexp" + "strings" "sync" ) @@ -78,10 +79,17 @@ func Build(path, IP, hostname, domainname string, extraContent []Record) error { //set main record var mainRec Record mainRec.IP = IP + // User might have provided a FQDN in hostname or split it across hostname + // and domainname. We want the FQDN and the bare hostname. + fqdn := hostname if domainname != "" { - mainRec.Hosts = fmt.Sprintf("%s.%s %s", hostname, domainname, hostname) + fqdn = fmt.Sprintf("%s.%s", fqdn, domainname) + } + parts := strings.SplitN(fqdn, ".", 2) + if len(parts) == 2 { + mainRec.Hosts = fmt.Sprintf("%s %s", fqdn, parts[0]) } else { - mainRec.Hosts = hostname + mainRec.Hosts = fqdn } if _, err := mainRec.WriteTo(content); err != nil { return err @@ -151,6 +159,10 @@ func Delete(path string, recs []Record) error { loop: for s.Scan() { b := s.Bytes() + if len(b) == 0 { + continue + } + if b[0] == '#' { buf.Write(b) buf.Write(eol) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/idm/idm.go b/components/engine/vendor/src/github.com/docker/libnetwork/idm/idm.go index 37ae79e558..3ed820ca91 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/idm/idm.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/idm/idm.go @@ -8,7 +8,7 @@ import ( "github.com/docker/libnetwork/datastore" ) -// Idm manages the reservation/release of numerical ids from a contiguos set +// Idm manages the reservation/release of numerical ids from a contiguous set type Idm struct { start uint64 end uint64 diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/ipam/allocator.go b/components/engine/vendor/src/github.com/docker/libnetwork/ipam/allocator.go index 4fa7e1e972..70fe06eba7 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/ipam/allocator.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/ipam/allocator.go @@ -8,6 +8,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/bitseq" "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/ipamapi" "github.com/docker/libnetwork/ipamutils" "github.com/docker/libnetwork/types" @@ -60,19 +61,9 @@ func NewAllocator(lcDs, glDs datastore.DataStore) (*Allocator, error) { if aspc.ds == nil { continue } - - a.addrSpaces[aspc.as] = &addrSpace{ - subnets: map[SubnetKey]*PoolData{}, - id: dsConfigKey + "/" + aspc.as, - scope: aspc.ds.Scope(), - ds: aspc.ds, - alloc: a, - } + a.initializeAddressSpace(aspc.as, aspc.ds) } - a.checkConsistency(localAddressSpace) - a.checkConsistency(globalAddressSpace) - return a, nil } @@ -118,25 +109,83 @@ func (a *Allocator) updateBitMasks(aSpace *addrSpace) error { return nil } -// Checks for and fixes damaged bitmask. Meant to be called in constructor only. +// Checks for and fixes damaged bitmask. func (a *Allocator) checkConsistency(as string) { + var sKeyList []SubnetKey + // Retrieve this address space's configuration and bitmasks from the datastore a.refresh(as) + a.Lock() aSpace, ok := a.addrSpaces[as] + a.Unlock() if !ok { return } a.updateBitMasks(aSpace) + + aSpace.Lock() for sk, pd := range aSpace.subnets { if pd.Range != nil { continue } - if err := a.addresses[sk].CheckConsistency(); err != nil { + sKeyList = append(sKeyList, sk) + } + aSpace.Unlock() + + for _, sk := range sKeyList { + a.Lock() + bm := a.addresses[sk] + a.Unlock() + if err := bm.CheckConsistency(); err != nil { log.Warnf("Error while running consistency check for %s: %v", sk, err) } } } +func (a *Allocator) initializeAddressSpace(as string, ds datastore.DataStore) error { + a.Lock() + if _, ok := a.addrSpaces[as]; ok { + a.Unlock() + return types.ForbiddenErrorf("tried to add an axisting address space: %s", as) + } + a.addrSpaces[as] = &addrSpace{ + subnets: map[SubnetKey]*PoolData{}, + id: dsConfigKey + "/" + as, + scope: ds.Scope(), + ds: ds, + alloc: a, + } + a.Unlock() + + a.checkConsistency(as) + + return nil +} + +// DiscoverNew informs the allocator about a new global scope datastore +func (a *Allocator) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { + if dType != discoverapi.DatastoreConfig { + return nil + } + + dsc, ok := data.(discoverapi.DatastoreConfigData) + if !ok { + return types.InternalErrorf("incorrect data in datastore update notification: %v", data) + } + + ds, err := datastore.NewDataStoreFromConfig(dsc) + if err != nil { + return err + } + + return a.initializeAddressSpace(globalAddressSpace, ds) +} + +// DiscoverDelete is a notification of no interest for the allocator +func (a *Allocator) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { + return nil +} + // GetDefaultAddressSpaces returns the local and global default address spaces func (a *Allocator) GetDefaultAddressSpaces() (string, string, error) { return localAddressSpace, globalAddressSpace, nil @@ -489,7 +538,7 @@ func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddres } else if prefAddress != nil { hostPart, e := types.GetHostPartIP(prefAddress, base.Mask) if e != nil { - return nil, types.InternalErrorf("failed to allocate preferred address %s: %v", prefAddress.String(), e) + return nil, types.InternalErrorf("failed to allocate requested address %s: %v", prefAddress.String(), e) } ordinal = ipToUint64(types.GetMinimalIP(hostPart)) err = bitmask.Set(ordinal) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/ipam/structures.go b/components/engine/vendor/src/github.com/docker/libnetwork/ipam/structures.go index 601eda4fba..09a77695dd 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/ipam/structures.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/ipam/structures.go @@ -40,7 +40,7 @@ type addrSpace struct { } // AddressRange specifies first and last ip ordinal which -// identify a range in a a pool of addresses +// identifies a range in a pool of addresses type AddressRange struct { Sub *net.IPNet Start, End uint64 @@ -85,7 +85,7 @@ func (s *SubnetKey) String() string { return k } -// FromString populate the SubnetKey object reading it from string +// FromString populates the SubnetKey object reading it from string func (s *SubnetKey) FromString(str string) error { if str == "" || !strings.Contains(str, "/") { return types.BadRequestErrorf("invalid string form for subnetkey: %s", str) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/ipam/utils.go b/components/engine/vendor/src/github.com/docker/libnetwork/ipam/utils.go index d524b47830..5117c55cc7 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/ipam/utils.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/ipam/utils.go @@ -62,7 +62,7 @@ func getAddressVersion(ip net.IP) ipVersion { } // Adds the ordinal IP to the current array -// 192.168.0.0 + 53 => 192.168.53 +// 192.168.0.0 + 53 => 192.168.0.53 func addIntToIP(array []byte, ordinal uint64) { for i := len(array) - 1; i >= 0; i-- { array[i] |= (byte)(ordinal & 0xff) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/ipamapi/contract.go b/components/engine/vendor/src/github.com/docker/libnetwork/ipamapi/contract.go index 5d561d81df..513e482349 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/ipamapi/contract.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/ipamapi/contract.go @@ -4,6 +4,7 @@ package ipamapi import ( "net" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/types" ) @@ -14,6 +15,8 @@ import ( const ( // DefaultIPAM is the name of the built-in default ipam driver DefaultIPAM = "default" + // NullIPAM is the name of the built-in null ipam driver + NullIPAM = "null" // PluginEndpointType represents the Endpoint Type used by Plugin system PluginEndpointType = "IpamDriver" // RequestAddressType represents the Address Type used when requesting an address @@ -32,7 +35,7 @@ type Callback interface { * IPAM Errors **************/ -// Weel-known errors returned by IPAM +// Well-known errors returned by IPAM var ( ErrIpamInternalError = types.InternalErrorf("IPAM Internal Error") ErrInvalidAddressSpace = types.BadRequestErrorf("Invalid Address Space") @@ -56,6 +59,8 @@ var ( // Ipam represents the interface the IPAM service plugins must implement // in order to allow injection/modification of IPAM database. type Ipam interface { + discoverapi.Discover + // GetDefaultAddressSpaces returns the default local and global address spaces for this ipam GetDefaultAddressSpaces() (string, string, error) // RequestPool returns an address pool along with its unique id. Address space is a mandatory field @@ -67,7 +72,7 @@ type Ipam interface { RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error) // ReleasePool releases the address pool identified by the passed id ReleasePool(poolID string) error - // Request address from the specified pool ID. Input options or preferred IP can be passed. + // Request address from the specified pool ID. Input options or required IP can be passed. RequestAddress(string, net.IP, map[string]string) (*net.IPNet, map[string]string, error) // Release the address from the specified pool ID ReleaseAddress(string, net.IP) error diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/ipams/builtin/builtin.go b/components/engine/vendor/src/github.com/docker/libnetwork/ipams/builtin/builtin_unix.go similarity index 96% rename from components/engine/vendor/src/github.com/docker/libnetwork/ipams/builtin/builtin.go rename to components/engine/vendor/src/github.com/docker/libnetwork/ipams/builtin/builtin_unix.go index 707e001f73..311183fc87 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/ipams/builtin/builtin.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/ipams/builtin/builtin_unix.go @@ -1,3 +1,5 @@ +// +build linux freebsd + package builtin import ( diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/ipams/builtin/builtin_windows.go b/components/engine/vendor/src/github.com/docker/libnetwork/ipams/builtin/builtin_windows.go new file mode 100644 index 0000000000..d24f5e63dc --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/libnetwork/ipams/builtin/builtin_windows.go @@ -0,0 +1,16 @@ +// +build windows + +package builtin + +import ( + "github.com/docker/libnetwork/ipamapi" + + windowsipam "github.com/docker/libnetwork/ipams/windowsipam" +) + +// Init registers the built-in ipam service with libnetwork +func Init(ic ipamapi.Callback, l, g interface{}) error { + initFunc := windowsipam.GetInit(ipamapi.DefaultIPAM) + + return initFunc(ic, l, g) +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/ipams/null/null.go b/components/engine/vendor/src/github.com/docker/libnetwork/ipams/null/null.go new file mode 100644 index 0000000000..60119a36ab --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/libnetwork/ipams/null/null.go @@ -0,0 +1,71 @@ +// Package null implements the null ipam driver. Null ipam driver satisfies ipamapi contract, +// but does not effectively reserve/allocate any address pool or address +package null + +import ( + "fmt" + "net" + + "github.com/docker/libnetwork/discoverapi" + "github.com/docker/libnetwork/ipamapi" + "github.com/docker/libnetwork/types" +) + +var ( + defaultAS = "null" + defaultPool, _ = types.ParseCIDR("0.0.0.0/0") + defaultPoolID = fmt.Sprintf("%s/%s", defaultAS, defaultPool.String()) +) + +type allocator struct{} + +func (a *allocator) GetDefaultAddressSpaces() (string, string, error) { + return defaultAS, defaultAS, nil +} + +func (a *allocator) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error) { + if addressSpace != defaultAS { + return "", nil, nil, types.BadRequestErrorf("unknown address space: %s", addressSpace) + } + if pool != "" { + return "", nil, nil, types.BadRequestErrorf("null ipam driver does not handle specific address pool requests") + } + if subPool != "" { + return "", nil, nil, types.BadRequestErrorf("null ipam driver does not handle specific address subpool requests") + } + if v6 { + return "", nil, nil, types.BadRequestErrorf("null ipam driver does not handle IPv6 address pool pool requests") + } + return defaultPoolID, defaultPool, nil, nil +} + +func (a *allocator) ReleasePool(poolID string) error { + return nil +} + +func (a *allocator) RequestAddress(poolID string, ip net.IP, opts map[string]string) (*net.IPNet, map[string]string, error) { + if poolID != defaultPoolID { + return nil, nil, types.BadRequestErrorf("unknown pool id: %s", poolID) + } + return nil, nil, nil +} + +func (a *allocator) ReleaseAddress(poolID string, ip net.IP) error { + if poolID != defaultPoolID { + return types.BadRequestErrorf("unknown pool id: %s", poolID) + } + return nil +} + +func (a *allocator) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { + return nil +} + +func (a *allocator) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { + return nil +} + +// Init registers a remote ipam when its plugin is activated +func Init(ic ipamapi.Callback, l, g interface{}) error { + return ic.RegisterIpamDriver(ipamapi.NullIPAM, &allocator{}) +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/ipams/remote/remote.go b/components/engine/vendor/src/github.com/docker/libnetwork/ipams/remote/remote.go index 581a9c8871..799a2e77f4 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/ipams/remote/remote.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/ipams/remote/remote.go @@ -6,6 +6,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/plugins" + "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/ipamapi" "github.com/docker/libnetwork/ipams/remote/api" "github.com/docker/libnetwork/types" @@ -124,3 +125,13 @@ func (a *allocator) ReleaseAddress(poolID string, address net.IP) error { res := &api.ReleaseAddressResponse{} return a.call("ReleaseAddress", req, res) } + +// DiscoverNew is a notification for a new discovery event, such as a new global datastore +func (a *allocator) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { + return nil +} + +// DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster +func (a *allocator) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { + return nil +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/ipams/windowsipam/windowsipam.go b/components/engine/vendor/src/github.com/docker/libnetwork/ipams/windowsipam/windowsipam.go new file mode 100644 index 0000000000..a92d5ccd77 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/libnetwork/ipams/windowsipam/windowsipam.go @@ -0,0 +1,99 @@ +package windowsipam + +import ( + "net" + + log "github.com/Sirupsen/logrus" + "github.com/docker/libnetwork/discoverapi" + "github.com/docker/libnetwork/ipamapi" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/types" +) + +const ( + localAddressSpace = "LocalDefault" + globalAddressSpace = "GlobalDefault" +) + +var ( + defaultPool, _ = types.ParseCIDR("0.0.0.0/0") +) + +type allocator struct { +} + +// GetInit registers the built-in ipam service with libnetwork +func GetInit(ipamName string) func(ic ipamapi.Callback, l, g interface{}) error { + return func(ic ipamapi.Callback, l, g interface{}) error { + return ic.RegisterIpamDriver(ipamName, &allocator{}) + } +} + +func (a *allocator) GetDefaultAddressSpaces() (string, string, error) { + return localAddressSpace, globalAddressSpace, nil +} + +// RequestPool returns an address pool along with its unique id. This is a null ipam driver. It allocates the +// subnet user asked and does not validate anything. Doesn't support subpool allocation +func (a *allocator) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error) { + log.Debugf("RequestPool(%s, %s, %s, %v, %t)", addressSpace, pool, subPool, options, v6) + if subPool != "" || v6 { + return "", nil, nil, types.InternalErrorf("This request is not supported by null ipam driver") + } + + var ipNet *net.IPNet + var err error + + if pool != "" { + _, ipNet, err = net.ParseCIDR(pool) + if err != nil { + return "", nil, nil, err + } + } else { + ipNet = defaultPool + } + + return ipNet.String(), ipNet, nil, nil +} + +// ReleasePool releases the address pool - always succeeds +func (a *allocator) ReleasePool(poolID string) error { + log.Debugf("ReleasePool(%s)", poolID) + return nil +} + +// RequestAddress returns an address from the specified pool ID. +// Always allocate the 0.0.0.0/32 ip if no preferred address was specified +func (a *allocator) RequestAddress(poolID string, prefAddress net.IP, opts map[string]string) (*net.IPNet, map[string]string, error) { + log.Debugf("RequestAddress(%s, %v, %v)", poolID, prefAddress, opts) + _, ipNet, err := net.ParseCIDR(poolID) + + if err != nil { + return nil, nil, err + } + + // TODO Windows: Remove this once the bug in docker daemon is fixed + // that causes it to throw an exception on nil gateway + if opts[ipamapi.RequestAddressType] == netlabel.Gateway { + return ipNet, nil, nil + } else if prefAddress == nil { + return nil, nil, nil + } + return &net.IPNet{IP: prefAddress, Mask: ipNet.Mask}, nil, nil +} + +// ReleaseAddress releases the address - always succeeds +func (a *allocator) ReleaseAddress(poolID string, address net.IP) error { + log.Debugf("ReleaseAddress(%s, %v)", poolID, address) + return nil +} + +// DiscoverNew informs the allocator about a new global scope datastore +func (a *allocator) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { + return nil +} + +// DiscoverDelete is a notification of no interest for the allocator +func (a *allocator) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { + return nil +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/iptables/iptables.go b/components/engine/vendor/src/github.com/docker/libnetwork/iptables/iptables.go index ca07893888..298c5bf472 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/iptables/iptables.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/iptables/iptables.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "os/exec" + "regexp" "strconv" "strings" "sync" @@ -36,6 +37,7 @@ const ( var ( iptablesPath string supportsXlock = false + supportsCOpt = false // used to lock iptables commands if xtables lock is not supported bestEffortLock sync.Mutex // ErrIptablesNotFound is returned when the rule is not found. @@ -60,7 +62,6 @@ func (e ChainError) Error() string { } func initCheck() error { - if iptablesPath == "" { path, err := exec.LookPath("iptables") if err != nil { @@ -68,6 +69,12 @@ func initCheck() error { } iptablesPath = path supportsXlock = exec.Command(iptablesPath, "--wait", "-L", "-n").Run() == nil + mj, mn, mc, err := GetVersion() + if err != nil { + logrus.Warnf("Failed to read iptables version: %v", err) + return nil + } + supportsCOpt = supportsCOption(mj, mn, mc) } return nil } @@ -299,20 +306,21 @@ func Exists(table Table, chain string, rule ...string) bool { table = Filter } - // iptables -C, --check option was added in v.1.4.11 - // http://ftp.netfilter.org/pub/iptables/changes-iptables-1.4.11.txt + initCheck() - // try -C - // if exit status is 0 then return true, the rule exists - if _, err := Raw(append([]string{ - "-t", string(table), "-C", chain}, rule...)...); err == nil { - return true + if supportsCOpt { + // if exit status is 0 then return true, the rule exists + _, err := Raw(append([]string{"-t", string(table), "-C", chain}, rule...)...) + return err == nil } - // parse "iptables -S" for the rule (this checks rules in a specific chain - // in a specific table) - ruleString := strings.Join(rule, " ") - ruleString = chain + " " + ruleString + // parse "iptables -S" for the rule (it checks rules in a specific chain + // in a specific table and it is very unreliable) + return existsRaw(table, chain, rule...) +} + +func existsRaw(table Table, chain string, rule ...string) bool { + ruleString := fmt.Sprintf("%s %s\n", chain, strings.Join(rule, " ")) existingRules, _ := exec.Command(iptablesPath, "-t", string(table), "-S", chain).Output() return strings.Contains(string(existingRules), ruleString) @@ -380,3 +388,25 @@ func ExistChain(chain string, table Table) bool { } return false } + +// GetVersion reads the iptables version numbers +func GetVersion() (major, minor, micro int, err error) { + out, err := Raw("--version") + if err == nil { + major, minor, micro = parseVersionNumbers(string(out)) + } + return +} + +func parseVersionNumbers(input string) (major, minor, micro int) { + re := regexp.MustCompile(`v\d*.\d*.\d*`) + line := re.FindString(input) + fmt.Sscanf(line, "v%d.%d.%d", &major, &minor, µ) + return +} + +// iptables -C, --check option was added in v.1.4.11 +// http://ftp.netfilter.org/pub/iptables/changes-iptables-1.4.11.txt +func supportsCOption(mj, mn, mc int) bool { + return mj > 1 || (mj == 1 && (mn > 4 || (mn == 4 && mc >= 11))) +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/netlabel/labels.go b/components/engine/vendor/src/github.com/docker/libnetwork/netlabel/labels.go index c6d7f13477..d44015f159 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/netlabel/labels.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/netlabel/labels.go @@ -56,6 +56,9 @@ var ( // GlobalKVProviderConfig constant represents the KV provider Config GlobalKVProviderConfig = MakeKVProviderConfig("global") + // GlobalKVClient constants represents the global kv store client + GlobalKVClient = MakeKVClient("global") + // LocalKVProvider constant represents the KV provider backend LocalKVProvider = MakeKVProvider("local") @@ -64,6 +67,9 @@ var ( // LocalKVProviderConfig constant represents the KV provider Config LocalKVProviderConfig = MakeKVProviderConfig("local") + + // LocalKVClient constants represents the local kv store client + LocalKVClient = MakeKVClient("local") ) // MakeKVProvider returns the kvprovider label for the scope @@ -81,6 +87,11 @@ func MakeKVProviderConfig(scope string) string { return DriverPrivatePrefix + scope + "kv_provider_config" } +// MakeKVClient returns the kv client label for the scope +func MakeKVClient(scope string) string { + return DriverPrivatePrefix + scope + "kv_client" +} + // Key extracts the key portion of the label func Key(label string) (key string) { if kv := strings.SplitN(label, "=", 2); len(kv) > 0 { diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/network.go b/components/engine/vendor/src/github.com/docker/libnetwork/network.go index aa32cb8d68..25dc39c3f5 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/network.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/network.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "net" - "strconv" "strings" "sync" @@ -62,6 +61,7 @@ type NetworkInfo interface { IpamInfo() ([]*IpamInfo, []*IpamInfo) DriverOptions() map[string]string Scope() string + IPv6Enabled() bool Internal() bool } @@ -466,7 +466,7 @@ func (n *network) UnmarshalJSON(b []byte) (err error) { return nil } -// NetworkOption is a option setter function type used to pass varios options to +// NetworkOption is an option setter function type used to pass various options to // NewNetwork method. The various setter functions of type NetworkOption are // provided by libnetwork, they look like NetworkOptionXXXX(...) type NetworkOption func(n *network) @@ -475,9 +475,17 @@ type NetworkOption func(n *network) // in a Dictionary of Key-Value pair func NetworkOptionGeneric(generic map[string]interface{}) NetworkOption { return func(n *network) { - n.generic = generic - if _, ok := generic[netlabel.EnableIPv6]; ok { - n.enableIPv6 = generic[netlabel.EnableIPv6].(bool) + if n.generic == nil { + n.generic = make(map[string]interface{}) + } + if val, ok := generic[netlabel.EnableIPv6]; ok { + n.enableIPv6 = val.(bool) + } + if val, ok := generic[netlabel.Internal]; ok { + n.internal = val.(bool) + } + for k, v := range generic { + n.generic[k] = v } } } @@ -489,14 +497,25 @@ func NetworkOptionPersist(persist bool) NetworkOption { } } +// NetworkOptionEnableIPv6 returns an option setter to explicitly configure IPv6 +func NetworkOptionEnableIPv6(enableIPv6 bool) NetworkOption { + return func(n *network) { + if n.generic == nil { + n.generic = make(map[string]interface{}) + } + n.enableIPv6 = enableIPv6 + n.generic[netlabel.EnableIPv6] = enableIPv6 + } +} + // NetworkOptionInternalNetwork returns an option setter to config the network // to be internal which disables default gateway service func NetworkOptionInternalNetwork() NetworkOption { return func(n *network) { - n.internal = true if n.generic == nil { n.generic = make(map[string]interface{}) } + n.internal = true n.generic[netlabel.Internal] = true } } @@ -525,13 +544,6 @@ func NetworkOptionDriverOpts(opts map[string]string) NetworkOption { } // Store the options n.generic[netlabel.GenericData] = opts - // Decode and store the endpoint options of libnetwork interest - if val, ok := opts[netlabel.EnableIPv6]; ok { - var err error - if n.enableIPv6, err = strconv.ParseBool(val); err != nil { - log.Warnf("Failed to parse %s' value: %s (%s)", netlabel.EnableIPv6, val, err.Error()) - } - } } } @@ -588,7 +600,7 @@ func (n *network) driver(load bool) (driverapi.Driver, error) { return nil, err } } else if !ok { - // dont fail if driver loading is not required + // don't fail if driver loading is not required return nil, nil } @@ -692,7 +704,7 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi ep.id = stringid.GenerateRandomID() // Initialize ep.network with a possibly stale copy of n. We need this to get network from - // store. But once we get it from store we will have the most uptodate copy possible. + // store. But once we get it from store we will have the most uptodate copy possibly. ep.network = n ep.locator = n.getController().clusterHostID() ep.network, err = ep.getNetworkFromStore() @@ -839,14 +851,25 @@ func (n *network) updateSvcRecord(ep *endpoint, localEps []*endpoint, isAdd bool if iface := ep.Iface(); iface.Address() != nil { myAliases := ep.MyAliases() if isAdd { - if !ep.isAnonymous() { + // If anonymous endpoint has an alias use the first alias + // for ip->name mapping. Not having the reverse mapping + // breaks some apps + if ep.isAnonymous() { + if len(myAliases) > 0 { + n.addSvcRecords(myAliases[0], iface.Address().IP, true) + } + } else { n.addSvcRecords(epName, iface.Address().IP, true) } for _, alias := range myAliases { n.addSvcRecords(alias, iface.Address().IP, false) } } else { - if !ep.isAnonymous() { + if ep.isAnonymous() { + if len(myAliases) > 0 { + n.deleteSvcRecords(myAliases[0], iface.Address().IP, true) + } + } else { n.deleteSvcRecords(epName, iface.Address().IP, true) } for _, alias := range myAliases { @@ -992,7 +1015,7 @@ func (n *network) ipamAllocateVersion(ipVer int, ipam ipamapi.Ipam) error { if ipVer == 6 { return nil } - *cfgList = []*IpamConf{&IpamConf{}} + *cfgList = []*IpamConf{{}} } *infoList = make([]*IpamInfo, len(*cfgList)) @@ -1237,3 +1260,10 @@ func (n *network) Internal() bool { return n.internal } + +func (n *network) IPv6Enabled() bool { + n.Lock() + defer n.Unlock() + + return n.enableIPv6 +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/osl/interface_linux.go b/components/engine/vendor/src/github.com/docker/libnetwork/osl/interface_linux.go index de74ee4852..205e3a3909 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/osl/interface_linux.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/osl/interface_linux.go @@ -21,6 +21,7 @@ type nwIface struct { dstName string master string dstMaster string + mac net.HardwareAddr address *net.IPNet addressIPv6 *net.IPNet routes []*net.IPNet @@ -64,6 +65,13 @@ func (i *nwIface) Master() string { return i.master } +func (i *nwIface) MacAddress() net.HardwareAddr { + i.Lock() + defer i.Unlock() + + return types.GetMacCopy(i.mac) +} + func (i *nwIface) Address() *net.IPNet { i.Lock() defer i.Unlock() @@ -304,6 +312,7 @@ func configureInterface(iface netlink.Link, i *nwIface) error { ErrMessage string }{ {setInterfaceName, fmt.Sprintf("error renaming interface %q to %q", ifaceName, i.DstName())}, + {setInterfaceMAC, fmt.Sprintf("error setting interface %q MAC to %q", ifaceName, i.MacAddress())}, {setInterfaceIP, fmt.Sprintf("error setting interface %q IP to %q", ifaceName, i.Address())}, {setInterfaceIPv6, fmt.Sprintf("error setting interface %q IPv6 to %q", ifaceName, i.AddressIPv6())}, {setInterfaceMaster, fmt.Sprintf("error setting interface %q master to %q", ifaceName, i.DstMaster())}, @@ -326,6 +335,13 @@ func setInterfaceMaster(iface netlink.Link, i *nwIface) error { LinkAttrs: netlink.LinkAttrs{Name: i.DstMaster()}}) } +func setInterfaceMAC(iface netlink.Link, i *nwIface) error { + if i.MacAddress() == nil { + return nil + } + return netlink.LinkSetHardwareAddr(iface, i.MacAddress()) +} + func setInterfaceIP(iface netlink.Link, i *nwIface) error { if i.Address() == nil { return nil diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/osl/options_linux.go b/components/engine/vendor/src/github.com/docker/libnetwork/osl/options_linux.go index 5295eb85c5..ea28e8b6be 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/osl/options_linux.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/osl/options_linux.go @@ -42,6 +42,12 @@ func (n *networkNamespace) Master(name string) IfaceOption { } } +func (n *networkNamespace) MacAddress(mac net.HardwareAddr) IfaceOption { + return func(i *nwIface) { + i.mac = mac + } +} + func (n *networkNamespace) Address(addr *net.IPNet) IfaceOption { return func(i *nwIface) { i.address = addr diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/osl/sandbox.go b/components/engine/vendor/src/github.com/docker/libnetwork/osl/sandbox.go index 3a824ae6ad..db49d43dce 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/osl/sandbox.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/osl/sandbox.go @@ -76,6 +76,9 @@ type IfaceOptionSetter interface { // Bridge returns an option setter to set if the interface is a bridge. Bridge(bool) IfaceOption + // MacAddress returns an option setter to set the MAC address. + MacAddress(net.HardwareAddr) IfaceOption + // Address returns an option setter to set IPv4 address. Address(*net.IPNet) IfaceOption diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy.go b/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy.go index 530703b259..ddde2744c2 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy.go @@ -85,7 +85,7 @@ func handleStopSignals(p proxy.Proxy) { s := make(chan os.Signal, 10) signal.Notify(s, os.Interrupt, syscall.SIGTERM, syscall.SIGSTOP) - for _ = range s { + for range s { p.Close() os.Exit(0) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/resolver.go b/components/engine/vendor/src/github.com/docker/libnetwork/resolver.go index 01d3483f08..a4dde792c7 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/resolver.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/resolver.go @@ -2,8 +2,11 @@ package libnetwork import ( "fmt" + "math/rand" "net" "strings" + "sync" + "time" log "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/iptables" @@ -31,23 +34,35 @@ type Resolver interface { } const ( - resolverIP = "127.0.0.11" - dnsPort = "53" - ptrIPv4domain = ".in-addr.arpa." - ptrIPv6domain = ".ip6.arpa." - respTTL = 1800 - maxExtDNS = 3 //max number of external servers to try + resolverIP = "127.0.0.11" + dnsPort = "53" + ptrIPv4domain = ".in-addr.arpa." + ptrIPv6domain = ".ip6.arpa." + respTTL = 600 + maxExtDNS = 3 //max number of external servers to try + extIOTimeout = 3 * time.Second + defaultRespSize = 512 ) +type extDNSEntry struct { + ipStr string + extConn net.Conn + extOnce sync.Once +} + // resolver implements the Resolver interface type resolver struct { - sb *sandbox - extDNS []string - server *dns.Server - conn *net.UDPConn - tcpServer *dns.Server - tcpListen *net.TCPListener - err error + sb *sandbox + extDNSList [maxExtDNS]extDNSEntry + server *dns.Server + conn *net.UDPConn + tcpServer *dns.Server + tcpListen *net.TCPListener + err error +} + +func init() { + rand.Seed(time.Now().Unix()) } // NewResolver creates a new instance of the Resolver @@ -136,7 +151,13 @@ func (r *resolver) Stop() { } func (r *resolver) SetExtServers(dns []string) { - r.extDNS = dns + l := len(dns) + if l > maxExtDNS { + l = maxExtDNS + } + for i := 0; i < l; i++ { + r.extDNSList[i].ipStr = dns[i] + } } func (r *resolver) NameServer() string { @@ -147,21 +168,40 @@ func (r *resolver) ResolverOptions() []string { return []string{"ndots:0"} } +func setCommonFlags(msg *dns.Msg) { + msg.RecursionAvailable = true +} + +func shuffleAddr(addr []net.IP) []net.IP { + for i := len(addr) - 1; i > 0; i-- { + r := rand.Intn(i + 1) + addr[i], addr[r] = addr[r], addr[i] + } + return addr +} + func (r *resolver) handleIPv4Query(name string, query *dns.Msg) (*dns.Msg, error) { addr := r.sb.ResolveName(name) if addr == nil { return nil, nil } - log.Debugf("Lookup for %s: IP %s", name, addr.String()) + log.Debugf("Lookup for %s: IP %v", name, addr) resp := new(dns.Msg) resp.SetReply(query) + setCommonFlags(resp) - rr := new(dns.A) - rr.Hdr = dns.RR_Header{Name: name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: respTTL} - rr.A = addr - resp.Answer = append(resp.Answer, rr) + if len(addr) > 1 { + addr = shuffleAddr(addr) + } + + for _, ip := range addr { + rr := new(dns.A) + rr.Hdr = dns.RR_Header{Name: name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: respTTL} + rr.A = ip + resp.Answer = append(resp.Answer, rr) + } return resp, nil } @@ -186,6 +226,7 @@ func (r *resolver) handlePTRQuery(ptr string, query *dns.Msg) (*dns.Msg, error) resp := new(dns.Msg) resp.SetReply(query) + setCommonFlags(resp) rr := new(dns.PTR) rr.Hdr = dns.RR_Header{Name: ptr, Rrtype: dns.TypePTR, Class: dns.ClassINET, Ttl: respTTL} @@ -194,12 +235,28 @@ func (r *resolver) handlePTRQuery(ptr string, query *dns.Msg) (*dns.Msg, error) return resp, nil } +func truncateResp(resp *dns.Msg, maxSize int, isTCP bool) { + if !isTCP { + resp.Truncated = true + } + + // trim the Answer RRs one by one till the whole message fits + // within the reply size + for resp.Len() > maxSize { + resp.Answer = resp.Answer[:len(resp.Answer)-1] + } +} + func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) { var ( - resp *dns.Msg - err error + extConn net.Conn + resp *dns.Msg + err error ) + if query == nil || len(query.Question) == 0 { + return + } name := query.Question[0].Name if query.Question[0].Qtype == dns.TypeA { resp, err = r.handleIPv4Query(name, query) @@ -212,27 +269,82 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) { return } - if resp == nil { - if len(r.extDNS) == 0 { - return + proto := w.LocalAddr().Network() + maxSize := 0 + if proto == "tcp" { + maxSize = dns.MaxMsgSize - 1 + } else if proto == "udp" { + optRR := query.IsEdns0() + if optRR != nil { + maxSize = int(optRR.UDPSize()) } - - num := maxExtDNS - if len(r.extDNS) < maxExtDNS { - num = len(r.extDNS) + if maxSize < defaultRespSize { + maxSize = defaultRespSize } - for i := 0; i < num; i++ { - log.Debugf("Querying ext dns %s:%s for %s[%d]", w.LocalAddr().Network(), r.extDNS[i], name, query.Question[0].Qtype) + } - c := &dns.Client{Net: w.LocalAddr().Network()} - addr := fmt.Sprintf("%s:%d", r.extDNS[i], 53) - - resp, _, err = c.Exchange(query, addr) - if err == nil { + if resp != nil { + if resp.Len() > maxSize { + truncateResp(resp, maxSize, proto == "tcp") + } + } else { + for i := 0; i < maxExtDNS; i++ { + extDNS := &r.extDNSList[i] + if extDNS.ipStr == "" { break } - log.Errorf("external resolution failed, %s", err) + log.Debugf("Querying ext dns %s:%s for %s[%d]", proto, extDNS.ipStr, name, query.Question[0].Qtype) + + extConnect := func() { + addr := fmt.Sprintf("%s:%d", extDNS.ipStr, 53) + extConn, err = net.DialTimeout(proto, addr, extIOTimeout) + } + + // For udp clients connection is persisted to reuse for further queries. + // Accessing extDNS.extConn be a race here between go rouines. Hence the + // connection setup is done in a Once block and fetch the extConn again + extConn = extDNS.extConn + if extConn == nil || proto == "tcp" { + if proto == "udp" { + extDNS.extOnce.Do(func() { + r.sb.execFunc(extConnect) + extDNS.extConn = extConn + }) + extConn = extDNS.extConn + } else { + r.sb.execFunc(extConnect) + } + if err != nil { + log.Debugf("Connect failed, %s", err) + continue + } + } + + // Timeout has to be set for every IO operation. + extConn.SetDeadline(time.Now().Add(extIOTimeout)) + co := &dns.Conn{Conn: extConn} + + defer func() { + if proto == "tcp" { + co.Close() + } + }() + err = co.WriteMsg(query) + if err != nil { + log.Debugf("Send to DNS server failed, %s", err) + continue + } + + resp, err = co.ReadMsg() + if err != nil { + log.Debugf("Read from DNS server failed, %s", err) + continue + } + + resp.Compress = true + break } + if resp == nil { return } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/sandbox.go b/components/engine/vendor/src/github.com/docker/libnetwork/sandbox.go index 71c8ebb753..5733deac18 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/sandbox.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/sandbox.go @@ -4,19 +4,14 @@ import ( "container/heap" "encoding/json" "fmt" - "io/ioutil" "net" - "os" - "path" - "path/filepath" "strings" "sync" log "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/etchosts" - "github.com/docker/libnetwork/netutils" + "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/osl" - "github.com/docker/libnetwork/resolvconf" "github.com/docker/libnetwork/types" ) @@ -43,7 +38,7 @@ type Sandbox interface { Delete() error // ResolveName searches for the service name in the networks to which the sandbox // is connected to. - ResolveName(name string) net.IP + ResolveName(name string) []net.IP // ResolveIP returns the service name for the passed in IP. IP is in reverse dotted // notation; the format used for DNS PTR records ResolveIP(name string) string @@ -51,7 +46,7 @@ type Sandbox interface { Endpoints() []Endpoint } -// SandboxOption is a option setter function type used to pass varios options to +// SandboxOption is an option setter function type used to pass various options to // NewNetContainer method. The various setter functions of type SandboxOption are // provided by libnetwork, they look like ContainerOptionXXXX(...) type SandboxOption func(sb *sandbox) @@ -124,6 +119,7 @@ type containerConfig struct { useDefaultSandBox bool useExternalKey bool prio int // higher the value, more the priority + exposedPorts []types.TransportPort } func (sb *sandbox) ID() string { @@ -142,18 +138,27 @@ func (sb *sandbox) Key() string { } func (sb *sandbox) Labels() map[string]interface{} { - return sb.config.generic + sb.Lock() + sb.Unlock() + opts := make(map[string]interface{}, len(sb.config.generic)) + for k, v := range sb.config.generic { + opts[k] = v + } + return opts } func (sb *sandbox) Statistics() (map[string]*types.InterfaceStatistics, error) { m := make(map[string]*types.InterfaceStatistics) - if sb.osSbox == nil { + sb.Lock() + osb := sb.osSbox + sb.Unlock() + if osb == nil { return m, nil } var err error - for _, i := range sb.osSbox.Info().Interfaces() { + for _, i := range osb.Info().Interfaces() { if m[i.DstName()], err = i.Statistics(); err != nil { return m, err } @@ -309,46 +314,6 @@ func (sb *sandbox) UnmarshalJSON(b []byte) (err error) { return nil } -func (sb *sandbox) startResolver() { - sb.resolverOnce.Do(func() { - var err error - sb.resolver = NewResolver(sb) - defer func() { - if err != nil { - sb.resolver = nil - } - }() - - err = sb.rebuildDNS() - if err != nil { - log.Errorf("Updating resolv.conf failed for container %s, %q", sb.ContainerID(), err) - return - } - sb.resolver.SetExtServers(sb.extDNS) - - sb.osSbox.InvokeFunc(sb.resolver.SetupFunc()) - if err = sb.resolver.Start(); err != nil { - log.Errorf("Resolver Setup/Start failed for container %s, %q", sb.ContainerID(), err) - } - }) -} - -func (sb *sandbox) setupResolutionFiles() error { - if err := sb.buildHostsFile(); err != nil { - return err - } - - if err := sb.updateParentHosts(); err != nil { - return err - } - - if err := sb.setupDNS(); err != nil { - return err - } - - return nil -} - func (sb *sandbox) Endpoints() []Endpoint { sb.Lock() defer sb.Unlock() @@ -372,6 +337,18 @@ func (sb *sandbox) getConnectedEndpoints() []*endpoint { return eps } +func (sb *sandbox) removeEndpoint(ep *endpoint) { + sb.Lock() + defer sb.Unlock() + + for i, e := range sb.endpoints { + if e == ep { + heap.Remove(&sb.endpoints, i) + return + } + } +} + func (sb *sandbox) getEndpoint(id string) *endpoint { sb.Lock() defer sb.Unlock() @@ -437,8 +414,12 @@ func (sb *sandbox) ResolveIP(ip string) string { return svc } -func (sb *sandbox) ResolveName(name string) net.IP { - var ip net.IP +func (sb *sandbox) execFunc(f func()) { + sb.osSbox.InvokeFunc(f) +} + +func (sb *sandbox) ResolveName(name string) []net.IP { + var ip []net.IP // Embedded server owns the docker network domain. Resolution should work // for both container_name and container_name.network_name @@ -486,7 +467,7 @@ func (sb *sandbox) ResolveName(name string) net.IP { return nil } -func (sb *sandbox) resolveName(req string, networkName string, epList []*endpoint, alias bool) net.IP { +func (sb *sandbox) resolveName(req string, networkName string, epList []*endpoint, alias bool) []net.IP { for _, ep := range epList { name := req n := ep.getNetwork() @@ -509,7 +490,7 @@ func (sb *sandbox) resolveName(req string, networkName string, epList []*endpoin } } else { // If it is a regular lookup and if the requested name is an alias - // dont perform a svc lookup for this endpoint. + // don't perform a svc lookup for this endpoint. ep.Lock() if _, ok := ep.aliases[req]; ok { ep.Unlock() @@ -527,7 +508,7 @@ func (sb *sandbox) resolveName(req string, networkName string, epList []*endpoin ip, ok := sr.svcMap[name] n.Unlock() if ok { - return ip[0] + return ip } } return nil @@ -652,6 +633,9 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error { if i.addrv6 != nil && i.addrv6.IP.To16() != nil { ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6)) } + if i.mac != nil { + ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac)) + } if err := sb.osSbox.AddInterface(i.srcName, i.dstPrefix, ifaceOptions...); err != nil { return fmt.Errorf("failed to add interface %s to sandbox: %v", i.srcName, err) @@ -667,14 +651,9 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error { } } - for _, gwep := range sb.getConnectedEndpoints() { - if len(gwep.Gateway()) > 0 { - if gwep != ep { - break - } - if err := sb.updateGateway(gwep); err != nil { - return err - } + if ep == sb.getGatewayEndpoint() { + if err := sb.updateGateway(ep); err != nil { + return err } } @@ -693,7 +672,7 @@ func (sb *sandbox) clearNetworkResources(origEp *endpoint) error { ep := sb.getEndpoint(origEp.id) if ep == nil { return fmt.Errorf("could not find the sandbox endpoint data for endpoint %s", - ep.name) + origEp.id) } sb.Lock() @@ -753,243 +732,6 @@ func (sb *sandbox) clearNetworkResources(origEp *endpoint) error { return nil } -const ( - defaultPrefix = "/var/lib/docker/network/files" - dirPerm = 0755 - filePerm = 0644 -) - -func (sb *sandbox) buildHostsFile() error { - if sb.config.hostsPath == "" { - sb.config.hostsPath = defaultPrefix + "/" + sb.id + "/hosts" - } - - dir, _ := filepath.Split(sb.config.hostsPath) - if err := createBasePath(dir); err != nil { - return err - } - - // This is for the host mode networking - if sb.config.originHostsPath != "" { - if err := copyFile(sb.config.originHostsPath, sb.config.hostsPath); err != nil && !os.IsNotExist(err) { - return types.InternalErrorf("could not copy source hosts file %s to %s: %v", sb.config.originHostsPath, sb.config.hostsPath, err) - } - return nil - } - - extraContent := make([]etchosts.Record, 0, len(sb.config.extraHosts)) - for _, extraHost := range sb.config.extraHosts { - extraContent = append(extraContent, etchosts.Record{Hosts: extraHost.name, IP: extraHost.IP}) - } - - return etchosts.Build(sb.config.hostsPath, "", sb.config.hostName, sb.config.domainName, extraContent) -} - -func (sb *sandbox) updateHostsFile(ifaceIP string) error { - var mhost string - - if sb.config.originHostsPath != "" { - return nil - } - - if sb.config.domainName != "" { - mhost = fmt.Sprintf("%s.%s %s", sb.config.hostName, sb.config.domainName, - sb.config.hostName) - } else { - mhost = sb.config.hostName - } - - extraContent := []etchosts.Record{{Hosts: mhost, IP: ifaceIP}} - - sb.addHostsEntries(extraContent) - return nil -} - -func (sb *sandbox) addHostsEntries(recs []etchosts.Record) { - if err := etchosts.Add(sb.config.hostsPath, recs); err != nil { - log.Warnf("Failed adding service host entries to the running container: %v", err) - } -} - -func (sb *sandbox) deleteHostsEntries(recs []etchosts.Record) { - if err := etchosts.Delete(sb.config.hostsPath, recs); err != nil { - log.Warnf("Failed deleting service host entries to the running container: %v", err) - } -} - -func (sb *sandbox) updateParentHosts() error { - var pSb Sandbox - - for _, update := range sb.config.parentUpdates { - sb.controller.WalkSandboxes(SandboxContainerWalker(&pSb, update.cid)) - if pSb == nil { - continue - } - if err := etchosts.Update(pSb.(*sandbox).config.hostsPath, update.ip, update.name); err != nil { - return err - } - } - - return nil -} - -func (sb *sandbox) setupDNS() error { - var newRC *resolvconf.File - - if sb.config.resolvConfPath == "" { - sb.config.resolvConfPath = defaultPrefix + "/" + sb.id + "/resolv.conf" - } - - sb.config.resolvConfHashFile = sb.config.resolvConfPath + ".hash" - - dir, _ := filepath.Split(sb.config.resolvConfPath) - if err := createBasePath(dir); err != nil { - return err - } - - // This is for the host mode networking - if sb.config.originResolvConfPath != "" { - if err := copyFile(sb.config.originResolvConfPath, sb.config.resolvConfPath); err != nil { - return fmt.Errorf("could not copy source resolv.conf file %s to %s: %v", sb.config.originResolvConfPath, sb.config.resolvConfPath, err) - } - return nil - } - - currRC, err := resolvconf.Get() - if err != nil { - return err - } - - if len(sb.config.dnsList) > 0 || len(sb.config.dnsSearchList) > 0 || len(sb.config.dnsOptionsList) > 0 { - var ( - err error - dnsList = resolvconf.GetNameservers(currRC.Content, netutils.IP) - dnsSearchList = resolvconf.GetSearchDomains(currRC.Content) - dnsOptionsList = resolvconf.GetOptions(currRC.Content) - ) - if len(sb.config.dnsList) > 0 { - dnsList = sb.config.dnsList - } - if len(sb.config.dnsSearchList) > 0 { - dnsSearchList = sb.config.dnsSearchList - } - if len(sb.config.dnsOptionsList) > 0 { - dnsOptionsList = sb.config.dnsOptionsList - } - newRC, err = resolvconf.Build(sb.config.resolvConfPath, dnsList, dnsSearchList, dnsOptionsList) - if err != nil { - return err - } - } else { - // Replace any localhost/127.* (at this point we have no info about ipv6, pass it as true) - if newRC, err = resolvconf.FilterResolvDNS(currRC.Content, true); err != nil { - return err - } - // No contention on container resolv.conf file at sandbox creation - if err := ioutil.WriteFile(sb.config.resolvConfPath, newRC.Content, filePerm); err != nil { - return types.InternalErrorf("failed to write unhaltered resolv.conf file content when setting up dns for sandbox %s: %v", sb.ID(), err) - } - } - - // Write hash - if err := ioutil.WriteFile(sb.config.resolvConfHashFile, []byte(newRC.Hash), filePerm); err != nil { - return types.InternalErrorf("failed to write resolv.conf hash file when setting up dns for sandbox %s: %v", sb.ID(), err) - } - - return nil -} - -func (sb *sandbox) updateDNS(ipv6Enabled bool) error { - var ( - currHash string - hashFile = sb.config.resolvConfHashFile - ) - - // This is for the host mode networking - if sb.config.originResolvConfPath != "" { - return nil - } - - if len(sb.config.dnsList) > 0 || len(sb.config.dnsSearchList) > 0 || len(sb.config.dnsOptionsList) > 0 { - return nil - } - - currRC, err := resolvconf.GetSpecific(sb.config.resolvConfPath) - if err != nil { - if !os.IsNotExist(err) { - return err - } - } else { - h, err := ioutil.ReadFile(hashFile) - if err != nil { - if !os.IsNotExist(err) { - return err - } - } else { - currHash = string(h) - } - } - - if currHash != "" && currHash != currRC.Hash { - // Seems the user has changed the container resolv.conf since the last time - // we checked so return without doing anything. - log.Infof("Skipping update of resolv.conf file with ipv6Enabled: %t because file was touched by user", ipv6Enabled) - return nil - } - - // replace any localhost/127.* and remove IPv6 nameservers if IPv6 disabled. - newRC, err := resolvconf.FilterResolvDNS(currRC.Content, ipv6Enabled) - if err != nil { - return err - } - err = ioutil.WriteFile(sb.config.resolvConfPath, newRC.Content, 0644) - if err != nil { - return err - } - - // write the new hash in a temp file and rename it to make the update atomic - dir := path.Dir(sb.config.resolvConfPath) - tmpHashFile, err := ioutil.TempFile(dir, "hash") - if err != nil { - return err - } - if err = ioutil.WriteFile(tmpHashFile.Name(), []byte(newRC.Hash), filePerm); err != nil { - return err - } - return os.Rename(tmpHashFile.Name(), hashFile) -} - -// Embedded DNS server has to be enabled for this sandbox. Rebuild the container's -// resolv.conf by doing the follwing -// - Save the external name servers in resolv.conf in the sandbox -// - Add only the embedded server's IP to container's resolv.conf -// - If the embedded server needs any resolv.conf options add it to the current list -func (sb *sandbox) rebuildDNS() error { - currRC, err := resolvconf.GetSpecific(sb.config.resolvConfPath) - if err != nil { - return err - } - - // localhost entries have already been filtered out from the list - // retain only the v4 servers in sb for forwarding the DNS queries - sb.extDNS = resolvconf.GetNameservers(currRC.Content, netutils.IPv4) - - var ( - dnsList = []string{sb.resolver.NameServer()} - dnsOptionsList = resolvconf.GetOptions(currRC.Content) - dnsSearchList = resolvconf.GetSearchDomains(currRC.Content) - ) - - // external v6 DNS servers has to be listed in resolv.conf - dnsList = append(dnsList, resolvconf.GetNameservers(currRC.Content, netutils.IPv6)...) - - // Resolver returns the options in the format resolv.conf expects - dnsOptionsList = append(dnsOptionsList, sb.resolver.ResolverOptions()...) - - _, err = resolvconf.Build(sb.config.resolvConfPath, dnsList, dnsSearchList, dnsOptionsList) - return err -} - // joinLeaveStart waits to ensure there are no joins or leaves in progress and // marks this join/leave in progress without race func (sb *sandbox) joinLeaveStart() { @@ -1022,6 +764,13 @@ func (sb *sandbox) joinLeaveEnd() { } } +func (sb *sandbox) hasPortConfigs() bool { + opts := sb.Labels() + _, hasExpPorts := opts[netlabel.ExposedPorts] + _, hasPortMaps := opts[netlabel.PortMap] + return hasExpPorts || hasPortMaps +} + // OptionHostname function returns an option setter for hostname option to // be passed to NewSandbox method. func OptionHostname(name string) SandboxOption { @@ -1131,7 +880,42 @@ func OptionUseExternalKey() SandboxOption { // net container creation method. Container Labels are a good example. func OptionGeneric(generic map[string]interface{}) SandboxOption { return func(sb *sandbox) { - sb.config.generic = generic + if sb.config.generic == nil { + sb.config.generic = make(map[string]interface{}, len(generic)) + } + for k, v := range generic { + sb.config.generic[k] = v + } + } +} + +// OptionExposedPorts function returns an option setter for the container exposed +// ports option to be passed to container Create method. +func OptionExposedPorts(exposedPorts []types.TransportPort) SandboxOption { + return func(sb *sandbox) { + if sb.config.generic == nil { + sb.config.generic = make(map[string]interface{}) + } + // Defensive copy + eps := make([]types.TransportPort, len(exposedPorts)) + copy(eps, exposedPorts) + // Store endpoint label and in generic because driver needs it + sb.config.exposedPorts = eps + sb.config.generic[netlabel.ExposedPorts] = eps + } +} + +// OptionPortMapping function returns an option setter for the mapping +// ports option to be passed to container Create method. +func OptionPortMapping(portBindings []types.PortBinding) SandboxOption { + return func(sb *sandbox) { + if sb.config.generic == nil { + sb.config.generic = make(map[string]interface{}) + } + // Store a copy of the bindings as generic data to pass to the driver + pbs := make([]types.PortBinding, len(portBindings)) + copy(pbs, portBindings) + sb.config.generic[netlabel.PortMap] = pbs } } @@ -1191,32 +975,3 @@ func (eh *epHeap) Pop() interface{} { *eh = old[0 : n-1] return x } - -func createBasePath(dir string) error { - return os.MkdirAll(dir, dirPerm) -} - -func createFile(path string) error { - var f *os.File - - dir, _ := filepath.Split(path) - err := createBasePath(dir) - if err != nil { - return err - } - - f, err = os.Create(path) - if err == nil { - f.Close() - } - - return err -} - -func copyFile(src, dst string) error { - sBytes, err := ioutil.ReadFile(src) - if err != nil { - return err - } - return ioutil.WriteFile(dst, sBytes, filePerm) -} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/sandbox_dns_unix.go b/components/engine/vendor/src/github.com/docker/libnetwork/sandbox_dns_unix.go new file mode 100644 index 0000000000..c8b595eb24 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/libnetwork/sandbox_dns_unix.go @@ -0,0 +1,323 @@ +// +build !windows + +package libnetwork + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + + log "github.com/Sirupsen/logrus" + "github.com/docker/libnetwork/etchosts" + "github.com/docker/libnetwork/netutils" + "github.com/docker/libnetwork/resolvconf" + "github.com/docker/libnetwork/types" +) + +const ( + defaultPrefix = "/var/lib/docker/network/files" + dirPerm = 0755 + filePerm = 0644 +) + +func (sb *sandbox) startResolver() { + sb.resolverOnce.Do(func() { + var err error + sb.resolver = NewResolver(sb) + defer func() { + if err != nil { + sb.resolver = nil + } + }() + + err = sb.rebuildDNS() + if err != nil { + log.Errorf("Updating resolv.conf failed for container %s, %q", sb.ContainerID(), err) + return + } + sb.resolver.SetExtServers(sb.extDNS) + + sb.osSbox.InvokeFunc(sb.resolver.SetupFunc()) + if err = sb.resolver.Start(); err != nil { + log.Errorf("Resolver Setup/Start failed for container %s, %q", sb.ContainerID(), err) + } + }) +} + +func (sb *sandbox) setupResolutionFiles() error { + if err := sb.buildHostsFile(); err != nil { + return err + } + + if err := sb.updateParentHosts(); err != nil { + return err + } + + if err := sb.setupDNS(); err != nil { + return err + } + + return nil +} + +func (sb *sandbox) buildHostsFile() error { + if sb.config.hostsPath == "" { + sb.config.hostsPath = defaultPrefix + "/" + sb.id + "/hosts" + } + + dir, _ := filepath.Split(sb.config.hostsPath) + if err := createBasePath(dir); err != nil { + return err + } + + // This is for the host mode networking + if sb.config.originHostsPath != "" { + if err := copyFile(sb.config.originHostsPath, sb.config.hostsPath); err != nil && !os.IsNotExist(err) { + return types.InternalErrorf("could not copy source hosts file %s to %s: %v", sb.config.originHostsPath, sb.config.hostsPath, err) + } + return nil + } + + extraContent := make([]etchosts.Record, 0, len(sb.config.extraHosts)) + for _, extraHost := range sb.config.extraHosts { + extraContent = append(extraContent, etchosts.Record{Hosts: extraHost.name, IP: extraHost.IP}) + } + + return etchosts.Build(sb.config.hostsPath, "", sb.config.hostName, sb.config.domainName, extraContent) +} + +func (sb *sandbox) updateHostsFile(ifaceIP string) error { + var mhost string + + if sb.config.originHostsPath != "" { + return nil + } + + if sb.config.domainName != "" { + mhost = fmt.Sprintf("%s.%s %s", sb.config.hostName, sb.config.domainName, + sb.config.hostName) + } else { + mhost = sb.config.hostName + } + + extraContent := []etchosts.Record{{Hosts: mhost, IP: ifaceIP}} + + sb.addHostsEntries(extraContent) + return nil +} + +func (sb *sandbox) addHostsEntries(recs []etchosts.Record) { + if err := etchosts.Add(sb.config.hostsPath, recs); err != nil { + log.Warnf("Failed adding service host entries to the running container: %v", err) + } +} + +func (sb *sandbox) deleteHostsEntries(recs []etchosts.Record) { + if err := etchosts.Delete(sb.config.hostsPath, recs); err != nil { + log.Warnf("Failed deleting service host entries to the running container: %v", err) + } +} + +func (sb *sandbox) updateParentHosts() error { + var pSb Sandbox + + for _, update := range sb.config.parentUpdates { + sb.controller.WalkSandboxes(SandboxContainerWalker(&pSb, update.cid)) + if pSb == nil { + continue + } + if err := etchosts.Update(pSb.(*sandbox).config.hostsPath, update.ip, update.name); err != nil { + return err + } + } + + return nil +} + +func (sb *sandbox) setupDNS() error { + var newRC *resolvconf.File + + if sb.config.resolvConfPath == "" { + sb.config.resolvConfPath = defaultPrefix + "/" + sb.id + "/resolv.conf" + } + + sb.config.resolvConfHashFile = sb.config.resolvConfPath + ".hash" + + dir, _ := filepath.Split(sb.config.resolvConfPath) + if err := createBasePath(dir); err != nil { + return err + } + + // This is for the host mode networking + if sb.config.originResolvConfPath != "" { + if err := copyFile(sb.config.originResolvConfPath, sb.config.resolvConfPath); err != nil { + return fmt.Errorf("could not copy source resolv.conf file %s to %s: %v", sb.config.originResolvConfPath, sb.config.resolvConfPath, err) + } + return nil + } + + currRC, err := resolvconf.Get() + if err != nil { + return err + } + + if len(sb.config.dnsList) > 0 || len(sb.config.dnsSearchList) > 0 || len(sb.config.dnsOptionsList) > 0 { + var ( + err error + dnsList = resolvconf.GetNameservers(currRC.Content, netutils.IP) + dnsSearchList = resolvconf.GetSearchDomains(currRC.Content) + dnsOptionsList = resolvconf.GetOptions(currRC.Content) + ) + if len(sb.config.dnsList) > 0 { + dnsList = sb.config.dnsList + } + if len(sb.config.dnsSearchList) > 0 { + dnsSearchList = sb.config.dnsSearchList + } + if len(sb.config.dnsOptionsList) > 0 { + dnsOptionsList = sb.config.dnsOptionsList + } + newRC, err = resolvconf.Build(sb.config.resolvConfPath, dnsList, dnsSearchList, dnsOptionsList) + if err != nil { + return err + } + } else { + // Replace any localhost/127.* (at this point we have no info about ipv6, pass it as true) + if newRC, err = resolvconf.FilterResolvDNS(currRC.Content, true); err != nil { + return err + } + // No contention on container resolv.conf file at sandbox creation + if err := ioutil.WriteFile(sb.config.resolvConfPath, newRC.Content, filePerm); err != nil { + return types.InternalErrorf("failed to write unhaltered resolv.conf file content when setting up dns for sandbox %s: %v", sb.ID(), err) + } + } + + // Write hash + if err := ioutil.WriteFile(sb.config.resolvConfHashFile, []byte(newRC.Hash), filePerm); err != nil { + return types.InternalErrorf("failed to write resolv.conf hash file when setting up dns for sandbox %s: %v", sb.ID(), err) + } + + return nil +} + +func (sb *sandbox) updateDNS(ipv6Enabled bool) error { + var ( + currHash string + hashFile = sb.config.resolvConfHashFile + ) + + // This is for the host mode networking + if sb.config.originResolvConfPath != "" { + return nil + } + + if len(sb.config.dnsList) > 0 || len(sb.config.dnsSearchList) > 0 || len(sb.config.dnsOptionsList) > 0 { + return nil + } + + currRC, err := resolvconf.GetSpecific(sb.config.resolvConfPath) + if err != nil { + if !os.IsNotExist(err) { + return err + } + } else { + h, err := ioutil.ReadFile(hashFile) + if err != nil { + if !os.IsNotExist(err) { + return err + } + } else { + currHash = string(h) + } + } + + if currHash != "" && currHash != currRC.Hash { + // Seems the user has changed the container resolv.conf since the last time + // we checked so return without doing anything. + log.Infof("Skipping update of resolv.conf file with ipv6Enabled: %t because file was touched by user", ipv6Enabled) + return nil + } + + // replace any localhost/127.* and remove IPv6 nameservers if IPv6 disabled. + newRC, err := resolvconf.FilterResolvDNS(currRC.Content, ipv6Enabled) + if err != nil { + return err + } + err = ioutil.WriteFile(sb.config.resolvConfPath, newRC.Content, 0644) + if err != nil { + return err + } + + // write the new hash in a temp file and rename it to make the update atomic + dir := path.Dir(sb.config.resolvConfPath) + tmpHashFile, err := ioutil.TempFile(dir, "hash") + if err != nil { + return err + } + if err = ioutil.WriteFile(tmpHashFile.Name(), []byte(newRC.Hash), filePerm); err != nil { + return err + } + return os.Rename(tmpHashFile.Name(), hashFile) +} + +// Embedded DNS server has to be enabled for this sandbox. Rebuild the container's +// resolv.conf by doing the follwing +// - Save the external name servers in resolv.conf in the sandbox +// - Add only the embedded server's IP to container's resolv.conf +// - If the embedded server needs any resolv.conf options add it to the current list +func (sb *sandbox) rebuildDNS() error { + currRC, err := resolvconf.GetSpecific(sb.config.resolvConfPath) + if err != nil { + return err + } + + // localhost entries have already been filtered out from the list + // retain only the v4 servers in sb for forwarding the DNS queries + sb.extDNS = resolvconf.GetNameservers(currRC.Content, netutils.IPv4) + + var ( + dnsList = []string{sb.resolver.NameServer()} + dnsOptionsList = resolvconf.GetOptions(currRC.Content) + dnsSearchList = resolvconf.GetSearchDomains(currRC.Content) + ) + + // external v6 DNS servers has to be listed in resolv.conf + dnsList = append(dnsList, resolvconf.GetNameservers(currRC.Content, netutils.IPv6)...) + + // Resolver returns the options in the format resolv.conf expects + dnsOptionsList = append(dnsOptionsList, sb.resolver.ResolverOptions()...) + + _, err = resolvconf.Build(sb.config.resolvConfPath, dnsList, dnsSearchList, dnsOptionsList) + return err +} + +func createBasePath(dir string) error { + return os.MkdirAll(dir, dirPerm) +} + +func createFile(path string) error { + var f *os.File + + dir, _ := filepath.Split(path) + err := createBasePath(dir) + if err != nil { + return err + } + + f, err = os.Create(path) + if err == nil { + f.Close() + } + + return err +} + +func copyFile(src, dst string) error { + sBytes, err := ioutil.ReadFile(src) + if err != nil { + return err + } + return ioutil.WriteFile(dst, sBytes, filePerm) +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/sandbox_dns_windows.go b/components/engine/vendor/src/github.com/docker/libnetwork/sandbox_dns_windows.go new file mode 100644 index 0000000000..ef90ddaeef --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/libnetwork/sandbox_dns_windows.go @@ -0,0 +1,32 @@ +// +build windows + +package libnetwork + +import ( + "github.com/docker/libnetwork/etchosts" +) + +// Stub implementations for DNS related functions + +func (sb *sandbox) startResolver() { +} + +func (sb *sandbox) setupResolutionFiles() error { + return nil +} + +func (sb *sandbox) updateHostsFile(ifaceIP string) error { + return nil +} + +func (sb *sandbox) addHostsEntries(recs []etchosts.Record) { + +} + +func (sb *sandbox) deleteHostsEntries(recs []etchosts.Record) { + +} + +func (sb *sandbox) updateDNS(ipv6Enabled bool) error { + return nil +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/sandbox_externalkey_unix.go b/components/engine/vendor/src/github.com/docker/libnetwork/sandbox_externalkey_unix.go index 74ae2af78e..3e5cee1c28 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/sandbox_externalkey_unix.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/sandbox_externalkey_unix.go @@ -1,4 +1,4 @@ -// +build !windows +// +build linux freebsd package libnetwork @@ -130,7 +130,7 @@ func (c *controller) acceptClientConnections(sock string, l net.Listener) { conn, err := l.Accept() if err != nil { if _, err1 := os.Stat(sock); os.IsNotExist(err1) { - logrus.Debugf("Unix socket %s doesnt exist. cannot accept client connections", sock) + logrus.Debugf("Unix socket %s doesn't exist. cannot accept client connections", sock) return } logrus.Errorf("Error accepting connection %v", err) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/store.go b/components/engine/vendor/src/github.com/docker/libnetwork/store.go index 89248800c9..182923ef6c 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/store.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/store.go @@ -14,6 +14,7 @@ func (c *controller) initStores() error { return nil } scopeConfigs := c.cfg.Scopes + c.stores = nil c.Unlock() for scope, scfg := range scopeConfigs { @@ -104,7 +105,8 @@ func (c *controller) getNetworksForScope(scope string) ([]*network, error) { ec := &endpointCnt{n: n} err = store.GetObject(datastore.Key(ec.Key()...), ec) if err != nil { - return nil, fmt.Errorf("could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err) + log.Warnf("Could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err) + continue } n.epCnt = ec @@ -138,11 +140,14 @@ func (c *controller) getNetworksFromStore() ([]*network, error) { ec := &endpointCnt{n: n} err = store.GetObject(datastore.Key(ec.Key()...), ec) if err != nil { - return nil, fmt.Errorf("could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err) + log.Warnf("could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err) + continue } + n.Lock() n.epCnt = ec n.scope = store.Scope() + n.Unlock() nl = append(nl, n) } } @@ -414,6 +419,9 @@ func (c *controller) watchLoop() { } func (c *controller) startWatch() { + if c.watchCh != nil { + return + } c.watchCh = make(chan *endpoint) c.unWatchCh = make(chan *endpoint) c.nmap = make(map[string]*netWatch) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/types/types.go b/components/engine/vendor/src/github.com/docker/libnetwork/types/types.go index 7ada9643c0..44ee563e69 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/types/types.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/types/types.go @@ -389,7 +389,7 @@ const ( // NEXTHOP indicates a StaticRoute with an IP next hop. NEXTHOP = iota - // CONNECTED indicates a StaticRoute with a interface for directly connected peers. + // CONNECTED indicates a StaticRoute with an interface for directly connected peers. CONNECTED ) @@ -458,25 +458,25 @@ type NotFoundError interface { NotFound() } -// ForbiddenError is an interface for errors which denote an valid request that cannot be honored +// ForbiddenError is an interface for errors which denote a valid request that cannot be honored type ForbiddenError interface { // Forbidden makes implementer into ForbiddenError type Forbidden() } -// NoServiceError is an interface for errors returned when the required service is not available +// NoServiceError is an interface for errors returned when the required service is not available type NoServiceError interface { // NoService makes implementer into NoServiceError type NoService() } -// TimeoutError is an interface for errors raised because of timeout +// TimeoutError is an interface for errors raised because of timeout type TimeoutError interface { // Timeout makes implementer into TimeoutError type Timeout() } -// NotImplementedError is an interface for errors raised because of requested functionality is not yet implemented +// NotImplementedError is an interface for errors raised because of requested functionality is not yet implemented type NotImplementedError interface { // NotImplemented makes implementer into NotImplementedError type NotImplemented() diff --git a/components/engine/vendor/src/github.com/docker/notary/Makefile b/components/engine/vendor/src/github.com/docker/notary/Makefile index 949d79864a..447514af09 100644 --- a/components/engine/vendor/src/github.com/docker/notary/Makefile +++ b/components/engine/vendor/src/github.com/docker/notary/Makefile @@ -42,14 +42,6 @@ GO_VERSION = $(shell go version | awk '{print $$3}') .DELETE_ON_ERROR: cover .DEFAULT: default -go_version: -ifeq (,$(findstring go1.5.,$(GO_VERSION))) - $(error Requires go version 1.5.x - found $(GO_VERSION)) -else - @echo -endif - - all: AUTHORS clean fmt vet fmt lint build test binaries AUTHORS: .git/HEAD @@ -71,7 +63,23 @@ ${PREFIX}/bin/notary-signer: NOTARY_VERSION $(shell find . -type f -name '*.go') @echo "+ $@" @godep go build -tags ${NOTARY_BUILDTAGS} -o $@ ${GO_LDFLAGS} ./cmd/notary-signer -vet: go_version +ifeq ($(shell uname -s),Darwin) +${PREFIX}/bin/static/notary-server: + @echo "notary-server: static builds not supported on OS X" + +${PREFIX}/bin/static/notary-signer: + @echo "notary-signer: static builds not supported on OS X" +else +${PREFIX}/bin/static/notary-server: NOTARY_VERSION $(shell find . -type f -name '*.go') + @echo "+ $@" + @godep go build -tags ${NOTARY_BUILDTAGS} -o $@ ${GO_LDFLAGS_STATIC} ./cmd/notary-server + +${PREFIX}/bin/static/notary-signer: NOTARY_VERSION $(shell find . -type f -name '*.go') + @echo "+ $@" + @godep go build -tags ${NOTARY_BUILDTAGS} -o $@ ${GO_LDFLAGS_STATIC} ./cmd/notary-signer +endif + +vet: @echo "+ $@" ifeq ($(shell uname -s), Darwin) @test -z "$(shell find . -iname *test*.go | grep -v _test.go | grep -v Godeps | xargs echo "This file should end with '_test':" | tee /dev/stderr)" @@ -88,14 +96,24 @@ lint: @echo "+ $@" @test -z "$$(golint ./... | grep -v .pb. | grep -v Godeps/_workspace/src/ | tee /dev/stderr)" -build: go_version +# Requires that the following: +# go get -u github.com/client9/misspell/cmd/misspell +# +# be run first + +# misspell target, don't include Godeps, binaries, python tests, or git files +misspell: + @echo "+ $@" + @test -z "$$(find . -name '*' | grep -v Godeps/_workspace/src/ | grep -v bin/ | grep -v misc/ | grep -v .git/ | xargs misspell | tee /dev/stderr)" + +build: @echo "+ $@" @go build -tags "${NOTARY_BUILDTAGS}" -v ${GO_LDFLAGS} ./... # When running `go test ./...`, it runs all the suites in parallel, which causes # problems when running with a yubikey test: TESTOPTS = -test: go_version +test: @echo Note: when testing with a yubikey plugged in, make sure to include 'TESTOPTS="-p 1"' @echo "+ $@ $(TESTOPTS)" @echo @@ -121,7 +139,7 @@ define gocover $(GO_EXC) test $(OPTS) $(TESTOPTS) -covermode="$(COVERMODE)" -coverprofile="$(COVERDIR)/$(subst /,-,$(1)).$(subst $(_space),.,$(NOTARY_BUILDTAGS)).coverage.txt" "$(1)" || exit 1; endef -gen-cover: go_version +gen-cover: @mkdir -p "$(COVERDIR)" $(foreach PKG,$(PKGS),$(call gocover,$(PKG))) rm -f "$(COVERDIR)"/*testutils*.coverage.txt @@ -150,7 +168,10 @@ covmerge: clean-protos: @rm proto/*.pb.go -binaries: go_version ${PREFIX}/bin/notary-server ${PREFIX}/bin/notary ${PREFIX}/bin/notary-signer +binaries: ${PREFIX}/bin/notary-server ${PREFIX}/bin/notary ${PREFIX}/bin/notary-signer + @echo "+ $@" + +static: ${PREFIX}/bin/static/notary-server ${PREFIX}/bin/static/notary-signer @echo "+ $@" define template @@ -158,7 +179,7 @@ mkdir -p ${PREFIX}/cross/$(1)/$(2); GOOS=$(1) GOARCH=$(2) CGO_ENABLED=0 go build -o ${PREFIX}/cross/$(1)/$(2)/notary -a -tags "static_build netgo" -installsuffix netgo ${GO_LDFLAGS_STATIC} ./cmd/notary; endef -cross: go_version +cross: $(foreach GOARCH,$(GOARCHS),$(foreach GOOS,$(GOOSES),$(call template,$(GOOS),$(GOARCH)))) diff --git a/components/engine/vendor/src/github.com/docker/notary/README.md b/components/engine/vendor/src/github.com/docker/notary/README.md index ed3ca49351..96cd5be5c6 100644 --- a/components/engine/vendor/src/github.com/docker/notary/README.md +++ b/components/engine/vendor/src/github.com/docker/notary/README.md @@ -1,4 +1,5 @@ -# Notary [![Circle CI](https://circleci.com/gh/docker/notary/tree/master.svg?style=shield)](https://circleci.com/gh/docker/notary/tree/master) +# Notary +[![Circle CI](https://circleci.com/gh/docker/notary/tree/master.svg?style=shield)](https://circleci.com/gh/docker/notary/tree/master) [![CodeCov](https://codecov.io/github/docker/notary/coverage.svg?branch=master)](https://codecov.io/github/docker/notary) The Notary project comprises a [server](cmd/notary-server) and a [client](cmd/notary) for running and interacting with trusted collections. diff --git a/components/engine/vendor/src/github.com/docker/notary/circle.yml b/components/engine/vendor/src/github.com/docker/notary/circle.yml index 163d610f24..6b98a161f1 100644 --- a/components/engine/vendor/src/github.com/docker/notary/circle.yml +++ b/components/engine/vendor/src/github.com/docker/notary/circle.yml @@ -6,7 +6,7 @@ machine: post: # Install many go versions - - gvm install go1.5.1 -B --name=stable + - gvm install go1.6 -B --name=stable environment: # Convenient shortcuts to "common" locations @@ -37,10 +37,11 @@ dependencies: pwd: $BASE_STABLE post: - # For the stable go version, additionally install linting tools + # For the stable go version, additionally install linting and misspell tools - > gvm use stable && - go get github.com/golang/lint/golint + go get github.com/golang/lint/golint && + go get -u github.com/client9/misspell/cmd/misspell test: pre: # Output the go versions we are going to test @@ -62,6 +63,10 @@ test: - gvm use stable && make lint: pwd: $BASE_STABLE + # MISSPELL + - gvm use stable && make misspell: + pwd: $BASE_STABLE + override: # Test stable, and report # hacking this to be parallel diff --git a/components/engine/vendor/src/github.com/docker/notary/client/changelist/change.go b/components/engine/vendor/src/github.com/docker/notary/client/changelist/change.go index 311857aa61..3307189c81 100644 --- a/components/engine/vendor/src/github.com/docker/notary/client/changelist/change.go +++ b/components/engine/vendor/src/github.com/docker/notary/client/changelist/change.go @@ -17,7 +17,7 @@ const ( // Types for TufChanges are namespaced by the Role they // are relevant for. The Root and Targets roles are the // only ones for which user action can cause a change, as -// all changes in Snapshot and Timestamp are programatically +// all changes in Snapshot and Timestamp are programmatically // generated base on Root and Targets changes. const ( TypeRootRole = "role" @@ -82,14 +82,13 @@ func (c TufChange) Content() []byte { // this includes creating a delegations. This format is used to avoid // unexpected race conditions between humans modifying the same delegation type TufDelegation struct { - NewName string `json:"new_name,omitempty"` - NewThreshold int `json:"threshold, omitempty"` - AddKeys data.KeyList `json:"add_keys, omitempty"` - RemoveKeys []string `json:"remove_keys,omitempty"` - AddPaths []string `json:"add_paths,omitempty"` - RemovePaths []string `json:"remove_paths,omitempty"` - AddPathHashPrefixes []string `json:"add_prefixes,omitempty"` - RemovePathHashPrefixes []string `json:"remove_prefixes,omitempty"` + NewName string `json:"new_name,omitempty"` + NewThreshold int `json:"threshold, omitempty"` + AddKeys data.KeyList `json:"add_keys, omitempty"` + RemoveKeys []string `json:"remove_keys,omitempty"` + AddPaths []string `json:"add_paths,omitempty"` + RemovePaths []string `json:"remove_paths,omitempty"` + ClearAllPaths bool `json:"clear_paths,omitempty"` } // ToNewRole creates a fresh role object from the TufDelegation data @@ -98,5 +97,5 @@ func (td TufDelegation) ToNewRole(scope string) (*data.Role, error) { if td.NewName != "" { name = td.NewName } - return data.NewRole(name, td.NewThreshold, td.AddKeys.IDs(), td.AddPaths, td.AddPathHashPrefixes) + return data.NewRole(name, td.NewThreshold, td.AddKeys.IDs(), td.AddPaths) } diff --git a/components/engine/vendor/src/github.com/docker/notary/client/client.go b/components/engine/vendor/src/github.com/docker/notary/client/client.go index b383c94dca..cbb4977132 100644 --- a/components/engine/vendor/src/github.com/docker/notary/client/client.go +++ b/components/engine/vendor/src/github.com/docker/notary/client/client.go @@ -9,6 +9,7 @@ import ( "net/url" "os" "path/filepath" + "strings" "time" "github.com/Sirupsen/logrus" @@ -20,23 +21,13 @@ import ( "github.com/docker/notary/tuf" tufclient "github.com/docker/notary/tuf/client" "github.com/docker/notary/tuf/data" - "github.com/docker/notary/tuf/keys" "github.com/docker/notary/tuf/signed" "github.com/docker/notary/tuf/store" -) - -const ( - maxSize = 5 << 20 + "github.com/docker/notary/tuf/utils" ) func init() { - data.SetDefaultExpiryTimes( - map[string]int{ - "root": 3650, - "targets": 1095, - "snapshot": 1095, - }, - ) + data.SetDefaultExpiryTimes(notary.NotaryDefaultExpiries) } // ErrRepoNotInitialized is returned when trying to publish an uninitialized @@ -118,7 +109,6 @@ func repositoryFromKeystores(baseDir, gun, baseURL string, rt http.RoundTripper, nRepo.tufRepoPath, "metadata", "json", - "", ) if err != nil { return nil, err @@ -218,11 +208,16 @@ func (r *NotaryRepository) Initialize(rootKeyID string, serverManagedRoles ...st return fmt.Errorf("invalid format for root key: %s", privKey.Algorithm()) } - kdb := keys.NewDB() - err = addKeyForRole(kdb, data.CanonicalRootRole, rootKey) - if err != nil { - return err - } + var ( + rootRole = data.NewBaseRole( + data.CanonicalRootRole, + notary.MinThreshold, + rootKey, + ) + timestampRole data.BaseRole + snapshotRole data.BaseRole + targetsRole data.BaseRole + ) // we want to create all the local keys first so we don't have to // make unnecessary network calls @@ -232,8 +227,19 @@ func (r *NotaryRepository) Initialize(rootKeyID string, serverManagedRoles ...st if err != nil { return err } - if err := addKeyForRole(kdb, role, key); err != nil { - return err + switch role { + case data.CanonicalSnapshotRole: + snapshotRole = data.NewBaseRole( + role, + notary.MinThreshold, + key, + ) + case data.CanonicalTargetsRole: + targetsRole = data.NewBaseRole( + role, + notary.MinThreshold, + key, + ) } } for _, role := range remotelyManagedKeys { @@ -244,14 +250,31 @@ func (r *NotaryRepository) Initialize(rootKeyID string, serverManagedRoles ...st } logrus.Debugf("got remote %s %s key with keyID: %s", role, key.Algorithm(), key.ID()) - if err := addKeyForRole(kdb, role, key); err != nil { - return err + switch role { + case data.CanonicalSnapshotRole: + snapshotRole = data.NewBaseRole( + role, + notary.MinThreshold, + key, + ) + case data.CanonicalTimestampRole: + timestampRole = data.NewBaseRole( + role, + notary.MinThreshold, + key, + ) } } - r.tufRepo = tuf.NewRepo(kdb, r.CryptoService) + r.tufRepo = tuf.NewRepo(r.CryptoService) - err = r.tufRepo.InitRoot(false) + err = r.tufRepo.InitRoot( + rootRole, + timestampRole, + snapshotRole, + targetsRole, + false, + ) if err != nil { logrus.Debug("Error on InitRoot: ", err.Error()) return err @@ -305,96 +328,6 @@ func addChange(cl *changelist.FileChangelist, c changelist.Change, roles ...stri return nil } -// AddDelegation creates a new changelist entry to add a delegation to the repository -// when the changelist gets applied at publish time. This does not do any validation -// other than checking the name of the delegation to add - all that will happen -// at publish time. -func (r *NotaryRepository) AddDelegation(name string, threshold int, - delegationKeys []data.PublicKey, paths []string) error { - - if !data.IsDelegation(name) { - return data.ErrInvalidRole{Role: name, Reason: "invalid delegation role name"} - } - - cl, err := changelist.NewFileChangelist(filepath.Join(r.tufRepoPath, "changelist")) - if err != nil { - return err - } - defer cl.Close() - - logrus.Debugf(`Adding delegation "%s" with threshold %d, and %d keys\n`, - name, threshold, len(delegationKeys)) - - tdJSON, err := json.Marshal(&changelist.TufDelegation{ - NewThreshold: threshold, - AddKeys: data.KeyList(delegationKeys), - AddPaths: paths, - }) - if err != nil { - return err - } - - template := changelist.NewTufChange( - changelist.ActionCreate, - name, - changelist.TypeTargetsDelegation, - "", // no path - tdJSON, - ) - - return addChange(cl, template, name) -} - -// RemoveDelegation creates a new changelist entry to remove a delegation from -// the repository when the changelist gets applied at publish time. -// This does not validate that the delegation exists, since one might exist -// after applying all changes. -func (r *NotaryRepository) RemoveDelegation(name string, keyIDs, paths []string, removeAll bool) error { - - if !data.IsDelegation(name) { - return data.ErrInvalidRole{Role: name, Reason: "invalid delegation role name"} - } - - cl, err := changelist.NewFileChangelist(filepath.Join(r.tufRepoPath, "changelist")) - if err != nil { - return err - } - defer cl.Close() - - logrus.Debugf(`Removing delegation "%s"\n`, name) - var template *changelist.TufChange - - // We use the Delete action only for force removal, Update is used for removing individual keys and paths - if removeAll { - template = changelist.NewTufChange( - changelist.ActionDelete, - name, - changelist.TypeTargetsDelegation, - "", // no path - nil, // deleting role, no data needed - ) - - } else { - tdJSON, err := json.Marshal(&changelist.TufDelegation{ - RemoveKeys: keyIDs, - RemovePaths: paths, - }) - if err != nil { - return err - } - - template = changelist.NewTufChange( - changelist.ActionUpdate, - name, - changelist.TypeTargetsDelegation, - "", // no path - tdJSON, - ) - } - - return addChange(cl, template, name) -} - // AddTarget creates new changelist entries to add a target to the given roles // in the repository when the changelist gets applied at publish time. // If roles are unspecified, the default role is "targets". @@ -452,10 +385,24 @@ func (r *NotaryRepository) ListTargets(roles ...string) ([]*TargetWithRole, erro } targets := make(map[string]*TargetWithRole) for _, role := range roles { - // we don't need to do anything special with removing role from - // roles because listSubtree always processes role and only excludes - // descendant delegations that appear in roles. - r.listSubtree(targets, role, roles...) + // Define an array of roles to skip for this walk (see IMPORTANT comment above) + skipRoles := utils.StrSliceRemove(roles, role) + + // Define a visitor function to populate the targets map in priority order + listVisitorFunc := func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} { + // We found targets so we should try to add them to our targets map + for targetName, targetMeta := range tgt.Signed.Targets { + // Follow the priority by not overriding previously set targets + // and check that this path is valid with this role + if _, ok := targets[targetName]; ok || !validRole.CheckPaths(targetName) { + continue + } + targets[targetName] = + &TargetWithRole{Target: Target{Name: targetName, Hashes: targetMeta.Hashes, Length: targetMeta.Length}, Role: validRole.Name} + } + return nil + } + r.tufRepo.WalkTargets("", role, listVisitorFunc, skipRoles...) } var targetList []*TargetWithRole @@ -466,34 +413,6 @@ func (r *NotaryRepository) ListTargets(roles ...string) ([]*TargetWithRole, erro return targetList, nil } -func (r *NotaryRepository) listSubtree(targets map[string]*TargetWithRole, role string, exclude ...string) { - excl := make(map[string]bool) - for _, r := range exclude { - excl[r] = true - } - roles := []string{role} - for len(roles) > 0 { - role = roles[0] - roles = roles[1:] - tgts, ok := r.tufRepo.Targets[role] - if !ok { - // not every role has to exist - continue - } - for name, meta := range tgts.Signed.Targets { - if _, ok := targets[name]; !ok { - targets[name] = &TargetWithRole{ - Target: Target{Name: name, Hashes: meta.Hashes, Length: meta.Length}, Role: role} - } - } - for _, d := range tgts.Signed.Delegations.Roles { - if !excl[d.Name] { - roles = append(roles, d.Name) - } - } - } -} - // GetTargetByName returns a target given a name. If no roles are passed // it uses the targets role and does a search of the entire delegation // graph, finding the first entry in a breadth first search of the delegations. @@ -502,7 +421,7 @@ func (r *NotaryRepository) listSubtree(targets map[string]*TargetWithRole, role // will be returned // See the IMPORTANT section on ListTargets above. Those roles also apply here. func (r *NotaryRepository) GetTargetByName(name string, roles ...string) (*TargetWithRole, error) { - c, err := r.Update(false) + _, err := r.Update(false) if err != nil { return nil, err } @@ -510,11 +429,30 @@ func (r *NotaryRepository) GetTargetByName(name string, roles ...string) (*Targe if len(roles) == 0 { roles = append(roles, data.CanonicalTargetsRole) } + var resultMeta data.FileMeta + var resultRoleName string + var foundTarget bool for _, role := range roles { - meta, foundRole := c.TargetMeta(role, name, roles...) - if meta != nil { - return &TargetWithRole{ - Target: Target{Name: name, Hashes: meta.Hashes, Length: meta.Length}, Role: foundRole}, nil + // Define an array of roles to skip for this walk (see IMPORTANT comment above) + skipRoles := utils.StrSliceRemove(roles, role) + + // Define a visitor function to find the specified target + getTargetVisitorFunc := func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} { + if tgt == nil { + return nil + } + // We found the target and validated path compatibility in our walk, + // so we should stop our walk and set the resultMeta and resultRoleName variables + if resultMeta, foundTarget = tgt.Signed.Targets[name]; foundTarget { + resultRoleName = validRole.Name + return tuf.StopWalk{} + } + return nil + } + err = r.tufRepo.WalkTargets(name, role, getTargetVisitorFunc, skipRoles...) + // Check that we didn't error, and that we assigned to our target + if err == nil && foundTarget { + return &TargetWithRole{Target: Target{Name: name, Hashes: resultMeta.Hashes, Length: resultMeta.Length}, Role: resultRoleName}, nil } } return nil, fmt.Errorf("No trust data for %s", name) @@ -532,45 +470,6 @@ func (r *NotaryRepository) GetChangelist() (changelist.Changelist, error) { return cl, nil } -// GetDelegationRoles returns the keys and roles of the repository's delegations -func (r *NotaryRepository) GetDelegationRoles() ([]*data.Role, error) { - // Update state of the repo to latest - if _, err := r.Update(false); err != nil { - return nil, err - } - - // All top level delegations (ex: targets/level1) are stored exclusively in targets.json - targets, ok := r.tufRepo.Targets[data.CanonicalTargetsRole] - if !ok { - return nil, store.ErrMetaNotFound{Resource: data.CanonicalTargetsRole} - } - - allDelegations := targets.Signed.Delegations.Roles - - // make a copy for traversing nested delegations - delegationsList := make([]*data.Role, len(allDelegations)) - copy(delegationsList, allDelegations) - - // Now traverse to lower level delegations (ex: targets/level1/level2) - for len(delegationsList) > 0 { - // Pop off first delegation to traverse - delegation := delegationsList[0] - delegationsList = delegationsList[1:] - - // Get metadata - delegationMeta, ok := r.tufRepo.Targets[delegation.Name] - // If we get an error, don't try to traverse further into this subtree because it doesn't exist or is malformed - if !ok { - continue - } - - // Add nested delegations to return list and exploration list - allDelegations = append(allDelegations, delegationMeta.Signed.Delegations.Roles...) - delegationsList = append(delegationsList, delegationMeta.Signed.Delegations.Roles...) - } - return allDelegations, nil -} - // RoleWithSignatures is a Role with its associated signatures type RoleWithSignatures struct { Signatures []data.Signature @@ -604,7 +503,7 @@ func (r *NotaryRepository) ListRoles() ([]RoleWithSignatures, error) { case data.CanonicalTimestampRole: roleWithSig.Signatures = r.tufRepo.Timestamp.Signatures default: - // If the role isn't a delegation, we should error -- this is only possible if we have invalid keyDB state + // If the role isn't a delegation, we should error -- this is only possible if we have invalid state if !data.IsDelegation(role.Name) { return nil, data.ErrInvalidRole{Role: role.Name, Reason: "invalid role name"} } @@ -705,7 +604,7 @@ func (r *NotaryRepository) Publish() error { r.tufRepo, data.CanonicalSnapshotRole) if err == nil { - // Only update the snapshot if we've sucessfully signed it. + // Only update the snapshot if we've successfully signed it. updatedFiles[data.CanonicalSnapshotRole] = snapshotJSON } else if _, ok := err.(signed.ErrNoKeys); ok { // If signing fails due to us not having the snapshot key, then @@ -743,11 +642,10 @@ func (r *NotaryRepository) Publish() error { // This can also be unified with some cache reading tools from tuf/client. // This assumes that bootstrapRepo is only used by Publish() func (r *NotaryRepository) bootstrapRepo() error { - kdb := keys.NewDB() - tufRepo := tuf.NewRepo(kdb, r.CryptoService) + tufRepo := tuf.NewRepo(r.CryptoService) logrus.Debugf("Loading trusted collection.") - rootJSON, err := r.fileStore.GetMeta("root", 0) + rootJSON, err := r.fileStore.GetMeta("root", -1) if err != nil { return err } @@ -760,7 +658,7 @@ func (r *NotaryRepository) bootstrapRepo() error { if err != nil { return err } - targetsJSON, err := r.fileStore.GetMeta("targets", 0) + targetsJSON, err := r.fileStore.GetMeta("targets", -1) if err != nil { return err } @@ -771,7 +669,7 @@ func (r *NotaryRepository) bootstrapRepo() error { } tufRepo.SetTargets("targets", targets) - snapshotJSON, err := r.fileStore.GetMeta("snapshot", 0) + snapshotJSON, err := r.fileStore.GetMeta("snapshot", -1) if err == nil { snapshot := &data.SignedSnapshot{} err = json.Unmarshal(snapshotJSON, snapshot) @@ -854,7 +752,10 @@ func (r *NotaryRepository) Update(forWrite bool) (*tufclient.Client, error) { } err = c.Update() if err != nil { - if notFound, ok := err.(store.ErrMetaNotFound); ok && notFound.Resource == data.CanonicalRootRole { + // notFound.Resource may include a checksum so when the role is root, + // it will be root.json or root..json. Therefore best we can + // do it match a "root." prefix + if notFound, ok := err.(store.ErrMetaNotFound); ok && strings.HasPrefix(notFound.Resource, data.CanonicalRootRole+".") { return nil, r.errRepositoryNotExist() } return nil, err @@ -876,7 +777,7 @@ func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (*tufclient.Cl // try to read root from cache first. We will trust this root // until we detect a problem during update which will cause // us to download a new root and perform a rotation. - rootJSON, cachedRootErr := r.fileStore.GetMeta("root", maxSize) + rootJSON, cachedRootErr := r.fileStore.GetMeta("root", -1) if cachedRootErr == nil { signedRoot, cachedRootErr = r.validateRoot(rootJSON) @@ -890,7 +791,8 @@ func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (*tufclient.Cl // checking for initialization of the repo). // if remote store successfully set up, try and get root from remote - tmpJSON, err := remote.GetMeta("root", maxSize) + // We don't have any local data to determine the size of root, so try the maximum (though it is restricted at 100MB) + tmpJSON, err := remote.GetMeta("root", -1) if err != nil { // we didn't have a root in cache and were unable to load one from // the server. Nothing we can do but error. @@ -912,8 +814,7 @@ func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (*tufclient.Cl } } - kdb := keys.NewDB() - r.tufRepo = tuf.NewRepo(kdb, r.CryptoService) + r.tufRepo = tuf.NewRepo(r.CryptoService) if signedRoot == nil { return nil, ErrRepoNotInitialized{} @@ -927,7 +828,6 @@ func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (*tufclient.Cl return tufclient.NewClient( r.tufRepo, remote, - kdb, r.fileStore, ), nil } @@ -1020,7 +920,7 @@ func (r *NotaryRepository) DeleteTrustData() error { if err := r.fileStore.RemoveAll(); err != nil { return fmt.Errorf("error clearing TUF repo data: %v", err) } - r.tufRepo = tuf.NewRepo(nil, nil) + r.tufRepo = tuf.NewRepo(nil) // Clear certificates certificates, err := r.CertStore.GetCertificatesByCN(r.gun) if err != nil { diff --git a/components/engine/vendor/src/github.com/docker/notary/client/delegations.go b/components/engine/vendor/src/github.com/docker/notary/client/delegations.go new file mode 100644 index 0000000000..c28e3d8469 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/notary/client/delegations.go @@ -0,0 +1,294 @@ +package client + +import ( + "encoding/json" + "fmt" + "path/filepath" + + "github.com/Sirupsen/logrus" + "github.com/docker/notary" + "github.com/docker/notary/client/changelist" + "github.com/docker/notary/tuf/data" + "github.com/docker/notary/tuf/store" + "github.com/docker/notary/tuf/utils" +) + +// AddDelegation creates changelist entries to add provided delegation public keys and paths. +// This method composes AddDelegationRoleAndKeys and AddDelegationPaths (each creates one changelist if called). +func (r *NotaryRepository) AddDelegation(name string, delegationKeys []data.PublicKey, paths []string) error { + if len(delegationKeys) > 0 { + err := r.AddDelegationRoleAndKeys(name, delegationKeys) + if err != nil { + return err + } + } + if len(paths) > 0 { + err := r.AddDelegationPaths(name, paths) + if err != nil { + return err + } + } + return nil +} + +// AddDelegationRoleAndKeys creates a changelist entry to add provided delegation public keys. +// This method is the simplest way to create a new delegation, because the delegation must have at least +// one key upon creation to be valid since we will reject the changelist while validating the threshold. +func (r *NotaryRepository) AddDelegationRoleAndKeys(name string, delegationKeys []data.PublicKey) error { + + if !data.IsDelegation(name) { + return data.ErrInvalidRole{Role: name, Reason: "invalid delegation role name"} + } + + cl, err := changelist.NewFileChangelist(filepath.Join(r.tufRepoPath, "changelist")) + if err != nil { + return err + } + defer cl.Close() + + logrus.Debugf(`Adding delegation "%s" with threshold %d, and %d keys\n`, + name, notary.MinThreshold, len(delegationKeys)) + + // Defaulting to threshold of 1, since we don't allow for larger thresholds at the moment. + tdJSON, err := json.Marshal(&changelist.TufDelegation{ + NewThreshold: notary.MinThreshold, + AddKeys: data.KeyList(delegationKeys), + }) + if err != nil { + return err + } + + template := newCreateDelegationChange(name, tdJSON) + return addChange(cl, template, name) +} + +// AddDelegationPaths creates a changelist entry to add provided paths to an existing delegation. +// This method cannot create a new delegation itself because the role must meet the key threshold upon creation. +func (r *NotaryRepository) AddDelegationPaths(name string, paths []string) error { + + if !data.IsDelegation(name) { + return data.ErrInvalidRole{Role: name, Reason: "invalid delegation role name"} + } + + cl, err := changelist.NewFileChangelist(filepath.Join(r.tufRepoPath, "changelist")) + if err != nil { + return err + } + defer cl.Close() + + logrus.Debugf(`Adding %s paths to delegation %s\n`, paths, name) + + tdJSON, err := json.Marshal(&changelist.TufDelegation{ + AddPaths: paths, + }) + if err != nil { + return err + } + + template := newCreateDelegationChange(name, tdJSON) + return addChange(cl, template, name) +} + +// RemoveDelegationKeysAndPaths creates changelist entries to remove provided delegation key IDs and paths. +// This method composes RemoveDelegationPaths and RemoveDelegationKeys (each creates one changelist if called). +func (r *NotaryRepository) RemoveDelegationKeysAndPaths(name string, keyIDs, paths []string) error { + if len(paths) > 0 { + err := r.RemoveDelegationPaths(name, paths) + if err != nil { + return err + } + } + if len(keyIDs) > 0 { + err := r.RemoveDelegationKeys(name, keyIDs) + if err != nil { + return err + } + } + return nil +} + +// RemoveDelegationRole creates a changelist to remove all paths and keys from a role, and delete the role in its entirety. +func (r *NotaryRepository) RemoveDelegationRole(name string) error { + + if !data.IsDelegation(name) { + return data.ErrInvalidRole{Role: name, Reason: "invalid delegation role name"} + } + + cl, err := changelist.NewFileChangelist(filepath.Join(r.tufRepoPath, "changelist")) + if err != nil { + return err + } + defer cl.Close() + + logrus.Debugf(`Removing delegation "%s"\n`, name) + + template := newDeleteDelegationChange(name, nil) + return addChange(cl, template, name) +} + +// RemoveDelegationPaths creates a changelist entry to remove provided paths from an existing delegation. +func (r *NotaryRepository) RemoveDelegationPaths(name string, paths []string) error { + + if !data.IsDelegation(name) { + return data.ErrInvalidRole{Role: name, Reason: "invalid delegation role name"} + } + + cl, err := changelist.NewFileChangelist(filepath.Join(r.tufRepoPath, "changelist")) + if err != nil { + return err + } + defer cl.Close() + + logrus.Debugf(`Removing %s paths from delegation "%s"\n`, paths, name) + + tdJSON, err := json.Marshal(&changelist.TufDelegation{ + RemovePaths: paths, + }) + if err != nil { + return err + } + + template := newUpdateDelegationChange(name, tdJSON) + return addChange(cl, template, name) +} + +// RemoveDelegationKeys creates a changelist entry to remove provided keys from an existing delegation. +// When this changelist is applied, if the specified keys are the only keys left in the role, +// the role itself will be deleted in its entirety. +func (r *NotaryRepository) RemoveDelegationKeys(name string, keyIDs []string) error { + + if !data.IsDelegation(name) { + return data.ErrInvalidRole{Role: name, Reason: "invalid delegation role name"} + } + + cl, err := changelist.NewFileChangelist(filepath.Join(r.tufRepoPath, "changelist")) + if err != nil { + return err + } + defer cl.Close() + + logrus.Debugf(`Removing %s keys from delegation "%s"\n`, keyIDs, name) + + tdJSON, err := json.Marshal(&changelist.TufDelegation{ + RemoveKeys: keyIDs, + }) + if err != nil { + return err + } + + template := newUpdateDelegationChange(name, tdJSON) + return addChange(cl, template, name) +} + +// ClearDelegationPaths creates a changelist entry to remove all paths from an existing delegation. +func (r *NotaryRepository) ClearDelegationPaths(name string) error { + + if !data.IsDelegation(name) { + return data.ErrInvalidRole{Role: name, Reason: "invalid delegation role name"} + } + + cl, err := changelist.NewFileChangelist(filepath.Join(r.tufRepoPath, "changelist")) + if err != nil { + return err + } + defer cl.Close() + + logrus.Debugf(`Removing all paths from delegation "%s"\n`, name) + + tdJSON, err := json.Marshal(&changelist.TufDelegation{ + ClearAllPaths: true, + }) + if err != nil { + return err + } + + template := newUpdateDelegationChange(name, tdJSON) + return addChange(cl, template, name) +} + +func newUpdateDelegationChange(name string, content []byte) *changelist.TufChange { + return changelist.NewTufChange( + changelist.ActionUpdate, + name, + changelist.TypeTargetsDelegation, + "", // no path for delegations + content, + ) +} + +func newCreateDelegationChange(name string, content []byte) *changelist.TufChange { + return changelist.NewTufChange( + changelist.ActionCreate, + name, + changelist.TypeTargetsDelegation, + "", // no path for delegations + content, + ) +} + +func newDeleteDelegationChange(name string, content []byte) *changelist.TufChange { + return changelist.NewTufChange( + changelist.ActionDelete, + name, + changelist.TypeTargetsDelegation, + "", // no path for delegations + content, + ) +} + +// GetDelegationRoles returns the keys and roles of the repository's delegations +// Also converts key IDs to canonical key IDs to keep consistent with signing prompts +func (r *NotaryRepository) GetDelegationRoles() ([]*data.Role, error) { + // Update state of the repo to latest + if _, err := r.Update(false); err != nil { + return nil, err + } + + // All top level delegations (ex: targets/level1) are stored exclusively in targets.json + _, ok := r.tufRepo.Targets[data.CanonicalTargetsRole] + if !ok { + return nil, store.ErrMetaNotFound{Resource: data.CanonicalTargetsRole} + } + + // make a copy for traversing nested delegations + allDelegations := []*data.Role{} + + // Define a visitor function to populate the delegations list and translate their key IDs to canonical IDs + delegationCanonicalListVisitor := func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} { + // For the return list, update with a copy that includes canonicalKeyIDs + // These aren't validated by the validRole + canonicalDelegations, err := translateDelegationsToCanonicalIDs(tgt.Signed.Delegations) + if err != nil { + return err + } + allDelegations = append(allDelegations, canonicalDelegations...) + return nil + } + err := r.tufRepo.WalkTargets("", "", delegationCanonicalListVisitor) + if err != nil { + return nil, err + } + return allDelegations, nil +} + +func translateDelegationsToCanonicalIDs(delegationInfo data.Delegations) ([]*data.Role, error) { + canonicalDelegations := make([]*data.Role, len(delegationInfo.Roles)) + copy(canonicalDelegations, delegationInfo.Roles) + delegationKeys := delegationInfo.Keys + for i, delegation := range canonicalDelegations { + canonicalKeyIDs := []string{} + for _, keyID := range delegation.KeyIDs { + pubKey, ok := delegationKeys[keyID] + if !ok { + return nil, fmt.Errorf("Could not translate canonical key IDs for %s", delegation.Name) + } + canonicalKeyID, err := utils.CanonicalKeyID(pubKey) + if err != nil { + return nil, fmt.Errorf("Could not translate canonical key IDs for %s: %v", delegation.Name, err) + } + canonicalKeyIDs = append(canonicalKeyIDs, canonicalKeyID) + } + canonicalDelegations[i].KeyIDs = canonicalKeyIDs + } + return canonicalDelegations, nil +} diff --git a/components/engine/vendor/src/github.com/docker/notary/client/helpers.go b/components/engine/vendor/src/github.com/docker/notary/client/helpers.go index a9fd590a9f..46648a11b5 100644 --- a/components/engine/vendor/src/github.com/docker/notary/client/helpers.go +++ b/components/engine/vendor/src/github.com/docker/notary/client/helpers.go @@ -12,8 +12,8 @@ import ( "github.com/docker/notary/client/changelist" tuf "github.com/docker/notary/tuf" "github.com/docker/notary/tuf/data" - "github.com/docker/notary/tuf/keys" "github.com/docker/notary/tuf/store" + "github.com/docker/notary/tuf/utils" ) // Use this to initialize remote HTTPStores from the config settings @@ -22,7 +22,6 @@ func getRemoteStore(baseURL, gun string, rt http.RoundTripper) (store.RemoteStor baseURL+"/v2/"+gun+"/_trust/tuf/", "", "json", - "", "key", rt, ) @@ -80,53 +79,51 @@ func changeTargetsDelegation(repo *tuf.Repo, c changelist.Change) error { if err != nil { return err } - r, err := repo.GetDelegation(c.Scope()) - if _, ok := err.(data.ErrNoSuchRole); err != nil && !ok { - // error that wasn't ErrNoSuchRole - return err - } - if err == nil { - // role existed, attempt to merge paths and keys - if err := r.AddPaths(td.AddPaths); err != nil { - return err - } - return repo.UpdateDelegations(r, td.AddKeys) - } - // create brand new role - r, err = td.ToNewRole(c.Scope()) + + // Try to create brand new role or update one + // First add the keys, then the paths. We can only add keys and paths in this scenario + err = repo.UpdateDelegationKeys(c.Scope(), td.AddKeys, []string{}, td.NewThreshold) if err != nil { return err } - return repo.UpdateDelegations(r, td.AddKeys) + return repo.UpdateDelegationPaths(c.Scope(), td.AddPaths, []string{}, false) case changelist.ActionUpdate: td := changelist.TufDelegation{} err := json.Unmarshal(c.Content(), &td) if err != nil { return err } - r, err := repo.GetDelegation(c.Scope()) + delgRole, err := repo.GetDelegationRole(c.Scope()) if err != nil { return err } + + // We need to translate the keys from canonical ID to TUF ID for compatibility + canonicalToTUFID := make(map[string]string) + for tufID, pubKey := range delgRole.Keys { + canonicalID, err := utils.CanonicalKeyID(pubKey) + if err != nil { + return err + } + canonicalToTUFID[canonicalID] = tufID + } + + removeTUFKeyIDs := []string{} + for _, canonID := range td.RemoveKeys { + removeTUFKeyIDs = append(removeTUFKeyIDs, canonicalToTUFID[canonID]) + } + // If we specify the only keys left delete the role, else just delete specified keys - if strings.Join(r.KeyIDs, ";") == strings.Join(td.RemoveKeys, ";") && len(td.AddKeys) == 0 { - r := data.Role{Name: c.Scope()} - return repo.DeleteDelegation(r) + if strings.Join(delgRole.ListKeyIDs(), ";") == strings.Join(removeTUFKeyIDs, ";") && len(td.AddKeys) == 0 { + return repo.DeleteDelegation(c.Scope()) } - // if we aren't deleting and the role exists, merge - if err := r.AddPaths(td.AddPaths); err != nil { + err = repo.UpdateDelegationKeys(c.Scope(), td.AddKeys, removeTUFKeyIDs, td.NewThreshold) + if err != nil { return err } - if err := r.AddPathHashPrefixes(td.AddPathHashPrefixes); err != nil { - return err - } - r.RemoveKeys(td.RemoveKeys) - r.RemovePaths(td.RemovePaths) - r.RemovePathHashPrefixes(td.RemovePathHashPrefixes) - return repo.UpdateDelegations(r, td.AddKeys) + return repo.UpdateDelegationPaths(c.Scope(), td.AddPaths, td.RemovePaths, td.ClearAllPaths) case changelist.ActionDelete: - r := data.Role{Name: c.Scope()} - return repo.DeleteDelegation(r) + return repo.DeleteDelegation(c.Scope()) default: return fmt.Errorf("unsupported action against delegations: %s", c.Action()) } @@ -239,19 +236,6 @@ func getRemoteKey(url, gun, role string, rt http.RoundTripper) (data.PublicKey, return pubKey, nil } -// add a key to a KeyDB, and create a role for the key and add it. -func addKeyForRole(kdb *keys.KeyDB, role string, key data.PublicKey) error { - theRole, err := data.NewRole(role, 1, []string{key.ID()}, nil, nil) - if err != nil { - return err - } - kdb.AddKey(key) - if err := kdb.AddRole(theRole); err != nil { - return err - } - return nil -} - // signs and serializes the metadata for a canonical role in a tuf repo to JSON func serializeCanonicalRole(tufRepo *tuf.Repo, role string) (out []byte, err error) { var s *data.Signed diff --git a/components/engine/vendor/src/github.com/docker/notary/const.go b/components/engine/vendor/src/github.com/docker/notary/const.go index a1140c0dc0..566c7f0fbd 100644 --- a/components/engine/vendor/src/github.com/docker/notary/const.go +++ b/components/engine/vendor/src/github.com/docker/notary/const.go @@ -1,7 +1,15 @@ package notary +import ( + "time" +) + // application wide constants const ( + // MaxDownloadSize is the maximum size we'll download for metadata if no limit is given + MaxDownloadSize int64 = 100 << 20 + // MaxTimestampSize is the maximum size of timestamp metadata - 1MiB. + MaxTimestampSize int64 = 1 << 20 // MinRSABitSize is the minimum bit size for RSA keys allowed in notary MinRSABitSize = 2048 // MinThreshold requires a minimum of one threshold for roles; currently we do not support a higher threshold @@ -14,4 +22,29 @@ const ( Sha256HexSize = 64 // TrustedCertsDir is the directory, under the notary repo base directory, where trusted certs are stored TrustedCertsDir = "trusted_certificates" + // PrivDir is the directory, under the notary repo base directory, where private keys are stored + PrivDir = "private" + // RootKeysSubdir is the subdirectory under PrivDir where root private keys are stored + RootKeysSubdir = "root_keys" + // NonRootKeysSubdir is the subdirectory under PrivDir where non-root private keys are stored + NonRootKeysSubdir = "tuf_keys" + + // Day is a duration of one day + Day = 24 * time.Hour + Year = 365 * Day + + // NotaryRootExpiry is the duration representing the expiry time of the Root role + NotaryRootExpiry = 10 * Year + NotaryTargetsExpiry = 3 * Year + NotarySnapshotExpiry = 3 * Year + NotaryTimestampExpiry = 14 * Day ) + +// NotaryDefaultExpiries is the construct used to configure the default expiry times of +// the various role files. +var NotaryDefaultExpiries = map[string]time.Duration{ + "root": NotaryRootExpiry, + "targets": NotaryTargetsExpiry, + "snapshot": NotarySnapshotExpiry, + "timestamp": NotaryTimestampExpiry, +} diff --git a/components/engine/vendor/src/github.com/docker/notary/cryptoservice/import_export.go b/components/engine/vendor/src/github.com/docker/notary/cryptoservice/import_export.go index f1d454e151..c4b19944a4 100644 --- a/components/engine/vendor/src/github.com/docker/notary/cryptoservice/import_export.go +++ b/components/engine/vendor/src/github.com/docker/notary/cryptoservice/import_export.go @@ -11,8 +11,10 @@ import ( "path/filepath" "strings" + "github.com/docker/notary" "github.com/docker/notary/passphrase" "github.com/docker/notary/trustmanager" + "github.com/docker/notary/tuf/data" ) const zipMadeByUNIX = 3 << 8 @@ -31,14 +33,17 @@ var ( ErrNoKeysFoundForGUN = errors.New("no keys found for specified GUN") ) -// ExportRootKey exports the specified root key to an io.Writer in PEM format. +// ExportKey exports the specified private key to an io.Writer in PEM format. // The key's existing encryption is preserved. -func (cs *CryptoService) ExportRootKey(dest io.Writer, keyID string) error { +func (cs *CryptoService) ExportKey(dest io.Writer, keyID, role string) error { var ( pemBytes []byte err error ) + if role != data.CanonicalRootRole { + keyID = filepath.Join(cs.gun, keyID) + } for _, ks := range cs.keyStores { pemBytes, err = ks.ExportKey(keyID) if err != nil { @@ -59,9 +64,9 @@ func (cs *CryptoService) ExportRootKey(dest io.Writer, keyID string) error { return nil } -// ExportRootKeyReencrypt exports the specified root key to an io.Writer in +// ExportKeyReencrypt exports the specified private key to an io.Writer in // PEM format. The key is reencrypted with a new passphrase. -func (cs *CryptoService) ExportRootKeyReencrypt(dest io.Writer, keyID string, newPassphraseRetriever passphrase.Retriever) error { +func (cs *CryptoService) ExportKeyReencrypt(dest io.Writer, keyID string, newPassphraseRetriever passphrase.Retriever) error { privateKey, role, err := cs.GetPrivateKey(keyID) if err != nil { return err @@ -103,14 +108,41 @@ func (cs *CryptoService) ImportRootKey(source io.Reader) error { if err != nil { return err } + return cs.ImportRoleKey(pemBytes, data.CanonicalRootRole, nil) +} - if err = checkRootKeyIsEncrypted(pemBytes); err != nil { - return err +// ImportRoleKey imports a private key in PEM format key from a byte array +// It prompts for the key's passphrase to verify the data and to determine +// the key ID. +func (cs *CryptoService) ImportRoleKey(pemBytes []byte, role string, newPassphraseRetriever passphrase.Retriever) error { + var alias string + var err error + if role == data.CanonicalRootRole { + alias = role + if err = checkRootKeyIsEncrypted(pemBytes); err != nil { + return err + } + } else { + // Parse the private key to get the key ID so that we can import it to the correct location + privKey, err := trustmanager.ParsePEMPrivateKey(pemBytes, "") + if err != nil { + privKey, _, err = trustmanager.GetPasswdDecryptBytes(newPassphraseRetriever, pemBytes, role, string(role)) + if err != nil { + return err + } + } + // Since we're importing a non-root role, we need to pass the path as an alias + alias = filepath.Join(notary.NonRootKeysSubdir, cs.gun, privKey.ID()) + // We also need to ensure that the role is properly set in the PEM headers + pemBytes, err = trustmanager.KeyToPEM(privKey, role) + if err != nil { + return err + } } for _, ks := range cs.keyStores { // don't redeclare err, we want the value carried out of the loop - if err = ks.ImportKey(pemBytes, "root"); err == nil { + if err = ks.ImportKey(pemBytes, alias); err == nil { return nil //bail on the first keystore we import to } } diff --git a/components/engine/vendor/src/github.com/docker/notary/docker-compose.yml b/components/engine/vendor/src/github.com/docker/notary/docker-compose.yml index 17b5798fa2..4f8705f384 100644 --- a/components/engine/vendor/src/github.com/docker/notary/docker-compose.yml +++ b/components/engine/vendor/src/github.com/docker/notary/docker-compose.yml @@ -1,27 +1,34 @@ -notaryserver: +server: build: . dockerfile: server.Dockerfile links: - - notarymysql - - notarysigner - ports: - - "8080" - - "4443:4443" + - mysql + - signer + - signer:notarysigner environment: - - SERVICE_NAME=notary - command: -config=fixtures/server-config.json -notarysigner: - volumes: - - /dev/bus/usb/003/010:/dev/bus/usb/002/010 - - /var/run/pcscd/pcscd.comm:/var/run/pcscd/pcscd.comm + - SERVICE_NAME=notary_server + ports: + - "8080" + - "4443:4443" + entrypoint: /bin/bash + command: -c "./migrations/migrate.sh && notary-server -config=fixtures/server-config.json" +signer: build: . dockerfile: signer.Dockerfile links: - - notarymysql - command: -config=fixtures/signer-config.json -notarymysql: + - mysql + environment: + - SERVICE_NAME=notary_signer + entrypoint: /bin/bash + command: -c "./migrations/migrate.sh && notary-signer -config=fixtures/signer-config.json" +mysql: volumes: - - notarymysql:/var/lib/mysql - build: ./notarymysql/ + - ./notarymysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + - notary_data:/var/lib/mysql + image: mariadb:10.1.10 ports: - "3306:3306" + environment: + - TERM=dumb + - MYSQL_ALLOW_EMPTY_PASSWORD="true" + command: mysqld --innodb_file_per_table diff --git a/components/engine/vendor/src/github.com/docker/notary/notarymysql/LICENSE b/components/engine/vendor/src/github.com/docker/notary/notarymysql/LICENSE deleted file mode 100644 index c8476ac066..0000000000 --- a/components/engine/vendor/src/github.com/docker/notary/notarymysql/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Sameer Naik - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/components/engine/vendor/src/github.com/docker/notary/server.Dockerfile b/components/engine/vendor/src/github.com/docker/notary/server.Dockerfile index a2273bc523..693064d7b5 100644 --- a/components/engine/vendor/src/github.com/docker/notary/server.Dockerfile +++ b/components/engine/vendor/src/github.com/docker/notary/server.Dockerfile @@ -1,4 +1,5 @@ -FROM golang:1.5.1 +FROM golang:1.5.3 +MAINTAINER David Lawrence "david.lawrence@docker.com" RUN apt-get update && apt-get install -y \ libltdl-dev \ @@ -7,13 +8,20 @@ RUN apt-get update && apt-get install -y \ EXPOSE 4443 +# Install DB migration tool +RUN go get github.com/mattes/migrate + ENV NOTARYPKG github.com/docker/notary ENV GOPATH /go/src/${NOTARYPKG}/Godeps/_workspace:$GOPATH + + +# Copy the local repo to the expected go path COPY . /go/src/github.com/docker/notary WORKDIR /go/src/${NOTARYPKG} +# Install notary-server RUN go install \ -tags pkcs11 \ -ldflags "-w -X ${NOTARYPKG}/version.GitCommit=`git rev-parse --short HEAD` -X ${NOTARYPKG}/version.NotaryVersion=`cat NOTARY_VERSION`" \ diff --git a/components/engine/vendor/src/github.com/docker/notary/signer.Dockerfile b/components/engine/vendor/src/github.com/docker/notary/signer.Dockerfile index 3ff8523448..837273bac5 100644 --- a/components/engine/vendor/src/github.com/docker/notary/signer.Dockerfile +++ b/components/engine/vendor/src/github.com/docker/notary/signer.Dockerfile @@ -1,29 +1,20 @@ -FROM dockersecurity/golang-softhsm2 -MAINTAINER Diogo Monica "diogo@docker.com" +FROM golang:1.5.3 +MAINTAINER David Lawrence "david.lawrence@docker.com" -# CHANGE-ME: Default values for SoftHSM2 PIN and SOPIN, used to initialize the first token -ENV NOTARY_SIGNER_PIN="1234" -ENV SOPIN="1234" -ENV LIBDIR="/usr/local/lib/softhsm/" -ENV NOTARY_SIGNER_DEFAULT_ALIAS="timestamp_1" -ENV NOTARY_SIGNER_TIMESTAMP_1="testpassword" - -# Install openSC and dependencies RUN apt-get update && apt-get install -y \ libltdl-dev \ - libpcsclite-dev \ - opensc \ - usbutils \ --no-install-recommends \ && rm -rf /var/lib/apt/lists/* -# Initialize the SoftHSM2 token on slod 0, using PIN and SOPIN varaibles -RUN softhsm2-util --init-token --slot 0 --label "test_token" --pin $NOTARY_SIGNER_PIN --so-pin $SOPIN +EXPOSE 4444 + +# Install DB migration tool +RUN go get github.com/mattes/migrate ENV NOTARYPKG github.com/docker/notary ENV GOPATH /go/src/${NOTARYPKG}/Godeps/_workspace:$GOPATH - -EXPOSE 4444 +ENV NOTARY_SIGNER_DEFAULT_ALIAS="timestamp_1" +ENV NOTARY_SIGNER_TIMESTAMP_1="testpassword" # Copy the local repo to the expected go path COPY . /go/src/github.com/docker/notary @@ -36,6 +27,5 @@ RUN go install \ -ldflags "-w -X ${NOTARYPKG}/version.GitCommit=`git rev-parse --short HEAD` -X ${NOTARYPKG}/version.NotaryVersion=`cat NOTARY_VERSION`" \ ${NOTARYPKG}/cmd/notary-signer - ENTRYPOINT [ "notary-signer" ] CMD [ "-config=fixtures/signer-config-local.json" ] diff --git a/components/engine/vendor/src/github.com/docker/notary/trustmanager/keyfilestore.go b/components/engine/vendor/src/github.com/docker/notary/trustmanager/keyfilestore.go index 0f9d821327..0a6b8ff4f8 100644 --- a/components/engine/vendor/src/github.com/docker/notary/trustmanager/keyfilestore.go +++ b/components/engine/vendor/src/github.com/docker/notary/trustmanager/keyfilestore.go @@ -8,16 +8,11 @@ import ( "sync" "github.com/Sirupsen/logrus" + "github.com/docker/notary" "github.com/docker/notary/passphrase" "github.com/docker/notary/tuf/data" ) -const ( - rootKeysSubdir = "root_keys" - nonRootKeysSubdir = "tuf_keys" - privDir = "private" -) - // KeyFileStore persists and manages private keys on disk type KeyFileStore struct { sync.Mutex @@ -37,7 +32,7 @@ type KeyMemoryStore struct { // NewKeyFileStore returns a new KeyFileStore creating a private directory to // hold the keys. func NewKeyFileStore(baseDir string, passphraseRetriever passphrase.Retriever) (*KeyFileStore, error) { - baseDir = filepath.Join(baseDir, privDir) + baseDir = filepath.Join(baseDir, notary.PrivDir) fileStore, err := NewPrivateSimpleFileStore(baseDir, keyExtension) if err != nil { return nil, err @@ -242,10 +237,10 @@ func listKeys(s LimitedFileStore) map[string]string { for _, f := range s.ListFiles() { // Remove the prefix of the directory from the filename var keyIDFull string - if strings.HasPrefix(f, rootKeysSubdir+"/") { - keyIDFull = strings.TrimPrefix(f, rootKeysSubdir+"/") + if strings.HasPrefix(f, notary.RootKeysSubdir+"/") { + keyIDFull = strings.TrimPrefix(f, notary.RootKeysSubdir+"/") } else { - keyIDFull = strings.TrimPrefix(f, nonRootKeysSubdir+"/") + keyIDFull = strings.TrimPrefix(f, notary.NonRootKeysSubdir+"/") } keyIDFull = strings.TrimSpace(keyIDFull) @@ -302,9 +297,9 @@ func removeKey(s LimitedFileStore, cachedKeys map[string]*cachedKey, name string // Assumes 2 subdirectories, 1 containing root keys and 1 containing tuf keys func getSubdir(alias string) string { if alias == "root" { - return rootKeysSubdir + return notary.RootKeysSubdir } - return nonRootKeysSubdir + return notary.NonRootKeysSubdir } // Given a key ID, gets the bytes and alias belonging to that key if the key @@ -327,7 +322,7 @@ func getRawKey(s LimitedFileStore, name string) ([]byte, string, error) { return keyBytes, role, nil } -// GetPasswdDecryptBytes gets the password to decript the given pem bytes. +// GetPasswdDecryptBytes gets the password to decrypt the given pem bytes. // Returns the password and private key func GetPasswdDecryptBytes(passphraseRetriever passphrase.Retriever, pemBytes []byte, name, alias string) (data.PrivateKey, string, error) { var ( diff --git a/components/engine/vendor/src/github.com/docker/notary/trustmanager/x509utils.go b/components/engine/vendor/src/github.com/docker/notary/trustmanager/x509utils.go index f39ca8eb22..601b15b8a9 100644 --- a/components/engine/vendor/src/github.com/docker/notary/trustmanager/x509utils.go +++ b/components/engine/vendor/src/github.com/docker/notary/trustmanager/x509utils.go @@ -470,12 +470,17 @@ func KeyToPEM(privKey data.PrivateKey, role string) ([]byte, error) { return nil, err } - block := &pem.Block{ - Type: bt, - Headers: map[string]string{ + headers := map[string]string{} + if role != "" { + headers = map[string]string{ "role": role, - }, - Bytes: privKey.Private(), + } + } + + block := &pem.Block{ + Type: bt, + Headers: headers, + Bytes: privKey.Private(), } return pem.EncodeToMemory(block), nil @@ -509,6 +514,19 @@ func EncryptPrivateKey(key data.PrivateKey, role, passphrase string) ([]byte, er return pem.EncodeToMemory(encryptedPEMBlock), nil } +// ReadRoleFromPEM returns the value from the role PEM header, if it exists +func ReadRoleFromPEM(pemBytes []byte) string { + pemBlock, _ := pem.Decode(pemBytes) + if pemBlock.Headers == nil { + return "" + } + role, ok := pemBlock.Headers["role"] + if !ok { + return "" + } + return role +} + // CertToKey transforms a single input certificate into its corresponding // PublicKey func CertToKey(cert *x509.Certificate) data.PublicKey { diff --git a/components/engine/vendor/src/github.com/docker/notary/trustmanager/yubikey/yubikeystore.go b/components/engine/vendor/src/github.com/docker/notary/trustmanager/yubikey/yubikeystore.go index a10048367a..3e292a2e9d 100644 --- a/components/engine/vendor/src/github.com/docker/notary/trustmanager/yubikey/yubikeystore.go +++ b/components/engine/vendor/src/github.com/docker/notary/trustmanager/yubikey/yubikeystore.go @@ -765,15 +765,15 @@ func (s *YubiKeyStore) ExportKey(keyID string) ([]byte, error) { // ImportKey imports a root key into a Yubikey func (s *YubiKeyStore) ImportKey(pemBytes []byte, keyPath string) error { logrus.Debugf("Attempting to import: %s key inside of YubiKeyStore", keyPath) + if keyPath != data.CanonicalRootRole { + return fmt.Errorf("yubikey only supports storing root keys") + } privKey, _, err := trustmanager.GetPasswdDecryptBytes( s.passRetriever, pemBytes, "", "imported root") if err != nil { logrus.Debugf("Failed to get and retrieve a key from: %s", keyPath) return err } - if keyPath != data.CanonicalRootRole { - return fmt.Errorf("yubikey only supports storing root keys") - } _, err = s.addKey(privKey.ID(), "root", privKey) return err } diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/README.md b/components/engine/vendor/src/github.com/docker/notary/tuf/README.md index ac8d6d1132..00a342e81e 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/README.md +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/README.md @@ -29,7 +29,7 @@ however in attempting to add delegations I found I was making such significant changes that I could not maintain backwards compatibility without the code becoming overly convoluted. -Some features such as pluggable verifiers have alreayd been merged upstream to flynn/go-tuf +Some features such as pluggable verifiers have already been merged upstream to flynn/go-tuf and we are in discussion with [titanous](https://github.com/titanous) about working to merge the 2 implementations. This implementation retains the same 3 Clause BSD license present on diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/client/client.go b/components/engine/vendor/src/github.com/docker/notary/tuf/client/client.go index 0eaa8c87e7..51aededc10 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/client/client.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/client/client.go @@ -3,38 +3,31 @@ package client import ( "bytes" "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" - "io" "path" - "strings" "github.com/Sirupsen/logrus" + "github.com/docker/notary" tuf "github.com/docker/notary/tuf" "github.com/docker/notary/tuf/data" - "github.com/docker/notary/tuf/keys" "github.com/docker/notary/tuf/signed" "github.com/docker/notary/tuf/store" "github.com/docker/notary/tuf/utils" ) -const maxSize int64 = 5 << 20 - // Client is a usability wrapper around a raw TUF repo type Client struct { local *tuf.Repo remote store.RemoteStore - keysDB *keys.KeyDB cache store.MetadataStore } -// NewClient initialized a Client with the given repo, remote source of content, key database, and cache -func NewClient(local *tuf.Repo, remote store.RemoteStore, keysDB *keys.KeyDB, cache store.MetadataStore) *Client { +// NewClient initialized a Client with the given repo, remote source of content, and cache +func NewClient(local *tuf.Repo, remote store.RemoteStore, cache store.MetadataStore) *Client { return &Client{ local: local, remote: remote, - keysDB: keysDB, cache: cache, } } @@ -131,11 +124,15 @@ func (c Client) checkRoot() error { func (c *Client) downloadRoot() error { logrus.Debug("Downloading Root...") role := data.CanonicalRootRole - size := maxSize + // We can't read an exact size for the root metadata without risking getting stuck in the TUF update cycle + // since it's possible that downloading timestamp/snapshot metadata may fail due to a signature mismatch + var size int64 = -1 var expectedSha256 []byte if c.local.Snapshot != nil { - size = c.local.Snapshot.Signed.Meta[role].Length - expectedSha256 = c.local.Snapshot.Signed.Meta[role].Hashes["sha256"] + if prevRootMeta, ok := c.local.Snapshot.Signed.Meta[role]; ok { + size = prevRootMeta.Length + expectedSha256 = prevRootMeta.Hashes["sha256"] + } } // if we're bootstrapping we may not have a cached root, an @@ -178,6 +175,7 @@ func (c *Client) downloadRoot() error { var s *data.Signed var raw []byte if download { + // use consistent download if we have the checksum. raw, s, err = c.downloadSigned(role, size, expectedSha256) if err != nil { return err @@ -201,34 +199,45 @@ func (c *Client) downloadRoot() error { func (c Client) verifyRoot(role string, s *data.Signed, minVersion int) error { // this will confirm that the root has been signed by the old root role - // as c.keysDB contains the root keys we bootstrapped with. + // with the root keys we bootstrapped with. // Still need to determine if there has been a root key update and // confirm signature with new root key logrus.Debug("verifying root with existing keys") - err := signed.Verify(s, role, minVersion, c.keysDB) + rootRole, err := c.local.GetBaseRole(role) if err != nil { + logrus.Debug("no previous root role loaded") + return err + } + // Verify using the rootRole loaded from the known root.json + if err = signed.Verify(s, rootRole, minVersion); err != nil { logrus.Debug("root did not verify with existing keys") return err } - // This will cause keyDB to get updated, overwriting any keyIDs associated - // with the roles in root.json logrus.Debug("updating known root roles and keys") root, err := data.RootFromSigned(s) if err != nil { logrus.Error(err.Error()) return err } + // replace the existing root.json with the new one (just in memory, we + // have another validation step before we fully accept the new root) err = c.local.SetRoot(root) if err != nil { logrus.Error(err.Error()) return err } - // verify again now that the old keys have been replaced with the new keys. + // Verify the new root again having loaded the rootRole out of this new + // file (verifies self-referential integrity) // TODO(endophage): be more intelligent and only re-verify if we detect // there has been a change in root keys logrus.Debug("verifying root with updated keys") - err = signed.Verify(s, role, minVersion, c.keysDB) + rootRole, err = c.local.GetBaseRole(role) + if err != nil { + logrus.Debug("root role with new keys not loaded") + return err + } + err = signed.Verify(s, rootRole, minVersion) if err != nil { logrus.Debug("root did not verify with new keys") return err @@ -248,11 +257,11 @@ func (c *Client) downloadTimestamp() error { // we're interacting with the repo. This will result in the // version being 0 var ( - saveToCache bool - old *data.Signed - version = 0 + old *data.Signed + ts *data.SignedTimestamp + version = 0 ) - cachedTS, err := c.cache.GetMeta(role, maxSize) + cachedTS, err := c.cache.GetMeta(role, notary.MaxTimestampSize) if err == nil { cached := &data.Signed{} err := json.Unmarshal(cachedTS, cached) @@ -266,49 +275,56 @@ func (c *Client) downloadTimestamp() error { } // unlike root, targets and snapshot, always try and download timestamps // from remote, only using the cache one if we couldn't reach remote. - raw, s, err := c.downloadSigned(role, maxSize, nil) - if err != nil || len(raw) == 0 { - if old == nil { - if err == nil { - // couldn't retrieve data from server and don't have valid - // data in cache. - return store.ErrMetaNotFound{Resource: data.CanonicalTimestampRole} - } - return err + raw, s, err := c.downloadSigned(role, notary.MaxTimestampSize, nil) + if err == nil { + ts, err = c.verifyTimestamp(s, version) + if err == nil { + logrus.Debug("successfully verified downloaded timestamp") + c.cache.SetMeta(role, raw) + c.local.SetTimestamp(ts) + return nil } - logrus.Debug(err.Error()) - logrus.Warn("Error while downloading remote metadata, using cached timestamp - this might not be the latest version available remotely") - s = old - } else { - saveToCache = true } - err = signed.Verify(s, role, version, c.keysDB) - if err != nil { + if old == nil { + // couldn't retrieve valid data from server and don't have unmarshallable data in cache. + logrus.Debug("no cached timestamp available") return err } - logrus.Debug("successfully verified timestamp") - if saveToCache { - c.cache.SetMeta(role, raw) - } - ts, err := data.TimestampFromSigned(s) + logrus.Debug(err.Error()) + logrus.Warn("Error while downloading remote metadata, using cached timestamp - this might not be the latest version available remotely") + ts, err = c.verifyTimestamp(old, version) if err != nil { return err } + logrus.Debug("successfully verified cached timestamp") c.local.SetTimestamp(ts) return nil } +// verifies that a timestamp is valid, and returned the SignedTimestamp object to add to the tuf repo +func (c *Client) verifyTimestamp(s *data.Signed, minVersion int) (*data.SignedTimestamp, error) { + timestampRole, err := c.local.GetBaseRole(data.CanonicalTimestampRole) + if err != nil { + logrus.Debug("no timestamp role loaded") + return nil, err + } + if err := signed.Verify(s, timestampRole, minVersion); err != nil { + return nil, err + } + return data.TimestampFromSigned(s) +} + // downloadSnapshot is responsible for downloading the snapshot.json func (c *Client) downloadSnapshot() error { logrus.Debug("Downloading Snapshot...") role := data.CanonicalSnapshotRole if c.local.Timestamp == nil { - return ErrMissingMeta{role: "snapshot"} + return tuf.ErrNotLoaded{Role: data.CanonicalTimestampRole} } size := c.local.Timestamp.Signed.Meta[role].Length expectedSha256, ok := c.local.Timestamp.Signed.Meta[role].Hashes["sha256"] if !ok { - return ErrMissingMeta{role: "snapshot"} + return data.ErrMissingMeta{Role: "snapshot"} } var download bool @@ -350,7 +366,12 @@ func (c *Client) downloadSnapshot() error { s = old } - err = signed.Verify(s, role, version, c.keysDB) + snapshotRole, err := c.local.GetBaseRole(role) + if err != nil { + logrus.Debug("no snapshot role loaded") + return err + } + err = signed.Verify(s, snapshotRole, version) if err != nil { return err } @@ -382,18 +403,14 @@ func (c *Client) downloadTargets(role string) error { return err } if c.local.Snapshot == nil { - return ErrMissingMeta{role: role} + return tuf.ErrNotLoaded{Role: data.CanonicalSnapshotRole} } snap := c.local.Snapshot.Signed root := c.local.Root.Signed - r := c.keysDB.GetRole(role) - if r == nil { - return fmt.Errorf("Invalid role: %s", role) - } - keyIDs := r.KeyIDs - s, err := c.getTargetsFile(role, keyIDs, snap.Meta, root.ConsistentSnapshot, r.Threshold) + + s, err := c.getTargetsFile(role, snap.Meta, root.ConsistentSnapshot) if err != nil { - if _, ok := err.(ErrMissingMeta); ok && role != data.CanonicalTargetsRole { + if _, ok := err.(data.ErrMissingMeta); ok && role != data.CanonicalTargetsRole { // if the role meta hasn't been published, // that's ok, continue continue @@ -401,7 +418,7 @@ func (c *Client) downloadTargets(role string) error { logrus.Error("Error getting targets file:", err) return err } - t, err := data.TargetsFromSigned(s) + t, err := data.TargetsFromSigned(s, role) if err != nil { return err } @@ -412,14 +429,19 @@ func (c *Client) downloadTargets(role string) error { // push delegated roles contained in the targets file onto the stack for _, r := range t.Signed.Delegations.Roles { - stack.Push(r.Name) + if path.Dir(r.Name) == role { + // only load children that are direct 1st generation descendants + // of the role we've just downloaded + stack.Push(r.Name) + } } } return nil } func (c *Client) downloadSigned(role string, size int64, expectedSha256 []byte) ([]byte, *data.Signed, error) { - raw, err := c.remote.GetMeta(role, size) + rolePath := utils.ConsistentName(role, expectedSha256) + raw, err := c.remote.GetMeta(rolePath, size) if err != nil { return nil, nil, err } @@ -437,15 +459,15 @@ func (c *Client) downloadSigned(role string, size int64, expectedSha256 []byte) return raw, s, nil } -func (c Client) getTargetsFile(role string, keyIDs []string, snapshotMeta data.Files, consistent bool, threshold int) (*data.Signed, error) { +func (c Client) getTargetsFile(role string, snapshotMeta data.Files, consistent bool) (*data.Signed, error) { // require role exists in snapshots roleMeta, ok := snapshotMeta[role] if !ok { - return nil, ErrMissingMeta{role: role} + return nil, data.ErrMissingMeta{Role: role} } expectedSha256, ok := snapshotMeta[role].Hashes["sha256"] if !ok { - return nil, ErrMissingMeta{role: role} + return nil, data.ErrMissingMeta{Role: role} } // try to get meta file from content addressed cache @@ -464,7 +486,7 @@ func (c Client) getTargetsFile(role string, keyIDs []string, snapshotMeta data.F } err := json.Unmarshal(raw, old) if err == nil { - targ, err := data.TargetsFromSigned(old) + targ, err := data.TargetsFromSigned(old, role) if err == nil { version = targ.Signed.Version } else { @@ -478,11 +500,7 @@ func (c Client) getTargetsFile(role string, keyIDs []string, snapshotMeta data.F size := snapshotMeta[role].Length var s *data.Signed if download { - rolePath, err := c.RoleTargetsPath(role, hex.EncodeToString(expectedSha256), consistent) - if err != nil { - return nil, err - } - raw, s, err = c.downloadSigned(rolePath, size, expectedSha256) + raw, s, err = c.downloadSigned(role, size, expectedSha256) if err != nil { return nil, err } @@ -490,9 +508,22 @@ func (c Client) getTargetsFile(role string, keyIDs []string, snapshotMeta data.F logrus.Debug("using cached ", role) s = old } - - err = signed.Verify(s, role, version, c.keysDB) - if err != nil { + var targetOrDelgRole data.BaseRole + if data.IsDelegation(role) { + delgRole, err := c.local.GetDelegationRole(role) + if err != nil { + logrus.Debugf("no %s delegation role loaded", role) + return nil, err + } + targetOrDelgRole = delgRole.BaseRole + } else { + targetOrDelgRole, err = c.local.GetBaseRole(role) + if err != nil { + logrus.Debugf("no %s role loaded", role) + return nil, err + } + } + if err = signed.Verify(s, targetOrDelgRole, version); err != nil { return nil, err } logrus.Debugf("successfully verified %s", role) @@ -505,73 +536,3 @@ func (c Client) getTargetsFile(role string, keyIDs []string, snapshotMeta data.F } return s, nil } - -// RoleTargetsPath generates the appropriate HTTP URL for the targets file, -// based on whether the repo is marked as consistent. -func (c Client) RoleTargetsPath(role string, hashSha256 string, consistent bool) (string, error) { - if consistent { - // Use path instead of filepath since we refer to the TUF role directly instead of its target files - dir := path.Dir(role) - if strings.Contains(role, "/") { - lastSlashIdx := strings.LastIndex(role, "/") - role = role[lastSlashIdx+1:] - } - role = path.Join( - dir, - fmt.Sprintf("%s.%s.json", hashSha256, role), - ) - } - return role, nil -} - -// TargetMeta ensures the repo is up to date. It assumes downloadTargets -// has already downloaded all delegated roles -func (c Client) TargetMeta(role, path string, excludeRoles ...string) (*data.FileMeta, string) { - excl := make(map[string]bool) - for _, r := range excludeRoles { - excl[r] = true - } - - pathDigest := sha256.Sum256([]byte(path)) - pathHex := hex.EncodeToString(pathDigest[:]) - - // FIFO list of targets delegations to inspect for target - roles := []string{role} - var ( - meta *data.FileMeta - curr string - ) - for len(roles) > 0 { - // have to do these lines here because of order of execution in for statement - curr = roles[0] - roles = roles[1:] - - meta = c.local.TargetMeta(curr, path) - if meta != nil { - // we found the target! - return meta, curr - } - delegations := c.local.TargetDelegations(curr, path, pathHex) - for _, d := range delegations { - if !excl[d.Name] { - roles = append(roles, d.Name) - } - } - } - return meta, "" -} - -// DownloadTarget downloads the target to dst from the remote -func (c Client) DownloadTarget(dst io.Writer, path string, meta *data.FileMeta) error { - reader, err := c.remote.GetTarget(path) - if err != nil { - return err - } - defer reader.Close() - r := io.TeeReader( - io.LimitReader(reader, meta.Length), - dst, - ) - err = utils.ValidateTarget(r, meta) - return err -} diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/client/errors.go b/components/engine/vendor/src/github.com/docker/notary/tuf/client/errors.go index 037b3df00b..ad0555127e 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/client/errors.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/client/errors.go @@ -13,15 +13,6 @@ func (e ErrChecksumMismatch) Error() string { return fmt.Sprintf("tuf: checksum for %s did not match", e.role) } -// ErrMissingMeta - couldn't find the FileMeta object for a role or target -type ErrMissingMeta struct { - role string -} - -func (e ErrMissingMeta) Error() string { - return fmt.Sprintf("tuf: sha256 checksum required for %s", e.role) -} - // ErrCorruptedCache - local data is incorrect type ErrCorruptedCache struct { file string diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/data/errors.go b/components/engine/vendor/src/github.com/docker/notary/tuf/data/errors.go new file mode 100644 index 0000000000..7ff5814c8e --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/data/errors.go @@ -0,0 +1,22 @@ +package data + +import "fmt" + +// ErrInvalidMetadata is the error to be returned when metadata is invalid +type ErrInvalidMetadata struct { + role string + msg string +} + +func (e ErrInvalidMetadata) Error() string { + return fmt.Sprintf("%s type metadata invalid: %s", e.role, e.msg) +} + +// ErrMissingMeta - couldn't find the FileMeta object for a role or target +type ErrMissingMeta struct { + Role string +} + +func (e ErrMissingMeta) Error() string { + return fmt.Sprintf("tuf: sha256 checksum required for %s", e.Role) +} diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/data/keys.go b/components/engine/vendor/src/github.com/docker/notary/tuf/data/keys.go index 9f94d5552f..25df598c16 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/data/keys.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/data/keys.go @@ -46,7 +46,7 @@ type Keys map[string]PublicKey // UnmarshalJSON implements the json.Unmarshaller interface func (ks *Keys) UnmarshalJSON(data []byte) error { - parsed := make(map[string]tufKey) + parsed := make(map[string]TUFKey) err := json.Unmarshal(data, &parsed) if err != nil { return err @@ -64,7 +64,7 @@ type KeyList []PublicKey // UnmarshalJSON implements the json.Unmarshaller interface func (ks *KeyList) UnmarshalJSON(data []byte) error { - parsed := make([]tufKey, 0, 1) + parsed := make([]TUFKey, 0, 1) err := json.Unmarshal(data, &parsed) if err != nil { return err @@ -86,64 +86,64 @@ func (ks KeyList) IDs() []string { return keyIDs } -func typedPublicKey(tk tufKey) PublicKey { +func typedPublicKey(tk TUFKey) PublicKey { switch tk.Algorithm() { case ECDSAKey: - return &ECDSAPublicKey{tufKey: tk} + return &ECDSAPublicKey{TUFKey: tk} case ECDSAx509Key: - return &ECDSAx509PublicKey{tufKey: tk} + return &ECDSAx509PublicKey{TUFKey: tk} case RSAKey: - return &RSAPublicKey{tufKey: tk} + return &RSAPublicKey{TUFKey: tk} case RSAx509Key: - return &RSAx509PublicKey{tufKey: tk} + return &RSAx509PublicKey{TUFKey: tk} case ED25519Key: - return &ED25519PublicKey{tufKey: tk} + return &ED25519PublicKey{TUFKey: tk} } - return &UnknownPublicKey{tufKey: tk} + return &UnknownPublicKey{TUFKey: tk} } -func typedPrivateKey(tk tufKey) (PrivateKey, error) { +func typedPrivateKey(tk TUFKey) (PrivateKey, error) { private := tk.Value.Private tk.Value.Private = nil switch tk.Algorithm() { case ECDSAKey: return NewECDSAPrivateKey( &ECDSAPublicKey{ - tufKey: tk, + TUFKey: tk, }, private, ) case ECDSAx509Key: return NewECDSAPrivateKey( &ECDSAx509PublicKey{ - tufKey: tk, + TUFKey: tk, }, private, ) case RSAKey: return NewRSAPrivateKey( &RSAPublicKey{ - tufKey: tk, + TUFKey: tk, }, private, ) case RSAx509Key: return NewRSAPrivateKey( &RSAx509PublicKey{ - tufKey: tk, + TUFKey: tk, }, private, ) case ED25519Key: return NewED25519PrivateKey( ED25519PublicKey{ - tufKey: tk, + TUFKey: tk, }, private, ) } return &UnknownPrivateKey{ - tufKey: tk, + TUFKey: tk, privateKey: privateKey{private: private}, }, nil } @@ -151,7 +151,7 @@ func typedPrivateKey(tk tufKey) (PrivateKey, error) { // NewPublicKey creates a new, correctly typed PublicKey, using the // UnknownPublicKey catchall for unsupported ciphers func NewPublicKey(alg string, public []byte) PublicKey { - tk := tufKey{ + tk := TUFKey{ Type: alg, Value: KeyPair{ Public: public, @@ -163,7 +163,7 @@ func NewPublicKey(alg string, public []byte) PublicKey { // NewPrivateKey creates a new, correctly typed PrivateKey, using the // UnknownPrivateKey catchall for unsupported ciphers func NewPrivateKey(pubKey PublicKey, private []byte) (PrivateKey, error) { - tk := tufKey{ + tk := TUFKey{ Type: pubKey.Algorithm(), Value: KeyPair{ Public: pubKey.Public(), @@ -175,7 +175,7 @@ func NewPrivateKey(pubKey PublicKey, private []byte) (PrivateKey, error) { // UnmarshalPublicKey is used to parse individual public keys in JSON func UnmarshalPublicKey(data []byte) (PublicKey, error) { - var parsed tufKey + var parsed TUFKey err := json.Unmarshal(data, &parsed) if err != nil { return nil, err @@ -185,7 +185,7 @@ func UnmarshalPublicKey(data []byte) (PublicKey, error) { // UnmarshalPrivateKey is used to parse individual private keys in JSON func UnmarshalPrivateKey(data []byte) (PrivateKey, error) { - var parsed tufKey + var parsed TUFKey err := json.Unmarshal(data, &parsed) if err != nil { return nil, err @@ -193,26 +193,26 @@ func UnmarshalPrivateKey(data []byte) (PrivateKey, error) { return typedPrivateKey(parsed) } -// tufKey is the structure used for both public and private keys in TUF. +// TUFKey is the structure used for both public and private keys in TUF. // Normally it would make sense to use a different structures for public and // private keys, but that would change the key ID algorithm (since the canonical // JSON would be different). This structure should normally be accessed through // the PublicKey or PrivateKey interfaces. -type tufKey struct { +type TUFKey struct { id string Type string `json:"keytype"` Value KeyPair `json:"keyval"` } // Algorithm returns the algorithm of the key -func (k tufKey) Algorithm() string { +func (k TUFKey) Algorithm() string { return k.Type } // ID efficiently generates if necessary, and caches the ID of the key -func (k *tufKey) ID() string { +func (k *TUFKey) ID() string { if k.id == "" { - pubK := tufKey{ + pubK := TUFKey{ Type: k.Algorithm(), Value: KeyPair{ Public: k.Public(), @@ -230,7 +230,7 @@ func (k *tufKey) ID() string { } // Public returns the public bytes -func (k tufKey) Public() []byte { +func (k TUFKey) Public() []byte { return k.Value.Public } @@ -239,42 +239,42 @@ func (k tufKey) Public() []byte { // ECDSAPublicKey represents an ECDSA key using a raw serialization // of the public key type ECDSAPublicKey struct { - tufKey + TUFKey } // ECDSAx509PublicKey represents an ECDSA key using an x509 cert // as the serialized format of the public key type ECDSAx509PublicKey struct { - tufKey + TUFKey } // RSAPublicKey represents an RSA key using a raw serialization // of the public key type RSAPublicKey struct { - tufKey + TUFKey } // RSAx509PublicKey represents an RSA key using an x509 cert // as the serialized format of the public key type RSAx509PublicKey struct { - tufKey + TUFKey } // ED25519PublicKey represents an ED25519 key using a raw serialization // of the public key type ED25519PublicKey struct { - tufKey + TUFKey } // UnknownPublicKey is a catchall for key types that are not supported type UnknownPublicKey struct { - tufKey + TUFKey } // NewECDSAPublicKey initializes a new public key with the ECDSAKey type func NewECDSAPublicKey(public []byte) *ECDSAPublicKey { return &ECDSAPublicKey{ - tufKey: tufKey{ + TUFKey: TUFKey{ Type: ECDSAKey, Value: KeyPair{ Public: public, @@ -287,7 +287,7 @@ func NewECDSAPublicKey(public []byte) *ECDSAPublicKey { // NewECDSAx509PublicKey initializes a new public key with the ECDSAx509Key type func NewECDSAx509PublicKey(public []byte) *ECDSAx509PublicKey { return &ECDSAx509PublicKey{ - tufKey: tufKey{ + TUFKey: TUFKey{ Type: ECDSAx509Key, Value: KeyPair{ Public: public, @@ -300,7 +300,7 @@ func NewECDSAx509PublicKey(public []byte) *ECDSAx509PublicKey { // NewRSAPublicKey initializes a new public key with the RSA type func NewRSAPublicKey(public []byte) *RSAPublicKey { return &RSAPublicKey{ - tufKey: tufKey{ + TUFKey: TUFKey{ Type: RSAKey, Value: KeyPair{ Public: public, @@ -313,7 +313,7 @@ func NewRSAPublicKey(public []byte) *RSAPublicKey { // NewRSAx509PublicKey initializes a new public key with the RSAx509Key type func NewRSAx509PublicKey(public []byte) *RSAx509PublicKey { return &RSAx509PublicKey{ - tufKey: tufKey{ + TUFKey: TUFKey{ Type: RSAx509Key, Value: KeyPair{ Public: public, @@ -326,7 +326,7 @@ func NewRSAx509PublicKey(public []byte) *RSAx509PublicKey { // NewED25519PublicKey initializes a new public key with the ED25519Key type func NewED25519PublicKey(public []byte) *ED25519PublicKey { return &ED25519PublicKey{ - tufKey: tufKey{ + TUFKey: TUFKey{ Type: ED25519Key, Value: KeyPair{ Public: public, @@ -367,7 +367,7 @@ type ED25519PrivateKey struct { // UnknownPrivateKey is a catchall for unsupported key types type UnknownPrivateKey struct { - tufKey + TUFKey privateKey } @@ -515,10 +515,10 @@ func (k UnknownPrivateKey) SignatureAlgorithm() SigAlgorithm { return "" } -// PublicKeyFromPrivate returns a new tufKey based on a private key, with +// PublicKeyFromPrivate returns a new TUFKey based on a private key, with // the private key bytes guaranteed to be nil. func PublicKeyFromPrivate(pk PrivateKey) PublicKey { - return typedPublicKey(tufKey{ + return typedPublicKey(TUFKey{ Type: pk.Algorithm(), Value: KeyPair{ Public: pk.Public(), diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/data/roles.go b/components/engine/vendor/src/github.com/docker/notary/tuf/data/roles.go index a505c92304..b1a2988bd0 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/data/roles.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/data/roles.go @@ -2,10 +2,11 @@ package data import ( "fmt" - "github.com/Sirupsen/logrus" "path" "regexp" "strings" + + "github.com/Sirupsen/logrus" ) // Canonical base role names @@ -85,32 +86,139 @@ func IsDelegation(role string) bool { isClean } +// BaseRole is an internal representation of a root/targets/snapshot/timestamp role, with its public keys included +type BaseRole struct { + Keys map[string]PublicKey + Name string + Threshold int +} + +// NewBaseRole creates a new BaseRole object with the provided parameters +func NewBaseRole(name string, threshold int, keys ...PublicKey) BaseRole { + r := BaseRole{ + Name: name, + Threshold: threshold, + Keys: make(map[string]PublicKey), + } + for _, k := range keys { + r.Keys[k.ID()] = k + } + return r +} + +// ListKeys retrieves the public keys valid for this role +func (b BaseRole) ListKeys() KeyList { + return listKeys(b.Keys) +} + +// ListKeyIDs retrieves the list of key IDs valid for this role +func (b BaseRole) ListKeyIDs() []string { + return listKeyIDs(b.Keys) +} + +// DelegationRole is an internal representation of a delegation role, with its public keys included +type DelegationRole struct { + BaseRole + Paths []string +} + +func listKeys(keyMap map[string]PublicKey) KeyList { + keys := KeyList{} + for _, key := range keyMap { + keys = append(keys, key) + } + return keys +} + +func listKeyIDs(keyMap map[string]PublicKey) []string { + keyIDs := []string{} + for id := range keyMap { + keyIDs = append(keyIDs, id) + } + return keyIDs +} + +// Restrict restricts the paths and path hash prefixes for the passed in delegation role, +// returning a copy of the role with validated paths as if it was a direct child +func (d DelegationRole) Restrict(child DelegationRole) (DelegationRole, error) { + if !d.IsParentOf(child) { + return DelegationRole{}, fmt.Errorf("%s is not a parent of %s", d.Name, child.Name) + } + return DelegationRole{ + BaseRole: BaseRole{ + Keys: child.Keys, + Name: child.Name, + Threshold: child.Threshold, + }, + Paths: RestrictDelegationPathPrefixes(d.Paths, child.Paths), + }, nil +} + +// IsParentOf returns whether the passed in delegation role is the direct child of this role, +// determined by delegation name. +// Ex: targets/a is a direct parent of targets/a/b, but targets/a is not a direct parent of targets/a/b/c +func (d DelegationRole) IsParentOf(child DelegationRole) bool { + return path.Dir(child.Name) == d.Name +} + +// CheckPaths checks if a given path is valid for the role +func (d DelegationRole) CheckPaths(path string) bool { + return checkPaths(path, d.Paths) +} + +func checkPaths(path string, permitted []string) bool { + for _, p := range permitted { + if strings.HasPrefix(path, p) { + return true + } + } + return false +} + +// RestrictDelegationPathPrefixes returns the list of valid delegationPaths that are prefixed by parentPaths +func RestrictDelegationPathPrefixes(parentPaths, delegationPaths []string) []string { + validPaths := []string{} + if len(delegationPaths) == 0 { + return validPaths + } + + // Validate each individual delegation path + for _, delgPath := range delegationPaths { + isPrefixed := false + for _, parentPath := range parentPaths { + if strings.HasPrefix(delgPath, parentPath) { + isPrefixed = true + break + } + } + // If the delegation path did not match prefix against any parent path, it is not valid + if isPrefixed { + validPaths = append(validPaths, delgPath) + } + } + return validPaths +} + // RootRole is a cut down role as it appears in the root.json +// Eventually should only be used for immediately before and after serialization/deserialization type RootRole struct { KeyIDs []string `json:"keyids"` Threshold int `json:"threshold"` } // Role is a more verbose role as they appear in targets delegations +// Eventually should only be used for immediately before and after serialization/deserialization type Role struct { RootRole - Name string `json:"name"` - Paths []string `json:"paths,omitempty"` - PathHashPrefixes []string `json:"path_hash_prefixes,omitempty"` - Email string `json:"email,omitempty"` + Name string `json:"name"` + Paths []string `json:"paths,omitempty"` } // NewRole creates a new Role object from the given parameters -func NewRole(name string, threshold int, keyIDs, paths, pathHashPrefixes []string) (*Role, error) { - if len(paths) > 0 && len(pathHashPrefixes) > 0 { - return nil, ErrInvalidRole{ - Role: name, - Reason: "roles may not have both Paths and PathHashPrefixes", - } - } +func NewRole(name string, threshold int, keyIDs, paths []string) (*Role, error) { if IsDelegation(name) { - if len(paths) == 0 && len(pathHashPrefixes) == 0 { - logrus.Debugf("role %s with no Paths and no PathHashPrefixes will never be able to publish content until one or more are added", name) + if len(paths) == 0 { + logrus.Debugf("role %s with no Paths will never be able to publish content until one or more are added", name) } } if threshold < 1 { @@ -124,52 +232,15 @@ func NewRole(name string, threshold int, keyIDs, paths, pathHashPrefixes []strin KeyIDs: keyIDs, Threshold: threshold, }, - Name: name, - Paths: paths, - PathHashPrefixes: pathHashPrefixes, + Name: name, + Paths: paths, }, nil } -// IsValid checks if the role has defined both paths and path hash prefixes, -// having both is invalid -func (r Role) IsValid() bool { - return !(len(r.Paths) > 0 && len(r.PathHashPrefixes) > 0) -} - -// ValidKey checks if the given id is a recognized signing key for the role -func (r Role) ValidKey(id string) bool { - for _, key := range r.KeyIDs { - if key == id { - return true - } - } - return false -} - // CheckPaths checks if a given path is valid for the role func (r Role) CheckPaths(path string) bool { - for _, p := range r.Paths { - if strings.HasPrefix(path, p) { - return true - } - } - return false -} - -// CheckPrefixes checks if a given hash matches the prefixes for the role -func (r Role) CheckPrefixes(hash string) bool { - for _, p := range r.PathHashPrefixes { - if strings.HasPrefix(hash, p) { - return true - } - } - return false -} - -// IsDelegation checks if the role is a delegation or a root role -func (r Role) IsDelegation() bool { - return IsDelegation(r.Name) + return checkPaths(path, r.Paths) } // AddKeys merges the ids into the current list of role key ids @@ -182,25 +253,10 @@ func (r *Role) AddPaths(paths []string) error { if len(paths) == 0 { return nil } - if len(r.PathHashPrefixes) > 0 { - return ErrInvalidRole{Role: r.Name, Reason: "attempted to add paths to role that already has hash prefixes"} - } r.Paths = mergeStrSlices(r.Paths, paths) return nil } -// AddPathHashPrefixes merges the prefixes into the list of role path hash prefixes -func (r *Role) AddPathHashPrefixes(prefixes []string) error { - if len(prefixes) == 0 { - return nil - } - if len(r.Paths) > 0 { - return ErrInvalidRole{Role: r.Name, Reason: "attempted to add hash prefixes to role that already has paths"} - } - r.PathHashPrefixes = mergeStrSlices(r.PathHashPrefixes, prefixes) - return nil -} - // RemoveKeys removes the ids from the current list of key ids func (r *Role) RemoveKeys(ids []string) { r.KeyIDs = subtractStrSlices(r.KeyIDs, ids) @@ -211,11 +267,6 @@ func (r *Role) RemovePaths(paths []string) { r.Paths = subtractStrSlices(r.Paths, paths) } -// RemovePathHashPrefixes removes the prefixes from the current list of path hash prefixes -func (r *Role) RemovePathHashPrefixes(prefixes []string) { - r.PathHashPrefixes = subtractStrSlices(r.PathHashPrefixes, prefixes) -} - func mergeStrSlices(orig, new []string) []string { have := make(map[string]bool) for _, e := range orig { diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/data/root.go b/components/engine/vendor/src/github.com/docker/notary/tuf/data/root.go index bd479206fd..3a9a1b1dec 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/data/root.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/data/root.go @@ -1,6 +1,7 @@ package data import ( + "fmt" "time" "github.com/docker/go/canonical/json" @@ -23,14 +24,57 @@ type Root struct { ConsistentSnapshot bool `json:"consistent_snapshot"` } +// isValidRootStructure returns an error, or nil, depending on whether the content of the struct +// is valid for root metadata. This does not check signatures or expiry, just that +// the metadata content is valid. +func isValidRootStructure(r Root) error { + expectedType := TUFTypes[CanonicalRootRole] + if r.Type != expectedType { + return ErrInvalidMetadata{ + role: CanonicalRootRole, msg: fmt.Sprintf("expected type %s, not %s", expectedType, r.Type)} + } + + // all the base roles MUST appear in the root.json - other roles are allowed, + // but other than the mirror role (not currently supported) are out of spec + for _, roleName := range BaseRoles { + roleObj, ok := r.Roles[roleName] + if !ok || roleObj == nil { + return ErrInvalidMetadata{ + role: CanonicalRootRole, msg: fmt.Sprintf("missing %s role specification", roleName)} + } + if err := isValidRootRoleStructure(CanonicalRootRole, roleName, *roleObj, r.Keys); err != nil { + return err + } + } + return nil +} + +func isValidRootRoleStructure(metaContainingRole, rootRoleName string, r RootRole, validKeys Keys) error { + if r.Threshold < 1 { + return ErrInvalidMetadata{ + role: metaContainingRole, + msg: fmt.Sprintf("invalid threshold specified for %s: %v ", rootRoleName, r.Threshold), + } + } + for _, keyID := range r.KeyIDs { + if _, ok := validKeys[keyID]; !ok { + return ErrInvalidMetadata{ + role: metaContainingRole, + msg: fmt.Sprintf("key ID %s specified in %s without corresponding key", keyID, rootRoleName), + } + } + } + return nil +} + // NewRoot initializes a new SignedRoot with a set of keys, roles, and the consistent flag func NewRoot(keys map[string]PublicKey, roles map[string]*RootRole, consistent bool) (*SignedRoot, error) { signedRoot := &SignedRoot{ Signatures: make([]Signature, 0), Signed: Root{ - Type: TUFTypes["root"], + Type: TUFTypes[CanonicalRootRole], Version: 0, - Expires: DefaultExpires("root"), + Expires: DefaultExpires(CanonicalRootRole), Keys: keys, Roles: roles, ConsistentSnapshot: consistent, @@ -41,6 +85,34 @@ func NewRoot(keys map[string]PublicKey, roles map[string]*RootRole, consistent b return signedRoot, nil } +// BuildBaseRole returns a copy of a BaseRole using the information in this SignedRoot for the specified role name. +// Will error for invalid role name or key metadata within this SignedRoot +func (r SignedRoot) BuildBaseRole(roleName string) (BaseRole, error) { + roleData, ok := r.Signed.Roles[roleName] + if !ok { + return BaseRole{}, ErrInvalidRole{Role: roleName, Reason: "role not found in root file"} + } + // Get all public keys for the base role from TUF metadata + keyIDs := roleData.KeyIDs + pubKeys := make(map[string]PublicKey) + for _, keyID := range keyIDs { + pubKey, ok := r.Signed.Keys[keyID] + if !ok { + return BaseRole{}, ErrInvalidRole{ + Role: roleName, + Reason: fmt.Sprintf("key with ID %s was not found in root metadata", keyID), + } + } + pubKeys[keyID] = pubKey + } + + return BaseRole{ + Name: roleName, + Keys: pubKeys, + Threshold: roleData.Threshold, + }, nil +} + // ToSigned partially serializes a SignedRoot for further signing func (r SignedRoot) ToSigned() (*Signed, error) { s, err := defaultSerializer.MarshalCanonical(r.Signed) @@ -70,11 +142,14 @@ func (r SignedRoot) MarshalJSON() ([]byte, error) { return defaultSerializer.Marshal(signed) } -// RootFromSigned fully unpacks a Signed object into a SignedRoot +// RootFromSigned fully unpacks a Signed object into a SignedRoot and ensures +// that it is a valid SignedRoot func RootFromSigned(s *Signed) (*SignedRoot, error) { r := Root{} - err := json.Unmarshal(s.Signed, &r) - if err != nil { + if err := defaultSerializer.Unmarshal(s.Signed, &r); err != nil { + return nil, err + } + if err := isValidRootStructure(r); err != nil { return nil, err } sigs := make([]Signature, len(s.Signatures)) diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/data/snapshot.go b/components/engine/vendor/src/github.com/docker/notary/tuf/data/snapshot.go index f13951ca83..9637dfff86 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/data/snapshot.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/data/snapshot.go @@ -2,6 +2,8 @@ package data import ( "bytes" + "crypto/sha256" + "fmt" "time" "github.com/Sirupsen/logrus" @@ -23,6 +25,30 @@ type Snapshot struct { Meta Files `json:"meta"` } +// isValidSnapshotStructure returns an error, or nil, depending on whether the content of the +// struct is valid for snapshot metadata. This does not check signatures or expiry, just that +// the metadata content is valid. +func isValidSnapshotStructure(s Snapshot) error { + expectedType := TUFTypes[CanonicalSnapshotRole] + if s.Type != expectedType { + return ErrInvalidMetadata{ + role: CanonicalSnapshotRole, msg: fmt.Sprintf("expected type %s, not %s", expectedType, s.Type)} + } + + for _, role := range []string{CanonicalRootRole, CanonicalTargetsRole} { + // Meta is a map of FileMeta, so if the role isn't in the map it returns + // an empty FileMeta, which has an empty map, and you can check on keys + // from an empty map. + if checksum, ok := s.Meta[role].Hashes["sha256"]; !ok || len(checksum) != sha256.Size { + return ErrInvalidMetadata{ + role: CanonicalSnapshotRole, + msg: fmt.Sprintf("missing or invalid %s sha256 checksum information", role), + } + } + } + return nil +} + // NewSnapshot initilizes a SignedSnapshot with a given top level root // and targets objects func NewSnapshot(root *Signed, targets *Signed) (*SignedSnapshot, error) { @@ -64,8 +90,8 @@ func (sp *SignedSnapshot) hashForRole(role string) []byte { } // ToSigned partially serializes a SignedSnapshot for further signing -func (sp SignedSnapshot) ToSigned() (*Signed, error) { - s, err := json.MarshalCanonical(sp.Signed) +func (sp *SignedSnapshot) ToSigned() (*Signed, error) { + s, err := defaultSerializer.MarshalCanonical(sp.Signed) if err != nil { return nil, err } @@ -88,6 +114,15 @@ func (sp *SignedSnapshot) AddMeta(role string, meta FileMeta) { sp.Dirty = true } +// GetMeta gets the metadata for a particular role, returning an error if it's +// not found +func (sp *SignedSnapshot) GetMeta(role string) (*FileMeta, error) { + if meta, ok := sp.Signed.Meta[role]; ok { + return &meta, nil + } + return nil, ErrMissingMeta{Role: role} +} + // DeleteMeta removes a role from the snapshot. If the role doesn't // exist in the snapshot, it's a noop. func (sp *SignedSnapshot) DeleteMeta(role string) { @@ -97,11 +132,22 @@ func (sp *SignedSnapshot) DeleteMeta(role string) { } } +// MarshalJSON returns the serialized form of SignedSnapshot as bytes +func (sp *SignedSnapshot) MarshalJSON() ([]byte, error) { + signed, err := sp.ToSigned() + if err != nil { + return nil, err + } + return defaultSerializer.Marshal(signed) +} + // SnapshotFromSigned fully unpacks a Signed object into a SignedSnapshot func SnapshotFromSigned(s *Signed) (*SignedSnapshot, error) { sp := Snapshot{} - err := json.Unmarshal(s.Signed, &sp) - if err != nil { + if err := defaultSerializer.Unmarshal(s.Signed, &sp); err != nil { + return nil, err + } + if err := isValidSnapshotStructure(sp); err != nil { return nil, err } sigs := make([]Signature, len(s.Signatures)) diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/data/targets.go b/components/engine/vendor/src/github.com/docker/notary/tuf/data/targets.go index a538d6afa5..fce4d177e0 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/data/targets.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/data/targets.go @@ -1,9 +1,9 @@ package data import ( - "crypto/sha256" - "encoding/hex" "errors" + "fmt" + "path" "github.com/docker/go/canonical/json" ) @@ -23,6 +23,33 @@ type Targets struct { Delegations Delegations `json:"delegations,omitempty"` } +// isValidTargetsStructure returns an error, or nil, depending on whether the content of the struct +// is valid for targets metadata. This does not check signatures or expiry, just that +// the metadata content is valid. +func isValidTargetsStructure(t Targets, roleName string) error { + if roleName != CanonicalTargetsRole && !IsDelegation(roleName) { + return ErrInvalidRole{Role: roleName} + } + + // even if it's a delegated role, the metadata type is "Targets" + expectedType := TUFTypes[CanonicalTargetsRole] + if t.Type != expectedType { + return ErrInvalidMetadata{ + role: roleName, msg: fmt.Sprintf("expected type %s, not %s", expectedType, t.Type)} + } + + for _, roleObj := range t.Delegations.Roles { + if !IsDelegation(roleObj.Name) || path.Dir(roleObj.Name) != roleName { + return ErrInvalidMetadata{ + role: roleName, msg: fmt.Sprintf("delegation role %s invalid", roleObj.Name)} + } + if err := isValidRootRoleStructure(roleName, roleObj.Name, roleObj.RootRole, t.Delegations.Keys); err != nil { + return err + } + } + return nil +} + // NewTargets intiializes a new empty SignedTargets object func NewTargets() *SignedTargets { return &SignedTargets{ @@ -51,30 +78,58 @@ func (t SignedTargets) GetMeta(path string) *FileMeta { return nil } -// GetDelegations filters the roles and associated keys that may be -// the signers for the given target path. If no appropriate roles -// can be found, it will simply return nil for the return values. -// The returned slice of Role will have order maintained relative -// to the role slice on Delegations per TUF spec proposal on using -// order to determine priority. -func (t SignedTargets) GetDelegations(path string) []*Role { - var roles []*Role - pathHashBytes := sha256.Sum256([]byte(path)) - pathHash := hex.EncodeToString(pathHashBytes[:]) - for _, r := range t.Signed.Delegations.Roles { - if !r.IsValid() { - // Role has both Paths and PathHashPrefixes. +// GetValidDelegations filters the delegation roles specified in the signed targets, and +// only returns roles that are direct children and restricts their paths +func (t SignedTargets) GetValidDelegations(parent DelegationRole) []DelegationRole { + roles := t.buildDelegationRoles() + result := []DelegationRole{} + for _, r := range roles { + validRole, err := parent.Restrict(r) + if err != nil { continue } - if r.CheckPaths(path) { - roles = append(roles, r) + result = append(result, validRole) + } + return result +} + +// BuildDelegationRole returns a copy of a DelegationRole using the information in this SignedTargets for the specified role name. +// Will error for invalid role name or key metadata within this SignedTargets. Path data is not validated. +func (t *SignedTargets) BuildDelegationRole(roleName string) (DelegationRole, error) { + for _, role := range t.Signed.Delegations.Roles { + if role.Name == roleName { + pubKeys := make(map[string]PublicKey) + for _, keyID := range role.KeyIDs { + pubKey, ok := t.Signed.Delegations.Keys[keyID] + if !ok { + // Couldn't retrieve all keys, so stop walking and return invalid role + return DelegationRole{}, ErrInvalidRole{Role: roleName, Reason: "delegation does not exist with all specified keys"} + } + pubKeys[keyID] = pubKey + } + return DelegationRole{ + BaseRole: BaseRole{ + Name: role.Name, + Keys: pubKeys, + Threshold: role.Threshold, + }, + Paths: role.Paths, + }, nil + } + } + return DelegationRole{}, ErrNoSuchRole{Role: roleName} +} + +// helper function to create DelegationRole structures from all delegations in a SignedTargets, +// these delegations are read directly from the SignedTargets and not modified or validated +func (t SignedTargets) buildDelegationRoles() []DelegationRole { + var roles []DelegationRole + for _, roleData := range t.Signed.Delegations.Roles { + delgRole, err := t.BuildDelegationRole(roleData.Name) + if err != nil { continue } - if r.CheckPrefixes(pathHash) { - roles = append(roles, r) - continue - } - //keysDB.AddRole(r) + roles = append(roles, delgRole) } return roles } @@ -93,8 +148,8 @@ func (t *SignedTargets) AddDelegation(role *Role, keys []*PublicKey) error { } // ToSigned partially serializes a SignedTargets for further signing -func (t SignedTargets) ToSigned() (*Signed, error) { - s, err := json.MarshalCanonical(t.Signed) +func (t *SignedTargets) ToSigned() (*Signed, error) { + s, err := defaultSerializer.MarshalCanonical(t.Signed) if err != nil { return nil, err } @@ -111,13 +166,25 @@ func (t SignedTargets) ToSigned() (*Signed, error) { }, nil } -// TargetsFromSigned fully unpacks a Signed object into a SignedTargets -func TargetsFromSigned(s *Signed) (*SignedTargets, error) { - t := Targets{} - err := json.Unmarshal(s.Signed, &t) +// MarshalJSON returns the serialized form of SignedTargets as bytes +func (t *SignedTargets) MarshalJSON() ([]byte, error) { + signed, err := t.ToSigned() if err != nil { return nil, err } + return defaultSerializer.Marshal(signed) +} + +// TargetsFromSigned fully unpacks a Signed object into a SignedTargets, given +// a role name (so it can validate the SignedTargets object) +func TargetsFromSigned(s *Signed, roleName string) (*SignedTargets, error) { + t := Targets{} + if err := defaultSerializer.Unmarshal(s.Signed, &t); err != nil { + return nil, err + } + if err := isValidTargetsStructure(t, roleName); err != nil { + return nil, err + } sigs := make([]Signature, len(s.Signatures)) copy(sigs, s.Signatures) return &SignedTargets{ diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/data/timestamp.go b/components/engine/vendor/src/github.com/docker/notary/tuf/data/timestamp.go index f68252ca5b..bc961f7a63 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/data/timestamp.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/data/timestamp.go @@ -2,6 +2,8 @@ package data import ( "bytes" + "crypto/sha256" + "fmt" "time" "github.com/docker/go/canonical/json" @@ -22,6 +24,26 @@ type Timestamp struct { Meta Files `json:"meta"` } +// isValidTimestampStructure returns an error, or nil, depending on whether the content of the struct +// is valid for timestamp metadata. This does not check signatures or expiry, just that +// the metadata content is valid. +func isValidTimestampStructure(t Timestamp) error { + expectedType := TUFTypes[CanonicalTimestampRole] + if t.Type != expectedType { + return ErrInvalidMetadata{ + role: CanonicalTimestampRole, msg: fmt.Sprintf("expected type %s, not %s", expectedType, t.Type)} + } + + // Meta is a map of FileMeta, so if the role isn't in the map it returns + // an empty FileMeta, which has an empty map, and you can check on keys + // from an empty map. + if cs, ok := t.Meta[CanonicalSnapshotRole].Hashes["sha256"]; !ok || len(cs) != sha256.Size { + return ErrInvalidMetadata{ + role: CanonicalTimestampRole, msg: "missing or invalid snapshot sha256 checksum information"} + } + return nil +} + // NewTimestamp initializes a timestamp with an existing snapshot func NewTimestamp(snapshot *Signed) (*SignedTimestamp, error) { snapshotJSON, err := json.Marshal(snapshot) @@ -47,8 +69,8 @@ func NewTimestamp(snapshot *Signed) (*SignedTimestamp, error) { // ToSigned partially serializes a SignedTimestamp such that it can // be signed -func (ts SignedTimestamp) ToSigned() (*Signed, error) { - s, err := json.MarshalCanonical(ts.Signed) +func (ts *SignedTimestamp) ToSigned() (*Signed, error) { + s, err := defaultSerializer.MarshalCanonical(ts.Signed) if err != nil { return nil, err } @@ -65,12 +87,33 @@ func (ts SignedTimestamp) ToSigned() (*Signed, error) { }, nil } +// GetSnapshot gets the expected snapshot metadata hashes in the timestamp metadata, +// or nil if it doesn't exist +func (ts *SignedTimestamp) GetSnapshot() (*FileMeta, error) { + snapshotExpected, ok := ts.Signed.Meta[CanonicalSnapshotRole] + if !ok { + return nil, ErrMissingMeta{Role: CanonicalSnapshotRole} + } + return &snapshotExpected, nil +} + +// MarshalJSON returns the serialized form of SignedTimestamp as bytes +func (ts *SignedTimestamp) MarshalJSON() ([]byte, error) { + signed, err := ts.ToSigned() + if err != nil { + return nil, err + } + return defaultSerializer.Marshal(signed) +} + // TimestampFromSigned parsed a Signed object into a fully unpacked // SignedTimestamp func TimestampFromSigned(s *Signed) (*SignedTimestamp, error) { ts := Timestamp{} - err := json.Unmarshal(s.Signed, &ts) - if err != nil { + if err := defaultSerializer.Unmarshal(s.Signed, &ts); err != nil { + return nil, err + } + if err := isValidTimestampStructure(ts); err != nil { return nil, err } sigs := make([]Signature, len(s.Signatures)) diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/data/types.go b/components/engine/vendor/src/github.com/docker/notary/tuf/data/types.go index 6459b8e664..4de55c4e1c 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/data/types.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/data/types.go @@ -12,6 +12,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/go/canonical/json" + "github.com/docker/notary" ) // SigAlgorithm for types of signatures @@ -171,16 +172,16 @@ func NewDelegations() *Delegations { } } -// defines number of days in which something should expire -var defaultExpiryTimes = map[string]int{ - CanonicalRootRole: 365, - CanonicalTargetsRole: 90, - CanonicalSnapshotRole: 7, - CanonicalTimestampRole: 1, +// These values are recommended TUF expiry times. +var defaultExpiryTimes = map[string]time.Duration{ + CanonicalRootRole: notary.Year, + CanonicalTargetsRole: 90 * notary.Day, + CanonicalSnapshotRole: 7 * notary.Day, + CanonicalTimestampRole: notary.Day, } // SetDefaultExpiryTimes allows one to change the default expiries. -func SetDefaultExpiryTimes(times map[string]int) { +func SetDefaultExpiryTimes(times map[string]time.Duration) { for key, value := range times { if _, ok := defaultExpiryTimes[key]; !ok { logrus.Errorf("Attempted to set default expiry for an unknown role: %s", key) @@ -192,10 +193,10 @@ func SetDefaultExpiryTimes(times map[string]int) { // DefaultExpires gets the default expiry time for the given role func DefaultExpires(role string) time.Time { - var t time.Time - if t, ok := defaultExpiryTimes[role]; ok { - return time.Now().AddDate(0, 0, t) + if d, ok := defaultExpiryTimes[role]; ok { + return time.Now().Add(d) } + var t time.Time return t.UTC().Round(time.Second) } diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/keys/db.go b/components/engine/vendor/src/github.com/docker/notary/tuf/keys/db.go deleted file mode 100644 index 92d2ef863e..0000000000 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/keys/db.go +++ /dev/null @@ -1,78 +0,0 @@ -package keys - -import ( - "errors" - - "github.com/docker/notary/tuf/data" -) - -// Various basic key database errors -var ( - ErrWrongType = errors.New("tuf: invalid key type") - ErrExists = errors.New("tuf: key already in db") - ErrWrongID = errors.New("tuf: key id mismatch") - ErrInvalidKey = errors.New("tuf: invalid key") - ErrInvalidKeyID = errors.New("tuf: invalid key id") - ErrInvalidThreshold = errors.New("tuf: invalid role threshold") -) - -// KeyDB is an in memory database of public keys and role associations. -// It is populated when parsing TUF files and used during signature -// verification to look up the keys for a given role -type KeyDB struct { - roles map[string]*data.Role - keys map[string]data.PublicKey -} - -// NewDB initializes an empty KeyDB -func NewDB() *KeyDB { - return &KeyDB{ - roles: make(map[string]*data.Role), - keys: make(map[string]data.PublicKey), - } -} - -// AddKey adds a public key to the database -func (db *KeyDB) AddKey(k data.PublicKey) { - db.keys[k.ID()] = k -} - -// AddRole adds a role to the database. Any keys associated with the -// role must have already been added. -func (db *KeyDB) AddRole(r *data.Role) error { - if !data.ValidRole(r.Name) { - return data.ErrInvalidRole{Role: r.Name} - } - if r.Threshold < 1 { - return ErrInvalidThreshold - } - - // validate all key ids are in the keys maps - for _, id := range r.KeyIDs { - if _, ok := db.keys[id]; !ok { - return ErrInvalidKeyID - } - } - - db.roles[r.Name] = r - return nil -} - -// GetAllRoles gets all roles from the database -func (db *KeyDB) GetAllRoles() []*data.Role { - roles := []*data.Role{} - for _, role := range db.roles { - roles = append(roles, role) - } - return roles -} - -// GetKey pulls a key out of the database by its ID -func (db *KeyDB) GetKey(id string) data.PublicKey { - return db.keys[id] -} - -// GetRole retrieves a role based on its name -func (db *KeyDB) GetRole(name string) *data.Role { - return db.roles[name] -} diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/signed/verify.go b/components/engine/vendor/src/github.com/docker/notary/tuf/signed/verify.go index 9548e4e53d..4869b17074 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/signed/verify.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/signed/verify.go @@ -8,7 +8,6 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/go/canonical/json" "github.com/docker/notary/tuf/data" - "github.com/docker/notary/tuf/keys" ) // Various basic signing errors @@ -57,18 +56,18 @@ func VerifyRoot(s *data.Signed, minVersion int, keys map[string]data.PublicKey) continue } // threshold of 1 so return on first success - return verifyMeta(s, "root", minVersion) + return verifyMeta(s, data.CanonicalRootRole, minVersion) } return ErrRoleThreshold{} } // Verify checks the signatures and metadata (expiry, version) for the signed role // data -func Verify(s *data.Signed, role string, minVersion int, db *keys.KeyDB) error { - if err := verifyMeta(s, role, minVersion); err != nil { +func Verify(s *data.Signed, role data.BaseRole, minVersion int) error { + if err := verifyMeta(s, role.Name, minVersion); err != nil { return err } - return VerifySignatures(s, role, db) + return VerifySignatures(s, role) } func verifyMeta(s *data.Signed, role string, minVersion int) error { @@ -96,21 +95,18 @@ func IsExpired(t time.Time) bool { } // VerifySignatures checks the we have sufficient valid signatures for the given role -func VerifySignatures(s *data.Signed, role string, db *keys.KeyDB) error { +func VerifySignatures(s *data.Signed, roleData data.BaseRole) error { if len(s.Signatures) == 0 { return ErrNoSignatures } - roleData := db.GetRole(role) - if roleData == nil { - return ErrUnknownRole - } - if roleData.Threshold < 1 { return ErrRoleThreshold{} } - logrus.Debugf("%s role has key IDs: %s", role, strings.Join(roleData.KeyIDs, ",")) + logrus.Debugf("%s role has key IDs: %s", roleData.Name, strings.Join(roleData.ListKeyIDs(), ",")) + // remarshal the signed part so we can verify the signature, since the signature has + // to be of a canonically marshalled signed object var decoded map[string]interface{} if err := json.Unmarshal(s.Signed, &decoded); err != nil { return err @@ -123,12 +119,8 @@ func VerifySignatures(s *data.Signed, role string, db *keys.KeyDB) error { valid := make(map[string]struct{}) for _, sig := range s.Signatures { logrus.Debug("verifying signature for key ID: ", sig.KeyID) - if !roleData.ValidKey(sig.KeyID) { - logrus.Debugf("continuing b/c keyid was invalid: %s for roledata %s\n", sig.KeyID, roleData) - continue - } - key := db.GetKey(sig.KeyID) - if key == nil { + key, ok := roleData.Keys[sig.KeyID] + if !ok { logrus.Debugf("continuing b/c keyid lookup was nil: %s\n", sig.KeyID) continue } @@ -153,28 +145,3 @@ func VerifySignatures(s *data.Signed, role string, db *keys.KeyDB) error { return nil } - -// Unmarshal unmarshals and verifys the raw bytes for a given role's metadata -func Unmarshal(b []byte, v interface{}, role string, minVersion int, db *keys.KeyDB) error { - s := &data.Signed{} - if err := json.Unmarshal(b, s); err != nil { - return err - } - if err := Verify(s, role, minVersion, db); err != nil { - return err - } - return json.Unmarshal(s.Signed, v) -} - -// UnmarshalTrusted unmarshals and verifies signatures only, not metadata, for a -// given role's metadata -func UnmarshalTrusted(b []byte, v interface{}, role string, db *keys.KeyDB) error { - s := &data.Signed{} - if err := json.Unmarshal(b, s); err != nil { - return err - } - if err := VerifySignatures(s, role, db); err != nil { - return err - } - return json.Unmarshal(s.Signed, v) -} diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/store/filestore.go b/components/engine/vendor/src/github.com/docker/notary/tuf/store/filestore.go index 52e7c8f289..44401707c4 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/store/filestore.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/store/filestore.go @@ -2,6 +2,7 @@ package store import ( "fmt" + "github.com/docker/notary" "io/ioutil" "os" "path" @@ -9,25 +10,19 @@ import ( ) // NewFilesystemStore creates a new store in a directory tree -func NewFilesystemStore(baseDir, metaSubDir, metaExtension, targetsSubDir string) (*FilesystemStore, error) { +func NewFilesystemStore(baseDir, metaSubDir, metaExtension string) (*FilesystemStore, error) { metaDir := path.Join(baseDir, metaSubDir) - targetsDir := path.Join(baseDir, targetsSubDir) // Make sure we can create the necessary dirs and they are writable err := os.MkdirAll(metaDir, 0700) if err != nil { return nil, err } - err = os.MkdirAll(targetsDir, 0700) - if err != nil { - return nil, err - } return &FilesystemStore{ baseDir: baseDir, metaDir: metaDir, metaExtension: metaExtension, - targetsDir: targetsDir, }, nil } @@ -36,7 +31,6 @@ type FilesystemStore struct { baseDir string metaDir string metaExtension string - targetsDir string } func (f *FilesystemStore) getPath(name string) string { @@ -44,7 +38,8 @@ func (f *FilesystemStore) getPath(name string) string { return filepath.Join(f.metaDir, fileName) } -// GetMeta returns the meta for the given name (a role) +// GetMeta returns the meta for the given name (a role) up to size bytes +// If size is -1, this corresponds to "infinite," but we cut off at 100MB func (f *FilesystemStore) GetMeta(name string, size int64) ([]byte, error) { meta, err := ioutil.ReadFile(f.getPath(name)) if err != nil { @@ -53,7 +48,14 @@ func (f *FilesystemStore) GetMeta(name string, size int64) ([]byte, error) { } return nil, err } - return meta, nil + if size == -1 { + size = notary.MaxDownloadSize + } + // Only return up to size bytes + if int64(len(meta)) < size { + return meta, nil + } + return meta[:size], nil } // SetMultiMeta sets the metadata for multiple roles in one operation diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/store/httpstore.go b/components/engine/vendor/src/github.com/docker/notary/tuf/store/httpstore.go index 7444a311b9..8b0d850114 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/store/httpstore.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/store/httpstore.go @@ -23,6 +23,7 @@ import ( "path" "github.com/Sirupsen/logrus" + "github.com/docker/notary" "github.com/docker/notary/tuf/validation" ) @@ -33,6 +34,9 @@ type ErrServerUnavailable struct { } func (err ErrServerUnavailable) Error() string { + if err.code == 401 { + return fmt.Sprintf("you are not authorized to perform this operation: server returned 401.") + } return fmt.Sprintf("unable to reach trust server at this time: %d.", err.code) } @@ -71,13 +75,12 @@ type HTTPStore struct { baseURL url.URL metaPrefix string metaExtension string - targetsPrefix string keyExtension string roundTrip http.RoundTripper } // NewHTTPStore initializes a new store against a URL and a number of configuration options -func NewHTTPStore(baseURL, metaPrefix, metaExtension, targetsPrefix, keyExtension string, roundTrip http.RoundTripper) (RemoteStore, error) { +func NewHTTPStore(baseURL, metaPrefix, metaExtension, keyExtension string, roundTrip http.RoundTripper) (RemoteStore, error) { base, err := url.Parse(baseURL) if err != nil { return nil, err @@ -92,7 +95,6 @@ func NewHTTPStore(baseURL, metaPrefix, metaExtension, targetsPrefix, keyExtensio baseURL: *base, metaPrefix: metaPrefix, metaExtension: metaExtension, - targetsPrefix: targetsPrefix, keyExtension: keyExtension, roundTrip: roundTrip, }, nil @@ -137,6 +139,7 @@ func translateStatusToError(resp *http.Response, resource string) error { // GetMeta downloads the named meta file with the given size. A short body // is acceptable because in the case of timestamp.json, the size is a cap, // not an exact length. +// If size is -1, this corresponds to "infinite," but we cut off at 100MB func (s HTTPStore) GetMeta(name string, size int64) ([]byte, error) { url, err := s.buildMetaURL(name) if err != nil { @@ -155,6 +158,9 @@ func (s HTTPStore) GetMeta(name string, size int64) ([]byte, error) { logrus.Debugf("received HTTP status %d when requesting %s.", resp.StatusCode, name) return nil, err } + if size == -1 { + size = notary.MaxDownloadSize + } if resp.ContentLength > size { return nil, ErrMaliciousServer{} } @@ -250,11 +256,6 @@ func (s HTTPStore) buildMetaURL(name string) (*url.URL, error) { return s.buildURL(uri) } -func (s HTTPStore) buildTargetsURL(name string) (*url.URL, error) { - uri := path.Join(s.targetsPrefix, name) - return s.buildURL(uri) -} - func (s HTTPStore) buildKeyURL(name string) (*url.URL, error) { filename := fmt.Sprintf("%s.%s", name, s.keyExtension) uri := path.Join(s.metaPrefix, filename) @@ -269,29 +270,6 @@ func (s HTTPStore) buildURL(uri string) (*url.URL, error) { return s.baseURL.ResolveReference(sub), nil } -// GetTarget returns a reader for the desired target or an error. -// N.B. The caller is responsible for closing the reader. -func (s HTTPStore) GetTarget(path string) (io.ReadCloser, error) { - url, err := s.buildTargetsURL(path) - if err != nil { - return nil, err - } - logrus.Debug("Attempting to download target: ", url.String()) - req, err := http.NewRequest("GET", url.String(), nil) - if err != nil { - return nil, err - } - resp, err := s.roundTrip.RoundTrip(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if err := translateStatusToError(resp, path); err != nil { - return nil, err - } - return resp.Body, nil -} - // GetKey retrieves a public key from the remote server func (s HTTPStore) GetKey(role string) ([]byte, error) { url, err := s.buildKeyURL(role) diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/store/interfaces.go b/components/engine/vendor/src/github.com/docker/notary/tuf/store/interfaces.go index 6d73da8a96..dd307168bf 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/store/interfaces.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/store/interfaces.go @@ -1,13 +1,5 @@ package store -import ( - "io" - - "github.com/docker/notary/tuf/data" -) - -type targetsWalkFunc func(path string, meta data.FileMeta) error - // MetadataStore must be implemented by anything that intends to interact // with a store of TUF files type MetadataStore interface { @@ -23,17 +15,9 @@ type PublicKeyStore interface { GetKey(role string) ([]byte, error) } -// TargetStore represents a collection of targets that can be walked similarly -// to walking a directory, passing a callback that receives the path and meta -// for each target -type TargetStore interface { - WalkStagedTargets(paths []string, targetsFn targetsWalkFunc) error -} - // LocalStore represents a local TUF sture type LocalStore interface { MetadataStore - TargetStore } // RemoteStore is similar to LocalStore with the added expectation that it should @@ -41,5 +25,4 @@ type LocalStore interface { type RemoteStore interface { MetadataStore PublicKeyStore - GetTarget(path string) (io.ReadCloser, error) } diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/store/memorystore.go b/components/engine/vendor/src/github.com/docker/notary/tuf/store/memorystore.go index 6072a8c446..493bb6f0b5 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/store/memorystore.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/store/memorystore.go @@ -1,38 +1,59 @@ package store import ( - "bytes" + "crypto/sha256" "fmt" - "io" + "github.com/docker/notary" "github.com/docker/notary/tuf/data" "github.com/docker/notary/tuf/utils" ) // NewMemoryStore returns a MetadataStore that operates entirely in memory. // Very useful for testing -func NewMemoryStore(meta map[string][]byte, files map[string][]byte) RemoteStore { +func NewMemoryStore(meta map[string][]byte) *MemoryStore { + var consistent = make(map[string][]byte) if meta == nil { meta = make(map[string][]byte) + } else { + // add all seed meta to consistent + for name, data := range meta { + checksum := sha256.Sum256(data) + path := utils.ConsistentName(name, checksum[:]) + consistent[path] = data + } } - if files == nil { - files = make(map[string][]byte) - } - return &memoryStore{ - meta: meta, - files: files, - keys: make(map[string][]data.PrivateKey), + return &MemoryStore{ + meta: meta, + consistent: consistent, + keys: make(map[string][]data.PrivateKey), } } -type memoryStore struct { - meta map[string][]byte - files map[string][]byte - keys map[string][]data.PrivateKey +// MemoryStore implements a mock RemoteStore entirely in memory. +// For testing purposes only. +type MemoryStore struct { + meta map[string][]byte + consistent map[string][]byte + keys map[string][]data.PrivateKey } -func (m *memoryStore) GetMeta(name string, size int64) ([]byte, error) { +// GetMeta returns up to size bytes of data references by name. +// If size is -1, this corresponds to "infinite," but we cut off at 100MB +// as we will always know the size for everything but a timestamp and +// sometimes a root, neither of which should be exceptionally large +func (m *MemoryStore) GetMeta(name string, size int64) ([]byte, error) { d, ok := m.meta[name] + if ok { + if size == -1 { + size = notary.MaxDownloadSize + } + if int64(len(d)) < size { + return d, nil + } + return d[:size], nil + } + d, ok = m.consistent[name] if ok { if int64(len(d)) < size { return d, nil @@ -42,12 +63,19 @@ func (m *memoryStore) GetMeta(name string, size int64) ([]byte, error) { return nil, ErrMetaNotFound{Resource: name} } -func (m *memoryStore) SetMeta(name string, meta []byte) error { +// SetMeta sets the metadata value for the given name +func (m *MemoryStore) SetMeta(name string, meta []byte) error { m.meta[name] = meta + + checksum := sha256.Sum256(meta) + path := utils.ConsistentName(name, checksum[:]) + m.consistent[path] = meta return nil } -func (m *memoryStore) SetMultiMeta(metas map[string][]byte) error { +// SetMultiMeta sets multiple pieces of metadata for multiple names +// in a single operation. +func (m *MemoryStore) SetMultiMeta(metas map[string][]byte) error { for role, blob := range metas { m.SetMeta(role, blob) } @@ -56,57 +84,23 @@ func (m *memoryStore) SetMultiMeta(metas map[string][]byte) error { // RemoveMeta removes the metadata for a single role - if the metadata doesn't // exist, no error is returned -func (m *memoryStore) RemoveMeta(name string) error { - delete(m.meta, name) - return nil -} - -func (m *memoryStore) GetTarget(path string) (io.ReadCloser, error) { - return &utils.NoopCloser{Reader: bytes.NewReader(m.files[path])}, nil -} - -func (m *memoryStore) WalkStagedTargets(paths []string, targetsFn targetsWalkFunc) error { - if len(paths) == 0 { - for path, dat := range m.files { - meta, err := data.NewFileMeta(bytes.NewReader(dat), "sha256") - if err != nil { - return err - } - if err = targetsFn(path, meta); err != nil { - return err - } - } - return nil - } - - for _, path := range paths { - dat, ok := m.files[path] - if !ok { - return ErrMetaNotFound{Resource: path} - } - meta, err := data.NewFileMeta(bytes.NewReader(dat), "sha256") - if err != nil { - return err - } - if err = targetsFn(path, meta); err != nil { - return err - } +func (m *MemoryStore) RemoveMeta(name string) error { + if meta, ok := m.meta[name]; ok { + checksum := sha256.Sum256(meta) + path := utils.ConsistentName(name, checksum[:]) + delete(m.meta, name) + delete(m.consistent, path) } return nil } -func (m *memoryStore) Commit(map[string][]byte, bool, map[string]data.Hashes) error { - return nil +// GetKey returns the public key for the given role +func (m *MemoryStore) GetKey(role string) ([]byte, error) { + return nil, fmt.Errorf("GetKey is not implemented for the MemoryStore") } -func (m *memoryStore) GetKey(role string) ([]byte, error) { - return nil, fmt.Errorf("GetKey is not implemented for the memoryStore") -} - -// Clear this existing memory store by setting this store as new empty one -func (m *memoryStore) RemoveAll() error { - m.meta = make(map[string][]byte) - m.files = make(map[string][]byte) - m.keys = make(map[string][]data.PrivateKey) +// RemoveAll clears the existing memory store by setting this store as new empty one +func (m *MemoryStore) RemoveAll() error { + *m = *NewMemoryStore(nil) return nil } diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/tuf.go b/components/engine/vendor/src/github.com/docker/notary/tuf/tuf.go index 96ab7da1d6..0a1912a5f5 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/tuf.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/tuf.go @@ -3,8 +3,6 @@ package tuf import ( "bytes" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "path" @@ -12,8 +10,8 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/docker/notary" "github.com/docker/notary/tuf/data" - "github.com/docker/notary/tuf/keys" "github.com/docker/notary/tuf/signed" "github.com/docker/notary/tuf/utils" ) @@ -40,15 +38,19 @@ func (e ErrLocalRootExpired) Error() string { } // ErrNotLoaded - attempted to access data that has not been loaded into -// the repo +// the repo. This means specifically that the relevant JSON file has not +// been loaded. type ErrNotLoaded struct { - role string + Role string } func (err ErrNotLoaded) Error() string { - return fmt.Sprintf("%s role has not been loaded", err.role) + return fmt.Sprintf("%s role has not been loaded", err.Role) } +// StopWalk - used by visitor functions to signal WalkTargets to stop walking +type StopWalk struct{} + // Repo is an in memory representation of the TUF Repo. // It operates at the data.Signed level, accepting and producing // data.Signed objects. Users of a Repo are responsible for @@ -59,16 +61,15 @@ type Repo struct { Targets map[string]*data.SignedTargets Snapshot *data.SignedSnapshot Timestamp *data.SignedTimestamp - keysDB *keys.KeyDB cryptoService signed.CryptoService } -// NewRepo initializes a Repo instance with a keysDB and a signer. -// If the Repo will only be used for reading, the signer should be nil. -func NewRepo(keysDB *keys.KeyDB, cryptoService signed.CryptoService) *Repo { +// NewRepo initializes a Repo instance with a CryptoService. +// If the Repo will only be used for reading, the CryptoService +// can be nil. +func NewRepo(cryptoService signed.CryptoService) *Repo { repo := &Repo{ Targets: make(map[string]*data.SignedTargets), - keysDB: keysDB, cryptoService: cryptoService, } return repo @@ -77,27 +78,15 @@ func NewRepo(keysDB *keys.KeyDB, cryptoService signed.CryptoService) *Repo { // AddBaseKeys is used to add keys to the role in root.json func (tr *Repo) AddBaseKeys(role string, keys ...data.PublicKey) error { if tr.Root == nil { - return ErrNotLoaded{role: "root"} + return ErrNotLoaded{Role: data.CanonicalRootRole} } ids := []string{} for _, k := range keys { // Store only the public portion tr.Root.Signed.Keys[k.ID()] = k - tr.keysDB.AddKey(k) tr.Root.Signed.Roles[role].KeyIDs = append(tr.Root.Signed.Roles[role].KeyIDs, k.ID()) ids = append(ids, k.ID()) } - r, err := data.NewRole( - role, - tr.Root.Signed.Roles[role].Threshold, - ids, - nil, - nil, - ) - if err != nil { - return err - } - tr.keysDB.AddRole(r) tr.Root.Dirty = true // also, whichever role was switched out needs to be re-signed @@ -121,8 +110,11 @@ func (tr *Repo) AddBaseKeys(role string, keys ...data.PublicKey) error { // ReplaceBaseKeys is used to replace all keys for the given role with the new keys func (tr *Repo) ReplaceBaseKeys(role string, keys ...data.PublicKey) error { - r := tr.keysDB.GetRole(role) - err := tr.RemoveBaseKeys(role, r.KeyIDs...) + r, err := tr.GetBaseRole(role) + if err != nil { + return err + } + err = tr.RemoveBaseKeys(role, r.ListKeyIDs()...) if err != nil { return err } @@ -132,7 +124,7 @@ func (tr *Repo) ReplaceBaseKeys(role string, keys ...data.PublicKey) error { // RemoveBaseKeys is used to remove keys from the roles in root.json func (tr *Repo) RemoveBaseKeys(role string, keyIDs ...string) error { if tr.Root == nil { - return ErrNotLoaded{role: "root"} + return ErrNotLoaded{Role: data.CanonicalRootRole} } var keep []string toDelete := make(map[string]struct{}) @@ -173,117 +165,253 @@ func (tr *Repo) RemoveBaseKeys(role string, keyIDs ...string) error { return nil } +// GetBaseRole gets a base role from this repo's metadata +func (tr *Repo) GetBaseRole(name string) (data.BaseRole, error) { + if !data.ValidRole(name) { + return data.BaseRole{}, data.ErrInvalidRole{Role: name, Reason: "invalid base role name"} + } + if tr.Root == nil { + return data.BaseRole{}, ErrNotLoaded{data.CanonicalRootRole} + } + // Find the role data public keys for the base role from TUF metadata + baseRole, err := tr.Root.BuildBaseRole(name) + if err != nil { + return data.BaseRole{}, err + } + + return baseRole, nil +} + +// GetDelegationRole gets a delegation role from this repo's metadata, walking from the targets role down to the delegation itself +func (tr *Repo) GetDelegationRole(name string) (data.DelegationRole, error) { + if !data.IsDelegation(name) { + return data.DelegationRole{}, data.ErrInvalidRole{Role: name, Reason: "invalid delegation name"} + } + if tr.Root == nil { + return data.DelegationRole{}, ErrNotLoaded{data.CanonicalRootRole} + } + _, ok := tr.Root.Signed.Roles[data.CanonicalTargetsRole] + if !ok { + return data.DelegationRole{}, ErrNotLoaded{data.CanonicalTargetsRole} + } + // Traverse target metadata, down to delegation itself + // Get all public keys for the base role from TUF metadata + _, ok = tr.Targets[data.CanonicalTargetsRole] + if !ok { + return data.DelegationRole{}, ErrNotLoaded{data.CanonicalTargetsRole} + } + + // Start with top level roles in targets. Walk the chain of ancestors + // until finding the desired role, or we run out of targets files to search. + var foundRole *data.DelegationRole + buildDelegationRoleVisitor := func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} { + // Try to find the delegation and build a DelegationRole structure + for _, role := range tgt.Signed.Delegations.Roles { + if role.Name == name { + delgRole, err := tgt.BuildDelegationRole(name) + if err != nil { + return err + } + foundRole = &delgRole + return StopWalk{} + } + } + return nil + } + + // Walk to the parent of this delegation, since that is where its role metadata exists + err := tr.WalkTargets("", path.Dir(name), buildDelegationRoleVisitor) + if err != nil { + return data.DelegationRole{}, err + } + + // We never found the delegation. In the context of this repo it is considered + // invalid. N.B. it may be that it existed at one point but an ancestor has since + // been modified/removed. + if foundRole == nil { + return data.DelegationRole{}, data.ErrInvalidRole{Role: name, Reason: "delegation does not exist"} + } + + return *foundRole, nil +} + // GetAllLoadedRoles returns a list of all role entries loaded in this TUF repo, could be empty func (tr *Repo) GetAllLoadedRoles() []*data.Role { - return tr.keysDB.GetAllRoles() + var res []*data.Role + if tr.Root == nil { + // if root isn't loaded, we should consider we have no loaded roles because we can't + // trust any other state that might be present + return res + } + for name, rr := range tr.Root.Signed.Roles { + res = append(res, &data.Role{ + RootRole: *rr, + Name: name, + }) + } + for _, delegate := range tr.Targets { + for _, r := range delegate.Signed.Delegations.Roles { + res = append(res, r) + } + } + return res } -// GetDelegation finds the role entry representing the provided -// role name or ErrInvalidRole -func (tr *Repo) GetDelegation(role string) (*data.Role, error) { - r := data.Role{Name: role} - if !r.IsDelegation() { - return nil, data.ErrInvalidRole{Role: role, Reason: "not a valid delegated role"} +// Walk to parent, and either create or update this delegation. We can only create a new delegation if we're given keys +// Ensure all updates are valid, by checking against parent ancestor paths and ensuring the keys meet the role threshold. +func delegationUpdateVisitor(roleName string, addKeys data.KeyList, removeKeys, addPaths, removePaths []string, clearAllPaths bool, newThreshold int) walkVisitorFunc { + return func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} { + var err error + // Validate the changes underneath this restricted validRole for adding paths, reject invalid path additions + if len(addPaths) != len(data.RestrictDelegationPathPrefixes(validRole.Paths, addPaths)) { + return data.ErrInvalidRole{Role: roleName, Reason: "invalid paths to add to role"} + } + // Try to find the delegation and amend it using our changelist + var delgRole *data.Role + for _, role := range tgt.Signed.Delegations.Roles { + if role.Name == roleName { + // Make a copy and operate on this role until we validate the changes + keyIDCopy := make([]string, len(role.KeyIDs)) + copy(keyIDCopy, role.KeyIDs) + pathsCopy := make([]string, len(role.Paths)) + copy(pathsCopy, role.Paths) + delgRole = &data.Role{ + RootRole: data.RootRole{ + KeyIDs: keyIDCopy, + Threshold: role.Threshold, + }, + Name: role.Name, + Paths: pathsCopy, + } + delgRole.RemovePaths(removePaths) + if clearAllPaths { + delgRole.Paths = []string{} + } + delgRole.AddPaths(addPaths) + delgRole.RemoveKeys(removeKeys) + break + } + } + // We didn't find the role earlier, so create it only if we have keys to add + if delgRole == nil { + if len(addKeys) > 0 { + delgRole, err = data.NewRole(roleName, newThreshold, addKeys.IDs(), addPaths) + if err != nil { + return err + } + } else { + // If we can't find the role and didn't specify keys to add, this is an error + return data.ErrInvalidRole{Role: roleName, Reason: "cannot create new delegation without keys"} + } + } + // Add the key IDs to the role and the keys themselves to the parent + for _, k := range addKeys { + if !utils.StrSliceContains(delgRole.KeyIDs, k.ID()) { + delgRole.KeyIDs = append(delgRole.KeyIDs, k.ID()) + } + } + // Make sure we have a valid role still + if len(delgRole.KeyIDs) < delgRole.Threshold { + return data.ErrInvalidRole{Role: roleName, Reason: "insufficient keys to meet threshold"} + } + // NOTE: this closure CANNOT error after this point, as we've committed to editing the SignedTargets metadata in the repo object. + // Any errors related to updating this delegation must occur before this point. + // If all of our changes were valid, we should edit the actual SignedTargets to match our copy + for _, k := range addKeys { + tgt.Signed.Delegations.Keys[k.ID()] = k + } + foundAt := utils.FindRoleIndex(tgt.Signed.Delegations.Roles, delgRole.Name) + if foundAt < 0 { + tgt.Signed.Delegations.Roles = append(tgt.Signed.Delegations.Roles, delgRole) + } else { + tgt.Signed.Delegations.Roles[foundAt] = delgRole + } + tgt.Dirty = true + utils.RemoveUnusedKeys(tgt) + return StopWalk{} } - - parent := path.Dir(role) - - // check the parent role - if parentRole := tr.keysDB.GetRole(parent); parentRole == nil { - return nil, data.ErrInvalidRole{Role: role, Reason: "parent role not found"} - } - - // check the parent role's metadata - p, ok := tr.Targets[parent] - if !ok { // the parent targetfile may not exist yet, so it can't be in the list - return nil, data.ErrNoSuchRole{Role: role} - } - - foundAt := utils.FindRoleIndex(p.Signed.Delegations.Roles, role) - if foundAt < 0 { - return nil, data.ErrNoSuchRole{Role: role} - } - return p.Signed.Delegations.Roles[foundAt], nil } -// UpdateDelegations updates the appropriate delegations, either adding +// UpdateDelegationKeys updates the appropriate delegations, either adding // a new delegation or updating an existing one. If keys are // provided, the IDs will be added to the role (if they do not exist // there already), and the keys will be added to the targets file. -func (tr *Repo) UpdateDelegations(role *data.Role, keys []data.PublicKey) error { - if !role.IsDelegation() || !role.IsValid() { - return data.ErrInvalidRole{Role: role.Name, Reason: "not a valid delegated role"} +func (tr *Repo) UpdateDelegationKeys(roleName string, addKeys data.KeyList, removeKeys []string, newThreshold int) error { + if !data.IsDelegation(roleName) { + return data.ErrInvalidRole{Role: roleName, Reason: "not a valid delegated role"} } - parent := path.Dir(role.Name) + parent := path.Dir(roleName) if err := tr.VerifyCanSign(parent); err != nil { return err } // check the parent role's metadata - p, ok := tr.Targets[parent] + _, ok := tr.Targets[parent] if !ok { // the parent targetfile may not exist yet - if not, then create it var err error - p, err = tr.InitTargets(parent) + _, err = tr.InitTargets(parent) if err != nil { return err } } - for _, k := range keys { - if !utils.StrSliceContains(role.KeyIDs, k.ID()) { - role.KeyIDs = append(role.KeyIDs, k.ID()) - } - p.Signed.Delegations.Keys[k.ID()] = k - tr.keysDB.AddKey(k) + // Walk to the parent of this delegation, since that is where its role metadata exists + // We do not have to verify that the walker reached its desired role in this scenario + // since we've already done another walk to the parent role in VerifyCanSign, and potentially made a targets file + err := tr.WalkTargets("", parent, delegationUpdateVisitor(roleName, addKeys, removeKeys, []string{}, []string{}, false, newThreshold)) + if err != nil { + return err + } + return nil +} + +// UpdateDelegationPaths updates the appropriate delegation's paths. +// It is not allowed to create a new delegation. +func (tr *Repo) UpdateDelegationPaths(roleName string, addPaths, removePaths []string, clearPaths bool) error { + if !data.IsDelegation(roleName) { + return data.ErrInvalidRole{Role: roleName, Reason: "not a valid delegated role"} + } + parent := path.Dir(roleName) + + if err := tr.VerifyCanSign(parent); err != nil { + return err } - // if the role has fewer keys than the threshold, it - // will never be able to create a valid targets file - // and should be considered invalid. - if len(role.KeyIDs) < role.Threshold { - return data.ErrInvalidRole{Role: role.Name, Reason: "insufficient keys to meet threshold"} + // check the parent role's metadata + _, ok := tr.Targets[parent] + if !ok { // the parent targetfile may not exist yet + // if not, this is an error because a delegation must exist to edit only paths + return data.ErrInvalidRole{Role: roleName, Reason: "no valid delegated role exists"} } - foundAt := utils.FindRoleIndex(p.Signed.Delegations.Roles, role.Name) - - if foundAt >= 0 { - p.Signed.Delegations.Roles[foundAt] = role - } else { - p.Signed.Delegations.Roles = append(p.Signed.Delegations.Roles, role) + // Walk to the parent of this delegation, since that is where its role metadata exists + // We do not have to verify that the walker reached its desired role in this scenario + // since we've already done another walk to the parent role in VerifyCanSign + err := tr.WalkTargets("", parent, delegationUpdateVisitor(roleName, data.KeyList{}, []string{}, addPaths, removePaths, clearPaths, notary.MinThreshold)) + if err != nil { + return err } - // We've made a change to parent. Set it to dirty - p.Dirty = true - - // We don't actually want to create the new delegation metadata yet. - // When we add a delegation, it may only be signable by a key we don't have - // (hence we are delegating signing). - - tr.keysDB.AddRole(role) - utils.RemoveUnusedKeys(p) - return nil } // DeleteDelegation removes a delegated targets role from its parent // targets object. It also deletes the delegation from the snapshot. // DeleteDelegation will only make use of the role Name field. -func (tr *Repo) DeleteDelegation(role data.Role) error { - if !role.IsDelegation() { - return data.ErrInvalidRole{Role: role.Name, Reason: "not a valid delegated role"} +func (tr *Repo) DeleteDelegation(roleName string) error { + if !data.IsDelegation(roleName) { + return data.ErrInvalidRole{Role: roleName, Reason: "not a valid delegated role"} } - // the role variable must not be used past this assignment for safety - name := role.Name - parent := path.Dir(name) + parent := path.Dir(roleName) if err := tr.VerifyCanSign(parent); err != nil { return err } // delete delegated data from Targets map and Snapshot - if they don't // exist, these are no-op - delete(tr.Targets, name) - tr.Snapshot.DeleteMeta(name) + delete(tr.Targets, roleName) + tr.Snapshot.DeleteMeta(roleName) p, ok := tr.Targets[parent] if !ok { @@ -292,7 +420,7 @@ func (tr *Repo) DeleteDelegation(role data.Role) error { return nil } - foundAt := utils.FindRoleIndex(p.Signed.Delegations.Roles, name) + foundAt := utils.FindRoleIndex(p.Signed.Delegations.Roles, roleName) if foundAt >= 0 { var roles []*data.Role @@ -311,53 +439,32 @@ func (tr *Repo) DeleteDelegation(role data.Role) error { return nil } -// InitRepo creates the base files for a repo. It inspects data.BaseRoles and -// data.ValidTypes to determine what the role names and filename should be. It -// also relies on the keysDB having already been populated with the keys and -// roles. -func (tr *Repo) InitRepo(consistent bool) error { - if err := tr.InitRoot(consistent); err != nil { - return err - } - if _, err := tr.InitTargets(data.CanonicalTargetsRole); err != nil { - return err - } - if err := tr.InitSnapshot(); err != nil { - return err - } - return tr.InitTimestamp() -} - -// InitRoot initializes an empty root file with the 4 core roles based -// on the current content of th ekey db -func (tr *Repo) InitRoot(consistent bool) error { +// InitRoot initializes an empty root file with the 4 core roles passed to the +// method, and the consistent flag. +func (tr *Repo) InitRoot(root, timestamp, snapshot, targets data.BaseRole, consistent bool) error { rootRoles := make(map[string]*data.RootRole) rootKeys := make(map[string]data.PublicKey) - for _, r := range data.BaseRoles { - role := tr.keysDB.GetRole(r) - if role == nil { - return data.ErrInvalidRole{Role: data.CanonicalRootRole, Reason: "root role not initialized in key database"} + + for _, r := range []data.BaseRole{root, timestamp, snapshot, targets} { + rootRoles[r.Name] = &data.RootRole{ + Threshold: r.Threshold, + KeyIDs: r.ListKeyIDs(), } - rootRoles[r] = &role.RootRole - for _, kid := range role.KeyIDs { - // don't need to check if GetKey returns nil, Key presence was - // checked by KeyDB when role was added. - key := tr.keysDB.GetKey(kid) - rootKeys[kid] = key + for kid, k := range r.Keys { + rootKeys[kid] = k } } - root, err := data.NewRoot(rootKeys, rootRoles, consistent) + r, err := data.NewRoot(rootKeys, rootRoles, consistent) if err != nil { return err } - tr.Root = root + tr.Root = r return nil } // InitTargets initializes an empty targets, and returns the new empty target func (tr *Repo) InitTargets(role string) (*data.SignedTargets, error) { - r := data.Role{Name: role} - if !r.IsDelegation() && role != data.CanonicalTargetsRole { + if !data.IsDelegation(role) && role != data.CanonicalTargetsRole { return nil, data.ErrInvalidRole{ Role: role, Reason: fmt.Sprintf("role is not a valid targets role name: %s", role), @@ -371,7 +478,7 @@ func (tr *Repo) InitTargets(role string) (*data.SignedTargets, error) { // InitSnapshot initializes a snapshot based on the current root and targets func (tr *Repo) InitSnapshot() error { if tr.Root == nil { - return ErrNotLoaded{role: "root"} + return ErrNotLoaded{Role: data.CanonicalRootRole} } root, err := tr.Root.ToSigned() if err != nil { @@ -379,7 +486,7 @@ func (tr *Repo) InitSnapshot() error { } if _, ok := tr.Targets[data.CanonicalTargetsRole]; !ok { - return ErrNotLoaded{role: "targets"} + return ErrNotLoaded{Role: data.CanonicalTargetsRole} } targets, err := tr.Targets[data.CanonicalTargetsRole].ToSigned() if err != nil { @@ -408,31 +515,8 @@ func (tr *Repo) InitTimestamp() error { return nil } -// SetRoot parses the Signed object into a SignedRoot object, sets -// the keys and roles in the KeyDB, and sets the Repo.Root field -// to the SignedRoot object. +// SetRoot sets the Repo.Root field to the SignedRoot object. func (tr *Repo) SetRoot(s *data.SignedRoot) error { - for _, key := range s.Signed.Keys { - logrus.Debug("Adding key ", key.ID()) - tr.keysDB.AddKey(key) - } - for roleName, role := range s.Signed.Roles { - logrus.Debugf("Adding role %s with keys %s", roleName, strings.Join(role.KeyIDs, ",")) - baseRole, err := data.NewRole( - roleName, - role.Threshold, - role.KeyIDs, - nil, - nil, - ) - if err != nil { - return err - } - err = tr.keysDB.AddRole(baseRole) - if err != nil { - return err - } - } tr.Root = s return nil } @@ -451,16 +535,9 @@ func (tr *Repo) SetSnapshot(s *data.SignedSnapshot) error { return nil } -// SetTargets parses the Signed object into a SignedTargets object, -// reads the delegated roles and keys into the KeyDB, and sets the -// SignedTargets object agaist the role in the Repo.Targets map. +// SetTargets sets the SignedTargets object agaist the role in the +// Repo.Targets map. func (tr *Repo) SetTargets(role string, s *data.SignedTargets) error { - for _, k := range s.Signed.Delegations.Keys { - tr.keysDB.AddKey(k) - } - for _, r := range s.Signed.Delegations.Roles { - tr.keysDB.AddRole(r) - } tr.Targets[role] = s return nil } @@ -479,15 +556,11 @@ func (tr Repo) TargetMeta(role, path string) *data.FileMeta { // TargetDelegations returns a slice of Roles that are valid publishers // for the target path provided. -func (tr Repo) TargetDelegations(role, path, pathHex string) []*data.Role { - if pathHex == "" { - pathDigest := sha256.Sum256([]byte(path)) - pathHex = hex.EncodeToString(pathDigest[:]) - } +func (tr Repo) TargetDelegations(role, path string) []*data.Role { var roles []*data.Role if t, ok := tr.Targets[role]; ok { for _, r := range t.Signed.Delegations.Roles { - if r.CheckPrefixes(pathHex) || r.CheckPaths(path) { + if r.CheckPaths(path) { roles = append(roles, r) } } @@ -495,50 +568,34 @@ func (tr Repo) TargetDelegations(role, path, pathHex string) []*data.Role { return roles } -// FindTarget attempts to find the target represented by the given -// path by starting at the top targets file and traversing -// appropriate delegations until the first entry is found or it -// runs out of locations to search. -// N.B. Multiple entries may exist in different delegated roles -// for the same target. Only the first one encountered is returned. -func (tr Repo) FindTarget(path string) *data.FileMeta { - pathDigest := sha256.Sum256([]byte(path)) - pathHex := hex.EncodeToString(pathDigest[:]) - - var walkTargets func(role string) *data.FileMeta - walkTargets = func(role string) *data.FileMeta { - if m := tr.TargetMeta(role, path); m != nil { - return m - } - // Depth first search of delegations based on order - // as presented in current targets file for role: - for _, r := range tr.TargetDelegations(role, path, pathHex) { - if m := walkTargets(r.Name); m != nil { - return m - } - } - return nil - } - - return walkTargets("targets") -} - // VerifyCanSign returns nil if the role exists and we have at least one // signing key for the role, false otherwise. This does not check that we have // enough signing keys to meet the threshold, since we want to support the use // case of multiple signers for a role. It returns an error if the role doesn't // exist or if there are no signing keys. func (tr *Repo) VerifyCanSign(roleName string) error { - role := tr.keysDB.GetRole(roleName) - if role == nil { + var ( + role data.BaseRole + err error + ) + // we only need the BaseRole part of a delegation because we're just + // checking KeyIDs + if data.IsDelegation(roleName) { + r, err := tr.GetDelegationRole(roleName) + if err != nil { + return err + } + role = r.BaseRole + } else { + role, err = tr.GetBaseRole(roleName) + } + if err != nil { return data.ErrInvalidRole{Role: roleName, Reason: "does not exist"} } - for _, keyID := range role.KeyIDs { - k := tr.keysDB.GetKey(keyID) - canonicalID, err := utils.CanonicalKeyID(k) + for keyID, k := range role.Keys { check := []string{keyID} - if err == nil { + if canonicalID, err := utils.CanonicalKeyID(k); err == nil { check = append(check, canonicalID) } for _, id := range check { @@ -548,45 +605,123 @@ func (tr *Repo) VerifyCanSign(roleName string) error { } } } - return signed.ErrNoKeys{KeyIDs: role.KeyIDs} + return signed.ErrNoKeys{KeyIDs: role.ListKeyIDs()} +} + +// used for walking the targets/delegations tree, potentially modifying the underlying SignedTargets for the repo +type walkVisitorFunc func(*data.SignedTargets, data.DelegationRole) interface{} + +// WalkTargets will apply the specified visitor function to iteratively walk the targets/delegation metadata tree, +// until receiving a StopWalk. The walk starts from the base "targets" role, and searches for the correct targetPath and/or rolePath +// to call the visitor function on. Any roles passed into skipRoles will be excluded from the walk, as well as roles in those subtrees +func (tr *Repo) WalkTargets(targetPath, rolePath string, visitTargets walkVisitorFunc, skipRoles ...string) error { + // Start with the base targets role, which implicitly has the "" targets path + targetsRole, err := tr.GetBaseRole(data.CanonicalTargetsRole) + if err != nil { + return err + } + // Make the targets role have the empty path, when we treat it as a delegation role + roles := []data.DelegationRole{ + { + BaseRole: targetsRole, + Paths: []string{""}, + }, + } + + for len(roles) > 0 { + role := roles[0] + roles = roles[1:] + + // Check the role metadata + signedTgt, ok := tr.Targets[role.Name] + if !ok { + // The role meta doesn't exist in the repo so continue onward + continue + } + + // We're at a prefix of the desired role subtree, so add its delegation role children and continue walking + if strings.HasPrefix(rolePath, role.Name+"/") { + roles = append(roles, signedTgt.GetValidDelegations(role)...) + continue + } + + // Determine whether to visit this role or not: + // If the paths validate against the specified targetPath and the rolePath is empty or is in the subtree + // Also check if we are choosing to skip visiting this role on this walk (see ListTargets and GetTargetByName priority) + if isValidPath(targetPath, role) && isAncestorRole(role.Name, rolePath) && !utils.StrSliceContains(skipRoles, role.Name) { + // If we had matching path or role name, visit this target and determine whether or not to keep walking + res := visitTargets(signedTgt, role) + switch typedRes := res.(type) { + case StopWalk: + // If the visitor function signalled a stop, return nil to finish the walk + return nil + case nil: + // If the visitor function signalled to continue, add this role's delegation to the walk + roles = append(roles, signedTgt.GetValidDelegations(role)...) + case error: + // Propagate any errors from the visitor + return typedRes + default: + // Return out with an error if we got a different result + return fmt.Errorf("unexpected return while walking: %v", res) + } + + } + } + return nil +} + +// helper function that returns whether the candidateChild role name is an ancestor or equal to the candidateAncestor role name +// Will return true if given an empty candidateAncestor role name +// The HasPrefix check is for determining whether the role name for candidateChild is a child (direct or further down the chain) +// of candidateAncestor, for ex: candidateAncestor targets/a and candidateChild targets/a/b/c +func isAncestorRole(candidateChild, candidateAncestor string) bool { + return candidateAncestor == "" || candidateAncestor == candidateChild || strings.HasPrefix(candidateChild, candidateAncestor+"/") +} + +// helper function that returns whether the delegation Role is valid against the given path +// Will return true if given an empty candidatePath +func isValidPath(candidatePath string, delgRole data.DelegationRole) bool { + return candidatePath == "" || delgRole.CheckPaths(candidatePath) } // AddTargets will attempt to add the given targets specifically to // the directed role. If the metadata for the role doesn't exist yet, // AddTargets will create one. func (tr *Repo) AddTargets(role string, targets data.Files) (data.Files, error) { - err := tr.VerifyCanSign(role) if err != nil { return nil, err } - // check the role's metadata - t, ok := tr.Targets[role] + // check existence of the role's metadata + _, ok := tr.Targets[role] if !ok { // the targetfile may not exist yet - if not, then create it var err error - t, err = tr.InitTargets(role) + _, err = tr.InitTargets(role) if err != nil { return nil, err } } - // VerifyCanSign already makes sure this is not nil - r := tr.keysDB.GetRole(role) - - invalid := make(data.Files) - for path, target := range targets { - pathDigest := sha256.Sum256([]byte(path)) - pathHex := hex.EncodeToString(pathDigest[:]) - if role == data.CanonicalTargetsRole || (r.CheckPaths(path) || r.CheckPrefixes(pathHex)) { - t.Signed.Targets[path] = target - } else { - invalid[path] = target + addedTargets := make(data.Files) + addTargetVisitor := func(targetPath string, targetMeta data.FileMeta) func(*data.SignedTargets, data.DelegationRole) interface{} { + return func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} { + // We've already validated the role's target path in our walk, so just modify the metadata + tgt.Signed.Targets[targetPath] = targetMeta + tgt.Dirty = true + // Also add to our new addedTargets map to keep track of every target we've added successfully + addedTargets[targetPath] = targetMeta + return StopWalk{} } } - t.Dirty = true - if len(invalid) > 0 { - return invalid, fmt.Errorf("Could not add all targets") + + // Walk the role tree while validating the target paths, and add all of our targets + for path, target := range targets { + tr.WalkTargets(path, role, addTargetVisitor(path, target)) + } + if len(addedTargets) != len(targets) { + return nil, fmt.Errorf("Could not add all targets") } return nil, nil } @@ -597,13 +732,23 @@ func (tr *Repo) RemoveTargets(role string, targets ...string) error { return err } + removeTargetVisitor := func(targetPath string) func(*data.SignedTargets, data.DelegationRole) interface{} { + return func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} { + // We've already validated the role path in our walk, so just modify the metadata + // We don't check against the target path against the valid role paths because it's + // possible we got into an invalid state and are trying to fix it + delete(tgt.Signed.Targets, targetPath) + tgt.Dirty = true + return StopWalk{} + } + } + // if the role exists but metadata does not yet, then our work is done - t, ok := tr.Targets[role] + _, ok := tr.Targets[role] if ok { for _, path := range targets { - delete(t.Signed.Targets, path) + tr.WalkTargets("", role, removeTargetVisitor(path)) } - t.Dirty = true } return nil @@ -644,12 +789,15 @@ func (tr *Repo) SignRoot(expires time.Time) (*data.Signed, error) { logrus.Debug("signing root...") tr.Root.Signed.Expires = expires tr.Root.Signed.Version++ - root := tr.keysDB.GetRole(data.CanonicalRootRole) + root, err := tr.GetBaseRole(data.CanonicalRootRole) + if err != nil { + return nil, err + } signed, err := tr.Root.ToSigned() if err != nil { return nil, err } - signed, err = tr.sign(signed, *root) + signed, err = tr.sign(signed, root) if err != nil { return nil, err } @@ -673,8 +821,22 @@ func (tr *Repo) SignTargets(role string, expires time.Time) (*data.Signed, error logrus.Debug("errored getting targets data.Signed object") return nil, err } - targets := tr.keysDB.GetRole(role) - signed, err = tr.sign(signed, *targets) + + var targets data.BaseRole + if role == data.CanonicalTargetsRole { + targets, err = tr.GetBaseRole(role) + } else { + tr, err := tr.GetDelegationRole(role) + if err != nil { + return nil, err + } + targets = tr.BaseRole + } + if err != nil { + return nil, err + } + + signed, err = tr.sign(signed, targets) if err != nil { logrus.Debug("errored signing ", role) return nil, err @@ -712,8 +874,11 @@ func (tr *Repo) SignSnapshot(expires time.Time) (*data.Signed, error) { if err != nil { return nil, err } - snapshot := tr.keysDB.GetRole(data.CanonicalSnapshotRole) - signed, err = tr.sign(signed, *snapshot) + snapshot, err := tr.GetBaseRole(data.CanonicalSnapshotRole) + if err != nil { + return nil, err + } + signed, err = tr.sign(signed, snapshot) if err != nil { return nil, err } @@ -738,8 +903,11 @@ func (tr *Repo) SignTimestamp(expires time.Time) (*data.Signed, error) { if err != nil { return nil, err } - timestamp := tr.keysDB.GetRole(data.CanonicalTimestampRole) - signed, err = tr.sign(signed, *timestamp) + timestamp, err := tr.GetBaseRole(data.CanonicalTimestampRole) + if err != nil { + return nil, err + } + signed, err = tr.sign(signed, timestamp) if err != nil { return nil, err } @@ -748,17 +916,10 @@ func (tr *Repo) SignTimestamp(expires time.Time) (*data.Signed, error) { return signed, nil } -func (tr Repo) sign(signedData *data.Signed, role data.Role) (*data.Signed, error) { - ks := make([]data.PublicKey, 0, len(role.KeyIDs)) - for _, kid := range role.KeyIDs { - k := tr.keysDB.GetKey(kid) - if k == nil { - continue - } - ks = append(ks, k) - } +func (tr Repo) sign(signedData *data.Signed, role data.BaseRole) (*data.Signed, error) { + ks := role.ListKeys() if len(ks) < 1 { - return nil, keys.ErrInvalidKey + return nil, signed.ErrNoKeys{} } err := signed.Sign(tr.cryptoService, signedData, ks...) if err != nil { diff --git a/components/engine/vendor/src/github.com/docker/notary/tuf/utils/utils.go b/components/engine/vendor/src/github.com/docker/notary/tuf/utils/utils.go index c09019c2e5..8190c5eccd 100644 --- a/components/engine/vendor/src/github.com/docker/notary/tuf/utils/utils.go +++ b/components/engine/vendor/src/github.com/docker/notary/tuf/utils/utils.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "crypto/sha512" "crypto/tls" + "encoding/hex" "fmt" "io" "net/http" @@ -61,6 +62,17 @@ func StrSliceContains(ss []string, s string) bool { return false } +// StrSliceRemove removes the the given string from the slice, returning a new slice +func StrSliceRemove(ss []string, s string) []string { + res := []string{} + for _, v := range ss { + if v != s { + res = append(res, v) + } + } + return res +} + // StrSliceContainsI checks if the given string appears in the slice // in a case insensitive manner func StrSliceContainsI(ss []string, s string) bool { @@ -146,3 +158,14 @@ func FindRoleIndex(rs []*data.Role, name string) int { } return -1 } + +// ConsistentName generates the appropriate HTTP URL path for the role, +// based on whether the repo is marked as consistent. The RemoteStore +// is responsible for adding file extensions. +func ConsistentName(role string, hashSha256 []byte) string { + if len(hashSha256) > 0 { + hash := hex.EncodeToString(hashSha256) + return fmt.Sprintf("%s.%s", role, hash) + } + return role +} diff --git a/components/engine/vendor/src/github.com/miekg/pkcs11/.travis.yml b/components/engine/vendor/src/github.com/miekg/pkcs11/.travis.yml new file mode 100644 index 0000000000..477573388d --- /dev/null +++ b/components/engine/vendor/src/github.com/miekg/pkcs11/.travis.yml @@ -0,0 +1,14 @@ +language: go +sudo: required +dist: trusty + +go: + - 1.5 + - 1.6 + +script: + - go test -v ./... + +before_script: + - sudo apt-get update + - sudo apt-get -y install libsofthsm diff --git a/components/engine/vendor/src/github.com/miekg/pkcs11/README.md b/components/engine/vendor/src/github.com/miekg/pkcs11/README.md index 6e7e550c21..f41c474394 100644 --- a/components/engine/vendor/src/github.com/miekg/pkcs11/README.md +++ b/components/engine/vendor/src/github.com/miekg/pkcs11/README.md @@ -1,13 +1,11 @@ -# PKCS#11 +# PKCS#11 [![Build Status](https://travis-ci.org/miekg/pkcs11.png?branch=master)](https://travis-ci.org/miekg/pkcs11) This is a Go implementation of the PKCS#11 API. It wraps the library closely, but uses Go idiom were it makes sense. It has been tested with SoftHSM. ## SoftHSM -* Make it use a custom configuration file - - export SOFTHSM_CONF=$PWD/softhsm.conf +* Make it use a custom configuration file `export SOFTHSM_CONF=$PWD/softhsm.conf` * Then use `softhsm` to init it @@ -22,16 +20,37 @@ were it makes sense. It has been tested with SoftHSM. A skeleton program would look somewhat like this (yes, pkcs#11 is verbose): p := pkcs11.New("/usr/lib/softhsm/libsofthsm.so") - p.Initialize() + err := p.Initialize() + if err != nil { + panic(err) + } + defer p.Destroy() defer p.Finalize() - slots, _ := p.GetSlotList(true) - session, _ := p.OpenSession(slots[0], pkcs11.CKF_SERIAL_SESSION|pkcs11.CKF_RW_SESSION) + + slots, err := p.GetSlotList(true) + if err != nil { + panic(err) + } + + session, err := p.OpenSession(slots[0], pkcs11.CKF_SERIAL_SESSION|pkcs11.CKF_RW_SESSION) + if err != nil { + panic(err) + } defer p.CloseSession(session) - p.Login(session, pkcs11.CKU_USER, "1234") + + err = p.Login(session, pkcs11.CKU_USER, "1234") + if err != nil { + panic(err) + } defer p.Logout(session) + p.DigestInit(session, []*pkcs11.Mechanism{pkcs11.NewMechanism(pkcs11.CKM_SHA_1, nil)}) hash, err := p.Digest(session, []byte("this is a string")) + if err != nil { + panic(err) + } + for _, d := range hash { fmt.Printf("%x", d) } @@ -41,8 +60,5 @@ Further examples are included in the tests. # TODO -* Fix/double check endian stuff, see types.go NewAttribute(); -* Kill C.Sizeof in that same function. -* Look at the memory copying in fast functions (sign, hash etc). -* Fix inconsistencies in naming? -* Add tests -- there are way too few +* Fix/double check endian stuff, see types.go NewAttribute() +* Look at the memory copying in fast functions (sign, hash etc) diff --git a/components/engine/vendor/src/github.com/miekg/pkcs11/const.go b/components/engine/vendor/src/github.com/miekg/pkcs11/const.go index 79a25e345d..5901b602cd 100644 --- a/components/engine/vendor/src/github.com/miekg/pkcs11/const.go +++ b/components/engine/vendor/src/github.com/miekg/pkcs11/const.go @@ -23,9 +23,9 @@ const ( CKO_VENDOR_DEFINED uint = 0x80000000 ) -// Generated with: awk '/#define CK[AFKMR]/{ print $2 "=" $3 }' pkcs11t.h +// Generated with: awk '/#define CK[AFKMRC]/{ print $2 "=" $3 }' pkcs11t.h -// All the flag (CKF_), attribute (CKA_), error code (CKR_), key type (CKK_) and +// All the flag (CKF_), attribute (CKA_), error code (CKR_), key type (CKK_), certificate type (CKC_) and // mechanism (CKM_) constants as defined in PKCS#11. const ( CKF_TOKEN_PRESENT = 0x00000001 @@ -83,6 +83,10 @@ const ( CKK_CAMELLIA = 0x00000025 CKK_ARIA = 0x00000026 CKK_VENDOR_DEFINED = 0x80000000 + CKC_X_509 = 0x00000000 + CKC_X_509_ATTR_CERT = 0x00000001 + CKC_WTLS = 0x00000002 + CKC_VENDOR_DEFINED = 0x80000000 CKF_ARRAY_ATTRIBUTE = 0x40000000 CKA_CLASS = 0x00000000 CKA_TOKEN = 0x00000001 @@ -117,11 +121,11 @@ const ( CKA_VERIFY = 0x0000010A CKA_VERIFY_RECOVER = 0x0000010B CKA_DERIVE = 0x0000010C - CKA_START_DATE = 0x00000110 // Use time.Time as a value. - CKA_END_DATE = 0x00000111 // Use time.Time as a value. + CKA_START_DATE = 0x00000110 + CKA_END_DATE = 0x00000111 CKA_MODULUS = 0x00000120 CKA_MODULUS_BITS = 0x00000121 - CKA_PUBLIC_EXPONENT = 0x00000122 // Use []byte slice as a value. + CKA_PUBLIC_EXPONENT = 0x00000122 CKA_PRIVATE_EXPONENT = 0x00000123 CKA_PRIME_1 = 0x00000124 CKA_PRIME_2 = 0x00000125 diff --git a/components/engine/vendor/src/github.com/miekg/pkcs11/pkcs11.go b/components/engine/vendor/src/github.com/miekg/pkcs11/pkcs11.go index fc0c1d96f6..424e74980a 100644 --- a/components/engine/vendor/src/github.com/miekg/pkcs11/pkcs11.go +++ b/components/engine/vendor/src/github.com/miekg/pkcs11/pkcs11.go @@ -1000,7 +1000,8 @@ func (c *Ctx) Logout(sh SessionHandle) error { /* CreateObject creates a new object. */ func (c *Ctx) CreateObject(sh SessionHandle, temp []*Attribute) (ObjectHandle, error) { var obj C.CK_OBJECT_HANDLE - t, tcount := cAttributeList(temp) + arena, t, tcount := cAttributeList(temp) + defer arena.Free() e := C.CreateObject(c.ctx, C.CK_SESSION_HANDLE(sh), t, tcount, C.CK_OBJECT_HANDLE_PTR(&obj)) e1 := toError(e) if e1 == nil { @@ -1012,7 +1013,8 @@ func (c *Ctx) CreateObject(sh SessionHandle, temp []*Attribute) (ObjectHandle, e /* CopyObject copies an object, creating a new object for the copy. */ func (c *Ctx) CopyObject(sh SessionHandle, o ObjectHandle, temp []*Attribute) (ObjectHandle, error) { var obj C.CK_OBJECT_HANDLE - t, tcount := cAttributeList(temp) + arena, t, tcount := cAttributeList(temp) + defer arena.Free() e := C.CopyObject(c.ctx, C.CK_SESSION_HANDLE(sh), C.CK_OBJECT_HANDLE(o), t, tcount, C.CK_OBJECT_HANDLE_PTR(&obj)) e1 := toError(e) @@ -1062,7 +1064,8 @@ func (c *Ctx) GetAttributeValue(sh SessionHandle, o ObjectHandle, a []*Attribute /* SetAttributeValue modifies the value of one or more object attributes */ func (c *Ctx) SetAttributeValue(sh SessionHandle, o ObjectHandle, a []*Attribute) error { - pa, palen := cAttributeList(a) + arena, pa, palen := cAttributeList(a) + defer arena.Free() e := C.SetAttributeValue(c.ctx, C.CK_SESSION_HANDLE(sh), C.CK_OBJECT_HANDLE(o), pa, palen) return toError(e) } @@ -1070,7 +1073,8 @@ func (c *Ctx) SetAttributeValue(sh SessionHandle, o ObjectHandle, a []*Attribute // FindObjectsInit initializes a search for token and session // objects that match a template. func (c *Ctx) FindObjectsInit(sh SessionHandle, temp []*Attribute) error { - t, tcount := cAttributeList(temp) + arena, t, tcount := cAttributeList(temp) + defer arena.Free() e := C.FindObjectsInit(c.ctx, C.CK_SESSION_HANDLE(sh), t, tcount) return toError(e) } @@ -1106,7 +1110,8 @@ func (c *Ctx) FindObjectsFinal(sh SessionHandle) error { /* EncryptInit initializes an encryption operation. */ func (c *Ctx) EncryptInit(sh SessionHandle, m []*Mechanism, o ObjectHandle) error { - mech, _ := cMechanismList(m) + arena, mech, _ := cMechanismList(m) + defer arena.Free() e := C.EncryptInit(c.ctx, C.CK_SESSION_HANDLE(sh), mech, C.CK_OBJECT_HANDLE(o)) return toError(e) } @@ -1158,7 +1163,8 @@ func (c *Ctx) EncryptFinal(sh SessionHandle) ([]byte, error) { /* DecryptInit initializes a decryption operation. */ func (c *Ctx) DecryptInit(sh SessionHandle, m []*Mechanism, o ObjectHandle) error { - mech, _ := cMechanismList(m) + arena, mech, _ := cMechanismList(m) + defer arena.Free() e := C.DecryptInit(c.ctx, C.CK_SESSION_HANDLE(sh), mech, C.CK_OBJECT_HANDLE(o)) return toError(e) } @@ -1210,7 +1216,8 @@ func (c *Ctx) DecryptFinal(sh SessionHandle) ([]byte, error) { /* DigestInit initializes a message-digesting operation. */ func (c *Ctx) DigestInit(sh SessionHandle, m []*Mechanism) error { - mech, _ := cMechanismList(m) + arena, mech, _ := cMechanismList(m) + defer arena.Free() e := C.DigestInit(c.ctx, C.CK_SESSION_HANDLE(sh), mech) return toError(e) } @@ -1270,7 +1277,8 @@ func (c *Ctx) DigestFinal(sh SessionHandle) ([]byte, error) { // the data, and plaintext cannot be recovered from the // signature. func (c *Ctx) SignInit(sh SessionHandle, m []*Mechanism, o ObjectHandle) error { - mech, _ := cMechanismList(m) // Only the first is used, but still use a list. + arena, mech, _ := cMechanismList(m) // Only the first is used, but still use a list. + defer arena.Free() e := C.SignInit(c.ctx, C.CK_SESSION_HANDLE(sh), mech, C.CK_OBJECT_HANDLE(o)) return toError(e) } @@ -1317,7 +1325,8 @@ func (c *Ctx) SignFinal(sh SessionHandle) ([]byte, error) { // SignRecoverInit initializes a signature operation, where // the data can be recovered from the signature. func (c *Ctx) SignRecoverInit(sh SessionHandle, m []*Mechanism, key ObjectHandle) error { - mech, _ := cMechanismList(m) + arena, mech, _ := cMechanismList(m) + defer arena.Free() e := C.SignRecoverInit(c.ctx, C.CK_SESSION_HANDLE(sh), mech, C.CK_OBJECT_HANDLE(key)) return toError(e) } @@ -1342,7 +1351,8 @@ func (c *Ctx) SignRecover(sh SessionHandle, data []byte) ([]byte, error) { // signature is an appendix to the data, and plaintext cannot // be recovered from the signature (e.g. DSA). func (c *Ctx) VerifyInit(sh SessionHandle, m []*Mechanism, key ObjectHandle) error { - mech, _ := cMechanismList(m) // only use one here + arena, mech, _ := cMechanismList(m) // only use one here + defer arena.Free() e := C.VerifyInit(c.ctx, C.CK_SESSION_HANDLE(sh), mech, C.CK_OBJECT_HANDLE(key)) return toError(e) } @@ -1373,7 +1383,8 @@ func (c *Ctx) VerifyFinal(sh SessionHandle, signature []byte) error { // VerifyRecoverInit initializes a signature verification // operation, where the data is recovered from the signature. func (c *Ctx) VerifyRecoverInit(sh SessionHandle, m []*Mechanism, key ObjectHandle) error { - mech, _ := cMechanismList(m) + arena, mech, _ := cMechanismList(m) + defer arena.Free() e := C.VerifyRecoverInit(c.ctx, C.CK_SESSION_HANDLE(sh), mech, C.CK_OBJECT_HANDLE(key)) return toError(e) } @@ -1458,8 +1469,10 @@ func (c *Ctx) DecryptVerifyUpdate(sh SessionHandle, cipher []byte) ([]byte, erro /* GenerateKey generates a secret key, creating a new key object. */ func (c *Ctx) GenerateKey(sh SessionHandle, m []*Mechanism, temp []*Attribute) (ObjectHandle, error) { var key C.CK_OBJECT_HANDLE - t, tcount := cAttributeList(temp) - mech, _ := cMechanismList(m) + attrarena, t, tcount := cAttributeList(temp) + defer attrarena.Free() + mecharena, mech, _ := cMechanismList(m) + defer mecharena.Free() e := C.GenerateKey(c.ctx, C.CK_SESSION_HANDLE(sh), mech, t, tcount, C.CK_OBJECT_HANDLE_PTR(&key)) e1 := toError(e) if e1 == nil { @@ -1474,9 +1487,12 @@ func (c *Ctx) GenerateKeyPair(sh SessionHandle, m []*Mechanism, public, private pubkey C.CK_OBJECT_HANDLE privkey C.CK_OBJECT_HANDLE ) - pub, pubcount := cAttributeList(public) - priv, privcount := cAttributeList(private) - mech, _ := cMechanismList(m) + pubarena, pub, pubcount := cAttributeList(public) + defer pubarena.Free() + privarena, priv, privcount := cAttributeList(private) + defer privarena.Free() + mecharena, mech, _ := cMechanismList(m) + defer mecharena.Free() e := C.GenerateKeyPair(c.ctx, C.CK_SESSION_HANDLE(sh), mech, pub, pubcount, priv, privcount, C.CK_OBJECT_HANDLE_PTR(&pubkey), C.CK_OBJECT_HANDLE_PTR(&privkey)) e1 := toError(e) if e1 == nil { @@ -1491,7 +1507,8 @@ func (c *Ctx) WrapKey(sh SessionHandle, m []*Mechanism, wrappingkey, key ObjectH wrappedkey C.CK_BYTE_PTR wrappedkeylen C.CK_ULONG ) - mech, _ := cMechanismList(m) + arena, mech, _ := cMechanismList(m) + defer arena.Free() e := C.WrapKey(c.ctx, C.CK_SESSION_HANDLE(sh), mech, C.CK_OBJECT_HANDLE(wrappingkey), C.CK_OBJECT_HANDLE(key), &wrappedkey, &wrappedkeylen) if toError(e) != nil { return nil, toError(e) @@ -1504,8 +1521,10 @@ func (c *Ctx) WrapKey(sh SessionHandle, m []*Mechanism, wrappingkey, key ObjectH /* UnwrapKey unwraps (decrypts) a wrapped key, creating a new key object. */ func (c *Ctx) UnwrapKey(sh SessionHandle, m []*Mechanism, unwrappingkey ObjectHandle, wrappedkey []byte, a []*Attribute) (ObjectHandle, error) { var key C.CK_OBJECT_HANDLE - ac, aclen := cAttributeList(a) - mech, _ := cMechanismList(m) + attrarena, ac, aclen := cAttributeList(a) + defer attrarena.Free() + mecharena, mech, _ := cMechanismList(m) + defer mecharena.Free() e := C.UnwrapKey(c.ctx, C.CK_SESSION_HANDLE(sh), mech, C.CK_OBJECT_HANDLE(unwrappingkey), C.CK_BYTE_PTR(unsafe.Pointer(&wrappedkey[0])), C.CK_ULONG(len(wrappedkey)), ac, aclen, &key) return ObjectHandle(key), toError(e) } @@ -1513,8 +1532,10 @@ func (c *Ctx) UnwrapKey(sh SessionHandle, m []*Mechanism, unwrappingkey ObjectHa // DeriveKey derives a key from a base key, creating a new key object. */ func (c *Ctx) DeriveKey(sh SessionHandle, m []*Mechanism, basekey ObjectHandle, a []*Attribute) (ObjectHandle, error) { var key C.CK_OBJECT_HANDLE - ac, aclen := cAttributeList(a) - mech, _ := cMechanismList(m) + attrarena, ac, aclen := cAttributeList(a) + defer attrarena.Free() + mecharena, mech, _ := cMechanismList(m) + defer mecharena.Free() e := C.DeriveKey(c.ctx, C.CK_SESSION_HANDLE(sh), mech, C.CK_OBJECT_HANDLE(basekey), ac, aclen, &key) return ObjectHandle(key), toError(e) } diff --git a/components/engine/vendor/src/github.com/miekg/pkcs11/types.go b/components/engine/vendor/src/github.com/miekg/pkcs11/types.go index 9a709cd851..e789bf1560 100644 --- a/components/engine/vendor/src/github.com/miekg/pkcs11/types.go +++ b/components/engine/vendor/src/github.com/miekg/pkcs11/types.go @@ -15,17 +15,13 @@ package pkcs11 #define CK_CALLBACK_FUNCTION(returnType, name) returnType (* name) #include +#include #include "pkcs11.h" CK_ULONG Index(CK_ULONG_PTR array, CK_ULONG i) { return array[i]; } - -CK_ULONG Sizeof() -{ - return sizeof(CK_ULONG); -} */ import "C" @@ -35,6 +31,21 @@ import ( "unsafe" ) +type arena []unsafe.Pointer + +func (a *arena) Allocate(obj []byte) (C.CK_VOID_PTR, C.CK_ULONG) { + cobj := C.calloc(C.size_t(len(obj)), 1) + *a = append(*a, cobj) + C.memmove(cobj, unsafe.Pointer(&obj[0]), C.size_t(len(obj))) + return C.CK_VOID_PTR(cobj), C.CK_ULONG(len(obj)) +} + +func (a arena) Free() { + for _, p := range a { + C.free(p) + } +} + // toList converts from a C style array to a []uint. func toList(clist C.CK_ULONG_PTR, size C.CK_ULONG) []uint { l := make([]uint, int(size)) @@ -53,6 +64,11 @@ func cBBool(x bool) C.CK_BBOOL { return C.CK_BBOOL(C.CK_FALSE) } +func uintToBytes(x uint64) []byte { + ul := C.CK_ULONG(x) + return C.GoBytes(unsafe.Pointer(&ul), C.int(unsafe.Sizeof(ul))) +} + // Error represents an PKCS#11 error. type Error uint @@ -156,46 +172,23 @@ func NewAttribute(typ uint, x interface{}) *Attribute { if x == nil { return a } - switch x.(type) { - case bool: // create bbool - if x.(bool) { + switch v := x.(type) { + case bool: + if v { a.Value = []byte{1} - break - } - a.Value = []byte{0} - case uint, int: - var y uint - if _, ok := x.(int); ok { - y = uint(x.(int)) - } - if _, ok := x.(uint); ok { - y = x.(uint) - } - // TODO(miek): ugly! - switch int(C.Sizeof()) { - case 4: - a.Value = make([]byte, 4) - a.Value[0] = byte(y) - a.Value[1] = byte(y >> 8) - a.Value[2] = byte(y >> 16) - a.Value[3] = byte(y >> 24) - case 8: - a.Value = make([]byte, 8) - a.Value[0] = byte(y) - a.Value[1] = byte(y >> 8) - a.Value[2] = byte(y >> 16) - a.Value[3] = byte(y >> 24) - a.Value[4] = byte(y >> 32) - a.Value[5] = byte(y >> 40) - a.Value[6] = byte(y >> 48) - a.Value[7] = byte(y >> 56) + } else { + a.Value = []byte{0} } + case int: + a.Value = uintToBytes(uint64(v)) + case uint: + a.Value = uintToBytes(uint64(v)) case string: - a.Value = []byte(x.(string)) - case []byte: // just copy - a.Value = x.([]byte) + a.Value = []byte(v) + case []byte: + a.Value = v case time.Time: // for CKA_DATE - a.Value = cDate(x.(time.Time)) + a.Value = cDate(v) default: panic("pkcs11: unhandled attribute type") } @@ -203,9 +196,10 @@ func NewAttribute(typ uint, x interface{}) *Attribute { } // cAttribute returns the start address and the length of an attribute list. -func cAttributeList(a []*Attribute) (C.CK_ATTRIBUTE_PTR, C.CK_ULONG) { +func cAttributeList(a []*Attribute) (arena, C.CK_ATTRIBUTE_PTR, C.CK_ULONG) { + var arena arena if len(a) == 0 { - return nil, 0 + return nil, nil, 0 } pa := make([]C.CK_ATTRIBUTE, len(a)) for i := 0; i < len(a); i++ { @@ -213,10 +207,9 @@ func cAttributeList(a []*Attribute) (C.CK_ATTRIBUTE_PTR, C.CK_ULONG) { if a[i].Value == nil { continue } - pa[i].pValue = C.CK_VOID_PTR((&a[i].Value[0])) - pa[i].ulValueLen = C.CK_ULONG(len(a[i].Value)) + pa[i].pValue, pa[i].ulValueLen = arena.Allocate(a[i].Value) } - return C.CK_ATTRIBUTE_PTR(&pa[0]), C.CK_ULONG(len(a)) + return arena, C.CK_ATTRIBUTE_PTR(&pa[0]), C.CK_ULONG(len(a)) } func cDate(t time.Time) []byte { @@ -250,9 +243,10 @@ func NewMechanism(mech uint, x interface{}) *Mechanism { return m } -func cMechanismList(m []*Mechanism) (C.CK_MECHANISM_PTR, C.CK_ULONG) { +func cMechanismList(m []*Mechanism) (arena, C.CK_MECHANISM_PTR, C.CK_ULONG) { + var arena arena if len(m) == 0 { - return nil, 0 + return nil, nil, 0 } pm := make([]C.CK_MECHANISM, len(m)) for i := 0; i < len(m); i++ { @@ -260,10 +254,9 @@ func cMechanismList(m []*Mechanism) (C.CK_MECHANISM_PTR, C.CK_ULONG) { if m[i].Parameter == nil { continue } - pm[i].pParameter = C.CK_VOID_PTR(&(m[i].Parameter[0])) - pm[i].ulParameterLen = C.CK_ULONG(len(m[i].Parameter)) + pm[i].pParameter, pm[i].ulParameterLen = arena.Allocate(m[i].Parameter) } - return C.CK_MECHANISM_PTR(&pm[0]), C.CK_ULONG(len(m)) + return arena, C.CK_MECHANISM_PTR(&pm[0]), C.CK_ULONG(len(m)) } // MechanismInfo provides information about a particular mechanism. diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/README.md b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/README.md index 295edb4f7e..fc6b4b0b18 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/README.md +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/README.md @@ -10,80 +10,165 @@ host system and which is (optionally) isolated from other containers in the syst #### Using libcontainer -To create a container you first have to initialize an instance of a factory -that will handle the creation and initialization for a container. - -Because containers are spawned in a two step process you will need to provide -arguments to a binary that will be executed as the init process for the container. -To use the current binary that is spawning the containers and acting as the parent -you can use `os.Args[0]` and we have a command called `init` setup. +Because containers are spawned in a two step process you will need a binary that +will be executed as the init process for the container. In libcontainer, we use +the current binary (/proc/self/exe) to be executed as the init process, and use +arg "init", we call the first step process "bootstrap", so you always need a "init" +function as the entry of "bootstrap". ```go -root, err := libcontainer.New("/var/lib/container", libcontainer.InitArgs(os.Args[0], "init")) +func init() { + if len(os.Args) > 1 && os.Args[1] == "init" { + runtime.GOMAXPROCS(1) + runtime.LockOSThread() + factory, _ := libcontainer.New("") + if err := factory.StartInitialization(); err != nil { + logrus.Fatal(err) + } + panic("--this line should have never been executed, congratulations--") + } +} +``` + +Then to create a container you first have to initialize an instance of a factory +that will handle the creation and initialization for a container. + +```go +factory, err := libcontainer.New("/var/lib/container", libcontainer.Cgroupfs, libcontainer.InitArgs(os.Args[0], "init")) if err != nil { - log.Fatal(err) + logrus.Fatal(err) + return } ``` Once you have an instance of the factory created we can create a configuration -struct describing how the container is to be created. A sample would look similar to this: +struct describing how the container is to be created. A sample would look similar to this: ```go +defaultMountFlags := syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV config := &configs.Config{ - Rootfs: rootfs, - Capabilities: []string{ - "CAP_CHOWN", - "CAP_DAC_OVERRIDE", - "CAP_FSETID", - "CAP_FOWNER", - "CAP_MKNOD", - "CAP_NET_RAW", - "CAP_SETGID", - "CAP_SETUID", - "CAP_SETFCAP", - "CAP_SETPCAP", - "CAP_NET_BIND_SERVICE", - "CAP_SYS_CHROOT", - "CAP_KILL", - "CAP_AUDIT_WRITE", - }, - Namespaces: configs.Namespaces([]configs.Namespace{ - {Type: configs.NEWNS}, - {Type: configs.NEWUTS}, - {Type: configs.NEWIPC}, - {Type: configs.NEWPID}, - {Type: configs.NEWNET}, - }), - Cgroups: &configs.Cgroup{ - Name: "test-container", - Parent: "system", - AllowAllDevices: false, - AllowedDevices: configs.DefaultAllowedDevices, - }, - - Devices: configs.DefaultAutoCreatedDevices, - Hostname: "testing", - Networks: []*configs.Network{ - { - Type: "loopback", - Address: "127.0.0.1/0", - Gateway: "localhost", - }, - }, - Rlimits: []configs.Rlimit{ - { - Type: syscall.RLIMIT_NOFILE, - Hard: uint64(1024), - Soft: uint64(1024), - }, - }, + Rootfs: "/your/path/to/rootfs", + Capabilities: []string{ + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_FSETID", + "CAP_FOWNER", + "CAP_MKNOD", + "CAP_NET_RAW", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETFCAP", + "CAP_SETPCAP", + "CAP_NET_BIND_SERVICE", + "CAP_SYS_CHROOT", + "CAP_KILL", + "CAP_AUDIT_WRITE", + }, + Namespaces: configs.Namespaces([]configs.Namespace{ + {Type: configs.NEWNS}, + {Type: configs.NEWUTS}, + {Type: configs.NEWIPC}, + {Type: configs.NEWPID}, + {Type: configs.NEWUSER}, + {Type: configs.NEWNET}, + }), + Cgroups: &configs.Cgroup{ + Name: "test-container", + Parent: "system", + Resources: &configs.Resources{ + MemorySwappiness: -1, + AllowAllDevices: false, + AllowedDevices: configs.DefaultAllowedDevices, + }, + }, + MaskPaths: []string{ + "/proc/kcore", + }, + ReadonlyPaths: []string{ + "/proc/sys", "/proc/sysrq-trigger", "/proc/irq", "/proc/bus", + }, + Devices: configs.DefaultAutoCreatedDevices, + Hostname: "testing", + Mounts: []*configs.Mount{ + { + Source: "proc", + Destination: "/proc", + Device: "proc", + Flags: defaultMountFlags, + }, + { + Source: "tmpfs", + Destination: "/dev", + Device: "tmpfs", + Flags: syscall.MS_NOSUID | syscall.MS_STRICTATIME, + Data: "mode=755", + }, + { + Source: "devpts", + Destination: "/dev/pts", + Device: "devpts", + Flags: syscall.MS_NOSUID | syscall.MS_NOEXEC, + Data: "newinstance,ptmxmode=0666,mode=0620,gid=5", + }, + { + Device: "tmpfs", + Source: "shm", + Destination: "/dev/shm", + Data: "mode=1777,size=65536k", + Flags: defaultMountFlags, + }, + { + Source: "mqueue", + Destination: "/dev/mqueue", + Device: "mqueue", + Flags: defaultMountFlags, + }, + { + Source: "sysfs", + Destination: "/sys", + Device: "sysfs", + Flags: defaultMountFlags | syscall.MS_RDONLY, + }, + }, + UidMappings: []configs.IDMap{ + { + ContainerID: 0, + Host: 1000, + size: 65536, + }, + }, + GidMappings: []configs.IDMap{ + { + ContainerID: 0, + Host: 1000, + size: 65536, + }, + }, + Networks: []*configs.Network{ + { + Type: "loopback", + Address: "127.0.0.1/0", + Gateway: "localhost", + }, + }, + Rlimits: []configs.Rlimit{ + { + Type: syscall.RLIMIT_NOFILE, + Hard: uint64(1025), + Soft: uint64(1025), + }, + }, } ``` Once you have the configuration populated you can create a container: ```go -container, err := root.Create("container-id", config) +container, err := factory.Create("container-id", config) +if err != nil { + logrus.Fatal(err) + return +} ``` To spawn bash as the initial process inside the container and have the @@ -91,23 +176,25 @@ processes pid returned in order to wait, signal, or kill the process: ```go process := &libcontainer.Process{ - Args: []string{"/bin/bash"}, - Env: []string{"PATH=/bin"}, - User: "daemon", - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, + Args: []string{"/bin/bash"}, + Env: []string{"PATH=/bin"}, + User: "daemon", + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, } err := container.Start(process) if err != nil { - log.Fatal(err) + logrus.Fatal(err) + container.Destroy() + return } // wait for the process to finish. -status, err := process.Wait() +_, err := process.Wait() if err != nil { - log.Fatal(err) + logrus.Fatal(err) } // destroy the container. @@ -124,7 +211,6 @@ processes, err := container.Processes() // it's processes. stats, err := container.Stats() - // pause all processes inside the container. container.Pause() diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/SPEC.md b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/SPEC.md index fad1dd72a2..221545c01d 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/SPEC.md +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/SPEC.md @@ -60,7 +60,7 @@ are required to be mounted within the rootfs that the runtime will setup. After a container's filesystems are mounted within the newly created mount namespace `/dev` will need to be populated with a set of device nodes. It is expected that a rootfs does not need to have any device nodes specified -for `/dev` witin the rootfs as the container will setup the correct devices +for `/dev` within the rootfs as the container will setup the correct devices that are required for executing a container's process. | Path | Mode | Access | @@ -142,6 +142,7 @@ system resources like cpu, memory, and device access. | perf_event | 1 | | freezer | 1 | | hugetlb | 1 | +| pids | 1 | All cgroup subsystem are joined so that statistics can be collected from @@ -199,7 +200,7 @@ provide a good default for security and flexibility for the applications. | CAP_SYS_BOOT | 0 | | CAP_LEASE | 0 | | CAP_WAKE_ALARM | 0 | -| CAP_BLOCK_SUSPE | 0 | +| CAP_BLOCK_SUSPEND | 0 | Additional security layers like [apparmor](https://wiki.ubuntu.com/AppArmor) diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/cgroups.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/cgroups.go index a08e905caa..c8f7796567 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/cgroups.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/cgroups.go @@ -15,6 +15,9 @@ type Manager interface { // Returns the PIDs inside the cgroup set GetPids() ([]int, error) + // Returns the PIDs inside the cgroup set & all sub-cgroups + GetAllPids() ([]int, error) + // Returns statistics for the cgroup set GetStats() (*Stats, error) diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go index 7d2da2dc07..758d119621 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go @@ -14,6 +14,7 @@ import ( "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/configs" + libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils" ) var ( @@ -23,12 +24,14 @@ var ( &MemoryGroup{}, &CpuGroup{}, &CpuacctGroup{}, + &PidsGroup{}, &BlkioGroup{}, &HugetlbGroup{}, &NetClsGroup{}, &NetPrioGroup{}, &PerfEventGroup{}, &FreezerGroup{}, + &NameGroup{GroupName: "name=systemd", Join: true}, } CgroupProcesses = "cgroup.procs" HugePageSizes, _ = cgroups.GetHugePageSize() @@ -93,11 +96,10 @@ func getCgroupRoot() (string, error) { } type cgroupData struct { - root string - parent string - name string - config *configs.Cgroup - pid int + root string + innerPath string + config *configs.Cgroup + pid int } func (m *Manager) Apply(pid int) (err error) { @@ -112,12 +114,23 @@ func (m *Manager) Apply(pid int) (err error) { return err } - paths := make(map[string]string) - defer func() { - if err != nil { - cgroups.RemovePaths(paths) + if c.Paths != nil { + paths := make(map[string]string) + for name, path := range c.Paths { + _, err := d.path(name) + if err != nil { + if cgroups.IsNotFound(err) { + continue + } + return err + } + paths[name] = path } - }() + m.Paths = paths + return cgroups.EnterPid(m.Paths, pid) + } + + paths := make(map[string]string) for _, sys := range subsystems { if err := sys.Apply(d); err != nil { return err @@ -135,17 +148,13 @@ func (m *Manager) Apply(pid int) (err error) { paths[sys.Name()] = p } m.Paths = paths - - if paths["cpu"] != "" { - if err := CheckCpushares(paths["cpu"], c.Resources.CpuShares); err != nil { - return err - } - } - return nil } func (m *Manager) Destroy() error { + if m.Cgroups.Paths != nil { + return nil + } m.mu.Lock() defer m.mu.Unlock() if err := cgroups.RemovePaths(m.Paths); err != nil { @@ -179,15 +188,28 @@ func (m *Manager) GetStats() (*cgroups.Stats, error) { } func (m *Manager) Set(container *configs.Config) error { - for name, path := range m.Paths { - sys, err := subsystems.Get(name) - if err == errSubsystemDoesNotExist || !cgroups.PathExists(path) { - continue + for _, sys := range subsystems { + // Generate fake cgroup data. + d, err := getCgroupData(container.Cgroups, -1) + if err != nil { + return err } + // Get the path, but don't error out if the cgroup wasn't found. + path, err := d.path(sys.Name()) + if err != nil && !cgroups.IsNotFound(err) { + return err + } + if err := sys.Set(path, container.Cgroups); err != nil { return err } } + + if m.Paths["cpu"] != "" { + if err := CheckCpushares(m.Paths["cpu"], container.Cgroups.Resources.CpuShares); err != nil { + return err + } + } return nil } @@ -217,41 +239,28 @@ func (m *Manager) Freeze(state configs.FreezerState) error { } func (m *Manager) GetPids() ([]int, error) { - d, err := getCgroupData(m.Cgroups, 0) + dir, err := getCgroupPath(m.Cgroups) if err != nil { return nil, err } - - dir, err := d.path("devices") - if err != nil { - return nil, err - } - return cgroups.GetPids(dir) } -// pathClean makes a path safe for use with filepath.Join. This is done by not -// only cleaning the path, but also (if the path is relative) adding a leading -// '/' and cleaning it (then removing the leading '/'). This ensures that a -// path resulting from prepending another path will always resolve to lexically -// be a subdirectory of the prefixed path. This is all done lexically, so paths -// that include symlinks won't be safe as a result of using pathClean. -func pathClean(path string) string { - // Ensure that all paths are cleaned (especially problematic ones like - // "/../../../../../" which can cause lots of issues). - path = filepath.Clean(path) +func (m *Manager) GetAllPids() ([]int, error) { + dir, err := getCgroupPath(m.Cgroups) + if err != nil { + return nil, err + } + return cgroups.GetAllPids(dir) +} - // If the path isn't absolute, we need to do more processing to fix paths - // such as "../../../..//some/path". We also shouldn't convert absolute - // paths to relative ones. - if !filepath.IsAbs(path) { - path = filepath.Clean(string(os.PathSeparator) + path) - // This can't fail, as (by definition) all paths are relative to root. - path, _ = filepath.Rel(string(os.PathSeparator), path) +func getCgroupPath(c *configs.Cgroup) (string, error) { + d, err := getCgroupData(c, 0) + if err != nil { + return "", err } - // Clean the path again for good measure. - return filepath.Clean(path) + return d.path("devices") } func getCgroupData(c *configs.Cgroup, pid int) (*cgroupData, error) { @@ -260,15 +269,25 @@ func getCgroupData(c *configs.Cgroup, pid int) (*cgroupData, error) { return nil, err } - // Clean the parent slice path. - c.Parent = pathClean(c.Parent) + if (c.Name != "" || c.Parent != "") && c.Path != "" { + return nil, fmt.Errorf("cgroup: either Path or Name and Parent should be used") + } + + // XXX: Do not remove this code. Path safety is important! -- cyphar + cgPath := libcontainerUtils.CleanPath(c.Path) + cgParent := libcontainerUtils.CleanPath(c.Parent) + cgName := libcontainerUtils.CleanPath(c.Name) + + innerPath := cgPath + if innerPath == "" { + innerPath = filepath.Join(cgParent, cgName) + } return &cgroupData{ - root: root, - parent: c.Parent, - name: c.Name, - config: c, - pid: pid, + root: root, + innerPath: innerPath, + config: c, + pid: pid, }, nil } @@ -296,11 +315,10 @@ func (raw *cgroupData) path(subsystem string) (string, error) { return "", err } - cgPath := filepath.Join(raw.parent, raw.name) // If the cgroup name/path is absolute do not look relative to the cgroup of the init process. - if filepath.IsAbs(cgPath) { + if filepath.IsAbs(raw.innerPath) { // Sometimes subsystems can be mounted togethger as 'cpu,cpuacct'. - return filepath.Join(raw.root, filepath.Base(mnt), cgPath), nil + return filepath.Join(raw.root, filepath.Base(mnt), raw.innerPath), nil } parentPath, err := raw.parentPath(subsystem, mnt, root) @@ -308,7 +326,7 @@ func (raw *cgroupData) path(subsystem string) (string, error) { return "", err } - return filepath.Join(parentPath, cgPath), nil + return filepath.Join(parentPath, raw.innerPath), nil } func (raw *cgroupData) join(subsystem string) (string, error) { diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/blkio.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/blkio.go index 518cb63f63..a142cb991d 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/blkio.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/blkio.go @@ -22,15 +22,10 @@ func (s *BlkioGroup) Name() string { } func (s *BlkioGroup) Apply(d *cgroupData) error { - dir, err := d.join("blkio") + _, err := d.join("blkio") if err != nil && !cgroups.IsNotFound(err) { return err } - - if err := s.Set(dir, d.config); err != nil { - return err - } - return nil } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpu.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpu.go index ad5f427ec2..a4ef28a60f 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpu.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpu.go @@ -22,15 +22,10 @@ func (s *CpuGroup) Name() string { func (s *CpuGroup) Apply(d *cgroupData) error { // We always want to join the cpu group, to allow fair cpu scheduling // on a container basis - dir, err := d.join("cpu") + _, err := d.join("cpu") if err != nil && !cgroups.IsNotFound(err) { return err } - - if err := s.Set(dir, d.config); err != nil { - return err - } - return nil } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpuset.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpuset.go index 8daacfc609..cbe62bd983 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpuset.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpuset.go @@ -12,6 +12,7 @@ import ( "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/configs" + libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils" ) type CpusetGroup struct { @@ -64,11 +65,6 @@ func (s *CpusetGroup) ApplyDir(dir string, cgroup *configs.Cgroup, pid int) erro if err := s.ensureParent(dir, root); err != nil { return err } - // the default values inherit from parent cgroup are already set in - // s.ensureParent, cover these if we have our own - if err := s.Set(dir, cgroup); err != nil { - return err - } // because we are not using d.join we need to place the pid into the procs file // unlike the other subsystems if err := writeFile(dir, "cgroup.procs", strconv.Itoa(pid)); err != nil { @@ -93,7 +89,7 @@ func (s *CpusetGroup) getSubsystemSettings(parent string) (cpus []byte, mems []b // it's parent. func (s *CpusetGroup) ensureParent(current, root string) error { parent := filepath.Dir(current) - if filepath.Clean(parent) == root { + if libcontainerUtils.CleanPath(parent) == root { return nil } // Avoid infinite recursion. diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/devices.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/devices.go index a9883eb4bb..5f78331094 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/devices.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/devices.go @@ -5,6 +5,7 @@ package fs import ( "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/configs" + "github.com/opencontainers/runc/libcontainer/system" ) type DevicesGroup struct { @@ -15,21 +16,33 @@ func (s *DevicesGroup) Name() string { } func (s *DevicesGroup) Apply(d *cgroupData) error { - dir, err := d.join("devices") + _, err := d.join("devices") if err != nil { // We will return error even it's `not found` error, devices // cgroup is hard requirement for container's security. return err } - - if err := s.Set(dir, d.config); err != nil { - return err - } - return nil } func (s *DevicesGroup) Set(path string, cgroup *configs.Cgroup) error { + if system.RunningInUserNS() { + return nil + } + + devices := cgroup.Resources.Devices + if len(devices) > 0 { + for _, dev := range devices { + file := "devices.deny" + if dev.Allow { + file = "devices.allow" + } + if err := writeFile(path, file, dev.CgroupString()); err != nil { + return err + } + } + return nil + } if !cgroup.Resources.AllowAllDevices { if err := writeFile(path, "devices.deny", "a"); err != nil { return err diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/freezer.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/freezer.go index 6aaad4e009..e70dfe3b95 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/freezer.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/freezer.go @@ -19,15 +19,10 @@ func (s *FreezerGroup) Name() string { } func (s *FreezerGroup) Apply(d *cgroupData) error { - dir, err := d.join("freezer") + _, err := d.join("freezer") if err != nil && !cgroups.IsNotFound(err) { return err } - - if err := s.Set(dir, d.config); err != nil { - return err - } - return nil } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/hugetlb.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/hugetlb.go index ca106da44d..2f9727719d 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/hugetlb.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/hugetlb.go @@ -19,15 +19,10 @@ func (s *HugetlbGroup) Name() string { } func (s *HugetlbGroup) Apply(d *cgroupData) error { - dir, err := d.join("hugetlb") + _, err := d.join("hugetlb") if err != nil && !cgroups.IsNotFound(err) { return err } - - if err := s.Set(dir, d.config); err != nil { - return err - } - return nil } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/memory.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/memory.go index 6b9687ca2f..2121f6d4d6 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/memory.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/memory.go @@ -32,8 +32,9 @@ func (s *MemoryGroup) Apply(d *cgroupData) (err error) { return err } } - - if err := s.Set(path, d.config); err != nil { + // We have to set kernel memory here, as we can't change it once + // processes have been attached. + if err := s.SetKernelMemory(path, d.config); err != nil { return err } } @@ -50,7 +51,17 @@ func (s *MemoryGroup) Apply(d *cgroupData) (err error) { if err != nil && !cgroups.IsNotFound(err) { return err } + return nil +} +func (s *MemoryGroup) SetKernelMemory(path string, cgroup *configs.Cgroup) error { + // This has to be done separately because it has special constraints (it + // can't be done after there are processes attached to the cgroup). + if cgroup.Resources.KernelMemory > 0 { + if err := writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(cgroup.Resources.KernelMemory, 10)); err != nil { + return err + } + } return nil } @@ -70,12 +81,6 @@ func (s *MemoryGroup) Set(path string, cgroup *configs.Cgroup) error { return err } } - if cgroup.Resources.KernelMemory > 0 { - if err := writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(cgroup.Resources.KernelMemory, 10)); err != nil { - return err - } - } - if cgroup.Resources.OomKillDisable { if err := writeFile(path, "memory.oom_control", "1"); err != nil { return err @@ -157,6 +162,7 @@ func getMemoryData(path, name string) (cgroups.MemoryData, error) { usage := strings.Join([]string{moduleName, "usage_in_bytes"}, ".") maxUsage := strings.Join([]string{moduleName, "max_usage_in_bytes"}, ".") failcnt := strings.Join([]string{moduleName, "failcnt"}, ".") + limit := strings.Join([]string{moduleName, "limit_in_bytes"}, ".") value, err := getCgroupParamUint(path, usage) if err != nil { @@ -182,6 +188,14 @@ func getMemoryData(path, name string) (cgroups.MemoryData, error) { return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", failcnt, err) } memoryData.Failcnt = value + value, err = getCgroupParamUint(path, limit) + if err != nil { + if moduleName != "memory" && os.IsNotExist(err) { + return cgroups.MemoryData{}, nil + } + return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", limit, err) + } + memoryData.Limit = value return memoryData, nil } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/name.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/name.go index 0e423f667d..d8cf1d87c0 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/name.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/name.go @@ -9,6 +9,7 @@ import ( type NameGroup struct { GroupName string + Join bool } func (s *NameGroup) Name() string { @@ -16,6 +17,10 @@ func (s *NameGroup) Name() string { } func (s *NameGroup) Apply(d *cgroupData) error { + if s.Join { + // ignore errors if the named cgroup does not exist + d.join(s.GroupName) + } return nil } @@ -24,6 +29,9 @@ func (s *NameGroup) Set(path string, cgroup *configs.Cgroup) error { } func (s *NameGroup) Remove(d *cgroupData) error { + if s.Join { + removePath(d.path(s.GroupName)) + } return nil } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_cls.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_cls.go index 6382373127..8a4054ba87 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_cls.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_cls.go @@ -15,15 +15,10 @@ func (s *NetClsGroup) Name() string { } func (s *NetClsGroup) Apply(d *cgroupData) error { - dir, err := d.join("net_cls") + _, err := d.join("net_cls") if err != nil && !cgroups.IsNotFound(err) { return err } - - if err := s.Set(dir, d.config); err != nil { - return err - } - return nil } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_prio.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_prio.go index 0dabaae729..d0ab2af894 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_prio.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_prio.go @@ -15,15 +15,10 @@ func (s *NetPrioGroup) Name() string { } func (s *NetPrioGroup) Apply(d *cgroupData) error { - dir, err := d.join("net_prio") + _, err := d.join("net_prio") if err != nil && !cgroups.IsNotFound(err) { return err } - - if err := s.Set(dir, d.config); err != nil { - return err - } - return nil } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/pids.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/pids.go new file mode 100644 index 0000000000..96cbb896cb --- /dev/null +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/pids.go @@ -0,0 +1,57 @@ +// +build linux + +package fs + +import ( + "fmt" + "strconv" + + "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/configs" +) + +type PidsGroup struct { +} + +func (s *PidsGroup) Name() string { + return "pids" +} + +func (s *PidsGroup) Apply(d *cgroupData) error { + _, err := d.join("pids") + if err != nil && !cgroups.IsNotFound(err) { + return err + } + return nil +} + +func (s *PidsGroup) Set(path string, cgroup *configs.Cgroup) error { + if cgroup.Resources.PidsLimit != 0 { + // "max" is the fallback value. + limit := "max" + + if cgroup.Resources.PidsLimit > 0 { + limit = strconv.FormatInt(cgroup.Resources.PidsLimit, 10) + } + + if err := writeFile(path, "pids.max", limit); err != nil { + return err + } + } + + return nil +} + +func (s *PidsGroup) Remove(d *cgroupData) error { + return removePath(d.path("pids")) +} + +func (s *PidsGroup) GetStats(path string, stats *cgroups.Stats) error { + value, err := getCgroupParamUint(path, "pids.current") + if err != nil { + return fmt.Errorf("failed to parse pids.current - %s", err) + } + + stats.PidsStats.Current = value + return nil +} diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/stats.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/stats.go index bda32b20c3..54ace4185d 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/stats.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/stats.go @@ -36,7 +36,9 @@ type MemoryData struct { Usage uint64 `json:"usage,omitempty"` MaxUsage uint64 `json:"max_usage,omitempty"` Failcnt uint64 `json:"failcnt"` + Limit uint64 `json:"limit"` } + type MemoryStats struct { // memory used for cache Cache uint64 `json:"cache,omitempty"` @@ -49,6 +51,11 @@ type MemoryStats struct { Stats map[string]uint64 `json:"stats,omitempty"` } +type PidsStats struct { + // number of pids in the cgroup + Current uint64 `json:"current,omitempty"` +} + type BlkioStatEntry struct { Major uint64 `json:"major,omitempty"` Minor uint64 `json:"minor,omitempty"` @@ -80,6 +87,7 @@ type HugetlbStats struct { type Stats struct { CpuStats CpuStats `json:"cpu_stats,omitempty"` MemoryStats MemoryStats `json:"memory_stats,omitempty"` + PidsStats PidsStats `json:"pids_stats,omitempty"` BlkioStats BlkioStats `json:"blkio_stats,omitempty"` // the map is in the format "size of hugepage: stats of the hugepage" HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"` diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/systemd/apply_nosystemd.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/systemd/apply_nosystemd.go index fa3485f1c0..7de9ae6050 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/systemd/apply_nosystemd.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/systemd/apply_nosystemd.go @@ -26,6 +26,10 @@ func (m *Manager) GetPids() ([]int, error) { return nil, fmt.Errorf("Systemd not supported") } +func (m *Manager) GetAllPids() ([]int, error) { + return nil, fmt.Errorf("Systemd not supported") +} + func (m *Manager) Destroy() error { return fmt.Errorf("Systemd not supported") } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/systemd/apply_systemd.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/systemd/apply_systemd.go index 93300cda38..3161639f21 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/systemd/apply_systemd.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/systemd/apply_systemd.go @@ -55,6 +55,7 @@ var subsystems = subsystemSet{ &fs.MemoryGroup{}, &fs.CpuGroup{}, &fs.CpuacctGroup{}, + &fs.PidsGroup{}, &fs.BlkioGroup{}, &fs.HugetlbGroup{}, &fs.PerfEventGroup{}, @@ -167,6 +168,23 @@ func (m *Manager) Apply(pid int) error { properties []systemdDbus.Property ) + if c.Paths != nil { + paths := make(map[string]string) + for name, path := range c.Paths { + _, err := getSubsystemPath(m.Cgroups, name) + if err != nil { + // Don't fail if a cgroup hierarchy was not found, just skip this subsystem + if cgroups.IsNotFound(err) { + continue + } + return err + } + paths[name] = path + } + m.Paths = paths + return cgroups.EnterPid(m.Paths, pid) + } + if c.Parent != "" { slice = c.Parent } @@ -233,7 +251,7 @@ func (m *Manager) Apply(pid int) error { return err } - // we need to manually join the freezer, net_cls, net_prio and cpuset cgroup in systemd + // we need to manually join the freezer, net_cls, net_prio, pids and cpuset cgroup in systemd // because it does not currently support it via the dbus api. if err := joinFreezer(c, pid); err != nil { return err @@ -246,6 +264,10 @@ func (m *Manager) Apply(pid int) error { return err } + if err := joinPids(c, pid); err != nil { + return err + } + if err := joinCpuset(c, pid); err != nil { return err } @@ -277,17 +299,13 @@ func (m *Manager) Apply(pid int) error { paths[s.Name()] = subsystemPath } m.Paths = paths - - if paths["cpu"] != "" { - if err := fs.CheckCpushares(paths["cpu"], c.Resources.CpuShares); err != nil { - return err - } - } - return nil } func (m *Manager) Destroy() error { + if m.Cgroups.Paths != nil { + return nil + } m.mu.Lock() defer m.mu.Unlock() theConn.StopUnit(getUnitName(m.Cgroups), "replace", nil) @@ -330,68 +348,74 @@ func join(c *configs.Cgroup, subsystem string, pid int) (string, error) { } func joinCpu(c *configs.Cgroup, pid int) error { - path, err := getSubsystemPath(c, "cpu") + _, err := join(c, "cpu", pid) if err != nil && !cgroups.IsNotFound(err) { return err } - if c.Resources.CpuQuota != 0 { - if err = writeFile(path, "cpu.cfs_quota_us", strconv.FormatInt(c.Resources.CpuQuota, 10)); err != nil { - return err - } - } - if c.Resources.CpuPeriod != 0 { - if err = writeFile(path, "cpu.cfs_period_us", strconv.FormatInt(c.Resources.CpuPeriod, 10)); err != nil { - return err - } - } - if c.Resources.CpuRtPeriod != 0 { - if err = writeFile(path, "cpu.rt_period_us", strconv.FormatInt(c.Resources.CpuRtPeriod, 10)); err != nil { - return err - } - } - if c.Resources.CpuRtRuntime != 0 { - if err = writeFile(path, "cpu.rt_runtime_us", strconv.FormatInt(c.Resources.CpuRtRuntime, 10)); err != nil { - return err - } - } - return nil } func joinFreezer(c *configs.Cgroup, pid int) error { - path, err := join(c, "freezer", pid) + _, err := join(c, "freezer", pid) if err != nil && !cgroups.IsNotFound(err) { return err } - freezer, err := subsystems.Get("freezer") - if err != nil { - return err - } - return freezer.Set(path, c) + return nil } func joinNetPrio(c *configs.Cgroup, pid int) error { - path, err := join(c, "net_prio", pid) + _, err := join(c, "net_prio", pid) if err != nil && !cgroups.IsNotFound(err) { return err } - netPrio, err := subsystems.Get("net_prio") - if err != nil { - return err - } - return netPrio.Set(path, c) + return nil } func joinNetCls(c *configs.Cgroup, pid int) error { - path, err := join(c, "net_cls", pid) + _, err := join(c, "net_cls", pid) if err != nil && !cgroups.IsNotFound(err) { return err } - netcls, err := subsystems.Get("net_cls") - if err != nil { + return nil +} + +func joinPids(c *configs.Cgroup, pid int) error { + _, err := join(c, "pids", pid) + if err != nil && !cgroups.IsNotFound(err) { return err } - return netcls.Set(path, c) + return nil +} + +// systemd represents slice heirarchy using `-`, so we need to follow suit when +// generating the path of slice. Essentially, test-a-b.slice becomes +// test.slice/test-a.slice/test-a-b.slice. +func expandSlice(slice string) (string, error) { + suffix := ".slice" + // Name has to end with ".slice", but can't be just ".slice". + if len(slice) < len(suffix) || !strings.HasSuffix(slice, suffix) { + return "", fmt.Errorf("invalid slice name: %s", slice) + } + + // Path-separators are not allowed. + if strings.Contains(slice, "/") { + return "", fmt.Errorf("invalid slice name: %s", slice) + } + + var path, prefix string + sliceName := strings.TrimSuffix(slice, suffix) + for _, component := range strings.Split(sliceName, "-") { + // test--a.slice isn't permitted, nor is -test.slice. + if component == "" { + return "", fmt.Errorf("invalid slice name: %s", slice) + } + + // Append the component to the path and to the prefix. + path += prefix + component + suffix + "/" + prefix += component + "-" + } + + return path, nil } func getSubsystemPath(c *configs.Cgroup, subsystem string) (string, error) { @@ -410,6 +434,11 @@ func getSubsystemPath(c *configs.Cgroup, subsystem string) (string, error) { slice = c.Parent } + slice, err = expandSlice(slice) + if err != nil { + return "", err + } + return filepath.Join(mountpoint, initPath, slice, getUnitName(c)), nil } @@ -440,6 +469,14 @@ func (m *Manager) GetPids() ([]int, error) { return cgroups.GetPids(path) } +func (m *Manager) GetAllPids() ([]int, error) { + path, err := getSubsystemPath(m.Cgroups, "devices") + if err != nil { + return nil, err + } + return cgroups.GetAllPids(path) +} + func (m *Manager) GetStats() (*cgroups.Stats, error) { m.mu.Lock() defer m.mu.Unlock() @@ -458,16 +495,23 @@ func (m *Manager) GetStats() (*cgroups.Stats, error) { } func (m *Manager) Set(container *configs.Config) error { - for name, path := range m.Paths { - sys, err := subsystems.Get(name) - if err == errSubsystemDoesNotExist || !cgroups.PathExists(path) { - continue + for _, sys := range subsystems { + // Get the subsystem path, but don't error out for not found cgroups. + path, err := getSubsystemPath(container.Cgroups, sys.Name()) + if err != nil && !cgroups.IsNotFound(err) { + return err } + if err := sys.Set(path, container.Cgroups); err != nil { return err } } + if m.Paths["cpu"] != "" { + if err := fs.CheckCpushares(m.Paths["cpu"], container.Cgroups.Resources.CpuShares); err != nil { + return err + } + } return nil } @@ -487,17 +531,13 @@ func getUnitName(c *configs.Cgroup) string { // because systemd will re-write the device settings if it needs to re-apply the cgroup context. // This happens at least for v208 when any sibling unit is started. func joinDevices(c *configs.Cgroup, pid int) error { - path, err := join(c, "devices", pid) + _, err := join(c, "devices", pid) // Even if it's `not found` error, we'll return err because devices cgroup // is hard requirement for container security. if err != nil { return err } - devices, err := subsystems.Get("devices") - if err != nil { - return err - } - return devices.Set(path, c) + return nil } func setKernelMemory(c *configs.Cgroup) error { @@ -510,52 +550,16 @@ func setKernelMemory(c *configs.Cgroup) error { return err } - if c.Resources.KernelMemory > 0 { - err = writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(c.Resources.KernelMemory, 10)) - if err != nil { - return err - } - } - - return nil + // This doesn't get called by manager.Set, so we need to do it here. + s := &fs.MemoryGroup{} + return s.SetKernelMemory(path, c) } func joinMemory(c *configs.Cgroup, pid int) error { - path, err := getSubsystemPath(c, "memory") + _, err := join(c, "memory", pid) if err != nil && !cgroups.IsNotFound(err) { return err } - - // -1 disables memoryswap - if c.Resources.MemorySwap > 0 { - err = writeFile(path, "memory.memsw.limit_in_bytes", strconv.FormatInt(c.Resources.MemorySwap, 10)) - if err != nil { - return err - } - } - if c.Resources.MemoryReservation > 0 { - err = writeFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(c.Resources.MemoryReservation, 10)) - if err != nil { - return err - } - } - if c.Resources.OomKillDisable { - if err := writeFile(path, "memory.oom_control", "1"); err != nil { - return err - } - } - - if c.Resources.MemorySwappiness >= 0 && c.Resources.MemorySwappiness <= 100 { - err = writeFile(path, "memory.swappiness", strconv.FormatInt(c.Resources.MemorySwappiness, 10)) - if err != nil { - return err - } - } else if c.Resources.MemorySwappiness == -1 { - return nil - } else { - return fmt.Errorf("invalid value:%d. valid memory swappiness range is 0-100", c.Resources.MemorySwappiness) - } - return nil } @@ -577,68 +581,25 @@ func joinCpuset(c *configs.Cgroup, pid int) error { // expects device path instead of major minor numbers, which is also confusing // for users. So we use fs work around for now. func joinBlkio(c *configs.Cgroup, pid int) error { - path, err := getSubsystemPath(c, "blkio") + _, err := join(c, "blkio", pid) if err != nil { return err } - // systemd doesn't directly support this in the dbus properties - if c.Resources.BlkioLeafWeight != 0 { - if err := writeFile(path, "blkio.leaf_weight", strconv.FormatUint(uint64(c.Resources.BlkioLeafWeight), 10)); err != nil { - return err - } - } - for _, wd := range c.Resources.BlkioWeightDevice { - if err := writeFile(path, "blkio.weight_device", wd.WeightString()); err != nil { - return err - } - if err := writeFile(path, "blkio.leaf_weight_device", wd.LeafWeightString()); err != nil { - return err - } - } - for _, td := range c.Resources.BlkioThrottleReadBpsDevice { - if err := writeFile(path, "blkio.throttle.read_bps_device", td.String()); err != nil { - return err - } - } - for _, td := range c.Resources.BlkioThrottleWriteBpsDevice { - if err := writeFile(path, "blkio.throttle.write_bps_device", td.String()); err != nil { - return err - } - } - for _, td := range c.Resources.BlkioThrottleReadIOPSDevice { - if err := writeFile(path, "blkio.throttle.read_iops_device", td.String()); err != nil { - return err - } - } - for _, td := range c.Resources.BlkioThrottleWriteIOPSDevice { - if err := writeFile(path, "blkio.throttle.write_iops_device", td.String()); err != nil { - return err - } - } - return nil } func joinHugetlb(c *configs.Cgroup, pid int) error { - path, err := join(c, "hugetlb", pid) + _, err := join(c, "hugetlb", pid) if err != nil && !cgroups.IsNotFound(err) { return err } - hugetlb, err := subsystems.Get("hugetlb") - if err != nil { - return err - } - return hugetlb.Set(path, c) + return nil } func joinPerfEvent(c *configs.Cgroup, pid int) error { - path, err := join(c, "perf_event", pid) + _, err := join(c, "perf_event", pid) if err != nil && !cgroups.IsNotFound(err) { return err } - perfEvent, err := subsystems.Get("perf_event") - if err != nil { - return err - } - return perfEvent.Set(path, c) + return nil } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/utils.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/utils.go index fbdb0cbdab..006800dcfa 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/utils.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/utils.go @@ -5,6 +5,7 @@ package cgroups import ( "bufio" "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -12,7 +13,6 @@ import ( "strings" "time" - "github.com/docker/docker/pkg/mount" "github.com/docker/go-units" ) @@ -84,10 +84,19 @@ func FindCgroupMountpointDir() (string, error) { // Safe as mountinfo encodes mountpoints with spaces as \040. index := strings.Index(text, " - ") postSeparatorFields := strings.Fields(text[index+3:]) - if len(postSeparatorFields) < 3 { - return "", fmt.Errorf("Error found less than 3 fields post '-' in %q", text) + numPostFields := len(postSeparatorFields) + + // This is an error as we can't detect if the mount is for "cgroup" + if numPostFields == 0 { + return "", fmt.Errorf("Found no fields post '-' in %q", text) } + if postSeparatorFields[0] == "cgroup" { + // Check that the mount is properly formated. + if numPostFields < 3 { + return "", fmt.Errorf("Error found less than 3 fields post '-' in %q", text) + } + return filepath.Dir(fields[4]), nil } } @@ -112,11 +121,45 @@ func (m Mount) GetThisCgroupDir(cgroups map[string]string) (string, error) { return getControllerPath(m.Subsystems[0], cgroups) } +func getCgroupMountsHelper(ss map[string]bool, mi io.Reader) ([]Mount, error) { + res := make([]Mount, 0, len(ss)) + scanner := bufio.NewScanner(mi) + for scanner.Scan() { + txt := scanner.Text() + sepIdx := strings.Index(txt, " - ") + if sepIdx == -1 { + return nil, fmt.Errorf("invalid mountinfo format") + } + if txt[sepIdx+3:sepIdx+9] != "cgroup" { + continue + } + fields := strings.Split(txt, " ") + m := Mount{ + Mountpoint: fields[4], + Root: fields[3], + } + for _, opt := range strings.Split(fields[len(fields)-1], ",") { + if strings.HasPrefix(opt, cgroupNamePrefix) { + m.Subsystems = append(m.Subsystems, opt[len(cgroupNamePrefix):]) + } + if ss[opt] { + m.Subsystems = append(m.Subsystems, opt) + } + } + res = append(res, m) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return res, nil +} + func GetCgroupMounts() ([]Mount, error) { - mounts, err := mount.GetMounts() + f, err := os.Open("/proc/self/mountinfo") if err != nil { return nil, err } + defer f.Close() all, err := GetAllSubsystems() if err != nil { @@ -127,24 +170,7 @@ func GetCgroupMounts() ([]Mount, error) { for _, s := range all { allMap[s] = true } - - res := []Mount{} - for _, mount := range mounts { - if mount.Fstype == "cgroup" { - m := Mount{Mountpoint: mount.Mountpoint, Root: mount.Root} - - for _, opt := range strings.Split(mount.VfsOpts, ",") { - if strings.HasPrefix(opt, cgroupNamePrefix) { - m.Subsystems = append(m.Subsystems, opt[len(cgroupNamePrefix):]) - } - if allMap[opt] { - m.Subsystems = append(m.Subsystems, opt) - } - } - res = append(res, m) - } - } - return res, nil + return getCgroupMountsHelper(allMap, f) } // Returns all the cgroup subsystems supported by the kernel @@ -323,9 +349,14 @@ func GetHugePageSize() ([]string, error) { return pageSizes, nil } -// GetPids returns all pids, that were added to cgroup at path and to all its -// subcgroups. +// GetPids returns all pids, that were added to cgroup at path. func GetPids(path string) ([]int, error) { + return readProcsFile(path) +} + +// GetAllPids returns all pids, that were added to cgroup at path and to all its +// subcgroups. +func GetAllPids(path string) ([]int, error) { var pids []int // collect pids from all sub-cgroups err := filepath.Walk(path, func(p string, info os.FileInfo, iErr error) error { diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/cgroup_unix.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/cgroup_unix.go index b9a06de26a..40a033f35b 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/cgroup_unix.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/cgroup_unix.go @@ -11,25 +11,38 @@ const ( ) type Cgroup struct { - Name string `json:"name"` + // Deprecated, use Path instead + Name string `json:"name,omitempty"` - // name of parent cgroup or slice - Parent string `json:"parent"` + // name of parent of cgroup or slice + // Deprecated, use Path instead + Parent string `json:"parent,omitempty"` + + // Path specifies the path to cgroups that are created and/or joined by the container. + // The path is assumed to be relative to the host system cgroup mountpoint. + Path string `json:"path"` // ScopePrefix decribes prefix for the scope name ScopePrefix string `json:"scope_prefix"` + // Paths represent the absolute cgroups paths to join. + // This takes precedence over Path. + Paths map[string]string + // Resources contains various cgroups settings to apply *Resources } type Resources struct { // If this is true allow access to any kind of device within the container. If false, allow access only to devices explicitly listed in the allowed_devices list. - AllowAllDevices bool `json:"allow_all_devices"` + // Deprecated + AllowAllDevices bool `json:"allow_all_devices,omitempty"` + // Deprecated + AllowedDevices []*Device `json:"allowed_devices,omitempty"` + // Deprecated + DeniedDevices []*Device `json:"denied_devices,omitempty"` - AllowedDevices []*Device `json:"allowed_devices"` - - DeniedDevices []*Device `json:"denied_devices"` + Devices []*Device `json:"devices"` // Memory limit (in bytes) Memory int64 `json:"memory"` @@ -37,7 +50,7 @@ type Resources struct { // Memory reservation or soft_limit (in bytes) MemoryReservation int64 `json:"memory_reservation"` - // Total memory usage (memory + swap); set `-1' to disable swap + // Total memory usage (memory + swap); set `-1` to enable unlimited swap MemorySwap int64 `json:"memory_swap"` // Kernel memory limit (in bytes) @@ -64,6 +77,9 @@ type Resources struct { // MEM to use CpusetMems string `json:"cpuset_mems"` + // Process limit; set <= `0' to disable limit. + PidsLimit int64 `json:"pids_limit"` + // Specifies per cgroup weight, range is from 10 to 1000. BlkioWeight uint16 `json:"blkio_weight"` diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/config.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/config.go index 069daae293..f6a163b734 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/config.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/config.go @@ -171,6 +171,9 @@ type Config struct { // A default action to be taken if no rules match is also given. Seccomp *Seccomp `json:"seccomp"` + // NoNewPrivileges controls whether processes in the container can gain additional privileges. + NoNewPrivileges bool `json:"no_new_privileges"` + // Hooks are a collection of actions to perform at various container lifecycle events. // Hooks are not able to be marshaled to json but they are also not needed to. Hooks *Hooks `json:"-"` diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/device.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/device.go index a52a024af1..8701bb212d 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/device.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/device.go @@ -35,6 +35,9 @@ type Device struct { // Gid of the device. Gid uint32 `json:"gid"` + + // Write the file to the allowed list + Allow bool `json:"allow"` } func (d *Device) CgroupString() string { diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/device_defaults.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/device_defaults.go index 0ce040fd34..e45299264c 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/device_defaults.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/configs/device_defaults.go @@ -82,20 +82,6 @@ var ( Minor: 1, Permissions: "rwm", }, - { - Path: "/dev/tty0", - Type: 'c', - Major: 4, - Minor: 0, - Permissions: "rwm", - }, - { - Path: "/dev/tty1", - Type: 'c', - Major: 4, - Minor: 1, - Permissions: "rwm", - }, // /dev/pts/ - pts namespaces are "coming soon" { Path: "", diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/container.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/container.go index 6292fd1852..68291231ec 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/container.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/container.go @@ -6,6 +6,7 @@ package libcontainer import ( "os" + "time" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -14,8 +15,11 @@ import ( type Status int const ( + // The container exists but has not been run yet + Created Status = iota + // The container exists and is running. - Running Status = iota + 1 + Running // The container exists, it is in the process of being paused. Pausing @@ -30,6 +34,25 @@ const ( Destroyed ) +func (s Status) String() string { + switch s { + case Created: + return "created" + case Running: + return "running" + case Pausing: + return "pausing" + case Paused: + return "paused" + case Checkpointed: + return "checkpointed" + case Destroyed: + return "destroyed" + default: + return "unknown" + } +} + // BaseState represents the platform agnostic pieces relating to a // running container's state type BaseState struct { @@ -39,9 +62,12 @@ type BaseState struct { // InitProcessPid is the init process id in the parent namespace. InitProcessPid int `json:"init_process_pid"` - // InitProcessStartTime is the init process start time. + // InitProcessStartTime is the init process start time in clock cycles since boot time. InitProcessStartTime string `json:"init_process_start"` + // Created is the unix timestamp for the creation time of the container in UTC + Created time.Time `json:"created"` + // Config is the container's configuration. Config configs.Config `json:"config"` } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/container_linux.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/container_linux.go index 916511ebf5..284e15ec33 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/container_linux.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/container_linux.go @@ -15,6 +15,7 @@ import ( "strings" "sync" "syscall" + "time" "github.com/Sirupsen/logrus" "github.com/golang/protobuf/proto" @@ -38,6 +39,8 @@ type linuxContainer struct { criuPath string m sync.Mutex criuVersion int + state containerState + created time.Time } // State represents a running container's state @@ -104,6 +107,12 @@ type Container interface { // errors: // Systemerror - System error. NotifyOOM() (<-chan struct{}, error) + + // NotifyMemoryPressure returns a read-only channel signaling when the container reaches a given pressure level + // + // errors: + // Systemerror - System error. + NotifyMemoryPressure(level PressureLevel) (<-chan struct{}, error) } // ID returns the container's unique ID @@ -129,7 +138,7 @@ func (c *linuxContainer) State() (*State, error) { } func (c *linuxContainer) Processes() ([]int, error) { - pids, err := c.cgroupManager.GetPids() + pids, err := c.cgroupManager.GetAllPids() if err != nil { return nil, newSystemError(err) } @@ -183,22 +192,30 @@ func (c *linuxContainer) Start(process *Process) error { } return newSystemError(err) } - if doInit { - c.updateState(parent) + // generate a timestamp indicating when the container was started + c.created = time.Now().UTC() + + c.state = &runningState{ + c: c, } - if c.config.Hooks != nil { - s := configs.HookState{ - Version: c.config.Version, - ID: c.id, - Pid: parent.pid(), - Root: c.config.Rootfs, + if doInit { + if err := c.updateState(parent); err != nil { + return err } - for _, hook := range c.config.Hooks.Poststart { - if err := hook.Run(s); err != nil { - if err := parent.terminate(); err != nil { - logrus.Warn(err) + if c.config.Hooks != nil { + s := configs.HookState{ + Version: c.config.Version, + ID: c.id, + Pid: parent.pid(), + Root: c.config.Rootfs, + } + for _, hook := range c.config.Hooks.Poststart { + if err := hook.Run(s); err != nil { + if err := parent.terminate(); err != nil { + logrus.Warn(err) + } + return newSystemError(err) } - return newSystemError(err) } } } @@ -251,7 +268,7 @@ func (c *linuxContainer) commandTemplate(p *Process, childPipe *os.File) (*exec. } func (c *linuxContainer) newInitProcess(p *Process, cmd *exec.Cmd, parentPipe, childPipe *os.File) (*initProcess, error) { - t := "_LIBCONTAINER_INITTYPE=standard" + t := "_LIBCONTAINER_INITTYPE=" + string(initStandard) cloneFlags := c.config.Namespaces.CloneFlags() if cloneFlags&syscall.CLONE_NEWUSER != 0 { if err := c.addUidGidMappings(cmd.SysProcAttr); err != nil { @@ -278,7 +295,7 @@ func (c *linuxContainer) newInitProcess(p *Process, cmd *exec.Cmd, parentPipe, c } func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, parentPipe, childPipe *os.File) (*setnsProcess, error) { - cmd.Env = append(cmd.Env, "_LIBCONTAINER_INITTYPE=setns") + cmd.Env = append(cmd.Env, "_LIBCONTAINER_INITTYPE="+string(initSetns)) // for setns process, we dont have to set cloneflags as the process namespaces // will only be set via setns syscall data, err := c.bootstrapData(0, c.initProcess.pid(), p.consolePath) @@ -321,54 +338,53 @@ func newPipe() (parent *os.File, child *os.File, err error) { func (c *linuxContainer) Destroy() error { c.m.Lock() defer c.m.Unlock() - status, err := c.currentStatus() - if err != nil { - return err - } - if status != Destroyed { - return newGenericError(fmt.Errorf("container is not destroyed"), ContainerNotStopped) - } - if !c.config.Namespaces.Contains(configs.NEWPID) { - if err := killCgroupProcesses(c.cgroupManager); err != nil { - logrus.Warn(err) - } - } - err = c.cgroupManager.Destroy() - if rerr := os.RemoveAll(c.root); err == nil { - err = rerr - } - c.initProcess = nil - if c.config.Hooks != nil { - s := configs.HookState{ - Version: c.config.Version, - ID: c.id, - Root: c.config.Rootfs, - } - for _, hook := range c.config.Hooks.Poststop { - if err := hook.Run(s); err != nil { - return err - } - } - } - return err + return c.state.destroy() } func (c *linuxContainer) Pause() error { c.m.Lock() defer c.m.Unlock() - return c.cgroupManager.Freeze(configs.Frozen) + status, err := c.currentStatus() + if err != nil { + return err + } + if status != Running { + return newGenericError(fmt.Errorf("container not running"), ContainerNotRunning) + } + if err := c.cgroupManager.Freeze(configs.Frozen); err != nil { + return err + } + return c.state.transition(&pausedState{ + c: c, + }) } func (c *linuxContainer) Resume() error { c.m.Lock() defer c.m.Unlock() - return c.cgroupManager.Freeze(configs.Thawed) + status, err := c.currentStatus() + if err != nil { + return err + } + if status != Paused { + return newGenericError(fmt.Errorf("container not paused"), ContainerNotPaused) + } + if err := c.cgroupManager.Freeze(configs.Thawed); err != nil { + return err + } + return c.state.transition(&runningState{ + c: c, + }) } func (c *linuxContainer) NotifyOOM() (<-chan struct{}, error) { return notifyOnOOM(c.cgroupManager.GetPaths()) } +func (c *linuxContainer) NotifyMemoryPressure(level PressureLevel) (<-chan struct{}, error) { + return notifyMemoryPressure(c.cgroupManager.GetPaths(), level) +} + // XXX debug support, remove when debugging done. func addArgsFromEnv(evar string, args *[]string) { if e := os.Getenv(evar); e != "" { @@ -460,7 +476,7 @@ func (c *linuxContainer) Checkpoint(criuOpts *CriuOpts) error { } if criuOpts.ImagesDirectory == "" { - criuOpts.ImagesDirectory = filepath.Join(c.root, "criu.image") + return fmt.Errorf("invalid directory to save checkpoint") } // Since a container can be C/R'ed multiple times, @@ -579,11 +595,9 @@ func (c *linuxContainer) addCriuRestoreMount(req *criurpc.CriuReq, m *configs.Mo func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { c.m.Lock() defer c.m.Unlock() - if err := c.checkCriuVersion("1.5.2"); err != nil { return err } - if criuOpts.WorkDirectory == "" { criuOpts.WorkDirectory = filepath.Join(c.root, "criu.work") } @@ -592,22 +606,19 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { if err := os.Mkdir(criuOpts.WorkDirectory, 0655); err != nil && !os.IsExist(err) { return err } - workDir, err := os.Open(criuOpts.WorkDirectory) if err != nil { return err } defer workDir.Close() - if criuOpts.ImagesDirectory == "" { - criuOpts.ImagesDirectory = filepath.Join(c.root, "criu.image") + return fmt.Errorf("invalid directory to restore checkpoint") } imageDir, err := os.Open(criuOpts.ImagesDirectory) if err != nil { return err } defer imageDir.Close() - // CRIU has a few requirements for a root directory: // * it must be a mount point // * its parent must not be overmounted @@ -618,18 +629,15 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { return err } defer os.Remove(root) - root, err = filepath.EvalSymlinks(root) if err != nil { return err } - err = syscall.Mount(c.config.Rootfs, root, "", syscall.MS_BIND|syscall.MS_REC, "") if err != nil { return err } defer syscall.Unmount(root, syscall.MNT_DETACH) - t := criurpc.CriuReqType_RESTORE req := &criurpc.CriuReq{ Type: &t, @@ -697,15 +705,13 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { fds []string fdJSON []byte ) - if fdJSON, err = ioutil.ReadFile(filepath.Join(criuOpts.ImagesDirectory, descriptorsFilename)); err != nil { return err } - if err = json.Unmarshal(fdJSON, &fds); err != nil { + if err := json.Unmarshal(fdJSON, &fds); err != nil { return err } - for i := range fds { if s := fds[i]; strings.Contains(s, "pipe:") { inheritFd := new(criurpc.InheritFd) @@ -714,12 +720,7 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { req.Opts.InheritFd = append(req.Opts.InheritFd, inheritFd) } } - - err = c.criuSwrk(process, req, criuOpts, true) - if err != nil { - return err - } - return nil + return c.criuSwrk(process, req, criuOpts, true) } func (c *linuxContainer) criuApplyCgroups(pid int, req *criurpc.CriuReq) error { @@ -914,46 +915,43 @@ func (c *linuxContainer) criuNotifications(resp *criurpc.CriuResp, process *Proc if notify == nil { return fmt.Errorf("invalid response: %s", resp.String()) } - switch { case notify.GetScript() == "post-dump": - if !opts.LeaveRunning { - f, err := os.Create(filepath.Join(c.root, "checkpoint")) - if err != nil { - return err - } - f.Close() + f, err := os.Create(filepath.Join(c.root, "checkpoint")) + if err != nil { + return err } - break - + f.Close() case notify.GetScript() == "network-unlock": if err := unlockNetwork(c.config); err != nil { return err } - break - case notify.GetScript() == "network-lock": if err := lockNetwork(c.config); err != nil { return err } - break - case notify.GetScript() == "post-restore": pid := notify.GetPid() r, err := newRestoredProcess(int(pid), fds) if err != nil { return err } - - // TODO: crosbymichael restore previous process information by saving the init process information in - // the container's state file or separate process state files. + process.ops = r + if err := c.state.transition(&restoredState{ + imageDir: opts.ImagesDirectory, + c: c, + }); err != nil { + return err + } if err := c.updateState(r); err != nil { return err } - process.ops = r - break + if err := os.Remove(filepath.Join(c.root, "checkpoint")); err != nil { + if !os.IsNotExist(err) { + logrus.Error(err) + } + } } - return nil } @@ -963,65 +961,108 @@ func (c *linuxContainer) updateState(process parentProcess) error { if err != nil { return err } + return c.saveState(state) +} + +func (c *linuxContainer) saveState(s *State) error { f, err := os.Create(filepath.Join(c.root, stateFilename)) if err != nil { return err } defer f.Close() - os.Remove(filepath.Join(c.root, "checkpoint")) - return utils.WriteJSON(f, state) + return utils.WriteJSON(f, s) +} + +func (c *linuxContainer) deleteState() error { + return os.Remove(filepath.Join(c.root, stateFilename)) } func (c *linuxContainer) currentStatus() (Status, error) { - if _, err := os.Stat(filepath.Join(c.root, "checkpoint")); err == nil { - return Checkpointed, nil + if err := c.refreshState(); err != nil { + return -1, err } + return c.state.status(), nil +} + +// refreshState needs to be called to verify that the current state on the +// container is what is true. Because consumers of libcontainer can use it +// out of process we need to verify the container's status based on runtime +// information and not rely on our in process info. +func (c *linuxContainer) refreshState() error { + paused, err := c.isPaused() + if err != nil { + return err + } + if paused { + return c.state.transition(&pausedState{c: c}) + } + running, err := c.isRunning() + if err != nil { + return err + } + if running { + return c.state.transition(&runningState{c: c}) + } + return c.state.transition(&stoppedState{c: c}) +} + +func (c *linuxContainer) isRunning() (bool, error) { if c.initProcess == nil { - return Destroyed, nil + return false, nil } // return Running if the init process is alive if err := syscall.Kill(c.initProcess.pid(), 0); err != nil { if err == syscall.ESRCH { - return Destroyed, nil + return false, nil } - return 0, newSystemError(err) + return false, newSystemError(err) } - if c.config.Cgroups != nil && c.config.Cgroups.Resources != nil && c.config.Cgroups.Resources.Freezer == configs.Frozen { - return Paused, nil + return true, nil +} + +func (c *linuxContainer) isPaused() (bool, error) { + data, err := ioutil.ReadFile(filepath.Join(c.cgroupManager.GetPaths()["freezer"], "freezer.state")) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, newSystemError(err) } - return Running, nil + return bytes.Equal(bytes.TrimSpace(data), []byte("FROZEN")), nil } func (c *linuxContainer) currentState() (*State, error) { - status, err := c.currentStatus() - if err != nil { - return nil, err - } - if status == Destroyed { - return nil, newGenericError(fmt.Errorf("container destroyed"), ContainerNotExists) - } - startTime, err := c.initProcess.startTime() - if err != nil { - return nil, newSystemError(err) + var ( + startTime string + externalDescriptors []string + pid = -1 + ) + if c.initProcess != nil { + pid = c.initProcess.pid() + startTime, _ = c.initProcess.startTime() + externalDescriptors = c.initProcess.externalDescriptors() } state := &State{ BaseState: BaseState{ ID: c.ID(), Config: *c.config, - InitProcessPid: c.initProcess.pid(), + InitProcessPid: pid, InitProcessStartTime: startTime, + Created: c.created, }, CgroupPaths: c.cgroupManager.GetPaths(), NamespacePaths: make(map[configs.NamespaceType]string), - ExternalDescriptors: c.initProcess.externalDescriptors(), + ExternalDescriptors: externalDescriptors, } - for _, ns := range c.config.Namespaces { - state.NamespacePaths[ns.Type] = ns.GetPath(c.initProcess.pid()) - } - for _, nsType := range configs.NamespaceTypes() { - if _, ok := state.NamespacePaths[nsType]; !ok { - ns := configs.Namespace{Type: nsType} - state.NamespacePaths[ns.Type] = ns.GetPath(c.initProcess.pid()) + if pid > 0 { + for _, ns := range c.config.Namespaces { + state.NamespacePaths[ns.Type] = ns.GetPath(pid) + } + for _, nsType := range configs.NamespaceTypes() { + if _, ok := state.NamespacePaths[nsType]; !ok { + ns := configs.Namespace{Type: nsType} + state.NamespacePaths[ns.Type] = ns.GetPath(pid) + } } } return state, nil diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/error.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/error.go index aa59d2aecc..b50aaae84e 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/error.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/error.go @@ -16,9 +16,10 @@ const ( ContainerPaused ContainerNotStopped ContainerNotRunning + ContainerNotPaused // Process errors - ProcessNotExecuted + NoProcessOps // Common errors ConfigInvalid @@ -46,6 +47,10 @@ func (c ErrorCode) String() string { return "Container is not running" case ConsoleExists: return "Console exists for process" + case ContainerNotPaused: + return "Container is not paused" + case NoProcessOps: + return "No process operations" default: return "Unknown error" } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/factory_linux.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/factory_linux.go index d03ce8642e..14e4f33a8c 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/factory_linux.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/factory_linux.go @@ -166,7 +166,7 @@ func (l *LinuxFactory) Create(id string, config *configs.Config) (Container, err if err := os.MkdirAll(containerRoot, 0700); err != nil { return nil, newGenericError(err, SystemError) } - return &linuxContainer{ + c := &linuxContainer{ id: id, root: containerRoot, config: config, @@ -174,7 +174,9 @@ func (l *LinuxFactory) Create(id string, config *configs.Config) (Container, err initArgs: l.InitArgs, criuPath: l.CriuPath, cgroupManager: l.NewCgroupsManager(config.Cgroups, nil), - }, nil + } + c.state = &stoppedState{c: c} + return c, nil } func (l *LinuxFactory) Load(id string) (Container, error) { @@ -191,7 +193,7 @@ func (l *LinuxFactory) Load(id string) (Container, error) { processStartTime: state.InitProcessStartTime, fds: state.ExternalDescriptors, } - return &linuxContainer{ + c := &linuxContainer{ initProcess: r, id: id, config: &state.Config, @@ -200,7 +202,13 @@ func (l *LinuxFactory) Load(id string) (Container, error) { criuPath: l.CriuPath, cgroupManager: l.NewCgroupsManager(state.Config.Cgroups, state.CgroupPaths), root: containerRoot, - }, nil + created: state.Created, + } + c.state = &createdState{c: c, s: Created} + if err := c.refreshState(); err != nil { + return nil, err + } + return c, nil } func (l *LinuxFactory) Type() string { @@ -222,18 +230,25 @@ func (l *LinuxFactory) StartInitialization() (err error) { // clear the current process's environment to clean any libcontainer // specific env vars. os.Clearenv() + var i initer defer func() { - // if we have an error during the initialization of the container's init then send it back to the - // parent process in the form of an initError. - if err != nil { - if err := utils.WriteJSON(pipe, newSystemError(err)); err != nil { + // We have an error during the initialization of the container's init, + // send it back to the parent process in the form of an initError. + // If container's init successed, syscall.Exec will not return, hence + // this defer function will never be called. + if _, ok := i.(*linuxStandardInit); ok { + // Synchronisation only necessary for standard init. + if err := utils.WriteJSON(pipe, syncT{procError}); err != nil { panic(err) } } + if err := utils.WriteJSON(pipe, newSystemError(err)); err != nil { + panic(err) + } // ensure that this pipe is always closed pipe.Close() }() - i, err := newContainerInit(it, pipe) + i, err = newContainerInit(it, pipe) if err != nil { return err } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/generic_error.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/generic_error.go index 6fbc2d75a5..75e980b1d7 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/generic_error.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/generic_error.go @@ -9,6 +9,18 @@ import ( "github.com/opencontainers/runc/libcontainer/stacktrace" ) +type syncType uint8 + +const ( + procReady syncType = iota + procError + procRun +) + +type syncT struct { + Type syncType `json:"type"` +} + var errorTemplate = template.Must(template.New("error").Parse(`Timestamp: {{.Timestamp}} Code: {{.ECode}} {{if .Message }} diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/init_linux.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/init_linux.go index ddb1186595..918f103016 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/init_linux.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/init_linux.go @@ -5,6 +5,7 @@ package libcontainer import ( "encoding/json" "fmt" + "io" "io/ioutil" "net" "os" @@ -73,6 +74,7 @@ func newContainerInit(t initType, pipe *os.File) (initer, error) { }, nil case initStandard: return &linuxStandardInit{ + pipe: pipe, parentPid: syscall.Getppid(), config: config, }, nil @@ -140,6 +142,27 @@ func finalizeNamespace(config *initConfig) error { return nil } +// syncParentReady sends to the given pipe a JSON payload which indicates that +// the init is ready to Exec the child process. It then waits for the parent to +// indicate that it is cleared to Exec. +func syncParentReady(pipe io.ReadWriter) error { + // Tell parent. + if err := utils.WriteJSON(pipe, syncT{procReady}); err != nil { + return err + } + // Wait for parent to give the all-clear. + var procSync syncT + if err := json.NewDecoder(pipe).Decode(&procSync); err != nil { + if err == io.EOF { + return fmt.Errorf("parent closed synchronisation channel") + } + if procSync.Type != procRun { + return fmt.Errorf("invalid synchronisation flag from parent") + } + } + return nil +} + // joinExistingNamespaces gets all the namespace paths specified for the container and // does a setns on the namespace fd so that the current process joins the namespace. func joinExistingNamespaces(namespaces []configs.Namespace) error { @@ -309,7 +332,7 @@ func killCgroupProcesses(m cgroups.Manager) error { if err := m.Freeze(configs.Frozen); err != nil { logrus.Warn(err) } - pids, err := m.GetPids() + pids, err := m.GetAllPids() if err != nil { m.Freeze(configs.Thawed) return err diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/keys/keyctl.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/keys/keyctl.go new file mode 100644 index 0000000000..c37ca21330 --- /dev/null +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/keys/keyctl.go @@ -0,0 +1,67 @@ +// +build linux + +package keyctl + +import ( + "fmt" + "syscall" + "strings" + "strconv" + "unsafe" +) + +const KEYCTL_JOIN_SESSION_KEYRING = 1 +const KEYCTL_SETPERM = 5 +const KEYCTL_DESCRIBE = 6 + +type KeySerial uint32 + +func JoinSessionKeyring(name string) (KeySerial, error) { + var _name *byte = nil + var err error + + if len(name) > 0 { + _name, err = syscall.BytePtrFromString(name) + if err != nil { + return KeySerial(0), err + } + } + + sessKeyId, _, errn := syscall.Syscall(syscall.SYS_KEYCTL, KEYCTL_JOIN_SESSION_KEYRING, uintptr(unsafe.Pointer(_name)), 0) + if errn != 0 { + return 0, fmt.Errorf("could not create session key: %v", errn) + } + return KeySerial(sessKeyId), nil +} + +// modify permissions on a keyring by reading the current permissions, +// anding the bits with the given mask (clearing permissions) and setting +// additional permission bits +func ModKeyringPerm(ringId KeySerial, mask, setbits uint32) error { + dest := make([]byte, 1024) + destBytes := unsafe.Pointer(&dest[0]) + + if _, _, err := syscall.Syscall6(syscall.SYS_KEYCTL, uintptr(KEYCTL_DESCRIBE), uintptr(ringId), uintptr(destBytes), uintptr(len(dest)), 0, 0); err != 0 { + return err + } + + res := strings.Split(string(dest), ";") + if len(res) < 5 { + return fmt.Errorf("Destination buffer for key description is too small") + } + + // parse permissions + perm64, err := strconv.ParseUint(res[3], 16, 32) + if err != nil { + return err + } + + perm := (uint32(perm64) & mask) | setbits + + if _, _, err := syscall.Syscall(syscall.SYS_KEYCTL, uintptr(KEYCTL_SETPERM), uintptr(ringId), uintptr(perm)); err != 0 { + return err + } + + return nil +} + diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/notify_linux.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/notify_linux.go index cf81e24d44..839a50c55a 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/notify_linux.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/notify_linux.go @@ -12,31 +12,32 @@ import ( const oomCgroupName = "memory" -// notifyOnOOM returns channel on which you can expect event about OOM, -// if process died without OOM this channel will be closed. -// s is current *libcontainer.State for container. -func notifyOnOOM(paths map[string]string) (<-chan struct{}, error) { - dir := paths[oomCgroupName] - if dir == "" { - return nil, fmt.Errorf("There is no path for %q in state", oomCgroupName) - } - oomControl, err := os.Open(filepath.Join(dir, "memory.oom_control")) +type PressureLevel uint + +const ( + LowPressure PressureLevel = iota + MediumPressure + CriticalPressure +) + +func registerMemoryEvent(cgDir string, evName string, arg string) (<-chan struct{}, error) { + evFile, err := os.Open(filepath.Join(cgDir, evName)) if err != nil { return nil, err } fd, _, syserr := syscall.RawSyscall(syscall.SYS_EVENTFD2, 0, syscall.FD_CLOEXEC, 0) if syserr != 0 { - oomControl.Close() + evFile.Close() return nil, syserr } eventfd := os.NewFile(fd, "eventfd") - eventControlPath := filepath.Join(dir, "cgroup.event_control") - data := fmt.Sprintf("%d %d", eventfd.Fd(), oomControl.Fd()) + eventControlPath := filepath.Join(cgDir, "cgroup.event_control") + data := fmt.Sprintf("%d %d %s", eventfd.Fd(), evFile.Fd(), arg) if err := ioutil.WriteFile(eventControlPath, []byte(data), 0700); err != nil { eventfd.Close() - oomControl.Close() + evFile.Close() return nil, err } ch := make(chan struct{}) @@ -44,7 +45,7 @@ func notifyOnOOM(paths map[string]string) (<-chan struct{}, error) { defer func() { close(ch) eventfd.Close() - oomControl.Close() + evFile.Close() }() buf := make([]byte, 8) for { @@ -61,3 +62,28 @@ func notifyOnOOM(paths map[string]string) (<-chan struct{}, error) { }() return ch, nil } + +// notifyOnOOM returns channel on which you can expect event about OOM, +// if process died without OOM this channel will be closed. +func notifyOnOOM(paths map[string]string) (<-chan struct{}, error) { + dir := paths[oomCgroupName] + if dir == "" { + return nil, fmt.Errorf("path %q missing", oomCgroupName) + } + + return registerMemoryEvent(dir, "memory.oom_control", "") +} + +func notifyMemoryPressure(paths map[string]string, level PressureLevel) (<-chan struct{}, error) { + dir := paths[oomCgroupName] + if dir == "" { + return nil, fmt.Errorf("path %q missing", oomCgroupName) + } + + if level > CriticalPressure { + return nil, fmt.Errorf("invalid pressure level %d", level) + } + + levelStr := []string{"low", "medium", "critical"}[level] + return registerMemoryEvent(dir, "memory.pressure_level", levelStr) +} diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c index 27e6e53d4d..6634afc424 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c @@ -17,6 +17,7 @@ #include #include +#include #include #include #include diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/process.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/process.go index 9661df80a8..8b4c558bd2 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/process.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/process.go @@ -55,7 +55,7 @@ type Process struct { // Wait releases any resources associated with the Process func (p Process) Wait() (*os.ProcessState, error) { if p.ops == nil { - return nil, newGenericError(fmt.Errorf("invalid process"), ProcessNotExecuted) + return nil, newGenericError(fmt.Errorf("invalid process"), NoProcessOps) } return p.ops.wait() } @@ -65,7 +65,7 @@ func (p Process) Pid() (int, error) { // math.MinInt32 is returned here, because it's invalid value // for the kill() system call. if p.ops == nil { - return math.MinInt32, newGenericError(fmt.Errorf("invalid process"), ProcessNotExecuted) + return math.MinInt32, newGenericError(fmt.Errorf("invalid process"), NoProcessOps) } return p.ops.pid(), nil } @@ -73,7 +73,7 @@ func (p Process) Pid() (int, error) { // Signal sends a signal to the Process. func (p Process) Signal(sig os.Signal) error { if p.ops == nil { - return newGenericError(fmt.Errorf("invalid process"), ProcessNotExecuted) + return newGenericError(fmt.Errorf("invalid process"), NoProcessOps) } return p.ops.signal(sig) } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/process_linux.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/process_linux.go index ee647369d3..aa9b9d0986 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/process_linux.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/process_linux.go @@ -5,6 +5,7 @@ package libcontainer import ( "encoding/json" "errors" + "fmt" "io" "os" "os/exec" @@ -87,6 +88,7 @@ func (p *setnsProcess) start() (err error) { if err := utils.WriteJSON(p.parentPipe, p.config); err != nil { return newSystemError(err) } + if err := syscall.Shutdown(int(p.parentPipe.Fd()), syscall.SHUT_WR); err != nil { return newSystemError(err) } @@ -96,6 +98,7 @@ func (p *setnsProcess) start() (err error) { if err := json.NewDecoder(p.parentPipe).Decode(&ierr); err != nil && err != io.EOF { return newSystemError(err) } + // Must be done after Shutdown so the child will exit and we can wait for it. if ierr != nil { p.wait() return newSystemError(ierr) @@ -199,7 +202,6 @@ func (p *initProcess) start() (err error) { return newSystemError(err) } p.setExternalDescriptors(fds) - // Do this before syncing with child so that no children // can escape the cgroup if err := p.manager.Apply(p.pid()); err != nil { @@ -230,13 +232,54 @@ func (p *initProcess) start() (err error) { if err := p.sendConfig(); err != nil { return newSystemError(err) } - // wait for the child process to fully complete and receive an error message - // if one was encoutered - var ierr *genericError - if err := json.NewDecoder(p.parentPipe).Decode(&ierr); err != nil && err != io.EOF { + var ( + procSync syncT + sentRun bool + ierr *genericError + ) + +loop: + for { + if err := json.NewDecoder(p.parentPipe).Decode(&procSync); err != nil { + if err == io.EOF { + break loop + } + return newSystemError(err) + } + switch procSync.Type { + case procReady: + if err := p.manager.Set(p.config.Config); err != nil { + return newSystemError(err) + } + // Sync with child. + if err := utils.WriteJSON(p.parentPipe, syncT{procRun}); err != nil { + return newSystemError(err) + } + sentRun = true + case procError: + // wait for the child process to fully complete and receive an error message + // if one was encoutered + if err := json.NewDecoder(p.parentPipe).Decode(&ierr); err != nil && err != io.EOF { + return newSystemError(err) + } + if ierr != nil { + break loop + } + // Programmer error. + panic("No error following JSON procError payload.") + default: + return newSystemError(fmt.Errorf("invalid JSON synchronisation payload from child")) + } + } + if !sentRun { + return newSystemError(fmt.Errorf("could not synchronise with container process")) + } + if err := syscall.Shutdown(int(p.parentPipe.Fd()), syscall.SHUT_WR); err != nil { return newSystemError(err) } + // Must be done after Shutdown so the child will exit and we can wait for it. if ierr != nil { + p.wait() return newSystemError(ierr) } return nil @@ -270,12 +313,10 @@ func (p *initProcess) startTime() (string, error) { } func (p *initProcess) sendConfig() error { - // send the state to the container's init process then shutdown writes for the parent - if err := utils.WriteJSON(p.parentPipe, p.config); err != nil { - return err - } - // shutdown writes for the parent side of the pipe - return syscall.Shutdown(int(p.parentPipe.Fd()), syscall.SHUT_WR) + // send the config to the container's init process, we don't use JSON Encode + // here because there might be a problem in JSON decoder in some cases, see: + // https://github.com/docker/docker/issues/14203#issuecomment-174177790 + return utils.WriteJSON(p.parentPipe, p.config) } func (p *initProcess) createNetworkInterfaces() error { diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/rootfs_linux.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/rootfs_linux.go index 5a2fad8818..a2cd43c94f 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/rootfs_linux.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/rootfs_linux.go @@ -18,6 +18,8 @@ import ( "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/configs" "github.com/opencontainers/runc/libcontainer/label" + "github.com/opencontainers/runc/libcontainer/system" + libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils" ) const defaultMountFlags = syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV @@ -293,12 +295,31 @@ func getCgroupMounts(m *configs.Mount) ([]*configs.Mount, error) { // checkMountDestination checks to ensure that the mount destination is not over the top of /proc. // dest is required to be an abs path and have any symlinks resolved before calling this function. func checkMountDestination(rootfs, dest string) error { - if filepath.Clean(rootfs) == filepath.Clean(dest) { + if libcontainerUtils.CleanPath(rootfs) == libcontainerUtils.CleanPath(dest) { return fmt.Errorf("mounting into / is prohibited") } invalidDestinations := []string{ "/proc", } + // White list, it should be sub directories of invalid destinations + validDestinations := []string{ + // These entries can be bind mounted by files emulated by fuse, + // so commands like top, free displays stats in container. + "/proc/cpuinfo", + "/proc/diskstats", + "/proc/meminfo", + "/proc/stat", + "/proc/net/dev", + } + for _, valid := range validDestinations { + path, err := filepath.Rel(filepath.Join(rootfs, valid), dest) + if err != nil { + return err + } + if path == "." { + return nil + } + } for _, invalid := range invalidDestinations { path, err := filepath.Rel(filepath.Join(rootfs, invalid), dest) if err != nil { @@ -321,7 +342,7 @@ func setupDevSymlinks(rootfs string) error { // kcore support can be toggled with CONFIG_PROC_KCORE; only create a symlink // in /dev if it exists in /proc. if _, err := os.Stat("/proc/kcore"); err == nil { - links = append(links, [2]string{"/proc/kcore", "/dev/kcore"}) + links = append(links, [2]string{"/proc/kcore", "/dev/core"}) } for _, link := range links { var ( @@ -365,11 +386,12 @@ func reOpenDevNull() error { // Create the device nodes in the container. func createDevices(config *configs.Config) error { + useBindMount := system.RunningInUserNS() || config.Namespaces.Contains(configs.NEWUSER) oldMask := syscall.Umask(0000) for _, node := range config.Devices { // containers running in a user namespace are not allowed to mknod // devices so we can just bind mount it from the host. - if err := createDeviceNode(config.Rootfs, node, config.Namespaces.Contains(configs.NEWUSER)); err != nil { + if err := createDeviceNode(config.Rootfs, node, useBindMount); err != nil { syscall.Umask(oldMask) return err } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/selinux/selinux.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/selinux/selinux.go index 2771bb50e0..88d612cade 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/selinux/selinux.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/selinux/selinux.go @@ -231,10 +231,14 @@ func ReserveLabel(scon string) { } } +func selinuxEnforcePath() string { + return fmt.Sprintf("%s/enforce", selinuxPath) +} + func SelinuxGetEnforce() int { var enforce int - enforceS, err := readCon(fmt.Sprintf("%s/enforce", selinuxPath)) + enforceS, err := readCon(selinuxEnforcePath()) if err != nil { return -1 } @@ -246,6 +250,10 @@ func SelinuxGetEnforce() int { return enforce } +func SelinuxSetEnforce(mode int) error { + return writeCon(selinuxEnforcePath(), fmt.Sprintf("%d", mode)) +} + func SelinuxGetEnforceMode() int { switch readConfig(selinuxTag) { case "enforcing": diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/setns_init_linux.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/setns_init_linux.go index 2bde44ffb4..29f5b26e05 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/setns_init_linux.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/setns_init_linux.go @@ -6,6 +6,7 @@ import ( "os" "github.com/opencontainers/runc/libcontainer/apparmor" + "github.com/opencontainers/runc/libcontainer/keys" "github.com/opencontainers/runc/libcontainer/label" "github.com/opencontainers/runc/libcontainer/seccomp" "github.com/opencontainers/runc/libcontainer/system" @@ -18,12 +19,21 @@ type linuxSetnsInit struct { } func (l *linuxSetnsInit) Init() error { + // do not inherit the parent's session keyring + if _, err := keyctl.JoinSessionKeyring("_ses"); err != nil { + return err + } if err := setupRlimits(l.config.Config); err != nil { return err } if err := setOomScoreAdj(l.config.Config.OomScoreAdj); err != nil { return err } + if l.config.Config.NoNewPrivileges { + if err := system.Prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return err + } + } if l.config.Config.Seccomp != nil { if err := seccomp.InitSeccomp(l.config.Config.Seccomp); err != nil { return err diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/standard_init_linux.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/standard_init_linux.go index ec1005789c..6240347aa4 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/standard_init_linux.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/standard_init_linux.go @@ -3,22 +3,41 @@ package libcontainer import ( + "io" "os" "syscall" "github.com/opencontainers/runc/libcontainer/apparmor" "github.com/opencontainers/runc/libcontainer/configs" + "github.com/opencontainers/runc/libcontainer/keys" "github.com/opencontainers/runc/libcontainer/label" "github.com/opencontainers/runc/libcontainer/seccomp" "github.com/opencontainers/runc/libcontainer/system" ) type linuxStandardInit struct { + pipe io.ReadWriter parentPid int config *initConfig } +// PR_SET_NO_NEW_PRIVS isn't exposed in Golang so we define it ourselves copying the value +// the kernel +const PR_SET_NO_NEW_PRIVS = 0x26 + func (l *linuxStandardInit) Init() error { + // do not inherit the parent's session keyring + sessKeyId, err := keyctl.JoinSessionKeyring("") + if err != nil { + return err + } + // make session keyring searcheable + // without user ns we need 'UID' search permissions + // with user ns we need 'other' search permissions + if err := keyctl.ModKeyringPerm(sessKeyId, 0xffffffff, 0x080008); err != nil { + return err + } + // join any namespaces via a path to the namespace fd if provided if err := joinExistingNamespaces(l.config.Config.Namespaces); err != nil { return err @@ -50,7 +69,6 @@ func (l *linuxStandardInit) Init() error { if err := setOomScoreAdj(l.config.Config.OomScoreAdj); err != nil { return err } - label.Init() // InitializeMountNamespace() can be executed only for a new mount namespace if l.config.Config.Namespaces.Contains(configs.NEWNS) { @@ -75,7 +93,6 @@ func (l *linuxStandardInit) Init() error { return err } } - for _, path := range l.config.Config.ReadonlyPaths { if err := remountReadonly(path); err != nil { return err @@ -90,6 +107,17 @@ func (l *linuxStandardInit) Init() error { if err != nil { return err } + if l.config.Config.NoNewPrivileges { + if err := system.Prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { + return err + } + } + // Tell our parent that we're ready to Execv. This must be done before the + // Seccomp rules have been applied, because we need to be able to read and + // write to a socket. + if err := syncParentReady(l.pipe); err != nil { + return err + } if l.config.Config.Seccomp != nil { if err := seccomp.InitSeccomp(l.config.Config.Seccomp); err != nil { return err @@ -109,5 +137,6 @@ func (l *linuxStandardInit) Init() error { if syscall.Getppid() != l.parentPid { return syscall.Kill(syscall.Getpid(), syscall.SIGKILL) } + return system.Execv(l.config.Args[0], l.config.Args[0:], os.Environ()) } diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/state_linux.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/state_linux.go new file mode 100644 index 0000000000..9ffe15a436 --- /dev/null +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/state_linux.go @@ -0,0 +1,226 @@ +// +build linux + +package libcontainer + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/Sirupsen/logrus" + "github.com/opencontainers/runc/libcontainer/configs" +) + +func newStateTransitionError(from, to containerState) error { + return &stateTransitionError{ + From: from.status().String(), + To: to.status().String(), + } +} + +// stateTransitionError is returned when an invalid state transition happens from one +// state to another. +type stateTransitionError struct { + From string + To string +} + +func (s *stateTransitionError) Error() string { + return fmt.Sprintf("invalid state transition from %s to %s", s.From, s.To) +} + +type containerState interface { + transition(containerState) error + destroy() error + status() Status +} + +func destroy(c *linuxContainer) error { + if !c.config.Namespaces.Contains(configs.NEWPID) { + if err := killCgroupProcesses(c.cgroupManager); err != nil { + logrus.Warn(err) + } + } + err := c.cgroupManager.Destroy() + if rerr := os.RemoveAll(c.root); err == nil { + err = rerr + } + c.initProcess = nil + if herr := runPoststopHooks(c); err == nil { + err = herr + } + c.state = &stoppedState{c: c} + return err +} + +func runPoststopHooks(c *linuxContainer) error { + if c.config.Hooks != nil { + s := configs.HookState{ + Version: c.config.Version, + ID: c.id, + Root: c.config.Rootfs, + } + for _, hook := range c.config.Hooks.Poststop { + if err := hook.Run(s); err != nil { + return err + } + } + } + return nil +} + +// stoppedState represents a container is a stopped/destroyed state. +type stoppedState struct { + c *linuxContainer +} + +func (b *stoppedState) status() Status { + return Destroyed +} + +func (b *stoppedState) transition(s containerState) error { + switch s.(type) { + case *runningState: + b.c.state = s + return nil + case *restoredState: + b.c.state = s + return nil + case *stoppedState: + return nil + } + return newStateTransitionError(b, s) +} + +func (b *stoppedState) destroy() error { + return destroy(b.c) +} + +// runningState represents a container that is currently running. +type runningState struct { + c *linuxContainer +} + +func (r *runningState) status() Status { + return Running +} + +func (r *runningState) transition(s containerState) error { + switch s.(type) { + case *stoppedState: + running, err := r.c.isRunning() + if err != nil { + return err + } + if running { + return newGenericError(fmt.Errorf("container still running"), ContainerNotStopped) + } + r.c.state = s + return nil + case *pausedState: + r.c.state = s + return nil + case *runningState: + return nil + } + return newStateTransitionError(r, s) +} + +func (r *runningState) destroy() error { + running, err := r.c.isRunning() + if err != nil { + return err + } + if running { + return newGenericError(fmt.Errorf("container is not destroyed"), ContainerNotStopped) + } + return destroy(r.c) +} + +// pausedState represents a container that is currently pause. It cannot be destroyed in a +// paused state and must transition back to running first. +type pausedState struct { + c *linuxContainer +} + +func (p *pausedState) status() Status { + return Paused +} + +func (p *pausedState) transition(s containerState) error { + switch s.(type) { + case *runningState, *stoppedState: + p.c.state = s + return nil + case *pausedState: + return nil + } + return newStateTransitionError(p, s) +} + +func (p *pausedState) destroy() error { + isRunning, err := p.c.isRunning() + if err != nil { + return err + } + if !isRunning { + if err := p.c.cgroupManager.Freeze(configs.Thawed); err != nil { + return err + } + return destroy(p.c) + } + return newGenericError(fmt.Errorf("container is paused"), ContainerPaused) +} + +// restoredState is the same as the running state but also has accociated checkpoint +// information that maybe need destroyed when the container is stopped and destory is called. +type restoredState struct { + imageDir string + c *linuxContainer +} + +func (r *restoredState) status() Status { + return Running +} + +func (r *restoredState) transition(s containerState) error { + switch s.(type) { + case *stoppedState: + return nil + case *runningState: + return nil + } + return newStateTransitionError(r, s) +} + +func (r *restoredState) destroy() error { + if _, err := os.Stat(filepath.Join(r.c.root, "checkpoint")); err != nil { + if !os.IsNotExist(err) { + return err + } + } + return destroy(r.c) +} + +// createdState is used whenever a container is restored, loaded, or setting additional +// processes inside and it should not be destroyed when it is exiting. +type createdState struct { + c *linuxContainer + s Status +} + +func (n *createdState) status() Status { + return n.s +} + +func (n *createdState) transition(s containerState) error { + n.c.state = s + return nil +} + +func (n *createdState) destroy() error { + if err := n.c.refreshState(); err != nil { + return err + } + return n.c.state.destroy() +} diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/system/linux.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/system/linux.go index 2cc3ef803a..babf55048b 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/system/linux.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/system/linux.go @@ -3,6 +3,9 @@ package system import ( + "bufio" + "fmt" + "os" "os/exec" "syscall" "unsafe" @@ -75,3 +78,45 @@ func Setctty() error { } return nil } + +/* + * Detect whether we are currently running in a user namespace. + * Copied from github.com/lxc/lxd/shared/util.go + */ +func RunningInUserNS() bool { + file, err := os.Open("/proc/self/uid_map") + if err != nil { + /* + * This kernel-provided file only exists if user namespaces are + * supported + */ + return false + } + defer file.Close() + + buf := bufio.NewReader(file) + l, _, err := buf.ReadLine() + if err != nil { + return false + } + + line := string(l) + var a, b, c int64 + fmt.Sscanf(line, "%d %d %d", &a, &b, &c) + /* + * We assume we are in the initial user namespace if we have a full + * range - 4294967295 uids starting at uid 0. + */ + if a == 0 && b == 0 && c == 4294967295 { + return false + } + return true +} + +func Prctl(option int, arg2, arg3, arg4, arg5 uintptr) (err error) { + _, _, e1 := syscall.Syscall6(syscall.SYS_PRCTL, uintptr(option), arg2, arg3, arg4, arg5, 0) + if e1 != 0 { + err = e1 + } + return +} diff --git a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/utils/utils.go b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/utils/utils.go index 1378006b0a..68ae3c477b 100644 --- a/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/utils/utils.go +++ b/components/engine/vendor/src/github.com/opencontainers/runc/libcontainer/utils/utils.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "io" + "os" "path/filepath" "syscall" ) @@ -54,3 +55,32 @@ func WriteJSON(w io.Writer, v interface{}) error { _, err = w.Write(data) return err } + +// CleanPath makes a path safe for use with filepath.Join. This is done by not +// only cleaning the path, but also (if the path is relative) adding a leading +// '/' and cleaning it (then removing the leading '/'). This ensures that a +// path resulting from prepending another path will always resolve to lexically +// be a subdirectory of the prefixed path. This is all done lexically, so paths +// that include symlinks won't be safe as a result of using CleanPath. +func CleanPath(path string) string { + // Deal with empty strings nicely. + if path == "" { + return "" + } + + // Ensure that all paths are cleaned (especially problematic ones like + // "/../../../../../" which can cause lots of issues). + path = filepath.Clean(path) + + // If the path isn't absolute, we need to do more processing to fix paths + // such as "../../../..//some/path". We also shouldn't convert absolute + // paths to relative ones. + if !filepath.IsAbs(path) { + path = filepath.Clean(string(os.PathSeparator) + path) + // This can't fail, as (by definition) all paths are relative to root. + path, _ = filepath.Rel(string(os.PathSeparator), path) + } + + // Clean the path again for good measure. + return filepath.Clean(path) +} diff --git a/components/engine/vendor/src/github.com/vdemeester/shakers/.gitignore b/components/engine/vendor/src/github.com/vdemeester/shakers/.gitignore index a9243971be..6a42174249 100644 --- a/components/engine/vendor/src/github.com/vdemeester/shakers/.gitignore +++ b/components/engine/vendor/src/github.com/vdemeester/shakers/.gitignore @@ -1,4 +1,2 @@ -Godeps/_workspace/bin -Godeps/_workspace/pkg - +vendor *.test diff --git a/components/engine/vendor/src/github.com/vdemeester/shakers/Dockerfile b/components/engine/vendor/src/github.com/vdemeester/shakers/Dockerfile index 12c09a0d2b..9864a041db 100644 --- a/components/engine/vendor/src/github.com/vdemeester/shakers/Dockerfile +++ b/components/engine/vendor/src/github.com/vdemeester/shakers/Dockerfile @@ -3,10 +3,14 @@ FROM golang:1.5 RUN go get golang.org/x/tools/cmd/cover RUN go get github.com/golang/lint/golint RUN go get golang.org/x/tools/cmd/vet +RUN go get github.com/Masterminds/glide WORKDIR /go/src/github.com/vdemeester/shakers # enable GO15VENDOREXPERIMENT ENV GO15VENDOREXPERIMENT 1 +COPY glide.yaml glide.yaml +RUN glide up + COPY . /go/src/github.com/vdemeester/shakers diff --git a/components/engine/vendor/src/github.com/vdemeester/shakers/LICENSE b/components/engine/vendor/src/github.com/vdemeester/shakers/LICENSE new file mode 100644 index 0000000000..e9e9e84587 --- /dev/null +++ b/components/engine/vendor/src/github.com/vdemeester/shakers/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2015-2016 Vincent Demeester + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/components/engine/vendor/src/github.com/vdemeester/shakers/Makefile b/components/engine/vendor/src/github.com/vdemeester/shakers/Makefile index 10b3be708e..74e60db684 100644 --- a/components/engine/vendor/src/github.com/vdemeester/shakers/Makefile +++ b/components/engine/vendor/src/github.com/vdemeester/shakers/Makefile @@ -9,10 +9,10 @@ DOCKER_RUN_SHAKERS := docker run $(if $(CIRCLECI),,--rm) -it $(SHAKERS_ENVS) $(S print-%: ; @echo $*=$($*) -default: binary +default: all -binary: build - $(DOCKER_RUN_SHAKERS) ./script/make.sh binary +all: build + $(DOCKER_RUN_SHAKERS) ./script/make.sh test-unit: build $(DOCKER_RUN_SHAKERS) ./script/make.sh test-unit @@ -35,6 +35,3 @@ build: shell: build $(DOCKER_RUN_SHAKERS) /bin/bash -run-dev: - go build - ./traefik diff --git a/components/engine/vendor/src/github.com/vdemeester/shakers/README.md b/components/engine/vendor/src/github.com/vdemeester/shakers/README.md index 05417f9e1c..165bc967ac 100644 --- a/components/engine/vendor/src/github.com/vdemeester/shakers/README.md +++ b/components/engine/vendor/src/github.com/vdemeester/shakers/README.md @@ -6,7 +6,7 @@ A collection of `go-check` Checkers to ease the use of it. ## Building and testing it You need either [docker](https://github.com/docker/docker), or `go` -and `godep` in order to build and test shakers. +and `glide` in order to build and test shakers. ### Using Docker and Makefile @@ -22,3 +22,9 @@ ok github.com/vdemeester/shakers 0.015s coverage: 96.0% of statements Test success ``` + +### Using glide and `GO15VENDOREXPERIMENT` + +- Get the dependencies with `glide up` (or use `go get` but you have no garantuees over the version of the dependencies) +- If you're using glide (and not standard `go get`) export `GO15VENDOREXPERIMENT` with `export GO15VENDOREXPERIMENT=1` +- Run tests with `go test .` diff --git a/components/engine/vendor/src/github.com/vishvananda/netlink/addr_linux.go b/components/engine/vendor/src/github.com/vishvananda/netlink/addr_linux.go index 9373e9c5a7..9e4f62f1d5 100644 --- a/components/engine/vendor/src/github.com/vishvananda/netlink/addr_linux.go +++ b/components/engine/vendor/src/github.com/vishvananda/netlink/addr_linux.go @@ -101,6 +101,10 @@ func AddrList(link Link, family int) ([]Addr, error) { continue } + if family != FAMILY_ALL && msg.Family != uint8(family) { + continue + } + attrs, err := nl.ParseRouteAttr(m[msg.Len():]) if err != nil { return nil, err diff --git a/components/engine/vendor/src/github.com/vishvananda/netlink/class.go b/components/engine/vendor/src/github.com/vishvananda/netlink/class.go index 35bdb33100..264e3ad003 100644 --- a/components/engine/vendor/src/github.com/vishvananda/netlink/class.go +++ b/components/engine/vendor/src/github.com/vishvananda/netlink/class.go @@ -56,6 +56,7 @@ func NewHtbClass(attrs ClassAttrs, cattrs HtbClassAttrs) *HtbClass { ceil := cattrs.Ceil / 8 buffer := cattrs.Buffer cbuffer := cattrs.Cbuffer + if ceil == 0 { ceil = rate } diff --git a/components/engine/vendor/src/github.com/vishvananda/netlink/class_linux.go b/components/engine/vendor/src/github.com/vishvananda/netlink/class_linux.go index 84828da101..4a52d2b997 100644 --- a/components/engine/vendor/src/github.com/vishvananda/netlink/class_linux.go +++ b/components/engine/vendor/src/github.com/vishvananda/netlink/class_linux.go @@ -1,6 +1,7 @@ package netlink import ( + "errors" "syscall" "github.com/vishvananda/netlink/nl" @@ -65,15 +66,32 @@ func classPayload(req *nl.NetlinkRequest, class Class) error { options := nl.NewRtAttr(nl.TCA_OPTIONS, nil) if htb, ok := class.(*HtbClass); ok { opt := nl.TcHtbCopt{} - opt.Rate.Rate = uint32(htb.Rate) - opt.Ceil.Rate = uint32(htb.Ceil) opt.Buffer = htb.Buffer opt.Cbuffer = htb.Cbuffer opt.Quantum = htb.Quantum opt.Level = htb.Level opt.Prio = htb.Prio // TODO: Handle Debug properly. For now default to 0 + /* Calculate {R,C}Tab and set Rate and Ceil */ + cell_log := -1 + ccell_log := -1 + linklayer := nl.LINKLAYER_ETHERNET + mtu := 1600 + var rtab [256]uint32 + var ctab [256]uint32 + tcrate := nl.TcRateSpec{Rate: uint32(htb.Rate)} + if CalcRtable(&tcrate, rtab, cell_log, uint32(mtu), linklayer) < 0 { + return errors.New("HTB: failed to calculate rate table.") + } + opt.Rate = tcrate + tcceil := nl.TcRateSpec{Rate: uint32(htb.Ceil)} + if CalcRtable(&tcceil, ctab, ccell_log, uint32(mtu), linklayer) < 0 { + return errors.New("HTB: failed to calculate ceil rate table.") + } + opt.Ceil = tcceil nl.NewRtAttrChild(options, nl.TCA_HTB_PARMS, opt.Serialize()) + nl.NewRtAttrChild(options, nl.TCA_HTB_RTAB, SerializeRtab(rtab)) + nl.NewRtAttrChild(options, nl.TCA_HTB_CTAB, SerializeRtab(ctab)) } req.AddData(options) return nil diff --git a/components/engine/vendor/src/github.com/vishvananda/netlink/link.go b/components/engine/vendor/src/github.com/vishvananda/netlink/link.go index 544a97cb46..2934c0fb2a 100644 --- a/components/engine/vendor/src/github.com/vishvananda/netlink/link.go +++ b/components/engine/vendor/src/github.com/vishvananda/netlink/link.go @@ -204,6 +204,7 @@ type Vxlan struct { RSC bool L2miss bool L3miss bool + UDPCSum bool NoAge bool GBP bool Age int diff --git a/components/engine/vendor/src/github.com/vishvananda/netlink/link_linux.go b/components/engine/vendor/src/github.com/vishvananda/netlink/link_linux.go index 3aa9124881..b3d0472004 100644 --- a/components/engine/vendor/src/github.com/vishvananda/netlink/link_linux.go +++ b/components/engine/vendor/src/github.com/vishvananda/netlink/link_linux.go @@ -142,6 +142,54 @@ func LinkSetHardwareAddr(link Link, hwaddr net.HardwareAddr) error { return err } +// LinkSetVfHardwareAddr sets the hardware address of a vf for the link. +// Equivalent to: `ip link set $link vf $vf mac $hwaddr` +func LinkSetVfHardwareAddr(link Link, vf int, hwaddr net.HardwareAddr) error { + base := link.Attrs() + ensureIndex(base) + req := nl.NewNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) + + msg := nl.NewIfInfomsg(syscall.AF_UNSPEC) + msg.Index = int32(base.Index) + req.AddData(msg) + + data := nl.NewRtAttr(nl.IFLA_VFINFO_LIST, nil) + info := nl.NewRtAttrChild(data, nl.IFLA_VF_INFO, nil) + vfmsg := nl.VfMac{ + Vf: uint32(vf), + } + copy(vfmsg.Mac[:], []byte(hwaddr)) + nl.NewRtAttrChild(info, nl.IFLA_VF_MAC, vfmsg.Serialize()) + req.AddData(data) + + _, err := req.Execute(syscall.NETLINK_ROUTE, 0) + return err +} + +// LinkSetVfVlan sets the vlan of a vf for the link. +// Equivalent to: `ip link set $link vf $vf vlan $vlan` +func LinkSetVfVlan(link Link, vf, vlan int) error { + base := link.Attrs() + ensureIndex(base) + req := nl.NewNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) + + msg := nl.NewIfInfomsg(syscall.AF_UNSPEC) + msg.Index = int32(base.Index) + req.AddData(msg) + + data := nl.NewRtAttr(nl.IFLA_VFINFO_LIST, nil) + info := nl.NewRtAttrChild(data, nl.IFLA_VF_INFO, nil) + vfmsg := nl.VfVlan{ + Vf: uint32(vf), + Vlan: uint32(vlan), + } + nl.NewRtAttrChild(info, nl.IFLA_VF_VLAN, vfmsg.Serialize()) + req.AddData(data) + + _, err := req.Execute(syscall.NETLINK_ROUTE, 0) + return err +} + // LinkSetMaster sets the master of the link device. // Equivalent to: `ip link set $link master $master` func LinkSetMaster(link Link, master *Bridge) error { @@ -277,10 +325,12 @@ func addVxlanAttrs(vxlan *Vxlan, linkInfo *nl.RtAttr) { nl.NewRtAttrChild(data, nl.IFLA_VXLAN_L2MISS, boolAttr(vxlan.L2miss)) nl.NewRtAttrChild(data, nl.IFLA_VXLAN_L3MISS, boolAttr(vxlan.L3miss)) + if vxlan.UDPCSum { + nl.NewRtAttrChild(data, nl.IFLA_VXLAN_UDP_CSUM, boolAttr(vxlan.UDPCSum)) + } if vxlan.GBP { nl.NewRtAttrChild(data, nl.IFLA_VXLAN_GBP, boolAttr(vxlan.GBP)) } - if vxlan.NoAge { nl.NewRtAttrChild(data, nl.IFLA_VXLAN_AGEING, nl.Uint32Attr(0)) } else if vxlan.Age > 0 { @@ -815,6 +865,7 @@ func LinkList() ([]Link, error) { // LinkUpdate is used to pass information back from LinkSubscribe() type LinkUpdate struct { nl.IfInfomsg + Header syscall.NlMsghdr Link } @@ -844,7 +895,7 @@ func LinkSubscribe(ch chan<- LinkUpdate, done <-chan struct{}) error { if err != nil { return } - ch <- LinkUpdate{IfInfomsg: *ifmsg, Link: link} + ch <- LinkUpdate{IfInfomsg: *ifmsg, Header: m.Header, Link: link} } } }() @@ -935,6 +986,8 @@ func parseVxlanData(link Link, data []syscall.NetlinkRouteAttr) { vxlan.L2miss = int8(datum.Value[0]) != 0 case nl.IFLA_VXLAN_L3MISS: vxlan.L3miss = int8(datum.Value[0]) != 0 + case nl.IFLA_VXLAN_UDP_CSUM: + vxlan.UDPCSum = int8(datum.Value[0]) != 0 case nl.IFLA_VXLAN_GBP: vxlan.GBP = int8(datum.Value[0]) != 0 case nl.IFLA_VXLAN_AGEING: diff --git a/components/engine/vendor/src/github.com/vishvananda/netlink/nl/link_linux.go b/components/engine/vendor/src/github.com/vishvananda/netlink/nl/link_linux.go index 8554a5d4a0..b7f50646d2 100644 --- a/components/engine/vendor/src/github.com/vishvananda/netlink/nl/link_linux.go +++ b/components/engine/vendor/src/github.com/vishvananda/netlink/nl/link_linux.go @@ -1,7 +1,13 @@ package nl +import ( + "unsafe" +) + const ( DEFAULT_CHANGE = 0xFFFFFFFF + // doesn't exist in syscall + IFLA_VFINFO_LIST = 0x16 ) const ( @@ -182,3 +188,209 @@ const ( GRE_FLAGS = 0x00F8 GRE_VERSION = 0x0007 ) + +const ( + IFLA_VF_INFO_UNSPEC = iota + IFLA_VF_INFO + IFLA_VF_INFO_MAX = IFLA_VF_INFO +) + +const ( + IFLA_VF_UNSPEC = iota + IFLA_VF_MAC /* Hardware queue specific attributes */ + IFLA_VF_VLAN + IFLA_VF_TX_RATE /* Max TX Bandwidth Allocation */ + IFLA_VF_SPOOFCHK /* Spoof Checking on/off switch */ + IFLA_VF_LINK_STATE /* link state enable/disable/auto switch */ + IFLA_VF_RATE /* Min and Max TX Bandwidth Allocation */ + IFLA_VF_RSS_QUERY_EN /* RSS Redirection Table and Hash Key query + * on/off switch + */ + IFLA_VF_STATS /* network device statistics */ + IFLA_VF_MAX = IFLA_VF_STATS +) + +const ( + IFLA_VF_LINK_STATE_AUTO = iota /* link state of the uplink */ + IFLA_VF_LINK_STATE_ENABLE /* link always up */ + IFLA_VF_LINK_STATE_DISABLE /* link always down */ + IFLA_VF_LINK_STATE_MAX = IFLA_VF_LINK_STATE_DISABLE +) + +const ( + IFLA_VF_STATS_RX_PACKETS = iota + IFLA_VF_STATS_TX_PACKETS + IFLA_VF_STATS_RX_BYTES + IFLA_VF_STATS_TX_BYTES + IFLA_VF_STATS_BROADCAST + IFLA_VF_STATS_MULTICAST + IFLA_VF_STATS_MAX = IFLA_VF_STATS_MULTICAST +) + +const ( + SizeofVfMac = 0x24 + SizeofVfVlan = 0x0c + SizeofVfTxRate = 0x08 + SizeofVfRate = 0x0c + SizeofVfSpoofchk = 0x08 + SizeofVfLinkState = 0x08 + SizeofVfRssQueryEn = 0x08 +) + +// struct ifla_vf_mac { +// __u32 vf; +// __u8 mac[32]; /* MAX_ADDR_LEN */ +// }; + +type VfMac struct { + Vf uint32 + Mac [32]byte +} + +func (msg *VfMac) Len() int { + return SizeofVfMac +} + +func DeserializeVfMac(b []byte) *VfMac { + return (*VfMac)(unsafe.Pointer(&b[0:SizeofVfMac][0])) +} + +func (msg *VfMac) Serialize() []byte { + return (*(*[SizeofVfMac]byte)(unsafe.Pointer(msg)))[:] +} + +// struct ifla_vf_vlan { +// __u32 vf; +// __u32 vlan; /* 0 - 4095, 0 disables VLAN filter */ +// __u32 qos; +// }; + +type VfVlan struct { + Vf uint32 + Vlan uint32 + Qos uint32 +} + +func (msg *VfVlan) Len() int { + return SizeofVfVlan +} + +func DeserializeVfVlan(b []byte) *VfVlan { + return (*VfVlan)(unsafe.Pointer(&b[0:SizeofVfVlan][0])) +} + +func (msg *VfVlan) Serialize() []byte { + return (*(*[SizeofVfVlan]byte)(unsafe.Pointer(msg)))[:] +} + +// struct ifla_vf_tx_rate { +// __u32 vf; +// __u32 rate; /* Max TX bandwidth in Mbps, 0 disables throttling */ +// }; + +type VfTxRate struct { + Vf uint32 + Rate uint32 +} + +func (msg *VfTxRate) Len() int { + return SizeofVfTxRate +} + +func DeserializeVfTxRate(b []byte) *VfTxRate { + return (*VfTxRate)(unsafe.Pointer(&b[0:SizeofVfTxRate][0])) +} + +func (msg *VfTxRate) Serialize() []byte { + return (*(*[SizeofVfTxRate]byte)(unsafe.Pointer(msg)))[:] +} + +// struct ifla_vf_rate { +// __u32 vf; +// __u32 min_tx_rate; /* Min Bandwidth in Mbps */ +// __u32 max_tx_rate; /* Max Bandwidth in Mbps */ +// }; + +type VfRate struct { + Vf uint32 + MinTxRate uint32 + MaxTxRate uint32 +} + +func (msg *VfRate) Len() int { + return SizeofVfRate +} + +func DeserializeVfRate(b []byte) *VfRate { + return (*VfRate)(unsafe.Pointer(&b[0:SizeofVfRate][0])) +} + +func (msg *VfRate) Serialize() []byte { + return (*(*[SizeofVfRate]byte)(unsafe.Pointer(msg)))[:] +} + +// struct ifla_vf_spoofchk { +// __u32 vf; +// __u32 setting; +// }; + +type VfSpoofchk struct { + Vf uint32 + Setting uint32 +} + +func (msg *VfSpoofchk) Len() int { + return SizeofVfSpoofchk +} + +func DeserializeVfSpoofchk(b []byte) *VfSpoofchk { + return (*VfSpoofchk)(unsafe.Pointer(&b[0:SizeofVfSpoofchk][0])) +} + +func (msg *VfSpoofchk) Serialize() []byte { + return (*(*[SizeofVfSpoofchk]byte)(unsafe.Pointer(msg)))[:] +} + +// struct ifla_vf_link_state { +// __u32 vf; +// __u32 link_state; +// }; + +type VfLinkState struct { + Vf uint32 + LinkState uint32 +} + +func (msg *VfLinkState) Len() int { + return SizeofVfLinkState +} + +func DeserializeVfLinkState(b []byte) *VfLinkState { + return (*VfLinkState)(unsafe.Pointer(&b[0:SizeofVfLinkState][0])) +} + +func (msg *VfLinkState) Serialize() []byte { + return (*(*[SizeofVfLinkState]byte)(unsafe.Pointer(msg)))[:] +} + +// struct ifla_vf_rss_query_en { +// __u32 vf; +// __u32 setting; +// }; + +type VfRssQueryEn struct { + Vf uint32 + Setting uint32 +} + +func (msg *VfRssQueryEn) Len() int { + return SizeofVfRssQueryEn +} + +func DeserializeVfRssQueryEn(b []byte) *VfRssQueryEn { + return (*VfRssQueryEn)(unsafe.Pointer(&b[0:SizeofVfRssQueryEn][0])) +} + +func (msg *VfRssQueryEn) Serialize() []byte { + return (*(*[SizeofVfRssQueryEn]byte)(unsafe.Pointer(msg)))[:] +} diff --git a/components/engine/vendor/src/github.com/vishvananda/netlink/xfrm_state_linux.go b/components/engine/vendor/src/github.com/vishvananda/netlink/xfrm_state_linux.go index 5f44ec8525..fc8604b9d1 100644 --- a/components/engine/vendor/src/github.com/vishvananda/netlink/xfrm_state_linux.go +++ b/components/engine/vendor/src/github.com/vishvananda/netlink/xfrm_state_linux.go @@ -110,9 +110,6 @@ func XfrmStateDel(state *XfrmState) error { func XfrmStateList(family int) ([]XfrmState, error) { req := nl.NewNetlinkRequest(nl.XFRM_MSG_GETSA, syscall.NLM_F_DUMP) - msg := nl.NewIfInfomsg(family) - req.AddData(msg) - msgs, err := req.Execute(syscall.NETLINK_XFRM, nl.XFRM_MSG_NEWSA) if err != nil { return nil, err diff --git a/components/engine/vendor/src/golang.org/x/net/context/ctxhttp/cancelreq.go b/components/engine/vendor/src/golang.org/x/net/context/ctxhttp/cancelreq.go new file mode 100644 index 0000000000..48610e3627 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/context/ctxhttp/cancelreq.go @@ -0,0 +1,18 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.5 + +package ctxhttp + +import "net/http" + +func canceler(client *http.Client, req *http.Request) func() { + ch := make(chan struct{}) + req.Cancel = ch + + return func() { + close(ch) + } +} diff --git a/components/engine/vendor/src/golang.org/x/net/context/ctxhttp/cancelreq_go14.go b/components/engine/vendor/src/golang.org/x/net/context/ctxhttp/cancelreq_go14.go new file mode 100644 index 0000000000..56bcbadb85 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/context/ctxhttp/cancelreq_go14.go @@ -0,0 +1,23 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.5 + +package ctxhttp + +import "net/http" + +type requestCanceler interface { + CancelRequest(*http.Request) +} + +func canceler(client *http.Client, req *http.Request) func() { + rc, ok := client.Transport.(requestCanceler) + if !ok { + return func() {} + } + return func() { + rc.CancelRequest(req) + } +} diff --git a/components/engine/vendor/src/golang.org/x/net/context/ctxhttp/ctxhttp.go b/components/engine/vendor/src/golang.org/x/net/context/ctxhttp/ctxhttp.go new file mode 100644 index 0000000000..504dd63ed9 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/context/ctxhttp/ctxhttp.go @@ -0,0 +1,79 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ctxhttp provides helper functions for performing context-aware HTTP requests. +package ctxhttp // import "golang.org/x/net/context/ctxhttp" + +import ( + "io" + "net/http" + "net/url" + "strings" + + "golang.org/x/net/context" +) + +// Do sends an HTTP request with the provided http.Client and returns an HTTP response. +// If the client is nil, http.DefaultClient is used. +// If the context is canceled or times out, ctx.Err() will be returned. +func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) { + if client == nil { + client = http.DefaultClient + } + + // Request cancelation changed in Go 1.5, see cancelreq.go and cancelreq_go14.go. + cancel := canceler(client, req) + + type responseAndError struct { + resp *http.Response + err error + } + result := make(chan responseAndError, 1) + + go func() { + resp, err := client.Do(req) + result <- responseAndError{resp, err} + }() + + select { + case <-ctx.Done(): + cancel() + return nil, ctx.Err() + case r := <-result: + return r.resp, r.err + } +} + +// Get issues a GET request via the Do function. +func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + return Do(ctx, client, req) +} + +// Head issues a HEAD request via the Do function. +func Head(ctx context.Context, client *http.Client, url string) (*http.Response, error) { + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return nil, err + } + return Do(ctx, client, req) +} + +// Post issues a POST request via the Do function. +func Post(ctx context.Context, client *http.Client, url string, bodyType string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest("POST", url, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", bodyType) + return Do(ctx, client, req) +} + +// PostForm issues a POST request via the Do function. +func PostForm(ctx context.Context, client *http.Client, url string, data url.Values) (*http.Response, error) { + return Post(ctx, client, url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/.gitignore b/components/engine/vendor/src/golang.org/x/net/http2/.gitignore new file mode 100644 index 0000000000..190f12234a --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/.gitignore @@ -0,0 +1,2 @@ +*~ +h2i/h2i diff --git a/components/engine/vendor/src/golang.org/x/net/http2/Dockerfile b/components/engine/vendor/src/golang.org/x/net/http2/Dockerfile new file mode 100644 index 0000000000..b4e14d55a5 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/Dockerfile @@ -0,0 +1,44 @@ +# +# This Dockerfile builds a recent curl with HTTP/2 client support, using +# a recent nghttp2 build. +# +# See the Makefile for how to tag it. If Docker and that image is found, the +# Go tests use this curl binary for integration tests. +# + +FROM ubuntu:trusty + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y git-core build-essential wget + +RUN apt-get install -y --no-install-recommends \ + autotools-dev libtool pkg-config zlib1g-dev \ + libcunit1-dev libssl-dev libxml2-dev libevent-dev \ + automake autoconf + +# Note: setting NGHTTP2_VER before the git clone, so an old git clone isn't cached: +ENV NGHTTP2_VER af24f8394e43f4 +RUN cd /root && git clone https://github.com/tatsuhiro-t/nghttp2.git + +WORKDIR /root/nghttp2 +RUN git reset --hard $NGHTTP2_VER +RUN autoreconf -i +RUN automake +RUN autoconf +RUN ./configure +RUN make +RUN make install + +WORKDIR /root +RUN wget http://curl.haxx.se/download/curl-7.40.0.tar.gz +RUN tar -zxvf curl-7.40.0.tar.gz +WORKDIR /root/curl-7.40.0 +RUN ./configure --with-ssl --with-nghttp2=/usr/local +RUN make +RUN make install +RUN ldconfig + +CMD ["-h"] +ENTRYPOINT ["/usr/local/bin/curl"] + diff --git a/components/engine/vendor/src/golang.org/x/net/http2/Makefile b/components/engine/vendor/src/golang.org/x/net/http2/Makefile new file mode 100644 index 0000000000..55fd826f77 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/Makefile @@ -0,0 +1,3 @@ +curlimage: + docker build -t gohttp2/curl . + diff --git a/components/engine/vendor/src/golang.org/x/net/http2/README b/components/engine/vendor/src/golang.org/x/net/http2/README new file mode 100644 index 0000000000..360d5aa379 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/README @@ -0,0 +1,20 @@ +This is a work-in-progress HTTP/2 implementation for Go. + +It will eventually live in the Go standard library and won't require +any changes to your code to use. It will just be automatic. + +Status: + +* The server support is pretty good. A few things are missing + but are being worked on. +* The client work has just started but shares a lot of code + is coming along much quicker. + +Docs are at https://godoc.org/golang.org/x/net/http2 + +Demo test server at https://http2.golang.org/ + +Help & bug reports welcome! + +Contributing: https://golang.org/doc/contribute.html +Bugs: https://golang.org/issue/new?title=x/net/http2:+ diff --git a/components/engine/vendor/src/golang.org/x/net/http2/buffer.go b/components/engine/vendor/src/golang.org/x/net/http2/buffer.go new file mode 100644 index 0000000000..c43954cf04 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/buffer.go @@ -0,0 +1,76 @@ +// Copyright 2014 The Go Authors. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +package http2 + +import ( + "errors" +) + +// buffer is an io.ReadWriteCloser backed by a fixed size buffer. +// It never allocates, but moves old data as new data is written. +type buffer struct { + buf []byte + r, w int + closed bool + err error // err to return to reader +} + +var ( + errReadEmpty = errors.New("read from empty buffer") + errWriteClosed = errors.New("write on closed buffer") + errWriteFull = errors.New("write on full buffer") +) + +// Read copies bytes from the buffer into p. +// It is an error to read when no data is available. +func (b *buffer) Read(p []byte) (n int, err error) { + n = copy(p, b.buf[b.r:b.w]) + b.r += n + if b.closed && b.r == b.w { + err = b.err + } else if b.r == b.w && n == 0 { + err = errReadEmpty + } + return n, err +} + +// Len returns the number of bytes of the unread portion of the buffer. +func (b *buffer) Len() int { + return b.w - b.r +} + +// Write copies bytes from p into the buffer. +// It is an error to write more data than the buffer can hold. +func (b *buffer) Write(p []byte) (n int, err error) { + if b.closed { + return 0, errWriteClosed + } + + // Slide existing data to beginning. + if b.r > 0 && len(p) > len(b.buf)-b.w { + copy(b.buf, b.buf[b.r:b.w]) + b.w -= b.r + b.r = 0 + } + + // Write new data. + n = copy(b.buf[b.w:], p) + b.w += n + if n < len(p) { + err = errWriteFull + } + return n, err +} + +// Close marks the buffer as closed. Future calls to Write will +// return an error. Future calls to Read, once the buffer is +// empty, will return err. +func (b *buffer) Close(err error) { + if !b.closed { + b.closed = true + b.err = err + } +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/errors.go b/components/engine/vendor/src/golang.org/x/net/http2/errors.go new file mode 100644 index 0000000000..c885328a82 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/errors.go @@ -0,0 +1,78 @@ +// Copyright 2014 The Go Authors. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +package http2 + +import "fmt" + +// An ErrCode is an unsigned 32-bit error code as defined in the HTTP/2 spec. +type ErrCode uint32 + +const ( + ErrCodeNo ErrCode = 0x0 + ErrCodeProtocol ErrCode = 0x1 + ErrCodeInternal ErrCode = 0x2 + ErrCodeFlowControl ErrCode = 0x3 + ErrCodeSettingsTimeout ErrCode = 0x4 + ErrCodeStreamClosed ErrCode = 0x5 + ErrCodeFrameSize ErrCode = 0x6 + ErrCodeRefusedStream ErrCode = 0x7 + ErrCodeCancel ErrCode = 0x8 + ErrCodeCompression ErrCode = 0x9 + ErrCodeConnect ErrCode = 0xa + ErrCodeEnhanceYourCalm ErrCode = 0xb + ErrCodeInadequateSecurity ErrCode = 0xc + ErrCodeHTTP11Required ErrCode = 0xd +) + +var errCodeName = map[ErrCode]string{ + ErrCodeNo: "NO_ERROR", + ErrCodeProtocol: "PROTOCOL_ERROR", + ErrCodeInternal: "INTERNAL_ERROR", + ErrCodeFlowControl: "FLOW_CONTROL_ERROR", + ErrCodeSettingsTimeout: "SETTINGS_TIMEOUT", + ErrCodeStreamClosed: "STREAM_CLOSED", + ErrCodeFrameSize: "FRAME_SIZE_ERROR", + ErrCodeRefusedStream: "REFUSED_STREAM", + ErrCodeCancel: "CANCEL", + ErrCodeCompression: "COMPRESSION_ERROR", + ErrCodeConnect: "CONNECT_ERROR", + ErrCodeEnhanceYourCalm: "ENHANCE_YOUR_CALM", + ErrCodeInadequateSecurity: "INADEQUATE_SECURITY", + ErrCodeHTTP11Required: "HTTP_1_1_REQUIRED", +} + +func (e ErrCode) String() string { + if s, ok := errCodeName[e]; ok { + return s + } + return fmt.Sprintf("unknown error code 0x%x", uint32(e)) +} + +// ConnectionError is an error that results in the termination of the +// entire connection. +type ConnectionError ErrCode + +func (e ConnectionError) Error() string { return fmt.Sprintf("connection error: %s", ErrCode(e)) } + +// StreamError is an error that only affects one stream within an +// HTTP/2 connection. +type StreamError struct { + StreamID uint32 + Code ErrCode +} + +func (e StreamError) Error() string { + return fmt.Sprintf("stream error: stream ID %d; %v", e.StreamID, e.Code) +} + +// 6.9.1 The Flow Control Window +// "If a sender receives a WINDOW_UPDATE that causes a flow control +// window to exceed this maximum it MUST terminate either the stream +// or the connection, as appropriate. For streams, [...]; for the +// connection, a GOAWAY frame with a FLOW_CONTROL_ERROR code." +type goAwayFlowError struct{} + +func (goAwayFlowError) Error() string { return "connection exceeded flow control window size" } diff --git a/components/engine/vendor/src/golang.org/x/net/http2/flow.go b/components/engine/vendor/src/golang.org/x/net/http2/flow.go new file mode 100644 index 0000000000..540fc4283e --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/flow.go @@ -0,0 +1,51 @@ +// Copyright 2014 The Go Authors. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +// Flow control + +package http2 + +// flow is the flow control window's size. +type flow struct { + // n is the number of DATA bytes we're allowed to send. + // A flow is kept both on a conn and a per-stream. + n int32 + + // conn points to the shared connection-level flow that is + // shared by all streams on that conn. It is nil for the flow + // that's on the conn directly. + conn *flow +} + +func (f *flow) setConnFlow(cf *flow) { f.conn = cf } + +func (f *flow) available() int32 { + n := f.n + if f.conn != nil && f.conn.n < n { + n = f.conn.n + } + return n +} + +func (f *flow) take(n int32) { + if n > f.available() { + panic("internal error: took too much") + } + f.n -= n + if f.conn != nil { + f.conn.n -= n + } +} + +// add adds n bytes (positive or negative) to the flow control window. +// It returns false if the sum would exceed 2^31-1. +func (f *flow) add(n int32) bool { + remain := (1<<31 - 1) - f.n + if n > remain { + return false + } + f.n += n + return true +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/frame.go b/components/engine/vendor/src/golang.org/x/net/http2/frame.go new file mode 100644 index 0000000000..e8b872a19b --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/frame.go @@ -0,0 +1,1113 @@ +// Copyright 2014 The Go Authors. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +package http2 + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "sync" +) + +const frameHeaderLen = 9 + +var padZeros = make([]byte, 255) // zeros for padding + +// A FrameType is a registered frame type as defined in +// http://http2.github.io/http2-spec/#rfc.section.11.2 +type FrameType uint8 + +const ( + FrameData FrameType = 0x0 + FrameHeaders FrameType = 0x1 + FramePriority FrameType = 0x2 + FrameRSTStream FrameType = 0x3 + FrameSettings FrameType = 0x4 + FramePushPromise FrameType = 0x5 + FramePing FrameType = 0x6 + FrameGoAway FrameType = 0x7 + FrameWindowUpdate FrameType = 0x8 + FrameContinuation FrameType = 0x9 +) + +var frameName = map[FrameType]string{ + FrameData: "DATA", + FrameHeaders: "HEADERS", + FramePriority: "PRIORITY", + FrameRSTStream: "RST_STREAM", + FrameSettings: "SETTINGS", + FramePushPromise: "PUSH_PROMISE", + FramePing: "PING", + FrameGoAway: "GOAWAY", + FrameWindowUpdate: "WINDOW_UPDATE", + FrameContinuation: "CONTINUATION", +} + +func (t FrameType) String() string { + if s, ok := frameName[t]; ok { + return s + } + return fmt.Sprintf("UNKNOWN_FRAME_TYPE_%d", uint8(t)) +} + +// Flags is a bitmask of HTTP/2 flags. +// The meaning of flags varies depending on the frame type. +type Flags uint8 + +// Has reports whether f contains all (0 or more) flags in v. +func (f Flags) Has(v Flags) bool { + return (f & v) == v +} + +// Frame-specific FrameHeader flag bits. +const ( + // Data Frame + FlagDataEndStream Flags = 0x1 + FlagDataPadded Flags = 0x8 + + // Headers Frame + FlagHeadersEndStream Flags = 0x1 + FlagHeadersEndHeaders Flags = 0x4 + FlagHeadersPadded Flags = 0x8 + FlagHeadersPriority Flags = 0x20 + + // Settings Frame + FlagSettingsAck Flags = 0x1 + + // Ping Frame + FlagPingAck Flags = 0x1 + + // Continuation Frame + FlagContinuationEndHeaders Flags = 0x4 + + FlagPushPromiseEndHeaders Flags = 0x4 + FlagPushPromisePadded Flags = 0x8 +) + +var flagName = map[FrameType]map[Flags]string{ + FrameData: { + FlagDataEndStream: "END_STREAM", + FlagDataPadded: "PADDED", + }, + FrameHeaders: { + FlagHeadersEndStream: "END_STREAM", + FlagHeadersEndHeaders: "END_HEADERS", + FlagHeadersPadded: "PADDED", + FlagHeadersPriority: "PRIORITY", + }, + FrameSettings: { + FlagSettingsAck: "ACK", + }, + FramePing: { + FlagPingAck: "ACK", + }, + FrameContinuation: { + FlagContinuationEndHeaders: "END_HEADERS", + }, + FramePushPromise: { + FlagPushPromiseEndHeaders: "END_HEADERS", + FlagPushPromisePadded: "PADDED", + }, +} + +// a frameParser parses a frame given its FrameHeader and payload +// bytes. The length of payload will always equal fh.Length (which +// might be 0). +type frameParser func(fh FrameHeader, payload []byte) (Frame, error) + +var frameParsers = map[FrameType]frameParser{ + FrameData: parseDataFrame, + FrameHeaders: parseHeadersFrame, + FramePriority: parsePriorityFrame, + FrameRSTStream: parseRSTStreamFrame, + FrameSettings: parseSettingsFrame, + FramePushPromise: parsePushPromise, + FramePing: parsePingFrame, + FrameGoAway: parseGoAwayFrame, + FrameWindowUpdate: parseWindowUpdateFrame, + FrameContinuation: parseContinuationFrame, +} + +func typeFrameParser(t FrameType) frameParser { + if f := frameParsers[t]; f != nil { + return f + } + return parseUnknownFrame +} + +// A FrameHeader is the 9 byte header of all HTTP/2 frames. +// +// See http://http2.github.io/http2-spec/#FrameHeader +type FrameHeader struct { + valid bool // caller can access []byte fields in the Frame + + // Type is the 1 byte frame type. There are ten standard frame + // types, but extension frame types may be written by WriteRawFrame + // and will be returned by ReadFrame (as UnknownFrame). + Type FrameType + + // Flags are the 1 byte of 8 potential bit flags per frame. + // They are specific to the frame type. + Flags Flags + + // Length is the length of the frame, not including the 9 byte header. + // The maximum size is one byte less than 16MB (uint24), but only + // frames up to 16KB are allowed without peer agreement. + Length uint32 + + // StreamID is which stream this frame is for. Certain frames + // are not stream-specific, in which case this field is 0. + StreamID uint32 +} + +// Header returns h. It exists so FrameHeaders can be embedded in other +// specific frame types and implement the Frame interface. +func (h FrameHeader) Header() FrameHeader { return h } + +func (h FrameHeader) String() string { + var buf bytes.Buffer + buf.WriteString("[FrameHeader ") + buf.WriteString(h.Type.String()) + if h.Flags != 0 { + buf.WriteString(" flags=") + set := 0 + for i := uint8(0); i < 8; i++ { + if h.Flags&(1< 1 { + buf.WriteByte('|') + } + name := flagName[h.Type][Flags(1<>24), + byte(streamID>>16), + byte(streamID>>8), + byte(streamID)) +} + +func (f *Framer) endWrite() error { + // Now that we know the final size, fill in the FrameHeader in + // the space previously reserved for it. Abuse append. + length := len(f.wbuf) - frameHeaderLen + if length >= (1 << 24) { + return ErrFrameTooLarge + } + _ = append(f.wbuf[:0], + byte(length>>16), + byte(length>>8), + byte(length)) + n, err := f.w.Write(f.wbuf) + if err == nil && n != len(f.wbuf) { + err = io.ErrShortWrite + } + return err +} + +func (f *Framer) writeByte(v byte) { f.wbuf = append(f.wbuf, v) } +func (f *Framer) writeBytes(v []byte) { f.wbuf = append(f.wbuf, v...) } +func (f *Framer) writeUint16(v uint16) { f.wbuf = append(f.wbuf, byte(v>>8), byte(v)) } +func (f *Framer) writeUint32(v uint32) { + f.wbuf = append(f.wbuf, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) +} + +const ( + minMaxFrameSize = 1 << 14 + maxFrameSize = 1<<24 - 1 +) + +// NewFramer returns a Framer that writes frames to w and reads them from r. +func NewFramer(w io.Writer, r io.Reader) *Framer { + fr := &Framer{ + w: w, + r: r, + } + fr.getReadBuf = func(size uint32) []byte { + if cap(fr.readBuf) >= int(size) { + return fr.readBuf[:size] + } + fr.readBuf = make([]byte, size) + return fr.readBuf + } + fr.SetMaxReadFrameSize(maxFrameSize) + return fr +} + +// SetMaxReadFrameSize sets the maximum size of a frame +// that will be read by a subsequent call to ReadFrame. +// It is the caller's responsibility to advertise this +// limit with a SETTINGS frame. +func (fr *Framer) SetMaxReadFrameSize(v uint32) { + if v > maxFrameSize { + v = maxFrameSize + } + fr.maxReadSize = v +} + +// ErrFrameTooLarge is returned from Framer.ReadFrame when the peer +// sends a frame that is larger than declared with SetMaxReadFrameSize. +var ErrFrameTooLarge = errors.New("http2: frame too large") + +// ReadFrame reads a single frame. The returned Frame is only valid +// until the next call to ReadFrame. +// If the frame is larger than previously set with SetMaxReadFrameSize, +// the returned error is ErrFrameTooLarge. +func (fr *Framer) ReadFrame() (Frame, error) { + if fr.lastFrame != nil { + fr.lastFrame.invalidate() + } + fh, err := readFrameHeader(fr.headerBuf[:], fr.r) + if err != nil { + return nil, err + } + if fh.Length > fr.maxReadSize { + return nil, ErrFrameTooLarge + } + payload := fr.getReadBuf(fh.Length) + if _, err := io.ReadFull(fr.r, payload); err != nil { + return nil, err + } + f, err := typeFrameParser(fh.Type)(fh, payload) + if err != nil { + return nil, err + } + fr.lastFrame = f + return f, nil +} + +// A DataFrame conveys arbitrary, variable-length sequences of octets +// associated with a stream. +// See http://http2.github.io/http2-spec/#rfc.section.6.1 +type DataFrame struct { + FrameHeader + data []byte +} + +func (f *DataFrame) StreamEnded() bool { + return f.FrameHeader.Flags.Has(FlagDataEndStream) +} + +// Data returns the frame's data octets, not including any padding +// size byte or padding suffix bytes. +// The caller must not retain the returned memory past the next +// call to ReadFrame. +func (f *DataFrame) Data() []byte { + f.checkValid() + return f.data +} + +func parseDataFrame(fh FrameHeader, payload []byte) (Frame, error) { + if fh.StreamID == 0 { + // DATA frames MUST be associated with a stream. If a + // DATA frame is received whose stream identifier + // field is 0x0, the recipient MUST respond with a + // connection error (Section 5.4.1) of type + // PROTOCOL_ERROR. + return nil, ConnectionError(ErrCodeProtocol) + } + f := &DataFrame{ + FrameHeader: fh, + } + var padSize byte + if fh.Flags.Has(FlagDataPadded) { + var err error + payload, padSize, err = readByte(payload) + if err != nil { + return nil, err + } + } + if int(padSize) > len(payload) { + // If the length of the padding is greater than the + // length of the frame payload, the recipient MUST + // treat this as a connection error. + // Filed: https://github.com/http2/http2-spec/issues/610 + return nil, ConnectionError(ErrCodeProtocol) + } + f.data = payload[:len(payload)-int(padSize)] + return f, nil +} + +var errStreamID = errors.New("invalid streamid") + +func validStreamID(streamID uint32) bool { + return streamID != 0 && streamID&(1<<31) == 0 +} + +// WriteData writes a DATA frame. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *Framer) WriteData(streamID uint32, endStream bool, data []byte) error { + // TODO: ignoring padding for now. will add when somebody cares. + if !validStreamID(streamID) && !f.AllowIllegalWrites { + return errStreamID + } + var flags Flags + if endStream { + flags |= FlagDataEndStream + } + f.startWrite(FrameData, flags, streamID) + f.wbuf = append(f.wbuf, data...) + return f.endWrite() +} + +// A SettingsFrame conveys configuration parameters that affect how +// endpoints communicate, such as preferences and constraints on peer +// behavior. +// +// See http://http2.github.io/http2-spec/#SETTINGS +type SettingsFrame struct { + FrameHeader + p []byte +} + +func parseSettingsFrame(fh FrameHeader, p []byte) (Frame, error) { + if fh.Flags.Has(FlagSettingsAck) && fh.Length > 0 { + // When this (ACK 0x1) bit is set, the payload of the + // SETTINGS frame MUST be empty. Receipt of a + // SETTINGS frame with the ACK flag set and a length + // field value other than 0 MUST be treated as a + // connection error (Section 5.4.1) of type + // FRAME_SIZE_ERROR. + return nil, ConnectionError(ErrCodeFrameSize) + } + if fh.StreamID != 0 { + // SETTINGS frames always apply to a connection, + // never a single stream. The stream identifier for a + // SETTINGS frame MUST be zero (0x0). If an endpoint + // receives a SETTINGS frame whose stream identifier + // field is anything other than 0x0, the endpoint MUST + // respond with a connection error (Section 5.4.1) of + // type PROTOCOL_ERROR. + return nil, ConnectionError(ErrCodeProtocol) + } + if len(p)%6 != 0 { + // Expecting even number of 6 byte settings. + return nil, ConnectionError(ErrCodeFrameSize) + } + f := &SettingsFrame{FrameHeader: fh, p: p} + if v, ok := f.Value(SettingInitialWindowSize); ok && v > (1<<31)-1 { + // Values above the maximum flow control window size of 2^31 - 1 MUST + // be treated as a connection error (Section 5.4.1) of type + // FLOW_CONTROL_ERROR. + return nil, ConnectionError(ErrCodeFlowControl) + } + return f, nil +} + +func (f *SettingsFrame) IsAck() bool { + return f.FrameHeader.Flags.Has(FlagSettingsAck) +} + +func (f *SettingsFrame) Value(s SettingID) (v uint32, ok bool) { + f.checkValid() + buf := f.p + for len(buf) > 0 { + settingID := SettingID(binary.BigEndian.Uint16(buf[:2])) + if settingID == s { + return binary.BigEndian.Uint32(buf[2:6]), true + } + buf = buf[6:] + } + return 0, false +} + +// ForeachSetting runs fn for each setting. +// It stops and returns the first error. +func (f *SettingsFrame) ForeachSetting(fn func(Setting) error) error { + f.checkValid() + buf := f.p + for len(buf) > 0 { + if err := fn(Setting{ + SettingID(binary.BigEndian.Uint16(buf[:2])), + binary.BigEndian.Uint32(buf[2:6]), + }); err != nil { + return err + } + buf = buf[6:] + } + return nil +} + +// WriteSettings writes a SETTINGS frame with zero or more settings +// specified and the ACK bit not set. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *Framer) WriteSettings(settings ...Setting) error { + f.startWrite(FrameSettings, 0, 0) + for _, s := range settings { + f.writeUint16(uint16(s.ID)) + f.writeUint32(s.Val) + } + return f.endWrite() +} + +// WriteSettings writes an empty SETTINGS frame with the ACK bit set. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *Framer) WriteSettingsAck() error { + f.startWrite(FrameSettings, FlagSettingsAck, 0) + return f.endWrite() +} + +// A PingFrame is a mechanism for measuring a minimal round trip time +// from the sender, as well as determining whether an idle connection +// is still functional. +// See http://http2.github.io/http2-spec/#rfc.section.6.7 +type PingFrame struct { + FrameHeader + Data [8]byte +} + +func parsePingFrame(fh FrameHeader, payload []byte) (Frame, error) { + if len(payload) != 8 { + return nil, ConnectionError(ErrCodeFrameSize) + } + if fh.StreamID != 0 { + return nil, ConnectionError(ErrCodeProtocol) + } + f := &PingFrame{FrameHeader: fh} + copy(f.Data[:], payload) + return f, nil +} + +func (f *Framer) WritePing(ack bool, data [8]byte) error { + var flags Flags + if ack { + flags = FlagPingAck + } + f.startWrite(FramePing, flags, 0) + f.writeBytes(data[:]) + return f.endWrite() +} + +// A GoAwayFrame informs the remote peer to stop creating streams on this connection. +// See http://http2.github.io/http2-spec/#rfc.section.6.8 +type GoAwayFrame struct { + FrameHeader + LastStreamID uint32 + ErrCode ErrCode + debugData []byte +} + +// DebugData returns any debug data in the GOAWAY frame. Its contents +// are not defined. +// The caller must not retain the returned memory past the next +// call to ReadFrame. +func (f *GoAwayFrame) DebugData() []byte { + f.checkValid() + return f.debugData +} + +func parseGoAwayFrame(fh FrameHeader, p []byte) (Frame, error) { + if fh.StreamID != 0 { + return nil, ConnectionError(ErrCodeProtocol) + } + if len(p) < 8 { + return nil, ConnectionError(ErrCodeFrameSize) + } + return &GoAwayFrame{ + FrameHeader: fh, + LastStreamID: binary.BigEndian.Uint32(p[:4]) & (1<<31 - 1), + ErrCode: ErrCode(binary.BigEndian.Uint32(p[4:8])), + debugData: p[8:], + }, nil +} + +func (f *Framer) WriteGoAway(maxStreamID uint32, code ErrCode, debugData []byte) error { + f.startWrite(FrameGoAway, 0, 0) + f.writeUint32(maxStreamID & (1<<31 - 1)) + f.writeUint32(uint32(code)) + f.writeBytes(debugData) + return f.endWrite() +} + +// An UnknownFrame is the frame type returned when the frame type is unknown +// or no specific frame type parser exists. +type UnknownFrame struct { + FrameHeader + p []byte +} + +// Payload returns the frame's payload (after the header). It is not +// valid to call this method after a subsequent call to +// Framer.ReadFrame, nor is it valid to retain the returned slice. +// The memory is owned by the Framer and is invalidated when the next +// frame is read. +func (f *UnknownFrame) Payload() []byte { + f.checkValid() + return f.p +} + +func parseUnknownFrame(fh FrameHeader, p []byte) (Frame, error) { + return &UnknownFrame{fh, p}, nil +} + +// A WindowUpdateFrame is used to implement flow control. +// See http://http2.github.io/http2-spec/#rfc.section.6.9 +type WindowUpdateFrame struct { + FrameHeader + Increment uint32 +} + +func parseWindowUpdateFrame(fh FrameHeader, p []byte) (Frame, error) { + if len(p) != 4 { + return nil, ConnectionError(ErrCodeFrameSize) + } + inc := binary.BigEndian.Uint32(p[:4]) & 0x7fffffff // mask off high reserved bit + if inc == 0 { + // A receiver MUST treat the receipt of a + // WINDOW_UPDATE frame with an flow control window + // increment of 0 as a stream error (Section 5.4.2) of + // type PROTOCOL_ERROR; errors on the connection flow + // control window MUST be treated as a connection + // error (Section 5.4.1). + if fh.StreamID == 0 { + return nil, ConnectionError(ErrCodeProtocol) + } + return nil, StreamError{fh.StreamID, ErrCodeProtocol} + } + return &WindowUpdateFrame{ + FrameHeader: fh, + Increment: inc, + }, nil +} + +// WriteWindowUpdate writes a WINDOW_UPDATE frame. +// The increment value must be between 1 and 2,147,483,647, inclusive. +// If the Stream ID is zero, the window update applies to the +// connection as a whole. +func (f *Framer) WriteWindowUpdate(streamID, incr uint32) error { + // "The legal range for the increment to the flow control window is 1 to 2^31-1 (2,147,483,647) octets." + if (incr < 1 || incr > 2147483647) && !f.AllowIllegalWrites { + return errors.New("illegal window increment value") + } + f.startWrite(FrameWindowUpdate, 0, streamID) + f.writeUint32(incr) + return f.endWrite() +} + +// A HeadersFrame is used to open a stream and additionally carries a +// header block fragment. +type HeadersFrame struct { + FrameHeader + + // Priority is set if FlagHeadersPriority is set in the FrameHeader. + Priority PriorityParam + + headerFragBuf []byte // not owned +} + +func (f *HeadersFrame) HeaderBlockFragment() []byte { + f.checkValid() + return f.headerFragBuf +} + +func (f *HeadersFrame) HeadersEnded() bool { + return f.FrameHeader.Flags.Has(FlagHeadersEndHeaders) +} + +func (f *HeadersFrame) StreamEnded() bool { + return f.FrameHeader.Flags.Has(FlagHeadersEndStream) +} + +func (f *HeadersFrame) HasPriority() bool { + return f.FrameHeader.Flags.Has(FlagHeadersPriority) +} + +func parseHeadersFrame(fh FrameHeader, p []byte) (_ Frame, err error) { + hf := &HeadersFrame{ + FrameHeader: fh, + } + if fh.StreamID == 0 { + // HEADERS frames MUST be associated with a stream. If a HEADERS frame + // is received whose stream identifier field is 0x0, the recipient MUST + // respond with a connection error (Section 5.4.1) of type + // PROTOCOL_ERROR. + return nil, ConnectionError(ErrCodeProtocol) + } + var padLength uint8 + if fh.Flags.Has(FlagHeadersPadded) { + if p, padLength, err = readByte(p); err != nil { + return + } + } + if fh.Flags.Has(FlagHeadersPriority) { + var v uint32 + p, v, err = readUint32(p) + if err != nil { + return nil, err + } + hf.Priority.StreamDep = v & 0x7fffffff + hf.Priority.Exclusive = (v != hf.Priority.StreamDep) // high bit was set + p, hf.Priority.Weight, err = readByte(p) + if err != nil { + return nil, err + } + } + if len(p)-int(padLength) <= 0 { + return nil, StreamError{fh.StreamID, ErrCodeProtocol} + } + hf.headerFragBuf = p[:len(p)-int(padLength)] + return hf, nil +} + +// HeadersFrameParam are the parameters for writing a HEADERS frame. +type HeadersFrameParam struct { + // StreamID is the required Stream ID to initiate. + StreamID uint32 + // BlockFragment is part (or all) of a Header Block. + BlockFragment []byte + + // EndStream indicates that the header block is the last that + // the endpoint will send for the identified stream. Setting + // this flag causes the stream to enter one of "half closed" + // states. + EndStream bool + + // EndHeaders indicates that this frame contains an entire + // header block and is not followed by any + // CONTINUATION frames. + EndHeaders bool + + // PadLength is the optional number of bytes of zeros to add + // to this frame. + PadLength uint8 + + // Priority, if non-zero, includes stream priority information + // in the HEADER frame. + Priority PriorityParam +} + +// WriteHeaders writes a single HEADERS frame. +// +// This is a low-level header writing method. Encoding headers and +// splitting them into any necessary CONTINUATION frames is handled +// elsewhere. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *Framer) WriteHeaders(p HeadersFrameParam) error { + if !validStreamID(p.StreamID) && !f.AllowIllegalWrites { + return errStreamID + } + var flags Flags + if p.PadLength != 0 { + flags |= FlagHeadersPadded + } + if p.EndStream { + flags |= FlagHeadersEndStream + } + if p.EndHeaders { + flags |= FlagHeadersEndHeaders + } + if !p.Priority.IsZero() { + flags |= FlagHeadersPriority + } + f.startWrite(FrameHeaders, flags, p.StreamID) + if p.PadLength != 0 { + f.writeByte(p.PadLength) + } + if !p.Priority.IsZero() { + v := p.Priority.StreamDep + if !validStreamID(v) && !f.AllowIllegalWrites { + return errors.New("invalid dependent stream id") + } + if p.Priority.Exclusive { + v |= 1 << 31 + } + f.writeUint32(v) + f.writeByte(p.Priority.Weight) + } + f.wbuf = append(f.wbuf, p.BlockFragment...) + f.wbuf = append(f.wbuf, padZeros[:p.PadLength]...) + return f.endWrite() +} + +// A PriorityFrame specifies the sender-advised priority of a stream. +// See http://http2.github.io/http2-spec/#rfc.section.6.3 +type PriorityFrame struct { + FrameHeader + PriorityParam +} + +// PriorityParam are the stream prioritzation parameters. +type PriorityParam struct { + // StreamDep is a 31-bit stream identifier for the + // stream that this stream depends on. Zero means no + // dependency. + StreamDep uint32 + + // Exclusive is whether the dependency is exclusive. + Exclusive bool + + // Weight is the stream's zero-indexed weight. It should be + // set together with StreamDep, or neither should be set. Per + // the spec, "Add one to the value to obtain a weight between + // 1 and 256." + Weight uint8 +} + +func (p PriorityParam) IsZero() bool { + return p == PriorityParam{} +} + +func parsePriorityFrame(fh FrameHeader, payload []byte) (Frame, error) { + if fh.StreamID == 0 { + return nil, ConnectionError(ErrCodeProtocol) + } + if len(payload) != 5 { + return nil, ConnectionError(ErrCodeFrameSize) + } + v := binary.BigEndian.Uint32(payload[:4]) + streamID := v & 0x7fffffff // mask off high bit + return &PriorityFrame{ + FrameHeader: fh, + PriorityParam: PriorityParam{ + Weight: payload[4], + StreamDep: streamID, + Exclusive: streamID != v, // was high bit set? + }, + }, nil +} + +// WritePriority writes a PRIORITY frame. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *Framer) WritePriority(streamID uint32, p PriorityParam) error { + if !validStreamID(streamID) && !f.AllowIllegalWrites { + return errStreamID + } + f.startWrite(FramePriority, 0, streamID) + v := p.StreamDep + if p.Exclusive { + v |= 1 << 31 + } + f.writeUint32(v) + f.writeByte(p.Weight) + return f.endWrite() +} + +// A RSTStreamFrame allows for abnormal termination of a stream. +// See http://http2.github.io/http2-spec/#rfc.section.6.4 +type RSTStreamFrame struct { + FrameHeader + ErrCode ErrCode +} + +func parseRSTStreamFrame(fh FrameHeader, p []byte) (Frame, error) { + if len(p) != 4 { + return nil, ConnectionError(ErrCodeFrameSize) + } + if fh.StreamID == 0 { + return nil, ConnectionError(ErrCodeProtocol) + } + return &RSTStreamFrame{fh, ErrCode(binary.BigEndian.Uint32(p[:4]))}, nil +} + +// WriteRSTStream writes a RST_STREAM frame. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *Framer) WriteRSTStream(streamID uint32, code ErrCode) error { + if !validStreamID(streamID) && !f.AllowIllegalWrites { + return errStreamID + } + f.startWrite(FrameRSTStream, 0, streamID) + f.writeUint32(uint32(code)) + return f.endWrite() +} + +// A ContinuationFrame is used to continue a sequence of header block fragments. +// See http://http2.github.io/http2-spec/#rfc.section.6.10 +type ContinuationFrame struct { + FrameHeader + headerFragBuf []byte +} + +func parseContinuationFrame(fh FrameHeader, p []byte) (Frame, error) { + return &ContinuationFrame{fh, p}, nil +} + +func (f *ContinuationFrame) StreamEnded() bool { + return f.FrameHeader.Flags.Has(FlagDataEndStream) +} + +func (f *ContinuationFrame) HeaderBlockFragment() []byte { + f.checkValid() + return f.headerFragBuf +} + +func (f *ContinuationFrame) HeadersEnded() bool { + return f.FrameHeader.Flags.Has(FlagContinuationEndHeaders) +} + +// WriteContinuation writes a CONTINUATION frame. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *Framer) WriteContinuation(streamID uint32, endHeaders bool, headerBlockFragment []byte) error { + if !validStreamID(streamID) && !f.AllowIllegalWrites { + return errStreamID + } + var flags Flags + if endHeaders { + flags |= FlagContinuationEndHeaders + } + f.startWrite(FrameContinuation, flags, streamID) + f.wbuf = append(f.wbuf, headerBlockFragment...) + return f.endWrite() +} + +// A PushPromiseFrame is used to initiate a server stream. +// See http://http2.github.io/http2-spec/#rfc.section.6.6 +type PushPromiseFrame struct { + FrameHeader + PromiseID uint32 + headerFragBuf []byte // not owned +} + +func (f *PushPromiseFrame) HeaderBlockFragment() []byte { + f.checkValid() + return f.headerFragBuf +} + +func (f *PushPromiseFrame) HeadersEnded() bool { + return f.FrameHeader.Flags.Has(FlagPushPromiseEndHeaders) +} + +func parsePushPromise(fh FrameHeader, p []byte) (_ Frame, err error) { + pp := &PushPromiseFrame{ + FrameHeader: fh, + } + if pp.StreamID == 0 { + // PUSH_PROMISE frames MUST be associated with an existing, + // peer-initiated stream. The stream identifier of a + // PUSH_PROMISE frame indicates the stream it is associated + // with. If the stream identifier field specifies the value + // 0x0, a recipient MUST respond with a connection error + // (Section 5.4.1) of type PROTOCOL_ERROR. + return nil, ConnectionError(ErrCodeProtocol) + } + // The PUSH_PROMISE frame includes optional padding. + // Padding fields and flags are identical to those defined for DATA frames + var padLength uint8 + if fh.Flags.Has(FlagPushPromisePadded) { + if p, padLength, err = readByte(p); err != nil { + return + } + } + + p, pp.PromiseID, err = readUint32(p) + if err != nil { + return + } + pp.PromiseID = pp.PromiseID & (1<<31 - 1) + + if int(padLength) > len(p) { + // like the DATA frame, error out if padding is longer than the body. + return nil, ConnectionError(ErrCodeProtocol) + } + pp.headerFragBuf = p[:len(p)-int(padLength)] + return pp, nil +} + +// PushPromiseParam are the parameters for writing a PUSH_PROMISE frame. +type PushPromiseParam struct { + // StreamID is the required Stream ID to initiate. + StreamID uint32 + + // PromiseID is the required Stream ID which this + // Push Promises + PromiseID uint32 + + // BlockFragment is part (or all) of a Header Block. + BlockFragment []byte + + // EndHeaders indicates that this frame contains an entire + // header block and is not followed by any + // CONTINUATION frames. + EndHeaders bool + + // PadLength is the optional number of bytes of zeros to add + // to this frame. + PadLength uint8 +} + +// WritePushPromise writes a single PushPromise Frame. +// +// As with Header Frames, This is the low level call for writing +// individual frames. Continuation frames are handled elsewhere. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility to not call other Write methods concurrently. +func (f *Framer) WritePushPromise(p PushPromiseParam) error { + if !validStreamID(p.StreamID) && !f.AllowIllegalWrites { + return errStreamID + } + var flags Flags + if p.PadLength != 0 { + flags |= FlagPushPromisePadded + } + if p.EndHeaders { + flags |= FlagPushPromiseEndHeaders + } + f.startWrite(FramePushPromise, flags, p.StreamID) + if p.PadLength != 0 { + f.writeByte(p.PadLength) + } + if !validStreamID(p.PromiseID) && !f.AllowIllegalWrites { + return errStreamID + } + f.writeUint32(p.PromiseID) + f.wbuf = append(f.wbuf, p.BlockFragment...) + f.wbuf = append(f.wbuf, padZeros[:p.PadLength]...) + return f.endWrite() +} + +// WriteRawFrame writes a raw frame. This can be used to write +// extension frames unknown to this package. +func (f *Framer) WriteRawFrame(t FrameType, flags Flags, streamID uint32, payload []byte) error { + f.startWrite(t, flags, streamID) + f.writeBytes(payload) + return f.endWrite() +} + +func readByte(p []byte) (remain []byte, b byte, err error) { + if len(p) == 0 { + return nil, 0, io.ErrUnexpectedEOF + } + return p[1:], p[0], nil +} + +func readUint32(p []byte) (remain []byte, v uint32, err error) { + if len(p) < 4 { + return nil, 0, io.ErrUnexpectedEOF + } + return p[4:], binary.BigEndian.Uint32(p[:4]), nil +} + +type streamEnder interface { + StreamEnded() bool +} + +type headersEnder interface { + HeadersEnded() bool +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/gotrack.go b/components/engine/vendor/src/golang.org/x/net/http2/gotrack.go new file mode 100644 index 0000000000..7dc2ef90db --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/gotrack.go @@ -0,0 +1,173 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +// Defensive debug-only utility to track that functions run on the +// goroutine that they're supposed to. + +package http2 + +import ( + "bytes" + "errors" + "fmt" + "os" + "runtime" + "strconv" + "sync" +) + +var DebugGoroutines = os.Getenv("DEBUG_HTTP2_GOROUTINES") == "1" + +type goroutineLock uint64 + +func newGoroutineLock() goroutineLock { + if !DebugGoroutines { + return 0 + } + return goroutineLock(curGoroutineID()) +} + +func (g goroutineLock) check() { + if !DebugGoroutines { + return + } + if curGoroutineID() != uint64(g) { + panic("running on the wrong goroutine") + } +} + +func (g goroutineLock) checkNotOn() { + if !DebugGoroutines { + return + } + if curGoroutineID() == uint64(g) { + panic("running on the wrong goroutine") + } +} + +var goroutineSpace = []byte("goroutine ") + +func curGoroutineID() uint64 { + bp := littleBuf.Get().(*[]byte) + defer littleBuf.Put(bp) + b := *bp + b = b[:runtime.Stack(b, false)] + // Parse the 4707 out of "goroutine 4707 [" + b = bytes.TrimPrefix(b, goroutineSpace) + i := bytes.IndexByte(b, ' ') + if i < 0 { + panic(fmt.Sprintf("No space found in %q", b)) + } + b = b[:i] + n, err := parseUintBytes(b, 10, 64) + if err != nil { + panic(fmt.Sprintf("Failed to parse goroutine ID out of %q: %v", b, err)) + } + return n +} + +var littleBuf = sync.Pool{ + New: func() interface{} { + buf := make([]byte, 64) + return &buf + }, +} + +// parseUintBytes is like strconv.ParseUint, but using a []byte. +func parseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) { + var cutoff, maxVal uint64 + + if bitSize == 0 { + bitSize = int(strconv.IntSize) + } + + s0 := s + switch { + case len(s) < 1: + err = strconv.ErrSyntax + goto Error + + case 2 <= base && base <= 36: + // valid base; nothing to do + + case base == 0: + // Look for octal, hex prefix. + switch { + case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'): + base = 16 + s = s[2:] + if len(s) < 1 { + err = strconv.ErrSyntax + goto Error + } + case s[0] == '0': + base = 8 + default: + base = 10 + } + + default: + err = errors.New("invalid base " + strconv.Itoa(base)) + goto Error + } + + n = 0 + cutoff = cutoff64(base) + maxVal = 1<= base { + n = 0 + err = strconv.ErrSyntax + goto Error + } + + if n >= cutoff { + // n*base overflows + n = 1<<64 - 1 + err = strconv.ErrRange + goto Error + } + n *= uint64(base) + + n1 := n + uint64(v) + if n1 < n || n1 > maxVal { + // n+v overflows + n = 1<<64 - 1 + err = strconv.ErrRange + goto Error + } + n = n1 + } + + return n, nil + +Error: + return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err} +} + +// Return the first number n such that n*base >= 1<<64. +func cutoff64(base int) uint64 { + if base < 2 { + return 0 + } + return (1<<64-1)/uint64(base) + 1 +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/headermap.go b/components/engine/vendor/src/golang.org/x/net/http2/headermap.go new file mode 100644 index 0000000000..67c7c48357 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/headermap.go @@ -0,0 +1,80 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +package http2 + +import ( + "net/http" + "strings" +) + +var ( + commonLowerHeader = map[string]string{} // Go-Canonical-Case -> lower-case + commonCanonHeader = map[string]string{} // lower-case -> Go-Canonical-Case +) + +func init() { + for _, v := range []string{ + "accept", + "accept-charset", + "accept-encoding", + "accept-language", + "accept-ranges", + "age", + "access-control-allow-origin", + "allow", + "authorization", + "cache-control", + "content-disposition", + "content-encoding", + "content-language", + "content-length", + "content-location", + "content-range", + "content-type", + "cookie", + "date", + "etag", + "expect", + "expires", + "from", + "host", + "if-match", + "if-modified-since", + "if-none-match", + "if-unmodified-since", + "last-modified", + "link", + "location", + "max-forwards", + "proxy-authenticate", + "proxy-authorization", + "range", + "referer", + "refresh", + "retry-after", + "server", + "set-cookie", + "strict-transport-security", + "transfer-encoding", + "user-agent", + "vary", + "via", + "www-authenticate", + } { + chk := http.CanonicalHeaderKey(v) + commonLowerHeader[chk] = v + commonCanonHeader[v] = chk + } +} + +func lowerHeader(v string) string { + if s, ok := commonLowerHeader[v]; ok { + return s + } + return strings.ToLower(v) +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/hpack/encode.go b/components/engine/vendor/src/golang.org/x/net/http2/hpack/encode.go new file mode 100644 index 0000000000..19bd9f4fcb --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/hpack/encode.go @@ -0,0 +1,252 @@ +// Copyright 2014 The Go Authors. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +package hpack + +import ( + "io" +) + +const ( + uint32Max = ^uint32(0) + initialHeaderTableSize = 4096 +) + +type Encoder struct { + dynTab dynamicTable + // minSize is the minimum table size set by + // SetMaxDynamicTableSize after the previous Header Table Size + // Update. + minSize uint32 + // maxSizeLimit is the maximum table size this encoder + // supports. This will protect the encoder from too large + // size. + maxSizeLimit uint32 + // tableSizeUpdate indicates whether "Header Table Size + // Update" is required. + tableSizeUpdate bool + w io.Writer + buf []byte +} + +// NewEncoder returns a new Encoder which performs HPACK encoding. An +// encoded data is written to w. +func NewEncoder(w io.Writer) *Encoder { + e := &Encoder{ + minSize: uint32Max, + maxSizeLimit: initialHeaderTableSize, + tableSizeUpdate: false, + w: w, + } + e.dynTab.setMaxSize(initialHeaderTableSize) + return e +} + +// WriteField encodes f into a single Write to e's underlying Writer. +// This function may also produce bytes for "Header Table Size Update" +// if necessary. If produced, it is done before encoding f. +func (e *Encoder) WriteField(f HeaderField) error { + e.buf = e.buf[:0] + + if e.tableSizeUpdate { + e.tableSizeUpdate = false + if e.minSize < e.dynTab.maxSize { + e.buf = appendTableSize(e.buf, e.minSize) + } + e.minSize = uint32Max + e.buf = appendTableSize(e.buf, e.dynTab.maxSize) + } + + idx, nameValueMatch := e.searchTable(f) + if nameValueMatch { + e.buf = appendIndexed(e.buf, idx) + } else { + indexing := e.shouldIndex(f) + if indexing { + e.dynTab.add(f) + } + + if idx == 0 { + e.buf = appendNewName(e.buf, f, indexing) + } else { + e.buf = appendIndexedName(e.buf, f, idx, indexing) + } + } + n, err := e.w.Write(e.buf) + if err == nil && n != len(e.buf) { + err = io.ErrShortWrite + } + return err +} + +// searchTable searches f in both stable and dynamic header tables. +// The static header table is searched first. Only when there is no +// exact match for both name and value, the dynamic header table is +// then searched. If there is no match, i is 0. If both name and value +// match, i is the matched index and nameValueMatch becomes true. If +// only name matches, i points to that index and nameValueMatch +// becomes false. +func (e *Encoder) searchTable(f HeaderField) (i uint64, nameValueMatch bool) { + for idx, hf := range staticTable { + if !constantTimeStringCompare(hf.Name, f.Name) { + continue + } + if i == 0 { + i = uint64(idx + 1) + } + if f.Sensitive { + continue + } + if !constantTimeStringCompare(hf.Value, f.Value) { + continue + } + i = uint64(idx + 1) + nameValueMatch = true + return + } + + j, nameValueMatch := e.dynTab.search(f) + if nameValueMatch || (i == 0 && j != 0) { + i = j + uint64(len(staticTable)) + } + return +} + +// SetMaxDynamicTableSize changes the dynamic header table size to v. +// The actual size is bounded by the value passed to +// SetMaxDynamicTableSizeLimit. +func (e *Encoder) SetMaxDynamicTableSize(v uint32) { + if v > e.maxSizeLimit { + v = e.maxSizeLimit + } + if v < e.minSize { + e.minSize = v + } + e.tableSizeUpdate = true + e.dynTab.setMaxSize(v) +} + +// SetMaxDynamicTableSizeLimit changes the maximum value that can be +// specified in SetMaxDynamicTableSize to v. By default, it is set to +// 4096, which is the same size of the default dynamic header table +// size described in HPACK specification. If the current maximum +// dynamic header table size is strictly greater than v, "Header Table +// Size Update" will be done in the next WriteField call and the +// maximum dynamic header table size is truncated to v. +func (e *Encoder) SetMaxDynamicTableSizeLimit(v uint32) { + e.maxSizeLimit = v + if e.dynTab.maxSize > v { + e.tableSizeUpdate = true + e.dynTab.setMaxSize(v) + } +} + +// shouldIndex reports whether f should be indexed. +func (e *Encoder) shouldIndex(f HeaderField) bool { + return !f.Sensitive && f.size() <= e.dynTab.maxSize +} + +// appendIndexed appends index i, as encoded in "Indexed Header Field" +// representation, to dst and returns the extended buffer. +func appendIndexed(dst []byte, i uint64) []byte { + first := len(dst) + dst = appendVarInt(dst, 7, i) + dst[first] |= 0x80 + return dst +} + +// appendNewName appends f, as encoded in one of "Literal Header field +// - New Name" representation variants, to dst and returns the +// extended buffer. +// +// If f.Sensitive is true, "Never Indexed" representation is used. If +// f.Sensitive is false and indexing is true, "Inremental Indexing" +// representation is used. +func appendNewName(dst []byte, f HeaderField, indexing bool) []byte { + dst = append(dst, encodeTypeByte(indexing, f.Sensitive)) + dst = appendHpackString(dst, f.Name) + return appendHpackString(dst, f.Value) +} + +// appendIndexedName appends f and index i referring indexed name +// entry, as encoded in one of "Literal Header field - Indexed Name" +// representation variants, to dst and returns the extended buffer. +// +// If f.Sensitive is true, "Never Indexed" representation is used. If +// f.Sensitive is false and indexing is true, "Incremental Indexing" +// representation is used. +func appendIndexedName(dst []byte, f HeaderField, i uint64, indexing bool) []byte { + first := len(dst) + var n byte + if indexing { + n = 6 + } else { + n = 4 + } + dst = appendVarInt(dst, n, i) + dst[first] |= encodeTypeByte(indexing, f.Sensitive) + return appendHpackString(dst, f.Value) +} + +// appendTableSize appends v, as encoded in "Header Table Size Update" +// representation, to dst and returns the extended buffer. +func appendTableSize(dst []byte, v uint32) []byte { + first := len(dst) + dst = appendVarInt(dst, 5, uint64(v)) + dst[first] |= 0x20 + return dst +} + +// appendVarInt appends i, as encoded in variable integer form using n +// bit prefix, to dst and returns the extended buffer. +// +// See +// http://http2.github.io/http2-spec/compression.html#integer.representation +func appendVarInt(dst []byte, n byte, i uint64) []byte { + k := uint64((1 << n) - 1) + if i < k { + return append(dst, byte(i)) + } + dst = append(dst, byte(k)) + i -= k + for ; i >= 128; i >>= 7 { + dst = append(dst, byte(0x80|(i&0x7f))) + } + return append(dst, byte(i)) +} + +// appendHpackString appends s, as encoded in "String Literal" +// representation, to dst and returns the the extended buffer. +// +// s will be encoded in Huffman codes only when it produces strictly +// shorter byte string. +func appendHpackString(dst []byte, s string) []byte { + huffmanLength := HuffmanEncodeLength(s) + if huffmanLength < uint64(len(s)) { + first := len(dst) + dst = appendVarInt(dst, 7, huffmanLength) + dst = AppendHuffmanString(dst, s) + dst[first] |= 0x80 + } else { + dst = appendVarInt(dst, 7, uint64(len(s))) + dst = append(dst, s...) + } + return dst +} + +// encodeTypeByte returns type byte. If sensitive is true, type byte +// for "Never Indexed" representation is returned. If sensitive is +// false and indexing is true, type byte for "Incremental Indexing" +// representation is returned. Otherwise, type byte for "Without +// Indexing" is returned. +func encodeTypeByte(indexing, sensitive bool) byte { + if sensitive { + return 0x10 + } + if indexing { + return 0x40 + } + return 0 +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/hpack/hpack.go b/components/engine/vendor/src/golang.org/x/net/http2/hpack/hpack.go new file mode 100644 index 0000000000..c9e36f7427 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/hpack/hpack.go @@ -0,0 +1,445 @@ +// Copyright 2014 The Go Authors. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +// Package hpack implements HPACK, a compression format for +// efficiently representing HTTP header fields in the context of HTTP/2. +// +// See http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-09 +package hpack + +import ( + "bytes" + "errors" + "fmt" +) + +// A DecodingError is something the spec defines as a decoding error. +type DecodingError struct { + Err error +} + +func (de DecodingError) Error() string { + return fmt.Sprintf("decoding error: %v", de.Err) +} + +// An InvalidIndexError is returned when an encoder references a table +// entry before the static table or after the end of the dynamic table. +type InvalidIndexError int + +func (e InvalidIndexError) Error() string { + return fmt.Sprintf("invalid indexed representation index %d", int(e)) +} + +// A HeaderField is a name-value pair. Both the name and value are +// treated as opaque sequences of octets. +type HeaderField struct { + Name, Value string + + // Sensitive means that this header field should never be + // indexed. + Sensitive bool +} + +func (hf *HeaderField) size() uint32 { + // http://http2.github.io/http2-spec/compression.html#rfc.section.4.1 + // "The size of the dynamic table is the sum of the size of + // its entries. The size of an entry is the sum of its name's + // length in octets (as defined in Section 5.2), its value's + // length in octets (see Section 5.2), plus 32. The size of + // an entry is calculated using the length of the name and + // value without any Huffman encoding applied." + + // This can overflow if somebody makes a large HeaderField + // Name and/or Value by hand, but we don't care, because that + // won't happen on the wire because the encoding doesn't allow + // it. + return uint32(len(hf.Name) + len(hf.Value) + 32) +} + +// A Decoder is the decoding context for incremental processing of +// header blocks. +type Decoder struct { + dynTab dynamicTable + emit func(f HeaderField) + + // buf is the unparsed buffer. It's only written to + // saveBuf if it was truncated in the middle of a header + // block. Because it's usually not owned, we can only + // process it under Write. + buf []byte // usually not owned + saveBuf bytes.Buffer +} + +func NewDecoder(maxSize uint32, emitFunc func(f HeaderField)) *Decoder { + d := &Decoder{ + emit: emitFunc, + } + d.dynTab.allowedMaxSize = maxSize + d.dynTab.setMaxSize(maxSize) + return d +} + +// TODO: add method *Decoder.Reset(maxSize, emitFunc) to let callers re-use Decoders and their +// underlying buffers for garbage reasons. + +func (d *Decoder) SetMaxDynamicTableSize(v uint32) { + d.dynTab.setMaxSize(v) +} + +// SetAllowedMaxDynamicTableSize sets the upper bound that the encoded +// stream (via dynamic table size updates) may set the maximum size +// to. +func (d *Decoder) SetAllowedMaxDynamicTableSize(v uint32) { + d.dynTab.allowedMaxSize = v +} + +type dynamicTable struct { + // ents is the FIFO described at + // http://http2.github.io/http2-spec/compression.html#rfc.section.2.3.2 + // The newest (low index) is append at the end, and items are + // evicted from the front. + ents []HeaderField + size uint32 + maxSize uint32 // current maxSize + allowedMaxSize uint32 // maxSize may go up to this, inclusive +} + +func (dt *dynamicTable) setMaxSize(v uint32) { + dt.maxSize = v + dt.evict() +} + +// TODO: change dynamicTable to be a struct with a slice and a size int field, +// per http://http2.github.io/http2-spec/compression.html#rfc.section.4.1: +// +// +// Then make add increment the size. maybe the max size should move from Decoder to +// dynamicTable and add should return an ok bool if there was enough space. +// +// Later we'll need a remove operation on dynamicTable. + +func (dt *dynamicTable) add(f HeaderField) { + dt.ents = append(dt.ents, f) + dt.size += f.size() + dt.evict() +} + +// If we're too big, evict old stuff (front of the slice) +func (dt *dynamicTable) evict() { + base := dt.ents // keep base pointer of slice + for dt.size > dt.maxSize { + dt.size -= dt.ents[0].size() + dt.ents = dt.ents[1:] + } + + // Shift slice contents down if we evicted things. + if len(dt.ents) != len(base) { + copy(base, dt.ents) + dt.ents = base[:len(dt.ents)] + } +} + +// constantTimeStringCompare compares string a and b in a constant +// time manner. +func constantTimeStringCompare(a, b string) bool { + if len(a) != len(b) { + return false + } + + c := byte(0) + + for i := 0; i < len(a); i++ { + c |= a[i] ^ b[i] + } + + return c == 0 +} + +// Search searches f in the table. The return value i is 0 if there is +// no name match. If there is name match or name/value match, i is the +// index of that entry (1-based). If both name and value match, +// nameValueMatch becomes true. +func (dt *dynamicTable) search(f HeaderField) (i uint64, nameValueMatch bool) { + l := len(dt.ents) + for j := l - 1; j >= 0; j-- { + ent := dt.ents[j] + if !constantTimeStringCompare(ent.Name, f.Name) { + continue + } + if i == 0 { + i = uint64(l - j) + } + if f.Sensitive { + continue + } + if !constantTimeStringCompare(ent.Value, f.Value) { + continue + } + i = uint64(l - j) + nameValueMatch = true + return + } + return +} + +func (d *Decoder) maxTableIndex() int { + return len(d.dynTab.ents) + len(staticTable) +} + +func (d *Decoder) at(i uint64) (hf HeaderField, ok bool) { + if i < 1 { + return + } + if i > uint64(d.maxTableIndex()) { + return + } + if i <= uint64(len(staticTable)) { + return staticTable[i-1], true + } + dents := d.dynTab.ents + return dents[len(dents)-(int(i)-len(staticTable))], true +} + +// Decode decodes an entire block. +// +// TODO: remove this method and make it incremental later? This is +// easier for debugging now. +func (d *Decoder) DecodeFull(p []byte) ([]HeaderField, error) { + var hf []HeaderField + saveFunc := d.emit + defer func() { d.emit = saveFunc }() + d.emit = func(f HeaderField) { hf = append(hf, f) } + if _, err := d.Write(p); err != nil { + return nil, err + } + if err := d.Close(); err != nil { + return nil, err + } + return hf, nil +} + +func (d *Decoder) Close() error { + if d.saveBuf.Len() > 0 { + d.saveBuf.Reset() + return DecodingError{errors.New("truncated headers")} + } + return nil +} + +func (d *Decoder) Write(p []byte) (n int, err error) { + if len(p) == 0 { + // Prevent state machine CPU attacks (making us redo + // work up to the point of finding out we don't have + // enough data) + return + } + // Only copy the data if we have to. Optimistically assume + // that p will contain a complete header block. + if d.saveBuf.Len() == 0 { + d.buf = p + } else { + d.saveBuf.Write(p) + d.buf = d.saveBuf.Bytes() + d.saveBuf.Reset() + } + + for len(d.buf) > 0 { + err = d.parseHeaderFieldRepr() + if err != nil { + if err == errNeedMore { + err = nil + d.saveBuf.Write(d.buf) + } + break + } + } + + return len(p), err +} + +// errNeedMore is an internal sentinel error value that means the +// buffer is truncated and we need to read more data before we can +// continue parsing. +var errNeedMore = errors.New("need more data") + +type indexType int + +const ( + indexedTrue indexType = iota + indexedFalse + indexedNever +) + +func (v indexType) indexed() bool { return v == indexedTrue } +func (v indexType) sensitive() bool { return v == indexedNever } + +// returns errNeedMore if there isn't enough data available. +// any other error is fatal. +// consumes d.buf iff it returns nil. +// precondition: must be called with len(d.buf) > 0 +func (d *Decoder) parseHeaderFieldRepr() error { + b := d.buf[0] + switch { + case b&128 != 0: + // Indexed representation. + // High bit set? + // http://http2.github.io/http2-spec/compression.html#rfc.section.6.1 + return d.parseFieldIndexed() + case b&192 == 64: + // 6.2.1 Literal Header Field with Incremental Indexing + // 0b10xxxxxx: top two bits are 10 + // http://http2.github.io/http2-spec/compression.html#rfc.section.6.2.1 + return d.parseFieldLiteral(6, indexedTrue) + case b&240 == 0: + // 6.2.2 Literal Header Field without Indexing + // 0b0000xxxx: top four bits are 0000 + // http://http2.github.io/http2-spec/compression.html#rfc.section.6.2.2 + return d.parseFieldLiteral(4, indexedFalse) + case b&240 == 16: + // 6.2.3 Literal Header Field never Indexed + // 0b0001xxxx: top four bits are 0001 + // http://http2.github.io/http2-spec/compression.html#rfc.section.6.2.3 + return d.parseFieldLiteral(4, indexedNever) + case b&224 == 32: + // 6.3 Dynamic Table Size Update + // Top three bits are '001'. + // http://http2.github.io/http2-spec/compression.html#rfc.section.6.3 + return d.parseDynamicTableSizeUpdate() + } + + return DecodingError{errors.New("invalid encoding")} +} + +// (same invariants and behavior as parseHeaderFieldRepr) +func (d *Decoder) parseFieldIndexed() error { + buf := d.buf + idx, buf, err := readVarInt(7, buf) + if err != nil { + return err + } + hf, ok := d.at(idx) + if !ok { + return DecodingError{InvalidIndexError(idx)} + } + d.emit(HeaderField{Name: hf.Name, Value: hf.Value}) + d.buf = buf + return nil +} + +// (same invariants and behavior as parseHeaderFieldRepr) +func (d *Decoder) parseFieldLiteral(n uint8, it indexType) error { + buf := d.buf + nameIdx, buf, err := readVarInt(n, buf) + if err != nil { + return err + } + + var hf HeaderField + if nameIdx > 0 { + ihf, ok := d.at(nameIdx) + if !ok { + return DecodingError{InvalidIndexError(nameIdx)} + } + hf.Name = ihf.Name + } else { + hf.Name, buf, err = readString(buf) + if err != nil { + return err + } + } + hf.Value, buf, err = readString(buf) + if err != nil { + return err + } + d.buf = buf + if it.indexed() { + d.dynTab.add(hf) + } + hf.Sensitive = it.sensitive() + d.emit(hf) + return nil +} + +// (same invariants and behavior as parseHeaderFieldRepr) +func (d *Decoder) parseDynamicTableSizeUpdate() error { + buf := d.buf + size, buf, err := readVarInt(5, buf) + if err != nil { + return err + } + if size > uint64(d.dynTab.allowedMaxSize) { + return DecodingError{errors.New("dynamic table size update too large")} + } + d.dynTab.setMaxSize(uint32(size)) + d.buf = buf + return nil +} + +var errVarintOverflow = DecodingError{errors.New("varint integer overflow")} + +// readVarInt reads an unsigned variable length integer off the +// beginning of p. n is the parameter as described in +// http://http2.github.io/http2-spec/compression.html#rfc.section.5.1. +// +// n must always be between 1 and 8. +// +// The returned remain buffer is either a smaller suffix of p, or err != nil. +// The error is errNeedMore if p doesn't contain a complete integer. +func readVarInt(n byte, p []byte) (i uint64, remain []byte, err error) { + if n < 1 || n > 8 { + panic("bad n") + } + if len(p) == 0 { + return 0, p, errNeedMore + } + i = uint64(p[0]) + if n < 8 { + i &= (1 << uint64(n)) - 1 + } + if i < (1< 0 { + b := p[0] + p = p[1:] + i += uint64(b&127) << m + if b&128 == 0 { + return i, p, nil + } + m += 7 + if m >= 63 { // TODO: proper overflow check. making this up. + return 0, origP, errVarintOverflow + } + } + return 0, origP, errNeedMore +} + +func readString(p []byte) (s string, remain []byte, err error) { + if len(p) == 0 { + return "", p, errNeedMore + } + isHuff := p[0]&128 != 0 + strLen, p, err := readVarInt(7, p) + if err != nil { + return "", p, err + } + if uint64(len(p)) < strLen { + return "", p, errNeedMore + } + if !isHuff { + return string(p[:strLen]), p[strLen:], nil + } + + // TODO: optimize this garbage: + var buf bytes.Buffer + if _, err := HuffmanDecode(&buf, p[:strLen]); err != nil { + return "", nil, err + } + return buf.String(), p[strLen:], nil +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/hpack/huffman.go b/components/engine/vendor/src/golang.org/x/net/http2/hpack/huffman.go new file mode 100644 index 0000000000..9fe76f68ee --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/hpack/huffman.go @@ -0,0 +1,159 @@ +// Copyright 2014 The Go Authors. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +package hpack + +import ( + "bytes" + "io" + "sync" +) + +var bufPool = sync.Pool{ + New: func() interface{} { return new(bytes.Buffer) }, +} + +// HuffmanDecode decodes the string in v and writes the expanded +// result to w, returning the number of bytes written to w and the +// Write call's return value. At most one Write call is made. +func HuffmanDecode(w io.Writer, v []byte) (int, error) { + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() + defer bufPool.Put(buf) + + n := rootHuffmanNode + cur, nbits := uint(0), uint8(0) + for _, b := range v { + cur = cur<<8 | uint(b) + nbits += 8 + for nbits >= 8 { + n = n.children[byte(cur>>(nbits-8))] + if n.children == nil { + buf.WriteByte(n.sym) + nbits -= n.codeLen + n = rootHuffmanNode + } else { + nbits -= 8 + } + } + } + for nbits > 0 { + n = n.children[byte(cur<<(8-nbits))] + if n.children != nil || n.codeLen > nbits { + break + } + buf.WriteByte(n.sym) + nbits -= n.codeLen + n = rootHuffmanNode + } + return w.Write(buf.Bytes()) +} + +type node struct { + // children is non-nil for internal nodes + children []*node + + // The following are only valid if children is nil: + codeLen uint8 // number of bits that led to the output of sym + sym byte // output symbol +} + +func newInternalNode() *node { + return &node{children: make([]*node, 256)} +} + +var rootHuffmanNode = newInternalNode() + +func init() { + for i, code := range huffmanCodes { + if i > 255 { + panic("too many huffman codes") + } + addDecoderNode(byte(i), code, huffmanCodeLen[i]) + } +} + +func addDecoderNode(sym byte, code uint32, codeLen uint8) { + cur := rootHuffmanNode + for codeLen > 8 { + codeLen -= 8 + i := uint8(code >> codeLen) + if cur.children[i] == nil { + cur.children[i] = newInternalNode() + } + cur = cur.children[i] + } + shift := 8 - codeLen + start, end := int(uint8(code<> (nbits - rembits)) + dst[len(dst)-1] |= t + } + + return dst +} + +// HuffmanEncodeLength returns the number of bytes required to encode +// s in Huffman codes. The result is round up to byte boundary. +func HuffmanEncodeLength(s string) uint64 { + n := uint64(0) + for i := 0; i < len(s); i++ { + n += uint64(huffmanCodeLen[s[i]]) + } + return (n + 7) / 8 +} + +// appendByteToHuffmanCode appends Huffman code for c to dst and +// returns the extended buffer and the remaining bits in the last +// element. The appending is not byte aligned and the remaining bits +// in the last element of dst is given in rembits. +func appendByteToHuffmanCode(dst []byte, rembits uint8, c byte) ([]byte, uint8) { + code := huffmanCodes[c] + nbits := huffmanCodeLen[c] + + for { + if rembits > nbits { + t := uint8(code << (rembits - nbits)) + dst[len(dst)-1] |= t + rembits -= nbits + break + } + + t := uint8(code >> (nbits - rembits)) + dst[len(dst)-1] |= t + + nbits -= rembits + rembits = 8 + + if nbits == 0 { + break + } + + dst = append(dst, 0) + } + + return dst, rembits +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/hpack/tables.go b/components/engine/vendor/src/golang.org/x/net/http2/hpack/tables.go new file mode 100644 index 0000000000..f898e25126 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/hpack/tables.go @@ -0,0 +1,353 @@ +// Copyright 2014 The Go Authors. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +package hpack + +func pair(name, value string) HeaderField { + return HeaderField{Name: name, Value: value} +} + +// http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-07#appendix-B +var staticTable = []HeaderField{ + pair(":authority", ""), // index 1 (1-based) + pair(":method", "GET"), + pair(":method", "POST"), + pair(":path", "/"), + pair(":path", "/index.html"), + pair(":scheme", "http"), + pair(":scheme", "https"), + pair(":status", "200"), + pair(":status", "204"), + pair(":status", "206"), + pair(":status", "304"), + pair(":status", "400"), + pair(":status", "404"), + pair(":status", "500"), + pair("accept-charset", ""), + pair("accept-encoding", "gzip, deflate"), + pair("accept-language", ""), + pair("accept-ranges", ""), + pair("accept", ""), + pair("access-control-allow-origin", ""), + pair("age", ""), + pair("allow", ""), + pair("authorization", ""), + pair("cache-control", ""), + pair("content-disposition", ""), + pair("content-encoding", ""), + pair("content-language", ""), + pair("content-length", ""), + pair("content-location", ""), + pair("content-range", ""), + pair("content-type", ""), + pair("cookie", ""), + pair("date", ""), + pair("etag", ""), + pair("expect", ""), + pair("expires", ""), + pair("from", ""), + pair("host", ""), + pair("if-match", ""), + pair("if-modified-since", ""), + pair("if-none-match", ""), + pair("if-range", ""), + pair("if-unmodified-since", ""), + pair("last-modified", ""), + pair("link", ""), + pair("location", ""), + pair("max-forwards", ""), + pair("proxy-authenticate", ""), + pair("proxy-authorization", ""), + pair("range", ""), + pair("referer", ""), + pair("refresh", ""), + pair("retry-after", ""), + pair("server", ""), + pair("set-cookie", ""), + pair("strict-transport-security", ""), + pair("transfer-encoding", ""), + pair("user-agent", ""), + pair("vary", ""), + pair("via", ""), + pair("www-authenticate", ""), +} + +var huffmanCodes = []uint32{ + 0x1ff8, + 0x7fffd8, + 0xfffffe2, + 0xfffffe3, + 0xfffffe4, + 0xfffffe5, + 0xfffffe6, + 0xfffffe7, + 0xfffffe8, + 0xffffea, + 0x3ffffffc, + 0xfffffe9, + 0xfffffea, + 0x3ffffffd, + 0xfffffeb, + 0xfffffec, + 0xfffffed, + 0xfffffee, + 0xfffffef, + 0xffffff0, + 0xffffff1, + 0xffffff2, + 0x3ffffffe, + 0xffffff3, + 0xffffff4, + 0xffffff5, + 0xffffff6, + 0xffffff7, + 0xffffff8, + 0xffffff9, + 0xffffffa, + 0xffffffb, + 0x14, + 0x3f8, + 0x3f9, + 0xffa, + 0x1ff9, + 0x15, + 0xf8, + 0x7fa, + 0x3fa, + 0x3fb, + 0xf9, + 0x7fb, + 0xfa, + 0x16, + 0x17, + 0x18, + 0x0, + 0x1, + 0x2, + 0x19, + 0x1a, + 0x1b, + 0x1c, + 0x1d, + 0x1e, + 0x1f, + 0x5c, + 0xfb, + 0x7ffc, + 0x20, + 0xffb, + 0x3fc, + 0x1ffa, + 0x21, + 0x5d, + 0x5e, + 0x5f, + 0x60, + 0x61, + 0x62, + 0x63, + 0x64, + 0x65, + 0x66, + 0x67, + 0x68, + 0x69, + 0x6a, + 0x6b, + 0x6c, + 0x6d, + 0x6e, + 0x6f, + 0x70, + 0x71, + 0x72, + 0xfc, + 0x73, + 0xfd, + 0x1ffb, + 0x7fff0, + 0x1ffc, + 0x3ffc, + 0x22, + 0x7ffd, + 0x3, + 0x23, + 0x4, + 0x24, + 0x5, + 0x25, + 0x26, + 0x27, + 0x6, + 0x74, + 0x75, + 0x28, + 0x29, + 0x2a, + 0x7, + 0x2b, + 0x76, + 0x2c, + 0x8, + 0x9, + 0x2d, + 0x77, + 0x78, + 0x79, + 0x7a, + 0x7b, + 0x7ffe, + 0x7fc, + 0x3ffd, + 0x1ffd, + 0xffffffc, + 0xfffe6, + 0x3fffd2, + 0xfffe7, + 0xfffe8, + 0x3fffd3, + 0x3fffd4, + 0x3fffd5, + 0x7fffd9, + 0x3fffd6, + 0x7fffda, + 0x7fffdb, + 0x7fffdc, + 0x7fffdd, + 0x7fffde, + 0xffffeb, + 0x7fffdf, + 0xffffec, + 0xffffed, + 0x3fffd7, + 0x7fffe0, + 0xffffee, + 0x7fffe1, + 0x7fffe2, + 0x7fffe3, + 0x7fffe4, + 0x1fffdc, + 0x3fffd8, + 0x7fffe5, + 0x3fffd9, + 0x7fffe6, + 0x7fffe7, + 0xffffef, + 0x3fffda, + 0x1fffdd, + 0xfffe9, + 0x3fffdb, + 0x3fffdc, + 0x7fffe8, + 0x7fffe9, + 0x1fffde, + 0x7fffea, + 0x3fffdd, + 0x3fffde, + 0xfffff0, + 0x1fffdf, + 0x3fffdf, + 0x7fffeb, + 0x7fffec, + 0x1fffe0, + 0x1fffe1, + 0x3fffe0, + 0x1fffe2, + 0x7fffed, + 0x3fffe1, + 0x7fffee, + 0x7fffef, + 0xfffea, + 0x3fffe2, + 0x3fffe3, + 0x3fffe4, + 0x7ffff0, + 0x3fffe5, + 0x3fffe6, + 0x7ffff1, + 0x3ffffe0, + 0x3ffffe1, + 0xfffeb, + 0x7fff1, + 0x3fffe7, + 0x7ffff2, + 0x3fffe8, + 0x1ffffec, + 0x3ffffe2, + 0x3ffffe3, + 0x3ffffe4, + 0x7ffffde, + 0x7ffffdf, + 0x3ffffe5, + 0xfffff1, + 0x1ffffed, + 0x7fff2, + 0x1fffe3, + 0x3ffffe6, + 0x7ffffe0, + 0x7ffffe1, + 0x3ffffe7, + 0x7ffffe2, + 0xfffff2, + 0x1fffe4, + 0x1fffe5, + 0x3ffffe8, + 0x3ffffe9, + 0xffffffd, + 0x7ffffe3, + 0x7ffffe4, + 0x7ffffe5, + 0xfffec, + 0xfffff3, + 0xfffed, + 0x1fffe6, + 0x3fffe9, + 0x1fffe7, + 0x1fffe8, + 0x7ffff3, + 0x3fffea, + 0x3fffeb, + 0x1ffffee, + 0x1ffffef, + 0xfffff4, + 0xfffff5, + 0x3ffffea, + 0x7ffff4, + 0x3ffffeb, + 0x7ffffe6, + 0x3ffffec, + 0x3ffffed, + 0x7ffffe7, + 0x7ffffe8, + 0x7ffffe9, + 0x7ffffea, + 0x7ffffeb, + 0xffffffe, + 0x7ffffec, + 0x7ffffed, + 0x7ffffee, + 0x7ffffef, + 0x7fffff0, + 0x3ffffee, +} + +var huffmanCodeLen = []uint8{ + 13, 23, 28, 28, 28, 28, 28, 28, 28, 24, 30, 28, 28, 30, 28, 28, + 28, 28, 28, 28, 28, 28, 30, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 6, 10, 10, 12, 13, 6, 8, 11, 10, 10, 8, 11, 8, 6, 6, 6, + 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 8, 15, 6, 12, 10, + 13, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 8, 7, 8, 13, 19, 13, 14, 6, + 15, 5, 6, 5, 6, 5, 6, 6, 6, 5, 7, 7, 6, 6, 6, 5, + 6, 7, 6, 5, 5, 6, 7, 7, 7, 7, 7, 15, 11, 14, 13, 28, + 20, 22, 20, 20, 22, 22, 22, 23, 22, 23, 23, 23, 23, 23, 24, 23, + 24, 24, 22, 23, 24, 23, 23, 23, 23, 21, 22, 23, 22, 23, 23, 24, + 22, 21, 20, 22, 22, 23, 23, 21, 23, 22, 22, 24, 21, 22, 23, 23, + 21, 21, 22, 21, 23, 22, 23, 23, 20, 22, 22, 22, 23, 22, 22, 23, + 26, 26, 20, 19, 22, 23, 22, 25, 26, 26, 26, 27, 27, 26, 24, 25, + 19, 21, 26, 27, 27, 26, 27, 24, 21, 21, 26, 26, 28, 27, 27, 27, + 20, 24, 20, 21, 22, 21, 21, 23, 22, 22, 25, 25, 24, 24, 26, 23, + 26, 27, 26, 26, 27, 27, 27, 27, 27, 28, 27, 27, 27, 27, 27, 26, +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/http2.go b/components/engine/vendor/src/golang.org/x/net/http2/http2.go new file mode 100644 index 0000000000..35f9b26e28 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/http2.go @@ -0,0 +1,249 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +// Package http2 implements the HTTP/2 protocol. +// +// This is a work in progress. This package is low-level and intended +// to be used directly by very few people. Most users will use it +// indirectly through integration with the net/http package. See +// ConfigureServer. That ConfigureServer call will likely be automatic +// or available via an empty import in the future. +// +// See http://http2.github.io/ +package http2 + +import ( + "bufio" + "fmt" + "io" + "net/http" + "strconv" + "sync" +) + +var VerboseLogs = false + +const ( + // ClientPreface is the string that must be sent by new + // connections from clients. + ClientPreface = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + + // SETTINGS_MAX_FRAME_SIZE default + // http://http2.github.io/http2-spec/#rfc.section.6.5.2 + initialMaxFrameSize = 16384 + + // NextProtoTLS is the NPN/ALPN protocol negotiated during + // HTTP/2's TLS setup. + NextProtoTLS = "h2" + + // http://http2.github.io/http2-spec/#SettingValues + initialHeaderTableSize = 4096 + + initialWindowSize = 65535 // 6.9.2 Initial Flow Control Window Size + + defaultMaxReadFrameSize = 1 << 20 +) + +var ( + clientPreface = []byte(ClientPreface) +) + +type streamState int + +const ( + stateIdle streamState = iota + stateOpen + stateHalfClosedLocal + stateHalfClosedRemote + stateResvLocal + stateResvRemote + stateClosed +) + +var stateName = [...]string{ + stateIdle: "Idle", + stateOpen: "Open", + stateHalfClosedLocal: "HalfClosedLocal", + stateHalfClosedRemote: "HalfClosedRemote", + stateResvLocal: "ResvLocal", + stateResvRemote: "ResvRemote", + stateClosed: "Closed", +} + +func (st streamState) String() string { + return stateName[st] +} + +// Setting is a setting parameter: which setting it is, and its value. +type Setting struct { + // ID is which setting is being set. + // See http://http2.github.io/http2-spec/#SettingValues + ID SettingID + + // Val is the value. + Val uint32 +} + +func (s Setting) String() string { + return fmt.Sprintf("[%v = %d]", s.ID, s.Val) +} + +// Valid reports whether the setting is valid. +func (s Setting) Valid() error { + // Limits and error codes from 6.5.2 Defined SETTINGS Parameters + switch s.ID { + case SettingEnablePush: + if s.Val != 1 && s.Val != 0 { + return ConnectionError(ErrCodeProtocol) + } + case SettingInitialWindowSize: + if s.Val > 1<<31-1 { + return ConnectionError(ErrCodeFlowControl) + } + case SettingMaxFrameSize: + if s.Val < 16384 || s.Val > 1<<24-1 { + return ConnectionError(ErrCodeProtocol) + } + } + return nil +} + +// A SettingID is an HTTP/2 setting as defined in +// http://http2.github.io/http2-spec/#iana-settings +type SettingID uint16 + +const ( + SettingHeaderTableSize SettingID = 0x1 + SettingEnablePush SettingID = 0x2 + SettingMaxConcurrentStreams SettingID = 0x3 + SettingInitialWindowSize SettingID = 0x4 + SettingMaxFrameSize SettingID = 0x5 + SettingMaxHeaderListSize SettingID = 0x6 +) + +var settingName = map[SettingID]string{ + SettingHeaderTableSize: "HEADER_TABLE_SIZE", + SettingEnablePush: "ENABLE_PUSH", + SettingMaxConcurrentStreams: "MAX_CONCURRENT_STREAMS", + SettingInitialWindowSize: "INITIAL_WINDOW_SIZE", + SettingMaxFrameSize: "MAX_FRAME_SIZE", + SettingMaxHeaderListSize: "MAX_HEADER_LIST_SIZE", +} + +func (s SettingID) String() string { + if v, ok := settingName[s]; ok { + return v + } + return fmt.Sprintf("UNKNOWN_SETTING_%d", uint16(s)) +} + +func validHeader(v string) bool { + if len(v) == 0 { + return false + } + for _, r := range v { + // "Just as in HTTP/1.x, header field names are + // strings of ASCII characters that are compared in a + // case-insensitive fashion. However, header field + // names MUST be converted to lowercase prior to their + // encoding in HTTP/2. " + if r >= 127 || ('A' <= r && r <= 'Z') { + return false + } + } + return true +} + +var httpCodeStringCommon = map[int]string{} // n -> strconv.Itoa(n) + +func init() { + for i := 100; i <= 999; i++ { + if v := http.StatusText(i); v != "" { + httpCodeStringCommon[i] = strconv.Itoa(i) + } + } +} + +func httpCodeString(code int) string { + if s, ok := httpCodeStringCommon[code]; ok { + return s + } + return strconv.Itoa(code) +} + +// from pkg io +type stringWriter interface { + WriteString(s string) (n int, err error) +} + +// A gate lets two goroutines coordinate their activities. +type gate chan struct{} + +func (g gate) Done() { g <- struct{}{} } +func (g gate) Wait() { <-g } + +// A closeWaiter is like a sync.WaitGroup but only goes 1 to 0 (open to closed). +type closeWaiter chan struct{} + +// Init makes a closeWaiter usable. +// It exists because so a closeWaiter value can be placed inside a +// larger struct and have the Mutex and Cond's memory in the same +// allocation. +func (cw *closeWaiter) Init() { + *cw = make(chan struct{}) +} + +// Close marks the closeWaiter as closed and unblocks any waiters. +func (cw closeWaiter) Close() { + close(cw) +} + +// Wait waits for the closeWaiter to become closed. +func (cw closeWaiter) Wait() { + <-cw +} + +// bufferedWriter is a buffered writer that writes to w. +// Its buffered writer is lazily allocated as needed, to minimize +// idle memory usage with many connections. +type bufferedWriter struct { + w io.Writer // immutable + bw *bufio.Writer // non-nil when data is buffered +} + +func newBufferedWriter(w io.Writer) *bufferedWriter { + return &bufferedWriter{w: w} +} + +var bufWriterPool = sync.Pool{ + New: func() interface{} { + // TODO: pick something better? this is a bit under + // (3 x typical 1500 byte MTU) at least. + return bufio.NewWriterSize(nil, 4<<10) + }, +} + +func (w *bufferedWriter) Write(p []byte) (n int, err error) { + if w.bw == nil { + bw := bufWriterPool.Get().(*bufio.Writer) + bw.Reset(w.w) + w.bw = bw + } + return w.bw.Write(p) +} + +func (w *bufferedWriter) Flush() error { + bw := w.bw + if bw == nil { + return nil + } + err := bw.Flush() + bw.Reset(nil) + bufWriterPool.Put(bw) + w.bw = nil + return err +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/pipe.go b/components/engine/vendor/src/golang.org/x/net/http2/pipe.go new file mode 100644 index 0000000000..ce9aad5336 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/pipe.go @@ -0,0 +1,43 @@ +// Copyright 2014 The Go Authors. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +package http2 + +import ( + "sync" +) + +type pipe struct { + b buffer + c sync.Cond + m sync.Mutex +} + +// Read waits until data is available and copies bytes +// from the buffer into p. +func (r *pipe) Read(p []byte) (n int, err error) { + r.c.L.Lock() + defer r.c.L.Unlock() + for r.b.Len() == 0 && !r.b.closed { + r.c.Wait() + } + return r.b.Read(p) +} + +// Write copies bytes from p into the buffer and wakes a reader. +// It is an error to write more data than the buffer can hold. +func (w *pipe) Write(p []byte) (n int, err error) { + w.c.L.Lock() + defer w.c.L.Unlock() + defer w.c.Signal() + return w.b.Write(p) +} + +func (c *pipe) Close(err error) { + c.c.L.Lock() + defer c.c.L.Unlock() + defer c.c.Signal() + c.b.Close(err) +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/server.go b/components/engine/vendor/src/golang.org/x/net/http2/server.go new file mode 100644 index 0000000000..99cc673cc2 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/server.go @@ -0,0 +1,1780 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +// TODO: replace all <-sc.doneServing with reads from the stream's cw +// instead, and make sure that on close we close all open +// streams. then remove doneServing? + +// TODO: finish GOAWAY support. Consider each incoming frame type and +// whether it should be ignored during a shutdown race. + +// TODO: disconnect idle clients. GFE seems to do 4 minutes. make +// configurable? or maximum number of idle clients and remove the +// oldest? + +// TODO: turn off the serve goroutine when idle, so +// an idle conn only has the readFrames goroutine active. (which could +// also be optimized probably to pin less memory in crypto/tls). This +// would involve tracking when the serve goroutine is active (atomic +// int32 read/CAS probably?) and starting it up when frames arrive, +// and shutting it down when all handlers exit. the occasional PING +// packets could use time.AfterFunc to call sc.wakeStartServeLoop() +// (which is a no-op if already running) and then queue the PING write +// as normal. The serve loop would then exit in most cases (if no +// Handlers running) and not be woken up again until the PING packet +// returns. + +// TODO (maybe): add a mechanism for Handlers to going into +// half-closed-local mode (rw.(io.Closer) test?) but not exit their +// handler, and continue to be able to read from the +// Request.Body. This would be a somewhat semantic change from HTTP/1 +// (or at least what we expose in net/http), so I'd probably want to +// add it there too. For now, this package says that returning from +// the Handler ServeHTTP function means you're both done reading and +// done writing, without a way to stop just one or the other. + +package http2 + +import ( + "bufio" + "bytes" + "crypto/tls" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/net/http2/hpack" +) + +const ( + prefaceTimeout = 10 * time.Second + firstSettingsTimeout = 2 * time.Second // should be in-flight with preface anyway + handlerChunkWriteSize = 4 << 10 + defaultMaxStreams = 250 // TODO: make this 100 as the GFE seems to? +) + +var ( + errClientDisconnected = errors.New("client disconnected") + errClosedBody = errors.New("body closed by handler") + errStreamBroken = errors.New("http2: stream broken") +) + +var responseWriterStatePool = sync.Pool{ + New: func() interface{} { + rws := &responseWriterState{} + rws.bw = bufio.NewWriterSize(chunkWriter{rws}, handlerChunkWriteSize) + return rws + }, +} + +// Test hooks. +var ( + testHookOnConn func() + testHookGetServerConn func(*serverConn) + testHookOnPanicMu *sync.Mutex // nil except in tests + testHookOnPanic func(sc *serverConn, panicVal interface{}) (rePanic bool) +) + +// Server is an HTTP/2 server. +type Server struct { + // MaxHandlers limits the number of http.Handler ServeHTTP goroutines + // which may run at a time over all connections. + // Negative or zero no limit. + // TODO: implement + MaxHandlers int + + // MaxConcurrentStreams optionally specifies the number of + // concurrent streams that each client may have open at a + // time. This is unrelated to the number of http.Handler goroutines + // which may be active globally, which is MaxHandlers. + // If zero, MaxConcurrentStreams defaults to at least 100, per + // the HTTP/2 spec's recommendations. + MaxConcurrentStreams uint32 + + // MaxReadFrameSize optionally specifies the largest frame + // this server is willing to read. A valid value is between + // 16k and 16M, inclusive. If zero or otherwise invalid, a + // default value is used. + MaxReadFrameSize uint32 + + // PermitProhibitedCipherSuites, if true, permits the use of + // cipher suites prohibited by the HTTP/2 spec. + PermitProhibitedCipherSuites bool +} + +func (s *Server) maxReadFrameSize() uint32 { + if v := s.MaxReadFrameSize; v >= minMaxFrameSize && v <= maxFrameSize { + return v + } + return defaultMaxReadFrameSize +} + +func (s *Server) maxConcurrentStreams() uint32 { + if v := s.MaxConcurrentStreams; v > 0 { + return v + } + return defaultMaxStreams +} + +// ConfigureServer adds HTTP/2 support to a net/http Server. +// +// The configuration conf may be nil. +// +// ConfigureServer must be called before s begins serving. +func ConfigureServer(s *http.Server, conf *Server) { + if conf == nil { + conf = new(Server) + } + if s.TLSConfig == nil { + s.TLSConfig = new(tls.Config) + } + + // Note: not setting MinVersion to tls.VersionTLS12, + // as we don't want to interfere with HTTP/1.1 traffic + // on the user's server. We enforce TLS 1.2 later once + // we accept a connection. Ideally this should be done + // during next-proto selection, but using TLS <1.2 with + // HTTP/2 is still the client's bug. + + // Be sure we advertise tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + // at least. + // TODO: enable PreferServerCipherSuites? + if s.TLSConfig.CipherSuites != nil { + const requiredCipher = tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + haveRequired := false + for _, v := range s.TLSConfig.CipherSuites { + if v == requiredCipher { + haveRequired = true + break + } + } + if !haveRequired { + s.TLSConfig.CipherSuites = append(s.TLSConfig.CipherSuites, requiredCipher) + } + } + + haveNPN := false + for _, p := range s.TLSConfig.NextProtos { + if p == NextProtoTLS { + haveNPN = true + break + } + } + if !haveNPN { + s.TLSConfig.NextProtos = append(s.TLSConfig.NextProtos, NextProtoTLS) + } + // h2-14 is temporary (as of 2015-03-05) while we wait for all browsers + // to switch to "h2". + s.TLSConfig.NextProtos = append(s.TLSConfig.NextProtos, "h2-14") + + if s.TLSNextProto == nil { + s.TLSNextProto = map[string]func(*http.Server, *tls.Conn, http.Handler){} + } + protoHandler := func(hs *http.Server, c *tls.Conn, h http.Handler) { + if testHookOnConn != nil { + testHookOnConn() + } + conf.handleConn(hs, c, h) + } + s.TLSNextProto[NextProtoTLS] = protoHandler + s.TLSNextProto["h2-14"] = protoHandler // temporary; see above. +} + +func (srv *Server) handleConn(hs *http.Server, c net.Conn, h http.Handler) { + sc := &serverConn{ + srv: srv, + hs: hs, + conn: c, + remoteAddrStr: c.RemoteAddr().String(), + bw: newBufferedWriter(c), + handler: h, + streams: make(map[uint32]*stream), + readFrameCh: make(chan frameAndGate), + readFrameErrCh: make(chan error, 1), // must be buffered for 1 + wantWriteFrameCh: make(chan frameWriteMsg, 8), + wroteFrameCh: make(chan struct{}, 1), // buffered; one send in reading goroutine + bodyReadCh: make(chan bodyReadMsg), // buffering doesn't matter either way + doneServing: make(chan struct{}), + advMaxStreams: srv.maxConcurrentStreams(), + writeSched: writeScheduler{ + maxFrameSize: initialMaxFrameSize, + }, + initialWindowSize: initialWindowSize, + headerTableSize: initialHeaderTableSize, + serveG: newGoroutineLock(), + pushEnabled: true, + } + sc.flow.add(initialWindowSize) + sc.inflow.add(initialWindowSize) + sc.hpackEncoder = hpack.NewEncoder(&sc.headerWriteBuf) + sc.hpackDecoder = hpack.NewDecoder(initialHeaderTableSize, sc.onNewHeaderField) + + fr := NewFramer(sc.bw, c) + fr.SetMaxReadFrameSize(srv.maxReadFrameSize()) + sc.framer = fr + + if tc, ok := c.(*tls.Conn); ok { + sc.tlsState = new(tls.ConnectionState) + *sc.tlsState = tc.ConnectionState() + // 9.2 Use of TLS Features + // An implementation of HTTP/2 over TLS MUST use TLS + // 1.2 or higher with the restrictions on feature set + // and cipher suite described in this section. Due to + // implementation limitations, it might not be + // possible to fail TLS negotiation. An endpoint MUST + // immediately terminate an HTTP/2 connection that + // does not meet the TLS requirements described in + // this section with a connection error (Section + // 5.4.1) of type INADEQUATE_SECURITY. + if sc.tlsState.Version < tls.VersionTLS12 { + sc.rejectConn(ErrCodeInadequateSecurity, "TLS version too low") + return + } + + if sc.tlsState.ServerName == "" { + // Client must use SNI, but we don't enforce that anymore, + // since it was causing problems when connecting to bare IP + // addresses during development. + // + // TODO: optionally enforce? Or enforce at the time we receive + // a new request, and verify the the ServerName matches the :authority? + // But that precludes proxy situations, perhaps. + // + // So for now, do nothing here again. + } + + if !srv.PermitProhibitedCipherSuites && isBadCipher(sc.tlsState.CipherSuite) { + // "Endpoints MAY choose to generate a connection error + // (Section 5.4.1) of type INADEQUATE_SECURITY if one of + // the prohibited cipher suites are negotiated." + // + // We choose that. In my opinion, the spec is weak + // here. It also says both parties must support at least + // TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 so there's no + // excuses here. If we really must, we could allow an + // "AllowInsecureWeakCiphers" option on the server later. + // Let's see how it plays out first. + sc.rejectConn(ErrCodeInadequateSecurity, fmt.Sprintf("Prohibited TLS 1.2 Cipher Suite: %x", sc.tlsState.CipherSuite)) + return + } + } + + if hook := testHookGetServerConn; hook != nil { + hook(sc) + } + sc.serve() +} + +// isBadCipher reports whether the cipher is blacklisted by the HTTP/2 spec. +func isBadCipher(cipher uint16) bool { + switch cipher { + case tls.TLS_RSA_WITH_RC4_128_SHA, + tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, + tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: + // Reject cipher suites from Appendix A. + // "This list includes those cipher suites that do not + // offer an ephemeral key exchange and those that are + // based on the TLS null, stream or block cipher type" + return true + default: + return false + } +} + +func (sc *serverConn) rejectConn(err ErrCode, debug string) { + log.Printf("REJECTING conn: %v, %s", err, debug) + // ignoring errors. hanging up anyway. + sc.framer.WriteGoAway(0, err, []byte(debug)) + sc.bw.Flush() + sc.conn.Close() +} + +// frameAndGates coordinates the readFrames and serve +// goroutines. Because the Framer interface only permits the most +// recently-read Frame from being accessed, the readFrames goroutine +// blocks until it has a frame, passes it to serve, and then waits for +// serve to be done with it before reading the next one. +type frameAndGate struct { + f Frame + g gate +} + +type serverConn struct { + // Immutable: + srv *Server + hs *http.Server + conn net.Conn + bw *bufferedWriter // writing to conn + handler http.Handler + framer *Framer + hpackDecoder *hpack.Decoder + doneServing chan struct{} // closed when serverConn.serve ends + readFrameCh chan frameAndGate // written by serverConn.readFrames + readFrameErrCh chan error + wantWriteFrameCh chan frameWriteMsg // from handlers -> serve + wroteFrameCh chan struct{} // from writeFrameAsync -> serve, tickles more frame writes + bodyReadCh chan bodyReadMsg // from handlers -> serve + testHookCh chan func() // code to run on the serve loop + flow flow // conn-wide (not stream-specific) outbound flow control + inflow flow // conn-wide inbound flow control + tlsState *tls.ConnectionState // shared by all handlers, like net/http + remoteAddrStr string + + // Everything following is owned by the serve loop; use serveG.check(): + serveG goroutineLock // used to verify funcs are on serve() + pushEnabled bool + sawFirstSettings bool // got the initial SETTINGS frame after the preface + needToSendSettingsAck bool + unackedSettings int // how many SETTINGS have we sent without ACKs? + clientMaxStreams uint32 // SETTINGS_MAX_CONCURRENT_STREAMS from client (our PUSH_PROMISE limit) + advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client + curOpenStreams uint32 // client's number of open streams + maxStreamID uint32 // max ever seen + streams map[uint32]*stream + initialWindowSize int32 + headerTableSize uint32 + maxHeaderListSize uint32 // zero means unknown (default) + canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case + req requestParam // non-zero while reading request headers + writingFrame bool // started write goroutine but haven't heard back on wroteFrameCh + needsFrameFlush bool // last frame write wasn't a flush + writeSched writeScheduler + inGoAway bool // we've started to or sent GOAWAY + needToSendGoAway bool // we need to schedule a GOAWAY frame write + goAwayCode ErrCode + shutdownTimerCh <-chan time.Time // nil until used + shutdownTimer *time.Timer // nil until used + + // Owned by the writeFrameAsync goroutine: + headerWriteBuf bytes.Buffer + hpackEncoder *hpack.Encoder +} + +// requestParam is the state of the next request, initialized over +// potentially several frames HEADERS + zero or more CONTINUATION +// frames. +type requestParam struct { + // stream is non-nil if we're reading (HEADER or CONTINUATION) + // frames for a request (but not DATA). + stream *stream + header http.Header + method, path string + scheme, authority string + sawRegularHeader bool // saw a non-pseudo header already + invalidHeader bool // an invalid header was seen +} + +// stream represents a stream. This is the minimal metadata needed by +// the serve goroutine. Most of the actual stream state is owned by +// the http.Handler's goroutine in the responseWriter. Because the +// responseWriter's responseWriterState is recycled at the end of a +// handler, this struct intentionally has no pointer to the +// *responseWriter{,State} itself, as the Handler ending nils out the +// responseWriter's state field. +type stream struct { + // immutable: + id uint32 + body *pipe // non-nil if expecting DATA frames + cw closeWaiter // closed wait stream transitions to closed state + + // owned by serverConn's serve loop: + bodyBytes int64 // body bytes seen so far + declBodyBytes int64 // or -1 if undeclared + flow flow // limits writing from Handler to client + inflow flow // what the client is allowed to POST/etc to us + parent *stream // or nil + weight uint8 + state streamState + sentReset bool // only true once detached from streams map + gotReset bool // only true once detacted from streams map +} + +func (sc *serverConn) Framer() *Framer { return sc.framer } +func (sc *serverConn) CloseConn() error { return sc.conn.Close() } +func (sc *serverConn) Flush() error { return sc.bw.Flush() } +func (sc *serverConn) HeaderEncoder() (*hpack.Encoder, *bytes.Buffer) { + return sc.hpackEncoder, &sc.headerWriteBuf +} + +func (sc *serverConn) state(streamID uint32) (streamState, *stream) { + sc.serveG.check() + // http://http2.github.io/http2-spec/#rfc.section.5.1 + if st, ok := sc.streams[streamID]; ok { + return st.state, st + } + // "The first use of a new stream identifier implicitly closes all + // streams in the "idle" state that might have been initiated by + // that peer with a lower-valued stream identifier. For example, if + // a client sends a HEADERS frame on stream 7 without ever sending a + // frame on stream 5, then stream 5 transitions to the "closed" + // state when the first frame for stream 7 is sent or received." + if streamID <= sc.maxStreamID { + return stateClosed, nil + } + return stateIdle, nil +} + +func (sc *serverConn) vlogf(format string, args ...interface{}) { + if VerboseLogs { + sc.logf(format, args...) + } +} + +func (sc *serverConn) logf(format string, args ...interface{}) { + if lg := sc.hs.ErrorLog; lg != nil { + lg.Printf(format, args...) + } else { + log.Printf(format, args...) + } +} + +func (sc *serverConn) condlogf(err error, format string, args ...interface{}) { + if err == nil { + return + } + str := err.Error() + if err == io.EOF || strings.Contains(str, "use of closed network connection") { + // Boring, expected errors. + sc.vlogf(format, args...) + } else { + sc.logf(format, args...) + } +} + +func (sc *serverConn) onNewHeaderField(f hpack.HeaderField) { + sc.serveG.check() + sc.vlogf("got header field %+v", f) + switch { + case !validHeader(f.Name): + sc.req.invalidHeader = true + case strings.HasPrefix(f.Name, ":"): + if sc.req.sawRegularHeader { + sc.logf("pseudo-header after regular header") + sc.req.invalidHeader = true + return + } + var dst *string + switch f.Name { + case ":method": + dst = &sc.req.method + case ":path": + dst = &sc.req.path + case ":scheme": + dst = &sc.req.scheme + case ":authority": + dst = &sc.req.authority + default: + // 8.1.2.1 Pseudo-Header Fields + // "Endpoints MUST treat a request or response + // that contains undefined or invalid + // pseudo-header fields as malformed (Section + // 8.1.2.6)." + sc.logf("invalid pseudo-header %q", f.Name) + sc.req.invalidHeader = true + return + } + if *dst != "" { + sc.logf("duplicate pseudo-header %q sent", f.Name) + sc.req.invalidHeader = true + return + } + *dst = f.Value + case f.Name == "cookie": + sc.req.sawRegularHeader = true + if s, ok := sc.req.header["Cookie"]; ok && len(s) == 1 { + s[0] = s[0] + "; " + f.Value + } else { + sc.req.header.Add("Cookie", f.Value) + } + default: + sc.req.sawRegularHeader = true + sc.req.header.Add(sc.canonicalHeader(f.Name), f.Value) + } +} + +func (sc *serverConn) canonicalHeader(v string) string { + sc.serveG.check() + cv, ok := commonCanonHeader[v] + if ok { + return cv + } + cv, ok = sc.canonHeader[v] + if ok { + return cv + } + if sc.canonHeader == nil { + sc.canonHeader = make(map[string]string) + } + cv = http.CanonicalHeaderKey(v) + sc.canonHeader[v] = cv + return cv +} + +// readFrames is the loop that reads incoming frames. +// It's run on its own goroutine. +func (sc *serverConn) readFrames() { + g := make(gate, 1) + for { + f, err := sc.framer.ReadFrame() + if err != nil { + sc.readFrameErrCh <- err + close(sc.readFrameCh) + return + } + sc.readFrameCh <- frameAndGate{f, g} + // We can't read another frame until this one is + // processed, as the ReadFrame interface doesn't copy + // memory. The Frame accessor methods access the last + // frame's (shared) buffer. So we wait for the + // serve goroutine to tell us it's done: + g.Wait() + } +} + +// writeFrameAsync runs in its own goroutine and writes a single frame +// and then reports when it's done. +// At most one goroutine can be running writeFrameAsync at a time per +// serverConn. +func (sc *serverConn) writeFrameAsync(wm frameWriteMsg) { + err := wm.write.writeFrame(sc) + if ch := wm.done; ch != nil { + select { + case ch <- err: + default: + panic(fmt.Sprintf("unbuffered done channel passed in for type %T", wm.write)) + } + } + sc.wroteFrameCh <- struct{}{} // tickle frame selection scheduler +} + +func (sc *serverConn) closeAllStreamsOnConnClose() { + sc.serveG.check() + for _, st := range sc.streams { + sc.closeStream(st, errClientDisconnected) + } +} + +func (sc *serverConn) stopShutdownTimer() { + sc.serveG.check() + if t := sc.shutdownTimer; t != nil { + t.Stop() + } +} + +func (sc *serverConn) notePanic() { + if testHookOnPanicMu != nil { + testHookOnPanicMu.Lock() + defer testHookOnPanicMu.Unlock() + } + if testHookOnPanic != nil { + if e := recover(); e != nil { + if testHookOnPanic(sc, e) { + panic(e) + } + } + } +} + +func (sc *serverConn) serve() { + sc.serveG.check() + defer sc.notePanic() + defer sc.conn.Close() + defer sc.closeAllStreamsOnConnClose() + defer sc.stopShutdownTimer() + defer close(sc.doneServing) // unblocks handlers trying to send + + sc.vlogf("HTTP/2 connection from %v on %p", sc.conn.RemoteAddr(), sc.hs) + + sc.writeFrame(frameWriteMsg{ + write: writeSettings{ + {SettingMaxFrameSize, sc.srv.maxReadFrameSize()}, + {SettingMaxConcurrentStreams, sc.advMaxStreams}, + + // TODO: more actual settings, notably + // SettingInitialWindowSize, but then we also + // want to bump up the conn window size the + // same amount here right after the settings + }, + }) + sc.unackedSettings++ + + if err := sc.readPreface(); err != nil { + sc.condlogf(err, "error reading preface from client %v: %v", sc.conn.RemoteAddr(), err) + return + } + + go sc.readFrames() // closed by defer sc.conn.Close above + + settingsTimer := time.NewTimer(firstSettingsTimeout) + for { + select { + case wm := <-sc.wantWriteFrameCh: + sc.writeFrame(wm) + case <-sc.wroteFrameCh: + if sc.writingFrame != true { + panic("internal error: expected to be already writing a frame") + } + sc.writingFrame = false + sc.scheduleFrameWrite() + case fg, ok := <-sc.readFrameCh: + if !ok { + sc.readFrameCh = nil + } + if !sc.processFrameFromReader(fg, ok) { + return + } + if settingsTimer.C != nil { + settingsTimer.Stop() + settingsTimer.C = nil + } + case m := <-sc.bodyReadCh: + sc.noteBodyRead(m.st, m.n) + case <-settingsTimer.C: + sc.logf("timeout waiting for SETTINGS frames from %v", sc.conn.RemoteAddr()) + return + case <-sc.shutdownTimerCh: + sc.vlogf("GOAWAY close timer fired; closing conn from %v", sc.conn.RemoteAddr()) + return + case fn := <-sc.testHookCh: + fn() + } + } +} + +// readPreface reads the ClientPreface greeting from the peer +// or returns an error on timeout or an invalid greeting. +func (sc *serverConn) readPreface() error { + errc := make(chan error, 1) + go func() { + // Read the client preface + buf := make([]byte, len(ClientPreface)) + if _, err := io.ReadFull(sc.conn, buf); err != nil { + errc <- err + } else if !bytes.Equal(buf, clientPreface) { + errc <- fmt.Errorf("bogus greeting %q", buf) + } else { + errc <- nil + } + }() + timer := time.NewTimer(prefaceTimeout) // TODO: configurable on *Server? + defer timer.Stop() + select { + case <-timer.C: + return errors.New("timeout waiting for client preface") + case err := <-errc: + if err == nil { + sc.vlogf("client %v said hello", sc.conn.RemoteAddr()) + } + return err + } +} + +// writeDataFromHandler writes the data described in req to stream.id. +// +// The provided ch is used to avoid allocating new channels for each +// write operation. It's expected that the caller reuses writeData and ch +// over time. +// +// The flow control currently happens in the Handler where it waits +// for 1 or more bytes to be available to then write here. So at this +// point we know that we have flow control. But this might have to +// change when priority is implemented, so the serve goroutine knows +// the total amount of bytes waiting to be sent and can can have more +// scheduling decisions available. +func (sc *serverConn) writeDataFromHandler(stream *stream, writeData *writeData, ch chan error) error { + sc.writeFrameFromHandler(frameWriteMsg{ + write: writeData, + stream: stream, + done: ch, + }) + select { + case err := <-ch: + return err + case <-sc.doneServing: + return errClientDisconnected + case <-stream.cw: + return errStreamBroken + } +} + +// writeFrameFromHandler sends wm to sc.wantWriteFrameCh, but aborts +// if the connection has gone away. +// +// This must not be run from the serve goroutine itself, else it might +// deadlock writing to sc.wantWriteFrameCh (which is only mildly +// buffered and is read by serve itself). If you're on the serve +// goroutine, call writeFrame instead. +func (sc *serverConn) writeFrameFromHandler(wm frameWriteMsg) { + sc.serveG.checkNotOn() // NOT + select { + case sc.wantWriteFrameCh <- wm: + case <-sc.doneServing: + // Client has closed their connection to the server. + } +} + +// writeFrame schedules a frame to write and sends it if there's nothing +// already being written. +// +// There is no pushback here (the serve goroutine never blocks). It's +// the http.Handlers that block, waiting for their previous frames to +// make it onto the wire +// +// If you're not on the serve goroutine, use writeFrameFromHandler instead. +func (sc *serverConn) writeFrame(wm frameWriteMsg) { + sc.serveG.check() + sc.writeSched.add(wm) + sc.scheduleFrameWrite() +} + +// startFrameWrite starts a goroutine to write wm (in a separate +// goroutine since that might block on the network), and updates the +// serve goroutine's state about the world, updated from info in wm. +func (sc *serverConn) startFrameWrite(wm frameWriteMsg) { + sc.serveG.check() + if sc.writingFrame { + panic("internal error: can only be writing one frame at a time") + } + sc.writingFrame = true + + st := wm.stream + if st != nil { + switch st.state { + case stateHalfClosedLocal: + panic("internal error: attempt to send frame on half-closed-local stream") + case stateClosed: + if st.sentReset || st.gotReset { + // Skip this frame. But fake the frame write to reschedule: + sc.wroteFrameCh <- struct{}{} + return + } + panic(fmt.Sprintf("internal error: attempt to send a write %v on a closed stream", wm)) + } + } + + sc.needsFrameFlush = true + if endsStream(wm.write) { + if st == nil { + panic("internal error: expecting non-nil stream") + } + switch st.state { + case stateOpen: + // Here we would go to stateHalfClosedLocal in + // theory, but since our handler is done and + // the net/http package provides no mechanism + // for finishing writing to a ResponseWriter + // while still reading data (see possible TODO + // at top of this file), we go into closed + // state here anyway, after telling the peer + // we're hanging up on them. + st.state = stateHalfClosedLocal // won't last long, but necessary for closeStream via resetStream + errCancel := StreamError{st.id, ErrCodeCancel} + sc.resetStream(errCancel) + case stateHalfClosedRemote: + sc.closeStream(st, nil) + } + } + go sc.writeFrameAsync(wm) +} + +// scheduleFrameWrite tickles the frame writing scheduler. +// +// If a frame is already being written, nothing happens. This will be called again +// when the frame is done being written. +// +// If a frame isn't being written we need to send one, the best frame +// to send is selected, preferring first things that aren't +// stream-specific (e.g. ACKing settings), and then finding the +// highest priority stream. +// +// If a frame isn't being written and there's nothing else to send, we +// flush the write buffer. +func (sc *serverConn) scheduleFrameWrite() { + sc.serveG.check() + if sc.writingFrame { + return + } + if sc.needToSendGoAway { + sc.needToSendGoAway = false + sc.startFrameWrite(frameWriteMsg{ + write: &writeGoAway{ + maxStreamID: sc.maxStreamID, + code: sc.goAwayCode, + }, + }) + return + } + if sc.needToSendSettingsAck { + sc.needToSendSettingsAck = false + sc.startFrameWrite(frameWriteMsg{write: writeSettingsAck{}}) + return + } + if !sc.inGoAway { + if wm, ok := sc.writeSched.take(); ok { + sc.startFrameWrite(wm) + return + } + } + if sc.needsFrameFlush { + sc.startFrameWrite(frameWriteMsg{write: flushFrameWriter{}}) + sc.needsFrameFlush = false // after startFrameWrite, since it sets this true + return + } +} + +func (sc *serverConn) goAway(code ErrCode) { + sc.serveG.check() + if sc.inGoAway { + return + } + if code != ErrCodeNo { + sc.shutDownIn(250 * time.Millisecond) + } else { + // TODO: configurable + sc.shutDownIn(1 * time.Second) + } + sc.inGoAway = true + sc.needToSendGoAway = true + sc.goAwayCode = code + sc.scheduleFrameWrite() +} + +func (sc *serverConn) shutDownIn(d time.Duration) { + sc.serveG.check() + sc.shutdownTimer = time.NewTimer(d) + sc.shutdownTimerCh = sc.shutdownTimer.C +} + +func (sc *serverConn) resetStream(se StreamError) { + sc.serveG.check() + sc.writeFrame(frameWriteMsg{write: se}) + if st, ok := sc.streams[se.StreamID]; ok { + st.sentReset = true + sc.closeStream(st, se) + } +} + +// curHeaderStreamID returns the stream ID of the header block we're +// currently in the middle of reading. If this returns non-zero, the +// next frame must be a CONTINUATION with this stream id. +func (sc *serverConn) curHeaderStreamID() uint32 { + sc.serveG.check() + st := sc.req.stream + if st == nil { + return 0 + } + return st.id +} + +// processFrameFromReader processes the serve loop's read from readFrameCh from the +// frame-reading goroutine. +// processFrameFromReader returns whether the connection should be kept open. +func (sc *serverConn) processFrameFromReader(fg frameAndGate, fgValid bool) bool { + sc.serveG.check() + var clientGone bool + var err error + if !fgValid { + err = <-sc.readFrameErrCh + if err == ErrFrameTooLarge { + sc.goAway(ErrCodeFrameSize) + return true // goAway will close the loop + } + clientGone = err == io.EOF || strings.Contains(err.Error(), "use of closed network connection") + if clientGone { + // TODO: could we also get into this state if + // the peer does a half close + // (e.g. CloseWrite) because they're done + // sending frames but they're still wanting + // our open replies? Investigate. + // TODO: add CloseWrite to crypto/tls.Conn first + // so we have a way to test this? I suppose + // just for testing we could have a non-TLS mode. + return false + } + } + + if fgValid { + f := fg.f + sc.vlogf("got %v: %#v", f.Header(), f) + err = sc.processFrame(f) + fg.g.Done() // unblock the readFrames goroutine + if err == nil { + return true + } + } + + switch ev := err.(type) { + case StreamError: + sc.resetStream(ev) + return true + case goAwayFlowError: + sc.goAway(ErrCodeFlowControl) + return true + case ConnectionError: + sc.logf("%v: %v", sc.conn.RemoteAddr(), ev) + sc.goAway(ErrCode(ev)) + return true // goAway will handle shutdown + default: + if !fgValid { + sc.logf("disconnecting; error reading frame from client %s: %v", sc.conn.RemoteAddr(), err) + } else { + sc.logf("disconnection due to other error: %v", err) + } + } + return false +} + +func (sc *serverConn) processFrame(f Frame) error { + sc.serveG.check() + + // First frame received must be SETTINGS. + if !sc.sawFirstSettings { + if _, ok := f.(*SettingsFrame); !ok { + return ConnectionError(ErrCodeProtocol) + } + sc.sawFirstSettings = true + } + + if s := sc.curHeaderStreamID(); s != 0 { + if cf, ok := f.(*ContinuationFrame); !ok { + return ConnectionError(ErrCodeProtocol) + } else if cf.Header().StreamID != s { + return ConnectionError(ErrCodeProtocol) + } + } + + switch f := f.(type) { + case *SettingsFrame: + return sc.processSettings(f) + case *HeadersFrame: + return sc.processHeaders(f) + case *ContinuationFrame: + return sc.processContinuation(f) + case *WindowUpdateFrame: + return sc.processWindowUpdate(f) + case *PingFrame: + return sc.processPing(f) + case *DataFrame: + return sc.processData(f) + case *RSTStreamFrame: + return sc.processResetStream(f) + case *PriorityFrame: + return sc.processPriority(f) + case *PushPromiseFrame: + // A client cannot push. Thus, servers MUST treat the receipt of a PUSH_PROMISE + // frame as a connection error (Section 5.4.1) of type PROTOCOL_ERROR. + return ConnectionError(ErrCodeProtocol) + default: + log.Printf("Ignoring frame: %v", f.Header()) + return nil + } +} + +func (sc *serverConn) processPing(f *PingFrame) error { + sc.serveG.check() + if f.Flags.Has(FlagSettingsAck) { + // 6.7 PING: " An endpoint MUST NOT respond to PING frames + // containing this flag." + return nil + } + if f.StreamID != 0 { + // "PING frames are not associated with any individual + // stream. If a PING frame is received with a stream + // identifier field value other than 0x0, the recipient MUST + // respond with a connection error (Section 5.4.1) of type + // PROTOCOL_ERROR." + return ConnectionError(ErrCodeProtocol) + } + sc.writeFrame(frameWriteMsg{write: writePingAck{f}}) + return nil +} + +func (sc *serverConn) processWindowUpdate(f *WindowUpdateFrame) error { + sc.serveG.check() + switch { + case f.StreamID != 0: // stream-level flow control + st := sc.streams[f.StreamID] + if st == nil { + // "WINDOW_UPDATE can be sent by a peer that has sent a + // frame bearing the END_STREAM flag. This means that a + // receiver could receive a WINDOW_UPDATE frame on a "half + // closed (remote)" or "closed" stream. A receiver MUST + // NOT treat this as an error, see Section 5.1." + return nil + } + if !st.flow.add(int32(f.Increment)) { + return StreamError{f.StreamID, ErrCodeFlowControl} + } + default: // connection-level flow control + if !sc.flow.add(int32(f.Increment)) { + return goAwayFlowError{} + } + } + sc.scheduleFrameWrite() + return nil +} + +func (sc *serverConn) processResetStream(f *RSTStreamFrame) error { + sc.serveG.check() + + state, st := sc.state(f.StreamID) + if state == stateIdle { + // 6.4 "RST_STREAM frames MUST NOT be sent for a + // stream in the "idle" state. If a RST_STREAM frame + // identifying an idle stream is received, the + // recipient MUST treat this as a connection error + // (Section 5.4.1) of type PROTOCOL_ERROR. + return ConnectionError(ErrCodeProtocol) + } + if st != nil { + st.gotReset = true + sc.closeStream(st, StreamError{f.StreamID, f.ErrCode}) + } + return nil +} + +func (sc *serverConn) closeStream(st *stream, err error) { + sc.serveG.check() + if st.state == stateIdle || st.state == stateClosed { + panic(fmt.Sprintf("invariant; can't close stream in state %v", st.state)) + } + st.state = stateClosed + sc.curOpenStreams-- + delete(sc.streams, st.id) + if p := st.body; p != nil { + p.Close(err) + } + st.cw.Close() // signals Handler's CloseNotifier, unblocks writes, etc + sc.writeSched.forgetStream(st.id) +} + +func (sc *serverConn) processSettings(f *SettingsFrame) error { + sc.serveG.check() + if f.IsAck() { + sc.unackedSettings-- + if sc.unackedSettings < 0 { + // Why is the peer ACKing settings we never sent? + // The spec doesn't mention this case, but + // hang up on them anyway. + return ConnectionError(ErrCodeProtocol) + } + return nil + } + if err := f.ForeachSetting(sc.processSetting); err != nil { + return err + } + sc.needToSendSettingsAck = true + sc.scheduleFrameWrite() + return nil +} + +func (sc *serverConn) processSetting(s Setting) error { + sc.serveG.check() + if err := s.Valid(); err != nil { + return err + } + sc.vlogf("processing setting %v", s) + switch s.ID { + case SettingHeaderTableSize: + sc.headerTableSize = s.Val + sc.hpackEncoder.SetMaxDynamicTableSize(s.Val) + case SettingEnablePush: + sc.pushEnabled = s.Val != 0 + case SettingMaxConcurrentStreams: + sc.clientMaxStreams = s.Val + case SettingInitialWindowSize: + return sc.processSettingInitialWindowSize(s.Val) + case SettingMaxFrameSize: + sc.writeSched.maxFrameSize = s.Val + case SettingMaxHeaderListSize: + sc.maxHeaderListSize = s.Val + default: + // Unknown setting: "An endpoint that receives a SETTINGS + // frame with any unknown or unsupported identifier MUST + // ignore that setting." + } + return nil +} + +func (sc *serverConn) processSettingInitialWindowSize(val uint32) error { + sc.serveG.check() + // Note: val already validated to be within range by + // processSetting's Valid call. + + // "A SETTINGS frame can alter the initial flow control window + // size for all current streams. When the value of + // SETTINGS_INITIAL_WINDOW_SIZE changes, a receiver MUST + // adjust the size of all stream flow control windows that it + // maintains by the difference between the new value and the + // old value." + old := sc.initialWindowSize + sc.initialWindowSize = int32(val) + growth := sc.initialWindowSize - old // may be negative + for _, st := range sc.streams { + if !st.flow.add(growth) { + // 6.9.2 Initial Flow Control Window Size + // "An endpoint MUST treat a change to + // SETTINGS_INITIAL_WINDOW_SIZE that causes any flow + // control window to exceed the maximum size as a + // connection error (Section 5.4.1) of type + // FLOW_CONTROL_ERROR." + return ConnectionError(ErrCodeFlowControl) + } + } + return nil +} + +func (sc *serverConn) processData(f *DataFrame) error { + sc.serveG.check() + // "If a DATA frame is received whose stream is not in "open" + // or "half closed (local)" state, the recipient MUST respond + // with a stream error (Section 5.4.2) of type STREAM_CLOSED." + id := f.Header().StreamID + st, ok := sc.streams[id] + if !ok || st.state != stateOpen { + // This includes sending a RST_STREAM if the stream is + // in stateHalfClosedLocal (which currently means that + // the http.Handler returned, so it's done reading & + // done writing). Try to stop the client from sending + // more DATA. + return StreamError{id, ErrCodeStreamClosed} + } + if st.body == nil { + panic("internal error: should have a body in this state") + } + data := f.Data() + + // Sender sending more than they'd declared? + if st.declBodyBytes != -1 && st.bodyBytes+int64(len(data)) > st.declBodyBytes { + st.body.Close(fmt.Errorf("sender tried to send more than declared Content-Length of %d bytes", st.declBodyBytes)) + return StreamError{id, ErrCodeStreamClosed} + } + if len(data) > 0 { + // Check whether the client has flow control quota. + if int(st.inflow.available()) < len(data) { + return StreamError{id, ErrCodeFlowControl} + } + st.inflow.take(int32(len(data))) + wrote, err := st.body.Write(data) + if err != nil { + return StreamError{id, ErrCodeStreamClosed} + } + if wrote != len(data) { + panic("internal error: bad Writer") + } + st.bodyBytes += int64(len(data)) + } + if f.StreamEnded() { + if st.declBodyBytes != -1 && st.declBodyBytes != st.bodyBytes { + st.body.Close(fmt.Errorf("request declared a Content-Length of %d but only wrote %d bytes", + st.declBodyBytes, st.bodyBytes)) + } else { + st.body.Close(io.EOF) + } + st.state = stateHalfClosedRemote + } + return nil +} + +func (sc *serverConn) processHeaders(f *HeadersFrame) error { + sc.serveG.check() + id := f.Header().StreamID + if sc.inGoAway { + // Ignore. + return nil + } + // http://http2.github.io/http2-spec/#rfc.section.5.1.1 + if id%2 != 1 || id <= sc.maxStreamID || sc.req.stream != nil { + // Streams initiated by a client MUST use odd-numbered + // stream identifiers. [...] The identifier of a newly + // established stream MUST be numerically greater than all + // streams that the initiating endpoint has opened or + // reserved. [...] An endpoint that receives an unexpected + // stream identifier MUST respond with a connection error + // (Section 5.4.1) of type PROTOCOL_ERROR. + return ConnectionError(ErrCodeProtocol) + } + if id > sc.maxStreamID { + sc.maxStreamID = id + } + st := &stream{ + id: id, + state: stateOpen, + } + if f.StreamEnded() { + st.state = stateHalfClosedRemote + } + st.cw.Init() + + st.flow.conn = &sc.flow // link to conn-level counter + st.flow.add(sc.initialWindowSize) + st.inflow.conn = &sc.inflow // link to conn-level counter + st.inflow.add(initialWindowSize) // TODO: update this when we send a higher initial window size in the initial settings + + sc.streams[id] = st + if f.HasPriority() { + adjustStreamPriority(sc.streams, st.id, f.Priority) + } + sc.curOpenStreams++ + sc.req = requestParam{ + stream: st, + header: make(http.Header), + } + return sc.processHeaderBlockFragment(st, f.HeaderBlockFragment(), f.HeadersEnded()) +} + +func (sc *serverConn) processContinuation(f *ContinuationFrame) error { + sc.serveG.check() + st := sc.streams[f.Header().StreamID] + if st == nil || sc.curHeaderStreamID() != st.id { + return ConnectionError(ErrCodeProtocol) + } + return sc.processHeaderBlockFragment(st, f.HeaderBlockFragment(), f.HeadersEnded()) +} + +func (sc *serverConn) processHeaderBlockFragment(st *stream, frag []byte, end bool) error { + sc.serveG.check() + if _, err := sc.hpackDecoder.Write(frag); err != nil { + // TODO: convert to stream error I assume? + return err + } + if !end { + return nil + } + if err := sc.hpackDecoder.Close(); err != nil { + // TODO: convert to stream error I assume? + return err + } + defer sc.resetPendingRequest() + if sc.curOpenStreams > sc.advMaxStreams { + // "Endpoints MUST NOT exceed the limit set by their + // peer. An endpoint that receives a HEADERS frame + // that causes their advertised concurrent stream + // limit to be exceeded MUST treat this as a stream + // error (Section 5.4.2) of type PROTOCOL_ERROR or + // REFUSED_STREAM." + if sc.unackedSettings == 0 { + // They should know better. + return StreamError{st.id, ErrCodeProtocol} + } + // Assume it's a network race, where they just haven't + // received our last SETTINGS update. But actually + // this can't happen yet, because we don't yet provide + // a way for users to adjust server parameters at + // runtime. + return StreamError{st.id, ErrCodeRefusedStream} + } + + rw, req, err := sc.newWriterAndRequest() + if err != nil { + return err + } + st.body = req.Body.(*requestBody).pipe // may be nil + st.declBodyBytes = req.ContentLength + go sc.runHandler(rw, req) + return nil +} + +func (sc *serverConn) processPriority(f *PriorityFrame) error { + adjustStreamPriority(sc.streams, f.StreamID, f.PriorityParam) + return nil +} + +func adjustStreamPriority(streams map[uint32]*stream, streamID uint32, priority PriorityParam) { + st, ok := streams[streamID] + if !ok { + // TODO: not quite correct (this streamID might + // already exist in the dep tree, but be closed), but + // close enough for now. + return + } + st.weight = priority.Weight + parent := streams[priority.StreamDep] // might be nil + if parent == st { + // if client tries to set this stream to be the parent of itself + // ignore and keep going + return + } + + // section 5.3.3: If a stream is made dependent on one of its + // own dependencies, the formerly dependent stream is first + // moved to be dependent on the reprioritized stream's previous + // parent. The moved dependency retains its weight. + for piter := parent; piter != nil; piter = piter.parent { + if piter == st { + parent.parent = st.parent + break + } + } + st.parent = parent + if priority.Exclusive && (st.parent != nil || priority.StreamDep == 0) { + for _, openStream := range streams { + if openStream != st && openStream.parent == st.parent { + openStream.parent = st + } + } + } +} + +// resetPendingRequest zeros out all state related to a HEADERS frame +// and its zero or more CONTINUATION frames sent to start a new +// request. +func (sc *serverConn) resetPendingRequest() { + sc.serveG.check() + sc.req = requestParam{} +} + +func (sc *serverConn) newWriterAndRequest() (*responseWriter, *http.Request, error) { + sc.serveG.check() + rp := &sc.req + if rp.invalidHeader || rp.method == "" || rp.path == "" || + (rp.scheme != "https" && rp.scheme != "http") { + // See 8.1.2.6 Malformed Requests and Responses: + // + // Malformed requests or responses that are detected + // MUST be treated as a stream error (Section 5.4.2) + // of type PROTOCOL_ERROR." + // + // 8.1.2.3 Request Pseudo-Header Fields + // "All HTTP/2 requests MUST include exactly one valid + // value for the :method, :scheme, and :path + // pseudo-header fields" + return nil, nil, StreamError{rp.stream.id, ErrCodeProtocol} + } + var tlsState *tls.ConnectionState // nil if not scheme https + if rp.scheme == "https" { + tlsState = sc.tlsState + } + authority := rp.authority + if authority == "" { + authority = rp.header.Get("Host") + } + needsContinue := rp.header.Get("Expect") == "100-continue" + if needsContinue { + rp.header.Del("Expect") + } + bodyOpen := rp.stream.state == stateOpen + body := &requestBody{ + conn: sc, + stream: rp.stream, + needsContinue: needsContinue, + } + // TODO: handle asterisk '*' requests + test + url, err := url.ParseRequestURI(rp.path) + if err != nil { + // TODO: find the right error code? + return nil, nil, StreamError{rp.stream.id, ErrCodeProtocol} + } + req := &http.Request{ + Method: rp.method, + URL: url, + RemoteAddr: sc.remoteAddrStr, + Header: rp.header, + RequestURI: rp.path, + Proto: "HTTP/2.0", + ProtoMajor: 2, + ProtoMinor: 0, + TLS: tlsState, + Host: authority, + Body: body, + } + if bodyOpen { + body.pipe = &pipe{ + b: buffer{buf: make([]byte, initialWindowSize)}, // TODO: share/remove XXX + } + body.pipe.c.L = &body.pipe.m + + if vv, ok := rp.header["Content-Length"]; ok { + req.ContentLength, _ = strconv.ParseInt(vv[0], 10, 64) + } else { + req.ContentLength = -1 + } + } + + rws := responseWriterStatePool.Get().(*responseWriterState) + bwSave := rws.bw + *rws = responseWriterState{} // zero all the fields + rws.conn = sc + rws.bw = bwSave + rws.bw.Reset(chunkWriter{rws}) + rws.stream = rp.stream + rws.req = req + rws.body = body + rws.frameWriteCh = make(chan error, 1) + + rw := &responseWriter{rws: rws} + return rw, req, nil +} + +// Run on its own goroutine. +func (sc *serverConn) runHandler(rw *responseWriter, req *http.Request) { + defer rw.handlerDone() + // TODO: catch panics like net/http.Server + sc.handler.ServeHTTP(rw, req) +} + +// called from handler goroutines. +// h may be nil. +func (sc *serverConn) writeHeaders(st *stream, headerData *writeResHeaders, tempCh chan error) { + sc.serveG.checkNotOn() // NOT on + var errc chan error + if headerData.h != nil { + // If there's a header map (which we don't own), so we have to block on + // waiting for this frame to be written, so an http.Flush mid-handler + // writes out the correct value of keys, before a handler later potentially + // mutates it. + errc = tempCh + } + sc.writeFrameFromHandler(frameWriteMsg{ + write: headerData, + stream: st, + done: errc, + }) + if errc != nil { + select { + case <-errc: + // Ignore. Just for synchronization. + // Any error will be handled in the writing goroutine. + case <-sc.doneServing: + // Client has closed the connection. + } + } +} + +// called from handler goroutines. +func (sc *serverConn) write100ContinueHeaders(st *stream) { + sc.writeFrameFromHandler(frameWriteMsg{ + write: write100ContinueHeadersFrame{st.id}, + stream: st, + }) +} + +// A bodyReadMsg tells the server loop that the http.Handler read n +// bytes of the DATA from the client on the given stream. +type bodyReadMsg struct { + st *stream + n int +} + +// called from handler goroutines. +// Notes that the handler for the given stream ID read n bytes of its body +// and schedules flow control tokens to be sent. +func (sc *serverConn) noteBodyReadFromHandler(st *stream, n int) { + sc.serveG.checkNotOn() // NOT on + sc.bodyReadCh <- bodyReadMsg{st, n} +} + +func (sc *serverConn) noteBodyRead(st *stream, n int) { + sc.serveG.check() + sc.sendWindowUpdate(nil, n) // conn-level + if st.state != stateHalfClosedRemote && st.state != stateClosed { + // Don't send this WINDOW_UPDATE if the stream is closed + // remotely. + sc.sendWindowUpdate(st, n) + } +} + +// st may be nil for conn-level +func (sc *serverConn) sendWindowUpdate(st *stream, n int) { + sc.serveG.check() + // "The legal range for the increment to the flow control + // window is 1 to 2^31-1 (2,147,483,647) octets." + // A Go Read call on 64-bit machines could in theory read + // a larger Read than this. Very unlikely, but we handle it here + // rather than elsewhere for now. + const maxUint31 = 1<<31 - 1 + for n >= maxUint31 { + sc.sendWindowUpdate32(st, maxUint31) + n -= maxUint31 + } + sc.sendWindowUpdate32(st, int32(n)) +} + +// st may be nil for conn-level +func (sc *serverConn) sendWindowUpdate32(st *stream, n int32) { + sc.serveG.check() + if n == 0 { + return + } + if n < 0 { + panic("negative update") + } + var streamID uint32 + if st != nil { + streamID = st.id + } + sc.writeFrame(frameWriteMsg{ + write: writeWindowUpdate{streamID: streamID, n: uint32(n)}, + stream: st, + }) + var ok bool + if st == nil { + ok = sc.inflow.add(n) + } else { + ok = st.inflow.add(n) + } + if !ok { + panic("internal error; sent too many window updates without decrements?") + } +} + +type requestBody struct { + stream *stream + conn *serverConn + closed bool + pipe *pipe // non-nil if we have a HTTP entity message body + needsContinue bool // need to send a 100-continue +} + +func (b *requestBody) Close() error { + if b.pipe != nil { + b.pipe.Close(errClosedBody) + } + b.closed = true + return nil +} + +func (b *requestBody) Read(p []byte) (n int, err error) { + if b.needsContinue { + b.needsContinue = false + b.conn.write100ContinueHeaders(b.stream) + } + if b.pipe == nil { + return 0, io.EOF + } + n, err = b.pipe.Read(p) + if n > 0 { + b.conn.noteBodyReadFromHandler(b.stream, n) + } + return +} + +// responseWriter is the http.ResponseWriter implementation. It's +// intentionally small (1 pointer wide) to minimize garbage. The +// responseWriterState pointer inside is zeroed at the end of a +// request (in handlerDone) and calls on the responseWriter thereafter +// simply crash (caller's mistake), but the much larger responseWriterState +// and buffers are reused between multiple requests. +type responseWriter struct { + rws *responseWriterState +} + +// Optional http.ResponseWriter interfaces implemented. +var ( + _ http.CloseNotifier = (*responseWriter)(nil) + _ http.Flusher = (*responseWriter)(nil) + _ stringWriter = (*responseWriter)(nil) +) + +type responseWriterState struct { + // immutable within a request: + stream *stream + req *http.Request + body *requestBody // to close at end of request, if DATA frames didn't + conn *serverConn + + // TODO: adjust buffer writing sizes based on server config, frame size updates from peer, etc + bw *bufio.Writer // writing to a chunkWriter{this *responseWriterState} + + // mutated by http.Handler goroutine: + handlerHeader http.Header // nil until called + snapHeader http.Header // snapshot of handlerHeader at WriteHeader time + status int // status code passed to WriteHeader + wroteHeader bool // WriteHeader called (explicitly or implicitly). Not necessarily sent to user yet. + sentHeader bool // have we sent the header frame? + handlerDone bool // handler has finished + curWrite writeData + frameWriteCh chan error // re-used whenever we need to block on a frame being written + + closeNotifierMu sync.Mutex // guards closeNotifierCh + closeNotifierCh chan bool // nil until first used +} + +type chunkWriter struct{ rws *responseWriterState } + +func (cw chunkWriter) Write(p []byte) (n int, err error) { return cw.rws.writeChunk(p) } + +// writeChunk writes chunks from the bufio.Writer. But because +// bufio.Writer may bypass its chunking, sometimes p may be +// arbitrarily large. +// +// writeChunk is also responsible (on the first chunk) for sending the +// HEADER response. +func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) { + if !rws.wroteHeader { + rws.writeHeader(200) + } + if !rws.sentHeader { + rws.sentHeader = true + var ctype, clen string // implicit ones, if we can calculate it + if rws.handlerDone && rws.snapHeader.Get("Content-Length") == "" { + clen = strconv.Itoa(len(p)) + } + if rws.snapHeader.Get("Content-Type") == "" { + ctype = http.DetectContentType(p) + } + endStream := rws.handlerDone && len(p) == 0 + rws.conn.writeHeaders(rws.stream, &writeResHeaders{ + streamID: rws.stream.id, + httpResCode: rws.status, + h: rws.snapHeader, + endStream: endStream, + contentType: ctype, + contentLength: clen, + }, rws.frameWriteCh) + if endStream { + return 0, nil + } + } + if len(p) == 0 && !rws.handlerDone { + return 0, nil + } + curWrite := &rws.curWrite + curWrite.streamID = rws.stream.id + curWrite.p = p + curWrite.endStream = rws.handlerDone + if err := rws.conn.writeDataFromHandler(rws.stream, curWrite, rws.frameWriteCh); err != nil { + return 0, err + } + return len(p), nil +} + +func (w *responseWriter) Flush() { + rws := w.rws + if rws == nil { + panic("Header called after Handler finished") + } + if rws.bw.Buffered() > 0 { + if err := rws.bw.Flush(); err != nil { + // Ignore the error. The frame writer already knows. + return + } + } else { + // The bufio.Writer won't call chunkWriter.Write + // (writeChunk with zero bytes, so we have to do it + // ourselves to force the HTTP response header and/or + // final DATA frame (with END_STREAM) to be sent. + rws.writeChunk(nil) + } +} + +func (w *responseWriter) CloseNotify() <-chan bool { + rws := w.rws + if rws == nil { + panic("CloseNotify called after Handler finished") + } + rws.closeNotifierMu.Lock() + ch := rws.closeNotifierCh + if ch == nil { + ch = make(chan bool, 1) + rws.closeNotifierCh = ch + go func() { + rws.stream.cw.Wait() // wait for close + ch <- true + }() + } + rws.closeNotifierMu.Unlock() + return ch +} + +func (w *responseWriter) Header() http.Header { + rws := w.rws + if rws == nil { + panic("Header called after Handler finished") + } + if rws.handlerHeader == nil { + rws.handlerHeader = make(http.Header) + } + return rws.handlerHeader +} + +func (w *responseWriter) WriteHeader(code int) { + rws := w.rws + if rws == nil { + panic("WriteHeader called after Handler finished") + } + rws.writeHeader(code) +} + +func (rws *responseWriterState) writeHeader(code int) { + if !rws.wroteHeader { + rws.wroteHeader = true + rws.status = code + if len(rws.handlerHeader) > 0 { + rws.snapHeader = cloneHeader(rws.handlerHeader) + } + } +} + +func cloneHeader(h http.Header) http.Header { + h2 := make(http.Header, len(h)) + for k, vv := range h { + vv2 := make([]string, len(vv)) + copy(vv2, vv) + h2[k] = vv2 + } + return h2 +} + +// The Life Of A Write is like this: +// +// * Handler calls w.Write or w.WriteString -> +// * -> rws.bw (*bufio.Writer) -> +// * (Handler migth call Flush) +// * -> chunkWriter{rws} +// * -> responseWriterState.writeChunk(p []byte) +// * -> responseWriterState.writeChunk (most of the magic; see comment there) +func (w *responseWriter) Write(p []byte) (n int, err error) { + return w.write(len(p), p, "") +} + +func (w *responseWriter) WriteString(s string) (n int, err error) { + return w.write(len(s), nil, s) +} + +// either dataB or dataS is non-zero. +func (w *responseWriter) write(lenData int, dataB []byte, dataS string) (n int, err error) { + rws := w.rws + if rws == nil { + panic("Write called after Handler finished") + } + if !rws.wroteHeader { + w.WriteHeader(200) + } + if dataB != nil { + return rws.bw.Write(dataB) + } else { + return rws.bw.WriteString(dataS) + } +} + +func (w *responseWriter) handlerDone() { + rws := w.rws + if rws == nil { + panic("handlerDone called twice") + } + rws.handlerDone = true + w.Flush() + w.rws = nil + responseWriterStatePool.Put(rws) +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/transport.go b/components/engine/vendor/src/golang.org/x/net/http2/transport.go new file mode 100644 index 0000000000..73f358eefe --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/transport.go @@ -0,0 +1,553 @@ +// Copyright 2015 The Go Authors. +// See https://go.googlesource.com/go/+/master/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://go.googlesource.com/go/+/master/LICENSE + +package http2 + +import ( + "bufio" + "bytes" + "crypto/tls" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "strconv" + "strings" + "sync" + + "golang.org/x/net/http2/hpack" +) + +type Transport struct { + Fallback http.RoundTripper + + // TODO: remove this and make more general with a TLS dial hook, like http + InsecureTLSDial bool + + connMu sync.Mutex + conns map[string][]*clientConn // key is host:port +} + +type clientConn struct { + t *Transport + tconn *tls.Conn + tlsState *tls.ConnectionState + connKey []string // key(s) this connection is cached in, in t.conns + + readerDone chan struct{} // closed on error + readerErr error // set before readerDone is closed + hdec *hpack.Decoder + nextRes *http.Response + + mu sync.Mutex + closed bool + goAway *GoAwayFrame // if non-nil, the GoAwayFrame we received + streams map[uint32]*clientStream + nextStreamID uint32 + bw *bufio.Writer + werr error // first write error that has occurred + br *bufio.Reader + fr *Framer + // Settings from peer: + maxFrameSize uint32 + maxConcurrentStreams uint32 + initialWindowSize uint32 + hbuf bytes.Buffer // HPACK encoder writes into this + henc *hpack.Encoder +} + +type clientStream struct { + ID uint32 + resc chan resAndError + pw *io.PipeWriter + pr *io.PipeReader +} + +type stickyErrWriter struct { + w io.Writer + err *error +} + +func (sew stickyErrWriter) Write(p []byte) (n int, err error) { + if *sew.err != nil { + return 0, *sew.err + } + n, err = sew.w.Write(p) + *sew.err = err + return +} + +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL.Scheme != "https" { + if t.Fallback == nil { + return nil, errors.New("http2: unsupported scheme and no Fallback") + } + return t.Fallback.RoundTrip(req) + } + + host, port, err := net.SplitHostPort(req.URL.Host) + if err != nil { + host = req.URL.Host + port = "443" + } + + for { + cc, err := t.getClientConn(host, port) + if err != nil { + return nil, err + } + res, err := cc.roundTrip(req) + if shouldRetryRequest(err) { // TODO: or clientconn is overloaded (too many outstanding requests)? + continue + } + if err != nil { + return nil, err + } + return res, nil + } +} + +// CloseIdleConnections closes any connections which were previously +// connected from previous requests but are now sitting idle. +// It does not interrupt any connections currently in use. +func (t *Transport) CloseIdleConnections() { + t.connMu.Lock() + defer t.connMu.Unlock() + for _, vv := range t.conns { + for _, cc := range vv { + cc.closeIfIdle() + } + } +} + +var errClientConnClosed = errors.New("http2: client conn is closed") + +func shouldRetryRequest(err error) bool { + // TODO: or GOAWAY graceful shutdown stuff + return err == errClientConnClosed +} + +func (t *Transport) removeClientConn(cc *clientConn) { + t.connMu.Lock() + defer t.connMu.Unlock() + for _, key := range cc.connKey { + vv, ok := t.conns[key] + if !ok { + continue + } + newList := filterOutClientConn(vv, cc) + if len(newList) > 0 { + t.conns[key] = newList + } else { + delete(t.conns, key) + } + } +} + +func filterOutClientConn(in []*clientConn, exclude *clientConn) []*clientConn { + out := in[:0] + for _, v := range in { + if v != exclude { + out = append(out, v) + } + } + return out +} + +func (t *Transport) getClientConn(host, port string) (*clientConn, error) { + t.connMu.Lock() + defer t.connMu.Unlock() + + key := net.JoinHostPort(host, port) + + for _, cc := range t.conns[key] { + if cc.canTakeNewRequest() { + return cc, nil + } + } + if t.conns == nil { + t.conns = make(map[string][]*clientConn) + } + cc, err := t.newClientConn(host, port, key) + if err != nil { + return nil, err + } + t.conns[key] = append(t.conns[key], cc) + return cc, nil +} + +func (t *Transport) newClientConn(host, port, key string) (*clientConn, error) { + cfg := &tls.Config{ + ServerName: host, + NextProtos: []string{NextProtoTLS}, + InsecureSkipVerify: t.InsecureTLSDial, + } + tconn, err := tls.Dial("tcp", net.JoinHostPort(host, port), cfg) + if err != nil { + return nil, err + } + if err := tconn.Handshake(); err != nil { + return nil, err + } + if !t.InsecureTLSDial { + if err := tconn.VerifyHostname(cfg.ServerName); err != nil { + return nil, err + } + } + state := tconn.ConnectionState() + if p := state.NegotiatedProtocol; p != NextProtoTLS { + // TODO(bradfitz): fall back to Fallback + return nil, fmt.Errorf("bad protocol: %v", p) + } + if !state.NegotiatedProtocolIsMutual { + return nil, errors.New("could not negotiate protocol mutually") + } + if _, err := tconn.Write(clientPreface); err != nil { + return nil, err + } + + cc := &clientConn{ + t: t, + tconn: tconn, + connKey: []string{key}, // TODO: cert's validated hostnames too + tlsState: &state, + readerDone: make(chan struct{}), + nextStreamID: 1, + maxFrameSize: 16 << 10, // spec default + initialWindowSize: 65535, // spec default + maxConcurrentStreams: 1000, // "infinite", per spec. 1000 seems good enough. + streams: make(map[uint32]*clientStream), + } + cc.bw = bufio.NewWriter(stickyErrWriter{tconn, &cc.werr}) + cc.br = bufio.NewReader(tconn) + cc.fr = NewFramer(cc.bw, cc.br) + cc.henc = hpack.NewEncoder(&cc.hbuf) + + cc.fr.WriteSettings() + // TODO: re-send more conn-level flow control tokens when server uses all these. + cc.fr.WriteWindowUpdate(0, 1<<30) // um, 0x7fffffff doesn't work to Google? it hangs? + cc.bw.Flush() + if cc.werr != nil { + return nil, cc.werr + } + + // Read the obligatory SETTINGS frame + f, err := cc.fr.ReadFrame() + if err != nil { + return nil, err + } + sf, ok := f.(*SettingsFrame) + if !ok { + return nil, fmt.Errorf("expected settings frame, got: %T", f) + } + cc.fr.WriteSettingsAck() + cc.bw.Flush() + + sf.ForeachSetting(func(s Setting) error { + switch s.ID { + case SettingMaxFrameSize: + cc.maxFrameSize = s.Val + case SettingMaxConcurrentStreams: + cc.maxConcurrentStreams = s.Val + case SettingInitialWindowSize: + cc.initialWindowSize = s.Val + default: + // TODO(bradfitz): handle more + log.Printf("Unhandled Setting: %v", s) + } + return nil + }) + // TODO: figure out henc size + cc.hdec = hpack.NewDecoder(initialHeaderTableSize, cc.onNewHeaderField) + + go cc.readLoop() + return cc, nil +} + +func (cc *clientConn) setGoAway(f *GoAwayFrame) { + cc.mu.Lock() + defer cc.mu.Unlock() + cc.goAway = f +} + +func (cc *clientConn) canTakeNewRequest() bool { + cc.mu.Lock() + defer cc.mu.Unlock() + return cc.goAway == nil && + int64(len(cc.streams)+1) < int64(cc.maxConcurrentStreams) && + cc.nextStreamID < 2147483647 +} + +func (cc *clientConn) closeIfIdle() { + cc.mu.Lock() + if len(cc.streams) > 0 { + cc.mu.Unlock() + return + } + cc.closed = true + // TODO: do clients send GOAWAY too? maybe? Just Close: + cc.mu.Unlock() + + cc.tconn.Close() +} + +func (cc *clientConn) roundTrip(req *http.Request) (*http.Response, error) { + cc.mu.Lock() + + if cc.closed { + cc.mu.Unlock() + return nil, errClientConnClosed + } + + cs := cc.newStream() + hasBody := false // TODO + + // we send: HEADERS[+CONTINUATION] + (DATA?) + hdrs := cc.encodeHeaders(req) + first := true + for len(hdrs) > 0 { + chunk := hdrs + if len(chunk) > int(cc.maxFrameSize) { + chunk = chunk[:cc.maxFrameSize] + } + hdrs = hdrs[len(chunk):] + endHeaders := len(hdrs) == 0 + if first { + cc.fr.WriteHeaders(HeadersFrameParam{ + StreamID: cs.ID, + BlockFragment: chunk, + EndStream: !hasBody, + EndHeaders: endHeaders, + }) + first = false + } else { + cc.fr.WriteContinuation(cs.ID, endHeaders, chunk) + } + } + cc.bw.Flush() + werr := cc.werr + cc.mu.Unlock() + + if hasBody { + // TODO: write data. and it should probably be interleaved: + // go ... io.Copy(dataFrameWriter{cc, cs, ...}, req.Body) ... etc + } + + if werr != nil { + return nil, werr + } + + re := <-cs.resc + if re.err != nil { + return nil, re.err + } + res := re.res + res.Request = req + res.TLS = cc.tlsState + return res, nil +} + +// requires cc.mu be held. +func (cc *clientConn) encodeHeaders(req *http.Request) []byte { + cc.hbuf.Reset() + + // TODO(bradfitz): figure out :authority-vs-Host stuff between http2 and Go + host := req.Host + if host == "" { + host = req.URL.Host + } + + path := req.URL.Path + if path == "" { + path = "/" + } + + cc.writeHeader(":authority", host) // probably not right for all sites + cc.writeHeader(":method", req.Method) + cc.writeHeader(":path", path) + cc.writeHeader(":scheme", "https") + + for k, vv := range req.Header { + lowKey := strings.ToLower(k) + if lowKey == "host" { + continue + } + for _, v := range vv { + cc.writeHeader(lowKey, v) + } + } + return cc.hbuf.Bytes() +} + +func (cc *clientConn) writeHeader(name, value string) { + log.Printf("sending %q = %q", name, value) + cc.henc.WriteField(hpack.HeaderField{Name: name, Value: value}) +} + +type resAndError struct { + res *http.Response + err error +} + +// requires cc.mu be held. +func (cc *clientConn) newStream() *clientStream { + cs := &clientStream{ + ID: cc.nextStreamID, + resc: make(chan resAndError, 1), + } + cc.nextStreamID += 2 + cc.streams[cs.ID] = cs + return cs +} + +func (cc *clientConn) streamByID(id uint32, andRemove bool) *clientStream { + cc.mu.Lock() + defer cc.mu.Unlock() + cs := cc.streams[id] + if andRemove { + delete(cc.streams, id) + } + return cs +} + +// runs in its own goroutine. +func (cc *clientConn) readLoop() { + defer cc.t.removeClientConn(cc) + defer close(cc.readerDone) + + activeRes := map[uint32]*clientStream{} // keyed by streamID + // Close any response bodies if the server closes prematurely. + // TODO: also do this if we've written the headers but not + // gotten a response yet. + defer func() { + err := cc.readerErr + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + for _, cs := range activeRes { + cs.pw.CloseWithError(err) + } + }() + + // continueStreamID is the stream ID we're waiting for + // continuation frames for. + var continueStreamID uint32 + + for { + f, err := cc.fr.ReadFrame() + if err != nil { + cc.readerErr = err + return + } + log.Printf("Transport received %v: %#v", f.Header(), f) + + streamID := f.Header().StreamID + + _, isContinue := f.(*ContinuationFrame) + if isContinue { + if streamID != continueStreamID { + log.Printf("Protocol violation: got CONTINUATION with id %d; want %d", streamID, continueStreamID) + cc.readerErr = ConnectionError(ErrCodeProtocol) + return + } + } else if continueStreamID != 0 { + // Continue frames need to be adjacent in the stream + // and we were in the middle of headers. + log.Printf("Protocol violation: got %T for stream %d, want CONTINUATION for %d", f, streamID, continueStreamID) + cc.readerErr = ConnectionError(ErrCodeProtocol) + return + } + + if streamID%2 == 0 { + // Ignore streams pushed from the server for now. + // These always have an even stream id. + continue + } + streamEnded := false + if ff, ok := f.(streamEnder); ok { + streamEnded = ff.StreamEnded() + } + + cs := cc.streamByID(streamID, streamEnded) + if cs == nil { + log.Printf("Received frame for untracked stream ID %d", streamID) + continue + } + + switch f := f.(type) { + case *HeadersFrame: + cc.nextRes = &http.Response{ + Proto: "HTTP/2.0", + ProtoMajor: 2, + Header: make(http.Header), + } + cs.pr, cs.pw = io.Pipe() + cc.hdec.Write(f.HeaderBlockFragment()) + case *ContinuationFrame: + cc.hdec.Write(f.HeaderBlockFragment()) + case *DataFrame: + log.Printf("DATA: %q", f.Data()) + cs.pw.Write(f.Data()) + case *GoAwayFrame: + cc.t.removeClientConn(cc) + if f.ErrCode != 0 { + // TODO: deal with GOAWAY more. particularly the error code + log.Printf("transport got GOAWAY with error code = %v", f.ErrCode) + } + cc.setGoAway(f) + default: + log.Printf("Transport: unhandled response frame type %T", f) + } + headersEnded := false + if he, ok := f.(headersEnder); ok { + headersEnded = he.HeadersEnded() + if headersEnded { + continueStreamID = 0 + } else { + continueStreamID = streamID + } + } + + if streamEnded { + cs.pw.Close() + delete(activeRes, streamID) + } + if headersEnded { + if cs == nil { + panic("couldn't find stream") // TODO be graceful + } + // TODO: set the Body to one which notes the + // Close and also sends the server a + // RST_STREAM + cc.nextRes.Body = cs.pr + res := cc.nextRes + activeRes[streamID] = cs + cs.resc <- resAndError{res: res} + } + } +} + +func (cc *clientConn) onNewHeaderField(f hpack.HeaderField) { + // TODO: verifiy pseudo headers come before non-pseudo headers + // TODO: verifiy the status is set + log.Printf("Header field: %+v", f) + if f.Name == ":status" { + code, err := strconv.Atoi(f.Value) + if err != nil { + panic("TODO: be graceful") + } + cc.nextRes.Status = f.Value + " " + http.StatusText(code) + cc.nextRes.StatusCode = code + return + } + if strings.HasPrefix(f.Name, ":") { + // "Endpoints MUST NOT generate pseudo-header fields other than those defined in this document." + // TODO: treat as invalid? + return + } + cc.nextRes.Header.Add(http.CanonicalHeaderKey(f.Name), f.Value) +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/write.go b/components/engine/vendor/src/golang.org/x/net/http2/write.go new file mode 100644 index 0000000000..02f0743de6 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/write.go @@ -0,0 +1,204 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +package http2 + +import ( + "bytes" + "fmt" + "net/http" + "time" + + "golang.org/x/net/http2/hpack" +) + +// writeFramer is implemented by any type that is used to write frames. +type writeFramer interface { + writeFrame(writeContext) error +} + +// writeContext is the interface needed by the various frame writer +// types below. All the writeFrame methods below are scheduled via the +// frame writing scheduler (see writeScheduler in writesched.go). +// +// This interface is implemented by *serverConn. +// TODO: use it from the client code too, once it exists. +type writeContext interface { + Framer() *Framer + Flush() error + CloseConn() error + // HeaderEncoder returns an HPACK encoder that writes to the + // returned buffer. + HeaderEncoder() (*hpack.Encoder, *bytes.Buffer) +} + +// endsStream reports whether the given frame writer w will locally +// close the stream. +func endsStream(w writeFramer) bool { + switch v := w.(type) { + case *writeData: + return v.endStream + case *writeResHeaders: + return v.endStream + } + return false +} + +type flushFrameWriter struct{} + +func (flushFrameWriter) writeFrame(ctx writeContext) error { + return ctx.Flush() +} + +type writeSettings []Setting + +func (s writeSettings) writeFrame(ctx writeContext) error { + return ctx.Framer().WriteSettings([]Setting(s)...) +} + +type writeGoAway struct { + maxStreamID uint32 + code ErrCode +} + +func (p *writeGoAway) writeFrame(ctx writeContext) error { + err := ctx.Framer().WriteGoAway(p.maxStreamID, p.code, nil) + if p.code != 0 { + ctx.Flush() // ignore error: we're hanging up on them anyway + time.Sleep(50 * time.Millisecond) + ctx.CloseConn() + } + return err +} + +type writeData struct { + streamID uint32 + p []byte + endStream bool +} + +func (w *writeData) String() string { + return fmt.Sprintf("writeData(stream=%d, p=%d, endStream=%v)", w.streamID, len(w.p), w.endStream) +} + +func (w *writeData) writeFrame(ctx writeContext) error { + return ctx.Framer().WriteData(w.streamID, w.endStream, w.p) +} + +func (se StreamError) writeFrame(ctx writeContext) error { + return ctx.Framer().WriteRSTStream(se.StreamID, se.Code) +} + +type writePingAck struct{ pf *PingFrame } + +func (w writePingAck) writeFrame(ctx writeContext) error { + return ctx.Framer().WritePing(true, w.pf.Data) +} + +type writeSettingsAck struct{} + +func (writeSettingsAck) writeFrame(ctx writeContext) error { + return ctx.Framer().WriteSettingsAck() +} + +// writeResHeaders is a request to write a HEADERS and 0+ CONTINUATION frames +// for HTTP response headers from a server handler. +type writeResHeaders struct { + streamID uint32 + httpResCode int + h http.Header // may be nil + endStream bool + + contentType string + contentLength string +} + +func (w *writeResHeaders) writeFrame(ctx writeContext) error { + enc, buf := ctx.HeaderEncoder() + buf.Reset() + enc.WriteField(hpack.HeaderField{Name: ":status", Value: httpCodeString(w.httpResCode)}) + for k, vv := range w.h { + k = lowerHeader(k) + for _, v := range vv { + // TODO: more of "8.1.2.2 Connection-Specific Header Fields" + if k == "transfer-encoding" && v != "trailers" { + continue + } + enc.WriteField(hpack.HeaderField{Name: k, Value: v}) + } + } + if w.contentType != "" { + enc.WriteField(hpack.HeaderField{Name: "content-type", Value: w.contentType}) + } + if w.contentLength != "" { + enc.WriteField(hpack.HeaderField{Name: "content-length", Value: w.contentLength}) + } + + headerBlock := buf.Bytes() + if len(headerBlock) == 0 { + panic("unexpected empty hpack") + } + + // For now we're lazy and just pick the minimum MAX_FRAME_SIZE + // that all peers must support (16KB). Later we could care + // more and send larger frames if the peer advertised it, but + // there's little point. Most headers are small anyway (so we + // generally won't have CONTINUATION frames), and extra frames + // only waste 9 bytes anyway. + const maxFrameSize = 16384 + + first := true + for len(headerBlock) > 0 { + frag := headerBlock + if len(frag) > maxFrameSize { + frag = frag[:maxFrameSize] + } + headerBlock = headerBlock[len(frag):] + endHeaders := len(headerBlock) == 0 + var err error + if first { + first = false + err = ctx.Framer().WriteHeaders(HeadersFrameParam{ + StreamID: w.streamID, + BlockFragment: frag, + EndStream: w.endStream, + EndHeaders: endHeaders, + }) + } else { + err = ctx.Framer().WriteContinuation(w.streamID, endHeaders, frag) + } + if err != nil { + return err + } + } + return nil +} + +type write100ContinueHeadersFrame struct { + streamID uint32 +} + +func (w write100ContinueHeadersFrame) writeFrame(ctx writeContext) error { + enc, buf := ctx.HeaderEncoder() + buf.Reset() + enc.WriteField(hpack.HeaderField{Name: ":status", Value: "100"}) + return ctx.Framer().WriteHeaders(HeadersFrameParam{ + StreamID: w.streamID, + BlockFragment: buf.Bytes(), + EndStream: false, + EndHeaders: true, + }) +} + +type writeWindowUpdate struct { + streamID uint32 // or 0 for conn-level + n uint32 +} + +func (wu writeWindowUpdate) writeFrame(ctx writeContext) error { + return ctx.Framer().WriteWindowUpdate(wu.streamID, wu.n) +} diff --git a/components/engine/vendor/src/golang.org/x/net/http2/writesched.go b/components/engine/vendor/src/golang.org/x/net/http2/writesched.go new file mode 100644 index 0000000000..0e1b7486fb --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/http2/writesched.go @@ -0,0 +1,286 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// See https://code.google.com/p/go/source/browse/CONTRIBUTORS +// Licensed under the same terms as Go itself: +// https://code.google.com/p/go/source/browse/LICENSE + +package http2 + +import "fmt" + +// frameWriteMsg is a request to write a frame. +type frameWriteMsg struct { + // write is the interface value that does the writing, once the + // writeScheduler (below) has decided to select this frame + // to write. The write functions are all defined in write.go. + write writeFramer + + stream *stream // used for prioritization. nil for non-stream frames. + + // done, if non-nil, must be a buffered channel with space for + // 1 message and is sent the return value from write (or an + // earlier error) when the frame has been written. + done chan error +} + +// for debugging only: +func (wm frameWriteMsg) String() string { + var streamID uint32 + if wm.stream != nil { + streamID = wm.stream.id + } + var des string + if s, ok := wm.write.(fmt.Stringer); ok { + des = s.String() + } else { + des = fmt.Sprintf("%T", wm.write) + } + return fmt.Sprintf("[frameWriteMsg stream=%d, ch=%v, type: %v]", streamID, wm.done != nil, des) +} + +// writeScheduler tracks pending frames to write, priorities, and decides +// the next one to use. It is not thread-safe. +type writeScheduler struct { + // zero are frames not associated with a specific stream. + // They're sent before any stream-specific freams. + zero writeQueue + + // maxFrameSize is the maximum size of a DATA frame + // we'll write. Must be non-zero and between 16K-16M. + maxFrameSize uint32 + + // sq contains the stream-specific queues, keyed by stream ID. + // when a stream is idle, it's deleted from the map. + sq map[uint32]*writeQueue + + // canSend is a slice of memory that's reused between frame + // scheduling decisions to hold the list of writeQueues (from sq) + // which have enough flow control data to send. After canSend is + // built, the best is selected. + canSend []*writeQueue + + // pool of empty queues for reuse. + queuePool []*writeQueue +} + +func (ws *writeScheduler) putEmptyQueue(q *writeQueue) { + if len(q.s) != 0 { + panic("queue must be empty") + } + ws.queuePool = append(ws.queuePool, q) +} + +func (ws *writeScheduler) getEmptyQueue() *writeQueue { + ln := len(ws.queuePool) + if ln == 0 { + return new(writeQueue) + } + q := ws.queuePool[ln-1] + ws.queuePool = ws.queuePool[:ln-1] + return q +} + +func (ws *writeScheduler) empty() bool { return ws.zero.empty() && len(ws.sq) == 0 } + +func (ws *writeScheduler) add(wm frameWriteMsg) { + st := wm.stream + if st == nil { + ws.zero.push(wm) + } else { + ws.streamQueue(st.id).push(wm) + } +} + +func (ws *writeScheduler) streamQueue(streamID uint32) *writeQueue { + if q, ok := ws.sq[streamID]; ok { + return q + } + if ws.sq == nil { + ws.sq = make(map[uint32]*writeQueue) + } + q := ws.getEmptyQueue() + ws.sq[streamID] = q + return q +} + +// take returns the most important frame to write and removes it from the scheduler. +// It is illegal to call this if the scheduler is empty or if there are no connection-level +// flow control bytes available. +func (ws *writeScheduler) take() (wm frameWriteMsg, ok bool) { + if ws.maxFrameSize == 0 { + panic("internal error: ws.maxFrameSize not initialized or invalid") + } + + // If there any frames not associated with streams, prefer those first. + // These are usually SETTINGS, etc. + if !ws.zero.empty() { + return ws.zero.shift(), true + } + if len(ws.sq) == 0 { + return + } + + // Next, prioritize frames on streams that aren't DATA frames (no cost). + for id, q := range ws.sq { + if q.firstIsNoCost() { + return ws.takeFrom(id, q) + } + } + + // Now, all that remains are DATA frames with non-zero bytes to + // send. So pick the best one. + if len(ws.canSend) != 0 { + panic("should be empty") + } + for _, q := range ws.sq { + if n := ws.streamWritableBytes(q); n > 0 { + ws.canSend = append(ws.canSend, q) + } + } + if len(ws.canSend) == 0 { + return + } + defer ws.zeroCanSend() + + // TODO: find the best queue + q := ws.canSend[0] + + return ws.takeFrom(q.streamID(), q) +} + +// zeroCanSend is defered from take. +func (ws *writeScheduler) zeroCanSend() { + for i := range ws.canSend { + ws.canSend[i] = nil + } + ws.canSend = ws.canSend[:0] +} + +// streamWritableBytes returns the number of DATA bytes we could write +// from the given queue's stream, if this stream/queue were +// selected. It is an error to call this if q's head isn't a +// *writeData. +func (ws *writeScheduler) streamWritableBytes(q *writeQueue) int32 { + wm := q.head() + ret := wm.stream.flow.available() // max we can write + if ret == 0 { + return 0 + } + if int32(ws.maxFrameSize) < ret { + ret = int32(ws.maxFrameSize) + } + if ret == 0 { + panic("internal error: ws.maxFrameSize not initialized or invalid") + } + wd := wm.write.(*writeData) + if len(wd.p) < int(ret) { + ret = int32(len(wd.p)) + } + return ret +} + +func (ws *writeScheduler) takeFrom(id uint32, q *writeQueue) (wm frameWriteMsg, ok bool) { + wm = q.head() + // If the first item in this queue costs flow control tokens + // and we don't have enough, write as much as we can. + if wd, ok := wm.write.(*writeData); ok && len(wd.p) > 0 { + allowed := wm.stream.flow.available() // max we can write + if allowed == 0 { + // No quota available. Caller can try the next stream. + return frameWriteMsg{}, false + } + if int32(ws.maxFrameSize) < allowed { + allowed = int32(ws.maxFrameSize) + } + // TODO: further restrict the allowed size, because even if + // the peer says it's okay to write 16MB data frames, we might + // want to write smaller ones to properly weight competing + // streams' priorities. + + if len(wd.p) > int(allowed) { + wm.stream.flow.take(allowed) + chunk := wd.p[:allowed] + wd.p = wd.p[allowed:] + // Make up a new write message of a valid size, rather + // than shifting one off the queue. + return frameWriteMsg{ + stream: wm.stream, + write: &writeData{ + streamID: wd.streamID, + p: chunk, + // even if the original had endStream set, there + // arebytes remaining because len(wd.p) > allowed, + // so we know endStream is false: + endStream: false, + }, + // our caller is blocking on the final DATA frame, not + // these intermediates, so no need to wait: + done: nil, + }, true + } + wm.stream.flow.take(int32(len(wd.p))) + } + + q.shift() + if q.empty() { + ws.putEmptyQueue(q) + delete(ws.sq, id) + } + return wm, true +} + +func (ws *writeScheduler) forgetStream(id uint32) { + q, ok := ws.sq[id] + if !ok { + return + } + delete(ws.sq, id) + + // But keep it for others later. + for i := range q.s { + q.s[i] = frameWriteMsg{} + } + q.s = q.s[:0] + ws.putEmptyQueue(q) +} + +type writeQueue struct { + s []frameWriteMsg +} + +// streamID returns the stream ID for a non-empty stream-specific queue. +func (q *writeQueue) streamID() uint32 { return q.s[0].stream.id } + +func (q *writeQueue) empty() bool { return len(q.s) == 0 } + +func (q *writeQueue) push(wm frameWriteMsg) { + q.s = append(q.s, wm) +} + +// head returns the next item that would be removed by shift. +func (q *writeQueue) head() frameWriteMsg { + if len(q.s) == 0 { + panic("invalid use of queue") + } + return q.s[0] +} + +func (q *writeQueue) shift() frameWriteMsg { + if len(q.s) == 0 { + panic("invalid use of queue") + } + wm := q.s[0] + // TODO: less copy-happy queue. + copy(q.s, q.s[1:]) + q.s[len(q.s)-1] = frameWriteMsg{} + q.s = q.s[:len(q.s)-1] + return wm +} + +func (q *writeQueue) firstIsNoCost() bool { + if df, ok := q.s[0].write.(*writeData); ok { + return len(df.p) == 0 + } + return true +} diff --git a/components/engine/vendor/src/golang.org/x/net/internal/timeseries/timeseries.go b/components/engine/vendor/src/golang.org/x/net/internal/timeseries/timeseries.go new file mode 100644 index 0000000000..1119f34482 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/internal/timeseries/timeseries.go @@ -0,0 +1,525 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package timeseries implements a time series structure for stats collection. +package timeseries // import "golang.org/x/net/internal/timeseries" + +import ( + "fmt" + "log" + "time" +) + +const ( + timeSeriesNumBuckets = 64 + minuteHourSeriesNumBuckets = 60 +) + +var timeSeriesResolutions = []time.Duration{ + 1 * time.Second, + 10 * time.Second, + 1 * time.Minute, + 10 * time.Minute, + 1 * time.Hour, + 6 * time.Hour, + 24 * time.Hour, // 1 day + 7 * 24 * time.Hour, // 1 week + 4 * 7 * 24 * time.Hour, // 4 weeks + 16 * 7 * 24 * time.Hour, // 16 weeks +} + +var minuteHourSeriesResolutions = []time.Duration{ + 1 * time.Second, + 1 * time.Minute, +} + +// An Observable is a kind of data that can be aggregated in a time series. +type Observable interface { + Multiply(ratio float64) // Multiplies the data in self by a given ratio + Add(other Observable) // Adds the data from a different observation to self + Clear() // Clears the observation so it can be reused. + CopyFrom(other Observable) // Copies the contents of a given observation to self +} + +// Float attaches the methods of Observable to a float64. +type Float float64 + +// NewFloat returns a Float. +func NewFloat() Observable { + f := Float(0) + return &f +} + +// String returns the float as a string. +func (f *Float) String() string { return fmt.Sprintf("%g", f.Value()) } + +// Value returns the float's value. +func (f *Float) Value() float64 { return float64(*f) } + +func (f *Float) Multiply(ratio float64) { *f *= Float(ratio) } + +func (f *Float) Add(other Observable) { + o := other.(*Float) + *f += *o +} + +func (f *Float) Clear() { *f = 0 } + +func (f *Float) CopyFrom(other Observable) { + o := other.(*Float) + *f = *o +} + +// A Clock tells the current time. +type Clock interface { + Time() time.Time +} + +type defaultClock int + +var defaultClockInstance defaultClock + +func (defaultClock) Time() time.Time { return time.Now() } + +// Information kept per level. Each level consists of a circular list of +// observations. The start of the level may be derived from end and the +// len(buckets) * sizeInMillis. +type tsLevel struct { + oldest int // index to oldest bucketed Observable + newest int // index to newest bucketed Observable + end time.Time // end timestamp for this level + size time.Duration // duration of the bucketed Observable + buckets []Observable // collections of observations + provider func() Observable // used for creating new Observable +} + +func (l *tsLevel) Clear() { + l.oldest = 0 + l.newest = len(l.buckets) - 1 + l.end = time.Time{} + for i := range l.buckets { + if l.buckets[i] != nil { + l.buckets[i].Clear() + l.buckets[i] = nil + } + } +} + +func (l *tsLevel) InitLevel(size time.Duration, numBuckets int, f func() Observable) { + l.size = size + l.provider = f + l.buckets = make([]Observable, numBuckets) +} + +// Keeps a sequence of levels. Each level is responsible for storing data at +// a given resolution. For example, the first level stores data at a one +// minute resolution while the second level stores data at a one hour +// resolution. + +// Each level is represented by a sequence of buckets. Each bucket spans an +// interval equal to the resolution of the level. New observations are added +// to the last bucket. +type timeSeries struct { + provider func() Observable // make more Observable + numBuckets int // number of buckets in each level + levels []*tsLevel // levels of bucketed Observable + lastAdd time.Time // time of last Observable tracked + total Observable // convenient aggregation of all Observable + clock Clock // Clock for getting current time + pending Observable // observations not yet bucketed + pendingTime time.Time // what time are we keeping in pending + dirty bool // if there are pending observations +} + +// init initializes a level according to the supplied criteria. +func (ts *timeSeries) init(resolutions []time.Duration, f func() Observable, numBuckets int, clock Clock) { + ts.provider = f + ts.numBuckets = numBuckets + ts.clock = clock + ts.levels = make([]*tsLevel, len(resolutions)) + + for i := range resolutions { + if i > 0 && resolutions[i-1] >= resolutions[i] { + log.Print("timeseries: resolutions must be monotonically increasing") + break + } + newLevel := new(tsLevel) + newLevel.InitLevel(resolutions[i], ts.numBuckets, ts.provider) + ts.levels[i] = newLevel + } + + ts.Clear() +} + +// Clear removes all observations from the time series. +func (ts *timeSeries) Clear() { + ts.lastAdd = time.Time{} + ts.total = ts.resetObservation(ts.total) + ts.pending = ts.resetObservation(ts.pending) + ts.pendingTime = time.Time{} + ts.dirty = false + + for i := range ts.levels { + ts.levels[i].Clear() + } +} + +// Add records an observation at the current time. +func (ts *timeSeries) Add(observation Observable) { + ts.AddWithTime(observation, ts.clock.Time()) +} + +// AddWithTime records an observation at the specified time. +func (ts *timeSeries) AddWithTime(observation Observable, t time.Time) { + + smallBucketDuration := ts.levels[0].size + + if t.After(ts.lastAdd) { + ts.lastAdd = t + } + + if t.After(ts.pendingTime) { + ts.advance(t) + ts.mergePendingUpdates() + ts.pendingTime = ts.levels[0].end + ts.pending.CopyFrom(observation) + ts.dirty = true + } else if t.After(ts.pendingTime.Add(-1 * smallBucketDuration)) { + // The observation is close enough to go into the pending bucket. + // This compensates for clock skewing and small scheduling delays + // by letting the update stay in the fast path. + ts.pending.Add(observation) + ts.dirty = true + } else { + ts.mergeValue(observation, t) + } +} + +// mergeValue inserts the observation at the specified time in the past into all levels. +func (ts *timeSeries) mergeValue(observation Observable, t time.Time) { + for _, level := range ts.levels { + index := (ts.numBuckets - 1) - int(level.end.Sub(t)/level.size) + if 0 <= index && index < ts.numBuckets { + bucketNumber := (level.oldest + index) % ts.numBuckets + if level.buckets[bucketNumber] == nil { + level.buckets[bucketNumber] = level.provider() + } + level.buckets[bucketNumber].Add(observation) + } + } + ts.total.Add(observation) +} + +// mergePendingUpdates applies the pending updates into all levels. +func (ts *timeSeries) mergePendingUpdates() { + if ts.dirty { + ts.mergeValue(ts.pending, ts.pendingTime) + ts.pending = ts.resetObservation(ts.pending) + ts.dirty = false + } +} + +// advance cycles the buckets at each level until the latest bucket in +// each level can hold the time specified. +func (ts *timeSeries) advance(t time.Time) { + if !t.After(ts.levels[0].end) { + return + } + for i := 0; i < len(ts.levels); i++ { + level := ts.levels[i] + if !level.end.Before(t) { + break + } + + // If the time is sufficiently far, just clear the level and advance + // directly. + if !t.Before(level.end.Add(level.size * time.Duration(ts.numBuckets))) { + for _, b := range level.buckets { + ts.resetObservation(b) + } + level.end = time.Unix(0, (t.UnixNano()/level.size.Nanoseconds())*level.size.Nanoseconds()) + } + + for t.After(level.end) { + level.end = level.end.Add(level.size) + level.newest = level.oldest + level.oldest = (level.oldest + 1) % ts.numBuckets + ts.resetObservation(level.buckets[level.newest]) + } + + t = level.end + } +} + +// Latest returns the sum of the num latest buckets from the level. +func (ts *timeSeries) Latest(level, num int) Observable { + now := ts.clock.Time() + if ts.levels[0].end.Before(now) { + ts.advance(now) + } + + ts.mergePendingUpdates() + + result := ts.provider() + l := ts.levels[level] + index := l.newest + + for i := 0; i < num; i++ { + if l.buckets[index] != nil { + result.Add(l.buckets[index]) + } + if index == 0 { + index = ts.numBuckets + } + index-- + } + + return result +} + +// LatestBuckets returns a copy of the num latest buckets from level. +func (ts *timeSeries) LatestBuckets(level, num int) []Observable { + if level < 0 || level > len(ts.levels) { + log.Print("timeseries: bad level argument: ", level) + return nil + } + if num < 0 || num >= ts.numBuckets { + log.Print("timeseries: bad num argument: ", num) + return nil + } + + results := make([]Observable, num) + now := ts.clock.Time() + if ts.levels[0].end.Before(now) { + ts.advance(now) + } + + ts.mergePendingUpdates() + + l := ts.levels[level] + index := l.newest + + for i := 0; i < num; i++ { + result := ts.provider() + results[i] = result + if l.buckets[index] != nil { + result.CopyFrom(l.buckets[index]) + } + + if index == 0 { + index = ts.numBuckets + } + index -= 1 + } + return results +} + +// ScaleBy updates observations by scaling by factor. +func (ts *timeSeries) ScaleBy(factor float64) { + for _, l := range ts.levels { + for i := 0; i < ts.numBuckets; i++ { + l.buckets[i].Multiply(factor) + } + } + + ts.total.Multiply(factor) + ts.pending.Multiply(factor) +} + +// Range returns the sum of observations added over the specified time range. +// If start or finish times don't fall on bucket boundaries of the same +// level, then return values are approximate answers. +func (ts *timeSeries) Range(start, finish time.Time) Observable { + return ts.ComputeRange(start, finish, 1)[0] +} + +// Recent returns the sum of observations from the last delta. +func (ts *timeSeries) Recent(delta time.Duration) Observable { + now := ts.clock.Time() + return ts.Range(now.Add(-delta), now) +} + +// Total returns the total of all observations. +func (ts *timeSeries) Total() Observable { + ts.mergePendingUpdates() + return ts.total +} + +// ComputeRange computes a specified number of values into a slice using +// the observations recorded over the specified time period. The return +// values are approximate if the start or finish times don't fall on the +// bucket boundaries at the same level or if the number of buckets spanning +// the range is not an integral multiple of num. +func (ts *timeSeries) ComputeRange(start, finish time.Time, num int) []Observable { + if start.After(finish) { + log.Printf("timeseries: start > finish, %v>%v", start, finish) + return nil + } + + if num < 0 { + log.Printf("timeseries: num < 0, %v", num) + return nil + } + + results := make([]Observable, num) + + for _, l := range ts.levels { + if !start.Before(l.end.Add(-l.size * time.Duration(ts.numBuckets))) { + ts.extract(l, start, finish, num, results) + return results + } + } + + // Failed to find a level that covers the desired range. So just + // extract from the last level, even if it doesn't cover the entire + // desired range. + ts.extract(ts.levels[len(ts.levels)-1], start, finish, num, results) + + return results +} + +// RecentList returns the specified number of values in slice over the most +// recent time period of the specified range. +func (ts *timeSeries) RecentList(delta time.Duration, num int) []Observable { + if delta < 0 { + return nil + } + now := ts.clock.Time() + return ts.ComputeRange(now.Add(-delta), now, num) +} + +// extract returns a slice of specified number of observations from a given +// level over a given range. +func (ts *timeSeries) extract(l *tsLevel, start, finish time.Time, num int, results []Observable) { + ts.mergePendingUpdates() + + srcInterval := l.size + dstInterval := finish.Sub(start) / time.Duration(num) + dstStart := start + srcStart := l.end.Add(-srcInterval * time.Duration(ts.numBuckets)) + + srcIndex := 0 + + // Where should scanning start? + if dstStart.After(srcStart) { + advance := dstStart.Sub(srcStart) / srcInterval + srcIndex += int(advance) + srcStart = srcStart.Add(advance * srcInterval) + } + + // The i'th value is computed as show below. + // interval = (finish/start)/num + // i'th value = sum of observation in range + // [ start + i * interval, + // start + (i + 1) * interval ) + for i := 0; i < num; i++ { + results[i] = ts.resetObservation(results[i]) + dstEnd := dstStart.Add(dstInterval) + for srcIndex < ts.numBuckets && srcStart.Before(dstEnd) { + srcEnd := srcStart.Add(srcInterval) + if srcEnd.After(ts.lastAdd) { + srcEnd = ts.lastAdd + } + + if !srcEnd.Before(dstStart) { + srcValue := l.buckets[(srcIndex+l.oldest)%ts.numBuckets] + if !srcStart.Before(dstStart) && !srcEnd.After(dstEnd) { + // dst completely contains src. + if srcValue != nil { + results[i].Add(srcValue) + } + } else { + // dst partially overlaps src. + overlapStart := maxTime(srcStart, dstStart) + overlapEnd := minTime(srcEnd, dstEnd) + base := srcEnd.Sub(srcStart) + fraction := overlapEnd.Sub(overlapStart).Seconds() / base.Seconds() + + used := ts.provider() + if srcValue != nil { + used.CopyFrom(srcValue) + } + used.Multiply(fraction) + results[i].Add(used) + } + + if srcEnd.After(dstEnd) { + break + } + } + srcIndex++ + srcStart = srcStart.Add(srcInterval) + } + dstStart = dstStart.Add(dstInterval) + } +} + +// resetObservation clears the content so the struct may be reused. +func (ts *timeSeries) resetObservation(observation Observable) Observable { + if observation == nil { + observation = ts.provider() + } else { + observation.Clear() + } + return observation +} + +// TimeSeries tracks data at granularities from 1 second to 16 weeks. +type TimeSeries struct { + timeSeries +} + +// NewTimeSeries creates a new TimeSeries using the function provided for creating new Observable. +func NewTimeSeries(f func() Observable) *TimeSeries { + return NewTimeSeriesWithClock(f, defaultClockInstance) +} + +// NewTimeSeriesWithClock creates a new TimeSeries using the function provided for creating new Observable and the clock for +// assigning timestamps. +func NewTimeSeriesWithClock(f func() Observable, clock Clock) *TimeSeries { + ts := new(TimeSeries) + ts.timeSeries.init(timeSeriesResolutions, f, timeSeriesNumBuckets, clock) + return ts +} + +// MinuteHourSeries tracks data at granularities of 1 minute and 1 hour. +type MinuteHourSeries struct { + timeSeries +} + +// NewMinuteHourSeries creates a new MinuteHourSeries using the function provided for creating new Observable. +func NewMinuteHourSeries(f func() Observable) *MinuteHourSeries { + return NewMinuteHourSeriesWithClock(f, defaultClockInstance) +} + +// NewMinuteHourSeriesWithClock creates a new MinuteHourSeries using the function provided for creating new Observable and the clock for +// assigning timestamps. +func NewMinuteHourSeriesWithClock(f func() Observable, clock Clock) *MinuteHourSeries { + ts := new(MinuteHourSeries) + ts.timeSeries.init(minuteHourSeriesResolutions, f, + minuteHourSeriesNumBuckets, clock) + return ts +} + +func (ts *MinuteHourSeries) Minute() Observable { + return ts.timeSeries.Latest(0, 60) +} + +func (ts *MinuteHourSeries) Hour() Observable { + return ts.timeSeries.Latest(1, 60) +} + +func minTime(a, b time.Time) time.Time { + if a.Before(b) { + return a + } + return b +} + +func maxTime(a, b time.Time) time.Time { + if a.After(b) { + return a + } + return b +} diff --git a/components/engine/vendor/src/golang.org/x/net/proxy/direct.go b/components/engine/vendor/src/golang.org/x/net/proxy/direct.go new file mode 100644 index 0000000000..4c5ad88b1e --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/proxy/direct.go @@ -0,0 +1,18 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "net" +) + +type direct struct{} + +// Direct is a direct proxy: one that makes network connections directly. +var Direct = direct{} + +func (direct) Dial(network, addr string) (net.Conn, error) { + return net.Dial(network, addr) +} diff --git a/components/engine/vendor/src/golang.org/x/net/proxy/per_host.go b/components/engine/vendor/src/golang.org/x/net/proxy/per_host.go new file mode 100644 index 0000000000..f540b196f7 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/proxy/per_host.go @@ -0,0 +1,140 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "net" + "strings" +) + +// A PerHost directs connections to a default Dialer unless the hostname +// requested matches one of a number of exceptions. +type PerHost struct { + def, bypass Dialer + + bypassNetworks []*net.IPNet + bypassIPs []net.IP + bypassZones []string + bypassHosts []string +} + +// NewPerHost returns a PerHost Dialer that directs connections to either +// defaultDialer or bypass, depending on whether the connection matches one of +// the configured rules. +func NewPerHost(defaultDialer, bypass Dialer) *PerHost { + return &PerHost{ + def: defaultDialer, + bypass: bypass, + } +} + +// Dial connects to the address addr on the given network through either +// defaultDialer or bypass. +func (p *PerHost) Dial(network, addr string) (c net.Conn, err error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + + return p.dialerForRequest(host).Dial(network, addr) +} + +func (p *PerHost) dialerForRequest(host string) Dialer { + if ip := net.ParseIP(host); ip != nil { + for _, net := range p.bypassNetworks { + if net.Contains(ip) { + return p.bypass + } + } + for _, bypassIP := range p.bypassIPs { + if bypassIP.Equal(ip) { + return p.bypass + } + } + return p.def + } + + for _, zone := range p.bypassZones { + if strings.HasSuffix(host, zone) { + return p.bypass + } + if host == zone[1:] { + // For a zone "example.com", we match "example.com" + // too. + return p.bypass + } + } + for _, bypassHost := range p.bypassHosts { + if bypassHost == host { + return p.bypass + } + } + return p.def +} + +// AddFromString parses a string that contains comma-separated values +// specifying hosts that should use the bypass proxy. Each value is either an +// IP address, a CIDR range, a zone (*.example.com) or a hostname +// (localhost). A best effort is made to parse the string and errors are +// ignored. +func (p *PerHost) AddFromString(s string) { + hosts := strings.Split(s, ",") + for _, host := range hosts { + host = strings.TrimSpace(host) + if len(host) == 0 { + continue + } + if strings.Contains(host, "/") { + // We assume that it's a CIDR address like 127.0.0.0/8 + if _, net, err := net.ParseCIDR(host); err == nil { + p.AddNetwork(net) + } + continue + } + if ip := net.ParseIP(host); ip != nil { + p.AddIP(ip) + continue + } + if strings.HasPrefix(host, "*.") { + p.AddZone(host[1:]) + continue + } + p.AddHost(host) + } +} + +// AddIP specifies an IP address that will use the bypass proxy. Note that +// this will only take effect if a literal IP address is dialed. A connection +// to a named host will never match an IP. +func (p *PerHost) AddIP(ip net.IP) { + p.bypassIPs = append(p.bypassIPs, ip) +} + +// AddNetwork specifies an IP range that will use the bypass proxy. Note that +// this will only take effect if a literal IP address is dialed. A connection +// to a named host will never match. +func (p *PerHost) AddNetwork(net *net.IPNet) { + p.bypassNetworks = append(p.bypassNetworks, net) +} + +// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of +// "example.com" matches "example.com" and all of its subdomains. +func (p *PerHost) AddZone(zone string) { + if strings.HasSuffix(zone, ".") { + zone = zone[:len(zone)-1] + } + if !strings.HasPrefix(zone, ".") { + zone = "." + zone + } + p.bypassZones = append(p.bypassZones, zone) +} + +// AddHost specifies a hostname that will use the bypass proxy. +func (p *PerHost) AddHost(host string) { + if strings.HasSuffix(host, ".") { + host = host[:len(host)-1] + } + p.bypassHosts = append(p.bypassHosts, host) +} diff --git a/components/engine/vendor/src/golang.org/x/net/proxy/proxy.go b/components/engine/vendor/src/golang.org/x/net/proxy/proxy.go new file mode 100644 index 0000000000..78a8b7bee9 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/proxy/proxy.go @@ -0,0 +1,94 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package proxy provides support for a variety of protocols to proxy network +// data. +package proxy // import "golang.org/x/net/proxy" + +import ( + "errors" + "net" + "net/url" + "os" +) + +// A Dialer is a means to establish a connection. +type Dialer interface { + // Dial connects to the given address via the proxy. + Dial(network, addr string) (c net.Conn, err error) +} + +// Auth contains authentication parameters that specific Dialers may require. +type Auth struct { + User, Password string +} + +// FromEnvironment returns the dialer specified by the proxy related variables in +// the environment. +func FromEnvironment() Dialer { + allProxy := os.Getenv("all_proxy") + if len(allProxy) == 0 { + return Direct + } + + proxyURL, err := url.Parse(allProxy) + if err != nil { + return Direct + } + proxy, err := FromURL(proxyURL, Direct) + if err != nil { + return Direct + } + + noProxy := os.Getenv("no_proxy") + if len(noProxy) == 0 { + return proxy + } + + perHost := NewPerHost(proxy, Direct) + perHost.AddFromString(noProxy) + return perHost +} + +// proxySchemes is a map from URL schemes to a function that creates a Dialer +// from a URL with such a scheme. +var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error) + +// RegisterDialerType takes a URL scheme and a function to generate Dialers from +// a URL with that scheme and a forwarding Dialer. Registered schemes are used +// by FromURL. +func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) { + if proxySchemes == nil { + proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error)) + } + proxySchemes[scheme] = f +} + +// FromURL returns a Dialer given a URL specification and an underlying +// Dialer for it to make network requests. +func FromURL(u *url.URL, forward Dialer) (Dialer, error) { + var auth *Auth + if u.User != nil { + auth = new(Auth) + auth.User = u.User.Username() + if p, ok := u.User.Password(); ok { + auth.Password = p + } + } + + switch u.Scheme { + case "socks5": + return SOCKS5("tcp", u.Host, auth, forward) + } + + // If the scheme doesn't match any of the built-in schemes, see if it + // was registered by another package. + if proxySchemes != nil { + if f, ok := proxySchemes[u.Scheme]; ok { + return f(u, forward) + } + } + + return nil, errors.New("proxy: unknown scheme: " + u.Scheme) +} diff --git a/components/engine/vendor/src/golang.org/x/net/proxy/socks5.go b/components/engine/vendor/src/golang.org/x/net/proxy/socks5.go new file mode 100644 index 0000000000..9b9628239a --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/proxy/socks5.go @@ -0,0 +1,210 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package proxy + +import ( + "errors" + "io" + "net" + "strconv" +) + +// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given address +// with an optional username and password. See RFC 1928. +func SOCKS5(network, addr string, auth *Auth, forward Dialer) (Dialer, error) { + s := &socks5{ + network: network, + addr: addr, + forward: forward, + } + if auth != nil { + s.user = auth.User + s.password = auth.Password + } + + return s, nil +} + +type socks5 struct { + user, password string + network, addr string + forward Dialer +} + +const socks5Version = 5 + +const ( + socks5AuthNone = 0 + socks5AuthPassword = 2 +) + +const socks5Connect = 1 + +const ( + socks5IP4 = 1 + socks5Domain = 3 + socks5IP6 = 4 +) + +var socks5Errors = []string{ + "", + "general failure", + "connection forbidden", + "network unreachable", + "host unreachable", + "connection refused", + "TTL expired", + "command not supported", + "address type not supported", +} + +// Dial connects to the address addr on the network net via the SOCKS5 proxy. +func (s *socks5) Dial(network, addr string) (net.Conn, error) { + switch network { + case "tcp", "tcp6", "tcp4": + default: + return nil, errors.New("proxy: no support for SOCKS5 proxy connections of type " + network) + } + + conn, err := s.forward.Dial(s.network, s.addr) + if err != nil { + return nil, err + } + closeConn := &conn + defer func() { + if closeConn != nil { + (*closeConn).Close() + } + }() + + host, portStr, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + + port, err := strconv.Atoi(portStr) + if err != nil { + return nil, errors.New("proxy: failed to parse port number: " + portStr) + } + if port < 1 || port > 0xffff { + return nil, errors.New("proxy: port number out of range: " + portStr) + } + + // the size here is just an estimate + buf := make([]byte, 0, 6+len(host)) + + buf = append(buf, socks5Version) + if len(s.user) > 0 && len(s.user) < 256 && len(s.password) < 256 { + buf = append(buf, 2 /* num auth methods */, socks5AuthNone, socks5AuthPassword) + } else { + buf = append(buf, 1 /* num auth methods */, socks5AuthNone) + } + + if _, err := conn.Write(buf); err != nil { + return nil, errors.New("proxy: failed to write greeting to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return nil, errors.New("proxy: failed to read greeting from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + if buf[0] != 5 { + return nil, errors.New("proxy: SOCKS5 proxy at " + s.addr + " has unexpected version " + strconv.Itoa(int(buf[0]))) + } + if buf[1] == 0xff { + return nil, errors.New("proxy: SOCKS5 proxy at " + s.addr + " requires authentication") + } + + if buf[1] == socks5AuthPassword { + buf = buf[:0] + buf = append(buf, 1 /* password protocol version */) + buf = append(buf, uint8(len(s.user))) + buf = append(buf, s.user...) + buf = append(buf, uint8(len(s.password))) + buf = append(buf, s.password...) + + if _, err := conn.Write(buf); err != nil { + return nil, errors.New("proxy: failed to write authentication request to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return nil, errors.New("proxy: failed to read authentication reply from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if buf[1] != 0 { + return nil, errors.New("proxy: SOCKS5 proxy at " + s.addr + " rejected username/password") + } + } + + buf = buf[:0] + buf = append(buf, socks5Version, socks5Connect, 0 /* reserved */) + + if ip := net.ParseIP(host); ip != nil { + if ip4 := ip.To4(); ip4 != nil { + buf = append(buf, socks5IP4) + ip = ip4 + } else { + buf = append(buf, socks5IP6) + } + buf = append(buf, ip...) + } else { + if len(host) > 255 { + return nil, errors.New("proxy: destination hostname too long: " + host) + } + buf = append(buf, socks5Domain) + buf = append(buf, byte(len(host))) + buf = append(buf, host...) + } + buf = append(buf, byte(port>>8), byte(port)) + + if _, err := conn.Write(buf); err != nil { + return nil, errors.New("proxy: failed to write connect request to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:4]); err != nil { + return nil, errors.New("proxy: failed to read connect reply from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + failure := "unknown error" + if int(buf[1]) < len(socks5Errors) { + failure = socks5Errors[buf[1]] + } + + if len(failure) > 0 { + return nil, errors.New("proxy: SOCKS5 proxy at " + s.addr + " failed to connect: " + failure) + } + + bytesToDiscard := 0 + switch buf[3] { + case socks5IP4: + bytesToDiscard = net.IPv4len + case socks5IP6: + bytesToDiscard = net.IPv6len + case socks5Domain: + _, err := io.ReadFull(conn, buf[:1]) + if err != nil { + return nil, errors.New("proxy: failed to read domain length from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + bytesToDiscard = int(buf[0]) + default: + return nil, errors.New("proxy: got unknown address type " + strconv.Itoa(int(buf[3])) + " from SOCKS5 proxy at " + s.addr) + } + + if cap(buf) < bytesToDiscard { + buf = make([]byte, bytesToDiscard) + } else { + buf = buf[:bytesToDiscard] + } + if _, err := io.ReadFull(conn, buf); err != nil { + return nil, errors.New("proxy: failed to read address from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + // Also need to discard the port number + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return nil, errors.New("proxy: failed to read port from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + closeConn = nil + return conn, nil +} diff --git a/components/engine/vendor/src/golang.org/x/net/trace/events.go b/components/engine/vendor/src/golang.org/x/net/trace/events.go new file mode 100644 index 0000000000..e66c7e3282 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/trace/events.go @@ -0,0 +1,524 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace + +import ( + "bytes" + "fmt" + "html/template" + "io" + "log" + "net/http" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "text/tabwriter" + "time" +) + +var eventsTmpl = template.Must(template.New("events").Funcs(template.FuncMap{ + "elapsed": elapsed, + "trimSpace": strings.TrimSpace, +}).Parse(eventsHTML)) + +const maxEventsPerLog = 100 + +type bucket struct { + MaxErrAge time.Duration + String string +} + +var buckets = []bucket{ + {0, "total"}, + {10 * time.Second, "errs<10s"}, + {1 * time.Minute, "errs<1m"}, + {10 * time.Minute, "errs<10m"}, + {1 * time.Hour, "errs<1h"}, + {10 * time.Hour, "errs<10h"}, + {24000 * time.Hour, "errors"}, +} + +// RenderEvents renders the HTML page typically served at /debug/events. +// It does not do any auth checking; see AuthRequest for the default auth check +// used by the handler registered on http.DefaultServeMux. +// req may be nil. +func RenderEvents(w http.ResponseWriter, req *http.Request, sensitive bool) { + now := time.Now() + data := &struct { + Families []string // family names + Buckets []bucket + Counts [][]int // eventLog count per family/bucket + + // Set when a bucket has been selected. + Family string + Bucket int + EventLogs eventLogs + Expanded bool + }{ + Buckets: buckets, + } + + data.Families = make([]string, 0, len(families)) + famMu.RLock() + for name := range families { + data.Families = append(data.Families, name) + } + famMu.RUnlock() + sort.Strings(data.Families) + + // Count the number of eventLogs in each family for each error age. + data.Counts = make([][]int, len(data.Families)) + for i, name := range data.Families { + // TODO(sameer): move this loop under the family lock. + f := getEventFamily(name) + data.Counts[i] = make([]int, len(data.Buckets)) + for j, b := range data.Buckets { + data.Counts[i][j] = f.Count(now, b.MaxErrAge) + } + } + + if req != nil { + var ok bool + data.Family, data.Bucket, ok = parseEventsArgs(req) + if !ok { + // No-op + } else { + data.EventLogs = getEventFamily(data.Family).Copy(now, buckets[data.Bucket].MaxErrAge) + } + if data.EventLogs != nil { + defer data.EventLogs.Free() + sort.Sort(data.EventLogs) + } + if exp, err := strconv.ParseBool(req.FormValue("exp")); err == nil { + data.Expanded = exp + } + } + + famMu.RLock() + defer famMu.RUnlock() + if err := eventsTmpl.Execute(w, data); err != nil { + log.Printf("net/trace: Failed executing template: %v", err) + } +} + +func parseEventsArgs(req *http.Request) (fam string, b int, ok bool) { + fam, bStr := req.FormValue("fam"), req.FormValue("b") + if fam == "" || bStr == "" { + return "", 0, false + } + b, err := strconv.Atoi(bStr) + if err != nil || b < 0 || b >= len(buckets) { + return "", 0, false + } + return fam, b, true +} + +// An EventLog provides a log of events associated with a specific object. +type EventLog interface { + // Printf formats its arguments with fmt.Sprintf and adds the + // result to the event log. + Printf(format string, a ...interface{}) + + // Errorf is like Printf, but it marks this event as an error. + Errorf(format string, a ...interface{}) + + // Finish declares that this event log is complete. + // The event log should not be used after calling this method. + Finish() +} + +// NewEventLog returns a new EventLog with the specified family name +// and title. +func NewEventLog(family, title string) EventLog { + el := newEventLog() + el.ref() + el.Family, el.Title = family, title + el.Start = time.Now() + el.events = make([]logEntry, 0, maxEventsPerLog) + el.stack = make([]uintptr, 32) + n := runtime.Callers(2, el.stack) + el.stack = el.stack[:n] + + getEventFamily(family).add(el) + return el +} + +func (el *eventLog) Finish() { + getEventFamily(el.Family).remove(el) + el.unref() // matches ref in New +} + +var ( + famMu sync.RWMutex + families = make(map[string]*eventFamily) // family name => family +) + +func getEventFamily(fam string) *eventFamily { + famMu.Lock() + defer famMu.Unlock() + f := families[fam] + if f == nil { + f = &eventFamily{} + families[fam] = f + } + return f +} + +type eventFamily struct { + mu sync.RWMutex + eventLogs eventLogs +} + +func (f *eventFamily) add(el *eventLog) { + f.mu.Lock() + f.eventLogs = append(f.eventLogs, el) + f.mu.Unlock() +} + +func (f *eventFamily) remove(el *eventLog) { + f.mu.Lock() + defer f.mu.Unlock() + for i, el0 := range f.eventLogs { + if el == el0 { + copy(f.eventLogs[i:], f.eventLogs[i+1:]) + f.eventLogs = f.eventLogs[:len(f.eventLogs)-1] + return + } + } +} + +func (f *eventFamily) Count(now time.Time, maxErrAge time.Duration) (n int) { + f.mu.RLock() + defer f.mu.RUnlock() + for _, el := range f.eventLogs { + if el.hasRecentError(now, maxErrAge) { + n++ + } + } + return +} + +func (f *eventFamily) Copy(now time.Time, maxErrAge time.Duration) (els eventLogs) { + f.mu.RLock() + defer f.mu.RUnlock() + els = make(eventLogs, 0, len(f.eventLogs)) + for _, el := range f.eventLogs { + if el.hasRecentError(now, maxErrAge) { + el.ref() + els = append(els, el) + } + } + return +} + +type eventLogs []*eventLog + +// Free calls unref on each element of the list. +func (els eventLogs) Free() { + for _, el := range els { + el.unref() + } +} + +// eventLogs may be sorted in reverse chronological order. +func (els eventLogs) Len() int { return len(els) } +func (els eventLogs) Less(i, j int) bool { return els[i].Start.After(els[j].Start) } +func (els eventLogs) Swap(i, j int) { els[i], els[j] = els[j], els[i] } + +// A logEntry is a timestamped log entry in an event log. +type logEntry struct { + When time.Time + Elapsed time.Duration // since previous event in log + NewDay bool // whether this event is on a different day to the previous event + What string + IsErr bool +} + +// WhenString returns a string representation of the elapsed time of the event. +// It will include the date if midnight was crossed. +func (e logEntry) WhenString() string { + if e.NewDay { + return e.When.Format("2006/01/02 15:04:05.000000") + } + return e.When.Format("15:04:05.000000") +} + +// An eventLog represents an active event log. +type eventLog struct { + // Family is the top-level grouping of event logs to which this belongs. + Family string + + // Title is the title of this event log. + Title string + + // Timing information. + Start time.Time + + // Call stack where this event log was created. + stack []uintptr + + // Append-only sequence of events. + // + // TODO(sameer): change this to a ring buffer to avoid the array copy + // when we hit maxEventsPerLog. + mu sync.RWMutex + events []logEntry + LastErrorTime time.Time + discarded int + + refs int32 // how many buckets this is in +} + +func (el *eventLog) reset() { + // Clear all but the mutex. Mutexes may not be copied, even when unlocked. + el.Family = "" + el.Title = "" + el.Start = time.Time{} + el.stack = nil + el.events = nil + el.LastErrorTime = time.Time{} + el.discarded = 0 + el.refs = 0 +} + +func (el *eventLog) hasRecentError(now time.Time, maxErrAge time.Duration) bool { + if maxErrAge == 0 { + return true + } + el.mu.RLock() + defer el.mu.RUnlock() + return now.Sub(el.LastErrorTime) < maxErrAge +} + +// delta returns the elapsed time since the last event or the log start, +// and whether it spans midnight. +// L >= el.mu +func (el *eventLog) delta(t time.Time) (time.Duration, bool) { + if len(el.events) == 0 { + return t.Sub(el.Start), false + } + prev := el.events[len(el.events)-1].When + return t.Sub(prev), prev.Day() != t.Day() + +} + +func (el *eventLog) Printf(format string, a ...interface{}) { + el.printf(false, format, a...) +} + +func (el *eventLog) Errorf(format string, a ...interface{}) { + el.printf(true, format, a...) +} + +func (el *eventLog) printf(isErr bool, format string, a ...interface{}) { + e := logEntry{When: time.Now(), IsErr: isErr, What: fmt.Sprintf(format, a...)} + el.mu.Lock() + e.Elapsed, e.NewDay = el.delta(e.When) + if len(el.events) < maxEventsPerLog { + el.events = append(el.events, e) + } else { + // Discard the oldest event. + if el.discarded == 0 { + // el.discarded starts at two to count for the event it + // is replacing, plus the next one that we are about to + // drop. + el.discarded = 2 + } else { + el.discarded++ + } + // TODO(sameer): if this causes allocations on a critical path, + // change eventLog.What to be a fmt.Stringer, as in trace.go. + el.events[0].What = fmt.Sprintf("(%d events discarded)", el.discarded) + // The timestamp of the discarded meta-event should be + // the time of the last event it is representing. + el.events[0].When = el.events[1].When + copy(el.events[1:], el.events[2:]) + el.events[maxEventsPerLog-1] = e + } + if e.IsErr { + el.LastErrorTime = e.When + } + el.mu.Unlock() +} + +func (el *eventLog) ref() { + atomic.AddInt32(&el.refs, 1) +} + +func (el *eventLog) unref() { + if atomic.AddInt32(&el.refs, -1) == 0 { + freeEventLog(el) + } +} + +func (el *eventLog) When() string { + return el.Start.Format("2006/01/02 15:04:05.000000") +} + +func (el *eventLog) ElapsedTime() string { + elapsed := time.Since(el.Start) + return fmt.Sprintf("%.6f", elapsed.Seconds()) +} + +func (el *eventLog) Stack() string { + buf := new(bytes.Buffer) + tw := tabwriter.NewWriter(buf, 1, 8, 1, '\t', 0) + printStackRecord(tw, el.stack) + tw.Flush() + return buf.String() +} + +// printStackRecord prints the function + source line information +// for a single stack trace. +// Adapted from runtime/pprof/pprof.go. +func printStackRecord(w io.Writer, stk []uintptr) { + for _, pc := range stk { + f := runtime.FuncForPC(pc) + if f == nil { + continue + } + file, line := f.FileLine(pc) + name := f.Name() + // Hide runtime.goexit and any runtime functions at the beginning. + if strings.HasPrefix(name, "runtime.") { + continue + } + fmt.Fprintf(w, "# %s\t%s:%d\n", name, file, line) + } +} + +func (el *eventLog) Events() []logEntry { + el.mu.RLock() + defer el.mu.RUnlock() + return el.events +} + +// freeEventLogs is a freelist of *eventLog +var freeEventLogs = make(chan *eventLog, 1000) + +// newEventLog returns a event log ready to use. +func newEventLog() *eventLog { + select { + case el := <-freeEventLogs: + return el + default: + return new(eventLog) + } +} + +// freeEventLog adds el to freeEventLogs if there's room. +// This is non-blocking. +func freeEventLog(el *eventLog) { + el.reset() + select { + case freeEventLogs <- el: + default: + } +} + +const eventsHTML = ` + + + events + + + + +

/debug/events

+ +
+ {{range $i, $fam := .Families}} + + + + {{range $j, $bucket := $.Buckets}} + {{$n := index $.Counts $i $j}} + + {{end}} + + {{end}} +
{{$fam}} + {{if $n}}{{end}} + [{{$n}} {{$bucket.String}}] + {{if $n}}{{end}} +
+ +{{if $.EventLogs}} +
+

Family: {{$.Family}}

+ +{{if $.Expanded}}{{end}} +[Summary]{{if $.Expanded}}{{end}} + +{{if not $.Expanded}}{{end}} +[Expanded]{{if not $.Expanded}}{{end}} + + + + {{range $el := $.EventLogs}} + + + + + {{if $.Expanded}} + + + + + + {{range $el.Events}} + + + + + + {{end}} + {{end}} + {{end}} +
WhenElapsed
{{$el.When}}{{$el.ElapsedTime}}{{$el.Title}} +
{{$el.Stack|trimSpace}}
{{.WhenString}}{{elapsed .Elapsed}}.{{if .IsErr}}E{{else}}.{{end}}. {{.What}}
+{{end}} + + +` diff --git a/components/engine/vendor/src/golang.org/x/net/trace/histogram.go b/components/engine/vendor/src/golang.org/x/net/trace/histogram.go new file mode 100644 index 0000000000..bb42aa5320 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/trace/histogram.go @@ -0,0 +1,356 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package trace + +// This file implements histogramming for RPC statistics collection. + +import ( + "bytes" + "fmt" + "html/template" + "log" + "math" + + "golang.org/x/net/internal/timeseries" +) + +const ( + bucketCount = 38 +) + +// histogram keeps counts of values in buckets that are spaced +// out in powers of 2: 0-1, 2-3, 4-7... +// histogram implements timeseries.Observable +type histogram struct { + sum int64 // running total of measurements + sumOfSquares float64 // square of running total + buckets []int64 // bucketed values for histogram + value int // holds a single value as an optimization + valueCount int64 // number of values recorded for single value +} + +// AddMeasurement records a value measurement observation to the histogram. +func (h *histogram) addMeasurement(value int64) { + // TODO: assert invariant + h.sum += value + h.sumOfSquares += float64(value) * float64(value) + + bucketIndex := getBucket(value) + + if h.valueCount == 0 || (h.valueCount > 0 && h.value == bucketIndex) { + h.value = bucketIndex + h.valueCount++ + } else { + h.allocateBuckets() + h.buckets[bucketIndex]++ + } +} + +func (h *histogram) allocateBuckets() { + if h.buckets == nil { + h.buckets = make([]int64, bucketCount) + h.buckets[h.value] = h.valueCount + h.value = 0 + h.valueCount = -1 + } +} + +func log2(i int64) int { + n := 0 + for ; i >= 0x100; i >>= 8 { + n += 8 + } + for ; i > 0; i >>= 1 { + n += 1 + } + return n +} + +func getBucket(i int64) (index int) { + index = log2(i) - 1 + if index < 0 { + index = 0 + } + if index >= bucketCount { + index = bucketCount - 1 + } + return +} + +// Total returns the number of recorded observations. +func (h *histogram) total() (total int64) { + if h.valueCount >= 0 { + total = h.valueCount + } + for _, val := range h.buckets { + total += int64(val) + } + return +} + +// Average returns the average value of recorded observations. +func (h *histogram) average() float64 { + t := h.total() + if t == 0 { + return 0 + } + return float64(h.sum) / float64(t) +} + +// Variance returns the variance of recorded observations. +func (h *histogram) variance() float64 { + t := float64(h.total()) + if t == 0 { + return 0 + } + s := float64(h.sum) / t + return h.sumOfSquares/t - s*s +} + +// StandardDeviation returns the standard deviation of recorded observations. +func (h *histogram) standardDeviation() float64 { + return math.Sqrt(h.variance()) +} + +// PercentileBoundary estimates the value that the given fraction of recorded +// observations are less than. +func (h *histogram) percentileBoundary(percentile float64) int64 { + total := h.total() + + // Corner cases (make sure result is strictly less than Total()) + if total == 0 { + return 0 + } else if total == 1 { + return int64(h.average()) + } + + percentOfTotal := round(float64(total) * percentile) + var runningTotal int64 + + for i := range h.buckets { + value := h.buckets[i] + runningTotal += value + if runningTotal == percentOfTotal { + // We hit an exact bucket boundary. If the next bucket has data, it is a + // good estimate of the value. If the bucket is empty, we interpolate the + // midpoint between the next bucket's boundary and the next non-zero + // bucket. If the remaining buckets are all empty, then we use the + // boundary for the next bucket as the estimate. + j := uint8(i + 1) + min := bucketBoundary(j) + if runningTotal < total { + for h.buckets[j] == 0 { + j++ + } + } + max := bucketBoundary(j) + return min + round(float64(max-min)/2) + } else if runningTotal > percentOfTotal { + // The value is in this bucket. Interpolate the value. + delta := runningTotal - percentOfTotal + percentBucket := float64(value-delta) / float64(value) + bucketMin := bucketBoundary(uint8(i)) + nextBucketMin := bucketBoundary(uint8(i + 1)) + bucketSize := nextBucketMin - bucketMin + return bucketMin + round(percentBucket*float64(bucketSize)) + } + } + return bucketBoundary(bucketCount - 1) +} + +// Median returns the estimated median of the observed values. +func (h *histogram) median() int64 { + return h.percentileBoundary(0.5) +} + +// Add adds other to h. +func (h *histogram) Add(other timeseries.Observable) { + o := other.(*histogram) + if o.valueCount == 0 { + // Other histogram is empty + } else if h.valueCount >= 0 && o.valueCount > 0 && h.value == o.value { + // Both have a single bucketed value, aggregate them + h.valueCount += o.valueCount + } else { + // Two different values necessitate buckets in this histogram + h.allocateBuckets() + if o.valueCount >= 0 { + h.buckets[o.value] += o.valueCount + } else { + for i := range h.buckets { + h.buckets[i] += o.buckets[i] + } + } + } + h.sumOfSquares += o.sumOfSquares + h.sum += o.sum +} + +// Clear resets the histogram to an empty state, removing all observed values. +func (h *histogram) Clear() { + h.buckets = nil + h.value = 0 + h.valueCount = 0 + h.sum = 0 + h.sumOfSquares = 0 +} + +// CopyFrom copies from other, which must be a *histogram, into h. +func (h *histogram) CopyFrom(other timeseries.Observable) { + o := other.(*histogram) + if o.valueCount == -1 { + h.allocateBuckets() + copy(h.buckets, o.buckets) + } + h.sum = o.sum + h.sumOfSquares = o.sumOfSquares + h.value = o.value + h.valueCount = o.valueCount +} + +// Multiply scales the histogram by the specified ratio. +func (h *histogram) Multiply(ratio float64) { + if h.valueCount == -1 { + for i := range h.buckets { + h.buckets[i] = int64(float64(h.buckets[i]) * ratio) + } + } else { + h.valueCount = int64(float64(h.valueCount) * ratio) + } + h.sum = int64(float64(h.sum) * ratio) + h.sumOfSquares = h.sumOfSquares * ratio +} + +// New creates a new histogram. +func (h *histogram) New() timeseries.Observable { + r := new(histogram) + r.Clear() + return r +} + +func (h *histogram) String() string { + return fmt.Sprintf("%d, %f, %d, %d, %v", + h.sum, h.sumOfSquares, h.value, h.valueCount, h.buckets) +} + +// round returns the closest int64 to the argument +func round(in float64) int64 { + return int64(math.Floor(in + 0.5)) +} + +// bucketBoundary returns the first value in the bucket. +func bucketBoundary(bucket uint8) int64 { + if bucket == 0 { + return 0 + } + return 1 << bucket +} + +// bucketData holds data about a specific bucket for use in distTmpl. +type bucketData struct { + Lower, Upper int64 + N int64 + Pct, CumulativePct float64 + GraphWidth int +} + +// data holds data about a Distribution for use in distTmpl. +type data struct { + Buckets []*bucketData + Count, Median int64 + Mean, StandardDeviation float64 +} + +// maxHTMLBarWidth is the maximum width of the HTML bar for visualizing buckets. +const maxHTMLBarWidth = 350.0 + +// newData returns data representing h for use in distTmpl. +func (h *histogram) newData() *data { + // Force the allocation of buckets to simplify the rendering implementation + h.allocateBuckets() + // We scale the bars on the right so that the largest bar is + // maxHTMLBarWidth pixels in width. + maxBucket := int64(0) + for _, n := range h.buckets { + if n > maxBucket { + maxBucket = n + } + } + total := h.total() + barsizeMult := maxHTMLBarWidth / float64(maxBucket) + var pctMult float64 + if total == 0 { + pctMult = 1.0 + } else { + pctMult = 100.0 / float64(total) + } + + buckets := make([]*bucketData, len(h.buckets)) + runningTotal := int64(0) + for i, n := range h.buckets { + if n == 0 { + continue + } + runningTotal += n + var upperBound int64 + if i < bucketCount-1 { + upperBound = bucketBoundary(uint8(i + 1)) + } else { + upperBound = math.MaxInt64 + } + buckets[i] = &bucketData{ + Lower: bucketBoundary(uint8(i)), + Upper: upperBound, + N: n, + Pct: float64(n) * pctMult, + CumulativePct: float64(runningTotal) * pctMult, + GraphWidth: int(float64(n) * barsizeMult), + } + } + return &data{ + Buckets: buckets, + Count: total, + Median: h.median(), + Mean: h.average(), + StandardDeviation: h.standardDeviation(), + } +} + +func (h *histogram) html() template.HTML { + buf := new(bytes.Buffer) + if err := distTmpl.Execute(buf, h.newData()); err != nil { + buf.Reset() + log.Printf("net/trace: couldn't execute template: %v", err) + } + return template.HTML(buf.String()) +} + +// Input: data +var distTmpl = template.Must(template.New("distTmpl").Parse(` + + + + + + + +
Count: {{.Count}}Mean: {{printf "%.0f" .Mean}}StdDev: {{printf "%.0f" .StandardDeviation}}Median: {{.Median}}
+
+ +{{range $b := .Buckets}} +{{if $b}} + + + + + + + + + +{{end}} +{{end}} +
[{{.Lower}},{{.Upper}}){{.N}}{{printf "%#.3f" .Pct}}%{{printf "%#.3f" .CumulativePct}}%
+`)) diff --git a/components/engine/vendor/src/golang.org/x/net/trace/trace.go b/components/engine/vendor/src/golang.org/x/net/trace/trace.go new file mode 100644 index 0000000000..c44cb7ec9e --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/net/trace/trace.go @@ -0,0 +1,1057 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package trace implements tracing of requests and long-lived objects. +It exports HTTP interfaces on /debug/requests and /debug/events. + +A trace.Trace provides tracing for short-lived objects, usually requests. +A request handler might be implemented like this: + + func fooHandler(w http.ResponseWriter, req *http.Request) { + tr := trace.New("mypkg.Foo", req.URL.Path) + defer tr.Finish() + ... + tr.LazyPrintf("some event %q happened", str) + ... + if err := somethingImportant(); err != nil { + tr.LazyPrintf("somethingImportant failed: %v", err) + tr.SetError() + } + } + +The /debug/requests HTTP endpoint organizes the traces by family, +errors, and duration. It also provides histogram of request duration +for each family. + +A trace.EventLog provides tracing for long-lived objects, such as RPC +connections. + + // A Fetcher fetches URL paths for a single domain. + type Fetcher struct { + domain string + events trace.EventLog + } + + func NewFetcher(domain string) *Fetcher { + return &Fetcher{ + domain, + trace.NewEventLog("mypkg.Fetcher", domain), + } + } + + func (f *Fetcher) Fetch(path string) (string, error) { + resp, err := http.Get("http://" + f.domain + "/" + path) + if err != nil { + f.events.Errorf("Get(%q) = %v", path, err) + return "", err + } + f.events.Printf("Get(%q) = %s", path, resp.Status) + ... + } + + func (f *Fetcher) Close() error { + f.events.Finish() + return nil + } + +The /debug/events HTTP endpoint organizes the event logs by family and +by time since the last error. The expanded view displays recent log +entries and the log's call stack. +*/ +package trace // import "golang.org/x/net/trace" + +import ( + "bytes" + "fmt" + "html/template" + "io" + "log" + "net" + "net/http" + "runtime" + "sort" + "strconv" + "sync" + "sync/atomic" + "time" + + "golang.org/x/net/context" + "golang.org/x/net/internal/timeseries" +) + +// DebugUseAfterFinish controls whether to debug uses of Trace values after finishing. +// FOR DEBUGGING ONLY. This will slow down the program. +var DebugUseAfterFinish = false + +// AuthRequest determines whether a specific request is permitted to load the +// /debug/requests or /debug/events pages. +// +// It returns two bools; the first indicates whether the page may be viewed at all, +// and the second indicates whether sensitive events will be shown. +// +// AuthRequest may be replaced by a program to customise its authorisation requirements. +// +// The default AuthRequest function returns (true, true) iff the request comes from localhost/127.0.0.1/[::1]. +var AuthRequest = func(req *http.Request) (any, sensitive bool) { + host, _, err := net.SplitHostPort(req.RemoteAddr) + switch { + case err != nil: // Badly formed address; fail closed. + return false, false + case host == "localhost" || host == "127.0.0.1" || host == "::1": + return true, true + default: + return false, false + } +} + +func init() { + http.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) { + any, sensitive := AuthRequest(req) + if !any { + http.Error(w, "not allowed", http.StatusUnauthorized) + return + } + Render(w, req, sensitive) + }) + http.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) { + any, sensitive := AuthRequest(req) + if !any { + http.Error(w, "not allowed", http.StatusUnauthorized) + return + } + RenderEvents(w, req, sensitive) + }) +} + +// Render renders the HTML page typically served at /debug/requests. +// It does not do any auth checking; see AuthRequest for the default auth check +// used by the handler registered on http.DefaultServeMux. +// req may be nil. +func Render(w io.Writer, req *http.Request, sensitive bool) { + data := &struct { + Families []string + ActiveTraceCount map[string]int + CompletedTraces map[string]*family + + // Set when a bucket has been selected. + Traces traceList + Family string + Bucket int + Expanded bool + Traced bool + Active bool + ShowSensitive bool // whether to show sensitive events + + Histogram template.HTML + HistogramWindow string // e.g. "last minute", "last hour", "all time" + + // If non-zero, the set of traces is a partial set, + // and this is the total number. + Total int + }{ + CompletedTraces: completedTraces, + } + + data.ShowSensitive = sensitive + if req != nil { + // Allow show_sensitive=0 to force hiding of sensitive data for testing. + // This only goes one way; you can't use show_sensitive=1 to see things. + if req.FormValue("show_sensitive") == "0" { + data.ShowSensitive = false + } + + if exp, err := strconv.ParseBool(req.FormValue("exp")); err == nil { + data.Expanded = exp + } + if exp, err := strconv.ParseBool(req.FormValue("rtraced")); err == nil { + data.Traced = exp + } + } + + completedMu.RLock() + data.Families = make([]string, 0, len(completedTraces)) + for fam, _ := range completedTraces { + data.Families = append(data.Families, fam) + } + completedMu.RUnlock() + sort.Strings(data.Families) + + // We are careful here to minimize the time spent locking activeMu, + // since that lock is required every time an RPC starts and finishes. + data.ActiveTraceCount = make(map[string]int, len(data.Families)) + activeMu.RLock() + for fam, s := range activeTraces { + data.ActiveTraceCount[fam] = s.Len() + } + activeMu.RUnlock() + + var ok bool + data.Family, data.Bucket, ok = parseArgs(req) + switch { + case !ok: + // No-op + case data.Bucket == -1: + data.Active = true + n := data.ActiveTraceCount[data.Family] + data.Traces = getActiveTraces(data.Family) + if len(data.Traces) < n { + data.Total = n + } + case data.Bucket < bucketsPerFamily: + if b := lookupBucket(data.Family, data.Bucket); b != nil { + data.Traces = b.Copy(data.Traced) + } + default: + if f := getFamily(data.Family, false); f != nil { + var obs timeseries.Observable + f.LatencyMu.RLock() + switch o := data.Bucket - bucketsPerFamily; o { + case 0: + obs = f.Latency.Minute() + data.HistogramWindow = "last minute" + case 1: + obs = f.Latency.Hour() + data.HistogramWindow = "last hour" + case 2: + obs = f.Latency.Total() + data.HistogramWindow = "all time" + } + f.LatencyMu.RUnlock() + if obs != nil { + data.Histogram = obs.(*histogram).html() + } + } + } + + if data.Traces != nil { + defer data.Traces.Free() + sort.Sort(data.Traces) + } + + completedMu.RLock() + defer completedMu.RUnlock() + if err := pageTmpl.ExecuteTemplate(w, "Page", data); err != nil { + log.Printf("net/trace: Failed executing template: %v", err) + } +} + +func parseArgs(req *http.Request) (fam string, b int, ok bool) { + if req == nil { + return "", 0, false + } + fam, bStr := req.FormValue("fam"), req.FormValue("b") + if fam == "" || bStr == "" { + return "", 0, false + } + b, err := strconv.Atoi(bStr) + if err != nil || b < -1 { + return "", 0, false + } + + return fam, b, true +} + +func lookupBucket(fam string, b int) *traceBucket { + f := getFamily(fam, false) + if f == nil || b < 0 || b >= len(f.Buckets) { + return nil + } + return f.Buckets[b] +} + +type contextKeyT string + +var contextKey = contextKeyT("golang.org/x/net/trace.Trace") + +// NewContext returns a copy of the parent context +// and associates it with a Trace. +func NewContext(ctx context.Context, tr Trace) context.Context { + return context.WithValue(ctx, contextKey, tr) +} + +// FromContext returns the Trace bound to the context, if any. +func FromContext(ctx context.Context) (tr Trace, ok bool) { + tr, ok = ctx.Value(contextKey).(Trace) + return +} + +// Trace represents an active request. +type Trace interface { + // LazyLog adds x to the event log. It will be evaluated each time the + // /debug/requests page is rendered. Any memory referenced by x will be + // pinned until the trace is finished and later discarded. + LazyLog(x fmt.Stringer, sensitive bool) + + // LazyPrintf evaluates its arguments with fmt.Sprintf each time the + // /debug/requests page is rendered. Any memory referenced by a will be + // pinned until the trace is finished and later discarded. + LazyPrintf(format string, a ...interface{}) + + // SetError declares that this trace resulted in an error. + SetError() + + // SetRecycler sets a recycler for the trace. + // f will be called for each event passed to LazyLog at a time when + // it is no longer required, whether while the trace is still active + // and the event is discarded, or when a completed trace is discarded. + SetRecycler(f func(interface{})) + + // SetTraceInfo sets the trace info for the trace. + // This is currently unused. + SetTraceInfo(traceID, spanID uint64) + + // SetMaxEvents sets the maximum number of events that will be stored + // in the trace. This has no effect if any events have already been + // added to the trace. + SetMaxEvents(m int) + + // Finish declares that this trace is complete. + // The trace should not be used after calling this method. + Finish() +} + +type lazySprintf struct { + format string + a []interface{} +} + +func (l *lazySprintf) String() string { + return fmt.Sprintf(l.format, l.a...) +} + +// New returns a new Trace with the specified family and title. +func New(family, title string) Trace { + tr := newTrace() + tr.ref() + tr.Family, tr.Title = family, title + tr.Start = time.Now() + tr.events = make([]event, 0, maxEventsPerTrace) + + activeMu.RLock() + s := activeTraces[tr.Family] + activeMu.RUnlock() + if s == nil { + activeMu.Lock() + s = activeTraces[tr.Family] // check again + if s == nil { + s = new(traceSet) + activeTraces[tr.Family] = s + } + activeMu.Unlock() + } + s.Add(tr) + + // Trigger allocation of the completed trace structure for this family. + // This will cause the family to be present in the request page during + // the first trace of this family. We don't care about the return value, + // nor is there any need for this to run inline, so we execute it in its + // own goroutine, but only if the family isn't allocated yet. + completedMu.RLock() + if _, ok := completedTraces[tr.Family]; !ok { + go allocFamily(tr.Family) + } + completedMu.RUnlock() + + return tr +} + +func (tr *trace) Finish() { + tr.Elapsed = time.Now().Sub(tr.Start) + if DebugUseAfterFinish { + buf := make([]byte, 4<<10) // 4 KB should be enough + n := runtime.Stack(buf, false) + tr.finishStack = buf[:n] + } + + activeMu.RLock() + m := activeTraces[tr.Family] + activeMu.RUnlock() + m.Remove(tr) + + f := getFamily(tr.Family, true) + for _, b := range f.Buckets { + if b.Cond.match(tr) { + b.Add(tr) + } + } + // Add a sample of elapsed time as microseconds to the family's timeseries + h := new(histogram) + h.addMeasurement(tr.Elapsed.Nanoseconds() / 1e3) + f.LatencyMu.Lock() + f.Latency.Add(h) + f.LatencyMu.Unlock() + + tr.unref() // matches ref in New +} + +const ( + bucketsPerFamily = 9 + tracesPerBucket = 10 + maxActiveTraces = 20 // Maximum number of active traces to show. + maxEventsPerTrace = 10 + numHistogramBuckets = 38 +) + +var ( + // The active traces. + activeMu sync.RWMutex + activeTraces = make(map[string]*traceSet) // family -> traces + + // Families of completed traces. + completedMu sync.RWMutex + completedTraces = make(map[string]*family) // family -> traces +) + +type traceSet struct { + mu sync.RWMutex + m map[*trace]bool + + // We could avoid the entire map scan in FirstN by having a slice of all the traces + // ordered by start time, and an index into that from the trace struct, with a periodic + // repack of the slice after enough traces finish; we could also use a skip list or similar. + // However, that would shift some of the expense from /debug/requests time to RPC time, + // which is probably the wrong trade-off. +} + +func (ts *traceSet) Len() int { + ts.mu.RLock() + defer ts.mu.RUnlock() + return len(ts.m) +} + +func (ts *traceSet) Add(tr *trace) { + ts.mu.Lock() + if ts.m == nil { + ts.m = make(map[*trace]bool) + } + ts.m[tr] = true + ts.mu.Unlock() +} + +func (ts *traceSet) Remove(tr *trace) { + ts.mu.Lock() + delete(ts.m, tr) + ts.mu.Unlock() +} + +// FirstN returns the first n traces ordered by time. +func (ts *traceSet) FirstN(n int) traceList { + ts.mu.RLock() + defer ts.mu.RUnlock() + + if n > len(ts.m) { + n = len(ts.m) + } + trl := make(traceList, 0, n) + + // Fast path for when no selectivity is needed. + if n == len(ts.m) { + for tr := range ts.m { + tr.ref() + trl = append(trl, tr) + } + sort.Sort(trl) + return trl + } + + // Pick the oldest n traces. + // This is inefficient. See the comment in the traceSet struct. + for tr := range ts.m { + // Put the first n traces into trl in the order they occur. + // When we have n, sort trl, and thereafter maintain its order. + if len(trl) < n { + tr.ref() + trl = append(trl, tr) + if len(trl) == n { + // This is guaranteed to happen exactly once during this loop. + sort.Sort(trl) + } + continue + } + if tr.Start.After(trl[n-1].Start) { + continue + } + + // Find where to insert this one. + tr.ref() + i := sort.Search(n, func(i int) bool { return trl[i].Start.After(tr.Start) }) + trl[n-1].unref() + copy(trl[i+1:], trl[i:]) + trl[i] = tr + } + + return trl +} + +func getActiveTraces(fam string) traceList { + activeMu.RLock() + s := activeTraces[fam] + activeMu.RUnlock() + if s == nil { + return nil + } + return s.FirstN(maxActiveTraces) +} + +func getFamily(fam string, allocNew bool) *family { + completedMu.RLock() + f := completedTraces[fam] + completedMu.RUnlock() + if f == nil && allocNew { + f = allocFamily(fam) + } + return f +} + +func allocFamily(fam string) *family { + completedMu.Lock() + defer completedMu.Unlock() + f := completedTraces[fam] + if f == nil { + f = newFamily() + completedTraces[fam] = f + } + return f +} + +// family represents a set of trace buckets and associated latency information. +type family struct { + // traces may occur in multiple buckets. + Buckets [bucketsPerFamily]*traceBucket + + // latency time series + LatencyMu sync.RWMutex + Latency *timeseries.MinuteHourSeries +} + +func newFamily() *family { + return &family{ + Buckets: [bucketsPerFamily]*traceBucket{ + {Cond: minCond(0)}, + {Cond: minCond(50 * time.Millisecond)}, + {Cond: minCond(100 * time.Millisecond)}, + {Cond: minCond(200 * time.Millisecond)}, + {Cond: minCond(500 * time.Millisecond)}, + {Cond: minCond(1 * time.Second)}, + {Cond: minCond(10 * time.Second)}, + {Cond: minCond(100 * time.Second)}, + {Cond: errorCond{}}, + }, + Latency: timeseries.NewMinuteHourSeries(func() timeseries.Observable { return new(histogram) }), + } +} + +// traceBucket represents a size-capped bucket of historic traces, +// along with a condition for a trace to belong to the bucket. +type traceBucket struct { + Cond cond + + // Ring buffer implementation of a fixed-size FIFO queue. + mu sync.RWMutex + buf [tracesPerBucket]*trace + start int // < tracesPerBucket + length int // <= tracesPerBucket +} + +func (b *traceBucket) Add(tr *trace) { + b.mu.Lock() + defer b.mu.Unlock() + + i := b.start + b.length + if i >= tracesPerBucket { + i -= tracesPerBucket + } + if b.length == tracesPerBucket { + // "Remove" an element from the bucket. + b.buf[i].unref() + b.start++ + if b.start == tracesPerBucket { + b.start = 0 + } + } + b.buf[i] = tr + if b.length < tracesPerBucket { + b.length++ + } + tr.ref() +} + +// Copy returns a copy of the traces in the bucket. +// If tracedOnly is true, only the traces with trace information will be returned. +// The logs will be ref'd before returning; the caller should call +// the Free method when it is done with them. +// TODO(dsymonds): keep track of traced requests in separate buckets. +func (b *traceBucket) Copy(tracedOnly bool) traceList { + b.mu.RLock() + defer b.mu.RUnlock() + + trl := make(traceList, 0, b.length) + for i, x := 0, b.start; i < b.length; i++ { + tr := b.buf[x] + if !tracedOnly || tr.spanID != 0 { + tr.ref() + trl = append(trl, tr) + } + x++ + if x == b.length { + x = 0 + } + } + return trl +} + +func (b *traceBucket) Empty() bool { + b.mu.RLock() + defer b.mu.RUnlock() + return b.length == 0 +} + +// cond represents a condition on a trace. +type cond interface { + match(t *trace) bool + String() string +} + +type minCond time.Duration + +func (m minCond) match(t *trace) bool { return t.Elapsed >= time.Duration(m) } +func (m minCond) String() string { return fmt.Sprintf("≥%gs", time.Duration(m).Seconds()) } + +type errorCond struct{} + +func (e errorCond) match(t *trace) bool { return t.IsError } +func (e errorCond) String() string { return "errors" } + +type traceList []*trace + +// Free calls unref on each element of the list. +func (trl traceList) Free() { + for _, t := range trl { + t.unref() + } +} + +// traceList may be sorted in reverse chronological order. +func (trl traceList) Len() int { return len(trl) } +func (trl traceList) Less(i, j int) bool { return trl[i].Start.After(trl[j].Start) } +func (trl traceList) Swap(i, j int) { trl[i], trl[j] = trl[j], trl[i] } + +// An event is a timestamped log entry in a trace. +type event struct { + When time.Time + Elapsed time.Duration // since previous event in trace + NewDay bool // whether this event is on a different day to the previous event + Recyclable bool // whether this event was passed via LazyLog + What interface{} // string or fmt.Stringer + Sensitive bool // whether this event contains sensitive information +} + +// WhenString returns a string representation of the elapsed time of the event. +// It will include the date if midnight was crossed. +func (e event) WhenString() string { + if e.NewDay { + return e.When.Format("2006/01/02 15:04:05.000000") + } + return e.When.Format("15:04:05.000000") +} + +// discarded represents a number of discarded events. +// It is stored as *discarded to make it easier to update in-place. +type discarded int + +func (d *discarded) String() string { + return fmt.Sprintf("(%d events discarded)", int(*d)) +} + +// trace represents an active or complete request, +// either sent or received by this program. +type trace struct { + // Family is the top-level grouping of traces to which this belongs. + Family string + + // Title is the title of this trace. + Title string + + // Timing information. + Start time.Time + Elapsed time.Duration // zero while active + + // Trace information if non-zero. + traceID uint64 + spanID uint64 + + // Whether this trace resulted in an error. + IsError bool + + // Append-only sequence of events (modulo discards). + mu sync.RWMutex + events []event + + refs int32 // how many buckets this is in + recycler func(interface{}) + disc discarded // scratch space to avoid allocation + + finishStack []byte // where finish was called, if DebugUseAfterFinish is set +} + +func (tr *trace) reset() { + // Clear all but the mutex. Mutexes may not be copied, even when unlocked. + tr.Family = "" + tr.Title = "" + tr.Start = time.Time{} + tr.Elapsed = 0 + tr.traceID = 0 + tr.spanID = 0 + tr.IsError = false + tr.events = nil + tr.refs = 0 + tr.recycler = nil + tr.disc = 0 + tr.finishStack = nil +} + +// delta returns the elapsed time since the last event or the trace start, +// and whether it spans midnight. +// L >= tr.mu +func (tr *trace) delta(t time.Time) (time.Duration, bool) { + if len(tr.events) == 0 { + return t.Sub(tr.Start), false + } + prev := tr.events[len(tr.events)-1].When + return t.Sub(prev), prev.Day() != t.Day() +} + +func (tr *trace) addEvent(x interface{}, recyclable, sensitive bool) { + if DebugUseAfterFinish && tr.finishStack != nil { + buf := make([]byte, 4<<10) // 4 KB should be enough + n := runtime.Stack(buf, false) + log.Printf("net/trace: trace used after finish:\nFinished at:\n%s\nUsed at:\n%s", tr.finishStack, buf[:n]) + } + + /* + NOTE TO DEBUGGERS + + If you are here because your program panicked in this code, + it is almost definitely the fault of code using this package, + and very unlikely to be the fault of this code. + + The most likely scenario is that some code elsewhere is using + a requestz.Trace after its Finish method is called. + You can temporarily set the DebugUseAfterFinish var + to help discover where that is; do not leave that var set, + since it makes this package much less efficient. + */ + + e := event{When: time.Now(), What: x, Recyclable: recyclable, Sensitive: sensitive} + tr.mu.Lock() + e.Elapsed, e.NewDay = tr.delta(e.When) + if len(tr.events) < cap(tr.events) { + tr.events = append(tr.events, e) + } else { + // Discard the middle events. + di := int((cap(tr.events) - 1) / 2) + if d, ok := tr.events[di].What.(*discarded); ok { + (*d)++ + } else { + // disc starts at two to count for the event it is replacing, + // plus the next one that we are about to drop. + tr.disc = 2 + if tr.recycler != nil && tr.events[di].Recyclable { + go tr.recycler(tr.events[di].What) + } + tr.events[di].What = &tr.disc + } + // The timestamp of the discarded meta-event should be + // the time of the last event it is representing. + tr.events[di].When = tr.events[di+1].When + + if tr.recycler != nil && tr.events[di+1].Recyclable { + go tr.recycler(tr.events[di+1].What) + } + copy(tr.events[di+1:], tr.events[di+2:]) + tr.events[cap(tr.events)-1] = e + } + tr.mu.Unlock() +} + +func (tr *trace) LazyLog(x fmt.Stringer, sensitive bool) { + tr.addEvent(x, true, sensitive) +} + +func (tr *trace) LazyPrintf(format string, a ...interface{}) { + tr.addEvent(&lazySprintf{format, a}, false, false) +} + +func (tr *trace) SetError() { tr.IsError = true } + +func (tr *trace) SetRecycler(f func(interface{})) { + tr.recycler = f +} + +func (tr *trace) SetTraceInfo(traceID, spanID uint64) { + tr.traceID, tr.spanID = traceID, spanID +} + +func (tr *trace) SetMaxEvents(m int) { + // Always keep at least three events: first, discarded count, last. + if len(tr.events) == 0 && m > 3 { + tr.events = make([]event, 0, m) + } +} + +func (tr *trace) ref() { + atomic.AddInt32(&tr.refs, 1) +} + +func (tr *trace) unref() { + if atomic.AddInt32(&tr.refs, -1) == 0 { + if tr.recycler != nil { + // freeTrace clears tr, so we hold tr.recycler and tr.events here. + go func(f func(interface{}), es []event) { + for _, e := range es { + if e.Recyclable { + f(e.What) + } + } + }(tr.recycler, tr.events) + } + + freeTrace(tr) + } +} + +func (tr *trace) When() string { + return tr.Start.Format("2006/01/02 15:04:05.000000") +} + +func (tr *trace) ElapsedTime() string { + t := tr.Elapsed + if t == 0 { + // Active trace. + t = time.Since(tr.Start) + } + return fmt.Sprintf("%.6f", t.Seconds()) +} + +func (tr *trace) Events() []event { + tr.mu.RLock() + defer tr.mu.RUnlock() + return tr.events +} + +var traceFreeList = make(chan *trace, 1000) // TODO(dsymonds): Use sync.Pool? + +// newTrace returns a trace ready to use. +func newTrace() *trace { + select { + case tr := <-traceFreeList: + return tr + default: + return new(trace) + } +} + +// freeTrace adds tr to traceFreeList if there's room. +// This is non-blocking. +func freeTrace(tr *trace) { + if DebugUseAfterFinish { + return // never reuse + } + tr.reset() + select { + case traceFreeList <- tr: + default: + } +} + +func elapsed(d time.Duration) string { + b := []byte(fmt.Sprintf("%.6f", d.Seconds())) + + // For subsecond durations, blank all zeros before decimal point, + // and all zeros between the decimal point and the first non-zero digit. + if d < time.Second { + dot := bytes.IndexByte(b, '.') + for i := 0; i < dot; i++ { + b[i] = ' ' + } + for i := dot + 1; i < len(b); i++ { + if b[i] == '0' { + b[i] = ' ' + } else { + break + } + } + } + + return string(b) +} + +var pageTmpl = template.Must(template.New("Page").Funcs(template.FuncMap{ + "elapsed": elapsed, + "add": func(a, b int) int { return a + b }, +}).Parse(pageHTML)) + +const pageHTML = ` +{{template "Prolog" .}} +{{template "StatusTable" .}} +{{template "Epilog" .}} + +{{define "Prolog"}} + + + /debug/requests + + + + +

/debug/requests

+{{end}} {{/* end of Prolog */}} + +{{define "StatusTable"}} + + {{range $fam := .Families}} + + + + {{$n := index $.ActiveTraceCount $fam}} + + + {{$f := index $.CompletedTraces $fam}} + {{range $i, $b := $f.Buckets}} + {{$empty := $b.Empty}} + + {{end}} + + {{$nb := len $f.Buckets}} + + + + + + {{end}} +
{{$fam}} + {{if $n}}{{end}} + [{{$n}} active] + {{if $n}}{{end}} + + {{if not $empty}}{{end}} + [{{.Cond}}] + {{if not $empty}}{{end}} + + [minute] + + [hour] + + [total] +
+{{end}} {{/* end of StatusTable */}} + +{{define "Epilog"}} +{{if $.Traces}} +
+

Family: {{$.Family}}

+ +{{if or $.Expanded $.Traced}} + [Normal/Summary] +{{else}} + [Normal/Summary] +{{end}} + +{{if or (not $.Expanded) $.Traced}} + [Normal/Expanded] +{{else}} + [Normal/Expanded] +{{end}} + +{{if not $.Active}} + {{if or $.Expanded (not $.Traced)}} + [Traced/Summary] + {{else}} + [Traced/Summary] + {{end}} + {{if or (not $.Expanded) (not $.Traced)}} + [Traced/Expanded] + {{else}} + [Traced/Expanded] + {{end}} +{{end}} + +{{if $.Total}} +

Showing {{len $.Traces}} of {{$.Total}} traces.

+{{end}} + + + + + {{range $tr := $.Traces}} + + + + + {{/* TODO: include traceID/spanID */}} + + {{if $.Expanded}} + {{range $tr.Events}} + + + + + + {{end}} + {{end}} + {{end}} +
+ {{if $.Active}}Active{{else}}Completed{{end}} Requests +
WhenElapsed (s)
{{$tr.When}}{{$tr.ElapsedTime}}{{$tr.Title}}
{{.WhenString}}{{elapsed .Elapsed}}{{if or $.ShowSensitive (not .Sensitive)}}... {{.What}}{{else}}[redacted]{{end}}
+{{end}} {{/* if $.Traces */}} + +{{if $.Histogram}} +

Latency (µs) of {{$.Family}} over {{$.HistogramWindow}}

+{{$.Histogram}} +{{end}} {{/* if $.Histogram */}} + + + +{{end}} {{/* end of Epilog */}} +` diff --git a/components/engine/vendor/src/golang.org/x/oauth2/.travis.yml b/components/engine/vendor/src/golang.org/x/oauth2/.travis.yml new file mode 100644 index 0000000000..a035125c35 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/.travis.yml @@ -0,0 +1,14 @@ +language: go + +go: + - 1.3 + - 1.4 + +install: + - export GOPATH="$HOME/gopath" + - mkdir -p "$GOPATH/src/golang.org/x" + - mv "$TRAVIS_BUILD_DIR" "$GOPATH/src/golang.org/x/oauth2" + - go get -v -t -d golang.org/x/oauth2/... + +script: + - go test -v golang.org/x/oauth2/... diff --git a/components/engine/vendor/src/golang.org/x/oauth2/AUTHORS b/components/engine/vendor/src/golang.org/x/oauth2/AUTHORS new file mode 100644 index 0000000000..15167cd746 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/components/engine/vendor/src/golang.org/x/oauth2/CONTRIBUTING.md b/components/engine/vendor/src/golang.org/x/oauth2/CONTRIBUTING.md new file mode 100644 index 0000000000..46aa2b12dd --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing to Go + +Go is an open source project. + +It is the work of hundreds of contributors. We appreciate your help! + + +## Filing issues + +When [filing an issue](https://github.com/golang/oauth2/issues), make sure to answer these five questions: + +1. What version of Go are you using (`go version`)? +2. What operating system and processor architecture are you using? +3. What did you do? +4. What did you expect to see? +5. What did you see instead? + +General questions should go to the [golang-nuts mailing list](https://groups.google.com/group/golang-nuts) instead of the issue tracker. +The gophers there will answer or ask you to file an issue if you've tripped over a bug. + +## Contributing code + +Please read the [Contribution Guidelines](https://golang.org/doc/contribute.html) +before sending patches. + +**We do not accept GitHub pull requests** +(we use [Gerrit](https://code.google.com/p/gerrit/) instead for code review). + +Unless otherwise noted, the Go source files are distributed under +the BSD-style license found in the LICENSE file. + diff --git a/components/engine/vendor/src/golang.org/x/oauth2/CONTRIBUTORS b/components/engine/vendor/src/golang.org/x/oauth2/CONTRIBUTORS new file mode 100644 index 0000000000..1c4577e968 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/CONTRIBUTORS @@ -0,0 +1,3 @@ +# This source code was written by the Go contributors. +# The master list of contributors is in the main Go distribution, +# visible at http://tip.golang.org/CONTRIBUTORS. diff --git a/components/engine/vendor/src/golang.org/x/oauth2/LICENSE b/components/engine/vendor/src/golang.org/x/oauth2/LICENSE new file mode 100644 index 0000000000..d02f24fd52 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The oauth2 Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/components/engine/vendor/src/golang.org/x/oauth2/README.md b/components/engine/vendor/src/golang.org/x/oauth2/README.md new file mode 100644 index 0000000000..0d5141733f --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/README.md @@ -0,0 +1,64 @@ +# OAuth2 for Go + +[![Build Status](https://travis-ci.org/golang/oauth2.svg?branch=master)](https://travis-ci.org/golang/oauth2) + +oauth2 package contains a client implementation for OAuth 2.0 spec. + +## Installation + +~~~~ +go get golang.org/x/oauth2 +~~~~ + +See godoc for further documentation and examples. + +* [godoc.org/golang.org/x/oauth2](http://godoc.org/golang.org/x/oauth2) +* [godoc.org/golang.org/x/oauth2/google](http://godoc.org/golang.org/x/oauth2/google) + + +## App Engine + +In change 96e89be (March 2015) we removed the `oauth2.Context2` type in favor +of the [`context.Context`](https://golang.org/x/net/context#Context) type from +the `golang.org/x/net/context` package + +This means its no longer possible to use the "Classic App Engine" +`appengine.Context` type with the `oauth2` package. (You're using +Classic App Engine if you import the package `"appengine"`.) + +To work around this, you may use the new `"google.golang.org/appengine"` +package. This package has almost the same API as the `"appengine"` package, +but it can be fetched with `go get` and used on "Managed VMs" and well as +Classic App Engine. + +See the [new `appengine` package's readme](https://github.com/golang/appengine#updating-a-go-app-engine-app) +for information on updating your app. + +If you don't want to update your entire app to use the new App Engine packages, +you may use both sets of packages in parallel, using only the new packages +with the `oauth2` package. + + import ( + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + newappengine "google.golang.org/appengine" + newurlfetch "google.golang.org/appengine/urlfetch" + + "appengine" + ) + + func handler(w http.ResponseWriter, r *http.Request) { + var c appengine.Context = appengine.NewContext(r) + c.Infof("Logging a message with the old package") + + var ctx context.Context = newappengine.NewContext(r) + client := &http.Client{ + Transport: &oauth2.Transport{ + Source: google.AppEngineTokenSource(ctx, "scope"), + Base: &newurlfetch.Transport{Context: ctx}, + }, + } + client.Get("...") + } + diff --git a/components/engine/vendor/src/golang.org/x/oauth2/client_appengine.go b/components/engine/vendor/src/golang.org/x/oauth2/client_appengine.go new file mode 100644 index 0000000000..8962c49d1d --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/client_appengine.go @@ -0,0 +1,25 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +// App Engine hooks. + +package oauth2 + +import ( + "net/http" + + "golang.org/x/net/context" + "golang.org/x/oauth2/internal" + "google.golang.org/appengine/urlfetch" +) + +func init() { + internal.RegisterContextClientFunc(contextClientAppEngine) +} + +func contextClientAppEngine(ctx context.Context) (*http.Client, error) { + return urlfetch.Client(ctx), nil +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/google/appengine.go b/components/engine/vendor/src/golang.org/x/oauth2/google/appengine.go new file mode 100644 index 0000000000..dc993efb5e --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/google/appengine.go @@ -0,0 +1,86 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package google + +import ( + "sort" + "strings" + "sync" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2" +) + +// Set at init time by appenginevm_hook.go. If true, we are on App Engine Managed VMs. +var appengineVM bool + +// Set at init time by appengine_hook.go. If nil, we're not on App Engine. +var appengineTokenFunc func(c context.Context, scopes ...string) (token string, expiry time.Time, err error) + +// AppEngineTokenSource returns a token source that fetches tokens +// issued to the current App Engine application's service account. +// If you are implementing a 3-legged OAuth 2.0 flow on App Engine +// that involves user accounts, see oauth2.Config instead. +// +// The provided context must have come from appengine.NewContext. +func AppEngineTokenSource(ctx context.Context, scope ...string) oauth2.TokenSource { + if appengineTokenFunc == nil { + panic("google: AppEngineTokenSource can only be used on App Engine.") + } + scopes := append([]string{}, scope...) + sort.Strings(scopes) + return &appEngineTokenSource{ + ctx: ctx, + scopes: scopes, + key: strings.Join(scopes, " "), + } +} + +// aeTokens helps the fetched tokens to be reused until their expiration. +var ( + aeTokensMu sync.Mutex + aeTokens = make(map[string]*tokenLock) // key is space-separated scopes +) + +type tokenLock struct { + mu sync.Mutex // guards t; held while fetching or updating t + t *oauth2.Token +} + +type appEngineTokenSource struct { + ctx context.Context + scopes []string + key string // to aeTokens map; space-separated scopes +} + +func (ts *appEngineTokenSource) Token() (*oauth2.Token, error) { + if appengineTokenFunc == nil { + panic("google: AppEngineTokenSource can only be used on App Engine.") + } + + aeTokensMu.Lock() + tok, ok := aeTokens[ts.key] + if !ok { + tok = &tokenLock{} + aeTokens[ts.key] = tok + } + aeTokensMu.Unlock() + + tok.mu.Lock() + defer tok.mu.Unlock() + if tok.t.Valid() { + return tok.t, nil + } + access, exp, err := appengineTokenFunc(ts.ctx, ts.scopes...) + if err != nil { + return nil, err + } + tok.t = &oauth2.Token{ + AccessToken: access, + Expiry: exp, + } + return tok.t, nil +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/google/appengine_hook.go b/components/engine/vendor/src/golang.org/x/oauth2/google/appengine_hook.go new file mode 100644 index 0000000000..4f42c8b343 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/google/appengine_hook.go @@ -0,0 +1,13 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appengine + +package google + +import "google.golang.org/appengine" + +func init() { + appengineTokenFunc = appengine.AccessToken +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/google/appenginevm_hook.go b/components/engine/vendor/src/golang.org/x/oauth2/google/appenginevm_hook.go new file mode 100644 index 0000000000..633611cc3a --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/google/appenginevm_hook.go @@ -0,0 +1,14 @@ +// Copyright 2015 The oauth2 Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build appenginevm + +package google + +import "google.golang.org/appengine" + +func init() { + appengineVM = true + appengineTokenFunc = appengine.AccessToken +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/google/default.go b/components/engine/vendor/src/golang.org/x/oauth2/google/default.go new file mode 100644 index 0000000000..b952362977 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/google/default.go @@ -0,0 +1,155 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package google + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "runtime" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/jwt" + "google.golang.org/cloud/compute/metadata" +) + +// DefaultClient returns an HTTP Client that uses the +// DefaultTokenSource to obtain authentication credentials. +// +// This client should be used when developing services +// that run on Google App Engine or Google Compute Engine +// and use "Application Default Credentials." +// +// For more details, see: +// https://developers.google.com/accounts/docs/application-default-credentials +// +func DefaultClient(ctx context.Context, scope ...string) (*http.Client, error) { + ts, err := DefaultTokenSource(ctx, scope...) + if err != nil { + return nil, err + } + return oauth2.NewClient(ctx, ts), nil +} + +// DefaultTokenSource is a token source that uses +// "Application Default Credentials". +// +// It looks for credentials in the following places, +// preferring the first location found: +// +// 1. A JSON file whose path is specified by the +// GOOGLE_APPLICATION_CREDENTIALS environment variable. +// 2. A JSON file in a location known to the gcloud command-line tool. +// On Windows, this is %APPDATA%/gcloud/application_default_credentials.json. +// On other systems, $HOME/.config/gcloud/application_default_credentials.json. +// 3. On Google App Engine it uses the appengine.AccessToken function. +// 4. On Google Compute Engine and Google App Engine Managed VMs, it fetches +// credentials from the metadata server. +// (In this final case any provided scopes are ignored.) +// +// For more details, see: +// https://developers.google.com/accounts/docs/application-default-credentials +// +func DefaultTokenSource(ctx context.Context, scope ...string) (oauth2.TokenSource, error) { + // First, try the environment variable. + const envVar = "GOOGLE_APPLICATION_CREDENTIALS" + if filename := os.Getenv(envVar); filename != "" { + ts, err := tokenSourceFromFile(ctx, filename, scope) + if err != nil { + return nil, fmt.Errorf("google: error getting credentials using %v environment variable: %v", envVar, err) + } + return ts, nil + } + + // Second, try a well-known file. + filename := wellKnownFile() + _, err := os.Stat(filename) + if err == nil { + ts, err2 := tokenSourceFromFile(ctx, filename, scope) + if err2 == nil { + return ts, nil + } + err = err2 + } else if os.IsNotExist(err) { + err = nil // ignore this error + } + if err != nil { + return nil, fmt.Errorf("google: error getting credentials using well-known file (%v): %v", filename, err) + } + + // Third, if we're on Google App Engine use those credentials. + if appengineTokenFunc != nil && !appengineVM { + return AppEngineTokenSource(ctx, scope...), nil + } + + // Fourth, if we're on Google Compute Engine use the metadata server. + if metadata.OnGCE() { + return ComputeTokenSource(""), nil + } + + // None are found; return helpful error. + const url = "https://developers.google.com/accounts/docs/application-default-credentials" + return nil, fmt.Errorf("google: could not find default credentials. See %v for more information.", url) +} + +func wellKnownFile() string { + const f = "application_default_credentials.json" + if runtime.GOOS == "windows" { + return filepath.Join(os.Getenv("APPDATA"), "gcloud", f) + } + return filepath.Join(guessUnixHomeDir(), ".config", "gcloud", f) +} + +func tokenSourceFromFile(ctx context.Context, filename string, scopes []string) (oauth2.TokenSource, error) { + b, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + var d struct { + // Common fields + Type string + ClientID string `json:"client_id"` + + // User Credential fields + ClientSecret string `json:"client_secret"` + RefreshToken string `json:"refresh_token"` + + // Service Account fields + ClientEmail string `json:"client_email"` + PrivateKeyID string `json:"private_key_id"` + PrivateKey string `json:"private_key"` + } + if err := json.Unmarshal(b, &d); err != nil { + return nil, err + } + switch d.Type { + case "authorized_user": + cfg := &oauth2.Config{ + ClientID: d.ClientID, + ClientSecret: d.ClientSecret, + Scopes: append([]string{}, scopes...), // copy + Endpoint: Endpoint, + } + tok := &oauth2.Token{RefreshToken: d.RefreshToken} + return cfg.TokenSource(ctx, tok), nil + case "service_account": + cfg := &jwt.Config{ + Email: d.ClientEmail, + PrivateKey: []byte(d.PrivateKey), + Scopes: append([]string{}, scopes...), // copy + TokenURL: JWTTokenURL, + } + return cfg.TokenSource(ctx), nil + case "": + return nil, errors.New("missing 'type' field in credentials") + default: + return nil, fmt.Errorf("unknown credential type: %q", d.Type) + } +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/google/google.go b/components/engine/vendor/src/golang.org/x/oauth2/google/google.go new file mode 100644 index 0000000000..9a3d5feb1b --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/google/google.go @@ -0,0 +1,145 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package google provides support for making OAuth2 authorized and +// authenticated HTTP requests to Google APIs. +// It supports the Web server flow, client-side credentials, service accounts, +// Google Compute Engine service accounts, and Google App Engine service +// accounts. +// +// For more information, please read +// https://developers.google.com/accounts/docs/OAuth2 +// and +// https://developers.google.com/accounts/docs/application-default-credentials. +package google // import "golang.org/x/oauth2/google" + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/jwt" + "google.golang.org/cloud/compute/metadata" +) + +// Endpoint is Google's OAuth 2.0 endpoint. +var Endpoint = oauth2.Endpoint{ + AuthURL: "https://accounts.google.com/o/oauth2/auth", + TokenURL: "https://accounts.google.com/o/oauth2/token", +} + +// JWTTokenURL is Google's OAuth 2.0 token URL to use with the JWT flow. +const JWTTokenURL = "https://accounts.google.com/o/oauth2/token" + +// ConfigFromJSON uses a Google Developers Console client_credentials.json +// file to construct a config. +// client_credentials.json can be downloadable from https://console.developers.google.com, +// under "APIs & Auth" > "Credentials". Download the Web application credentials in the +// JSON format and provide the contents of the file as jsonKey. +func ConfigFromJSON(jsonKey []byte, scope ...string) (*oauth2.Config, error) { + type cred struct { + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + RedirectURIs []string `json:"redirect_uris"` + AuthURI string `json:"auth_uri"` + TokenURI string `json:"token_uri"` + } + var j struct { + Web *cred `json:"web"` + Installed *cred `json:"installed"` + } + if err := json.Unmarshal(jsonKey, &j); err != nil { + return nil, err + } + var c *cred + switch { + case j.Web != nil: + c = j.Web + case j.Installed != nil: + c = j.Installed + default: + return nil, fmt.Errorf("oauth2/google: no credentials found") + } + if len(c.RedirectURIs) < 1 { + return nil, errors.New("oauth2/google: missing redirect URL in the client_credentials.json") + } + return &oauth2.Config{ + ClientID: c.ClientID, + ClientSecret: c.ClientSecret, + RedirectURL: c.RedirectURIs[0], + Scopes: scope, + Endpoint: oauth2.Endpoint{ + AuthURL: c.AuthURI, + TokenURL: c.TokenURI, + }, + }, nil +} + +// JWTConfigFromJSON uses a Google Developers service account JSON key file to read +// the credentials that authorize and authenticate the requests. +// Create a service account on "Credentials" page under "APIs & Auth" for your +// project at https://console.developers.google.com to download a JSON key file. +func JWTConfigFromJSON(jsonKey []byte, scope ...string) (*jwt.Config, error) { + var key struct { + Email string `json:"client_email"` + PrivateKey string `json:"private_key"` + } + if err := json.Unmarshal(jsonKey, &key); err != nil { + return nil, err + } + return &jwt.Config{ + Email: key.Email, + PrivateKey: []byte(key.PrivateKey), + Scopes: scope, + TokenURL: JWTTokenURL, + }, nil +} + +// ComputeTokenSource returns a token source that fetches access tokens +// from Google Compute Engine (GCE)'s metadata server. It's only valid to use +// this token source if your program is running on a GCE instance. +// If no account is specified, "default" is used. +// Further information about retrieving access tokens from the GCE metadata +// server can be found at https://cloud.google.com/compute/docs/authentication. +func ComputeTokenSource(account string) oauth2.TokenSource { + return oauth2.ReuseTokenSource(nil, computeSource{account: account}) +} + +type computeSource struct { + account string +} + +func (cs computeSource) Token() (*oauth2.Token, error) { + if !metadata.OnGCE() { + return nil, errors.New("oauth2/google: can't get a token from the metadata service; not running on GCE") + } + acct := cs.account + if acct == "" { + acct = "default" + } + tokenJSON, err := metadata.Get("instance/service-accounts/" + acct + "/token") + if err != nil { + return nil, err + } + var res struct { + AccessToken string `json:"access_token"` + ExpiresInSec int `json:"expires_in"` + TokenType string `json:"token_type"` + } + err = json.NewDecoder(strings.NewReader(tokenJSON)).Decode(&res) + if err != nil { + return nil, fmt.Errorf("oauth2/google: invalid token JSON from metadata: %v", err) + } + if res.ExpiresInSec == 0 || res.AccessToken == "" { + return nil, fmt.Errorf("oauth2/google: incomplete token received from metadata") + } + return &oauth2.Token{ + AccessToken: res.AccessToken, + TokenType: res.TokenType, + Expiry: time.Now().Add(time.Duration(res.ExpiresInSec) * time.Second), + }, nil +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/google/jwt.go b/components/engine/vendor/src/golang.org/x/oauth2/google/jwt.go new file mode 100644 index 0000000000..b91991786f --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/google/jwt.go @@ -0,0 +1,71 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package google + +import ( + "crypto/rsa" + "fmt" + "time" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/internal" + "golang.org/x/oauth2/jws" +) + +// JWTAccessTokenSourceFromJSON uses a Google Developers service account JSON +// key file to read the credentials that authorize and authenticate the +// requests, and returns a TokenSource that does not use any OAuth2 flow but +// instead creates a JWT and sends that as the access token. +// The audience is typically a URL that specifies the scope of the credentials. +// +// Note that this is not a standard OAuth flow, but rather an +// optimization supported by a few Google services. +// Unless you know otherwise, you should use JWTConfigFromJSON instead. +func JWTAccessTokenSourceFromJSON(jsonKey []byte, audience string) (oauth2.TokenSource, error) { + cfg, err := JWTConfigFromJSON(jsonKey) + if err != nil { + return nil, fmt.Errorf("google: could not parse JSON key: %v", err) + } + pk, err := internal.ParseKey(cfg.PrivateKey) + if err != nil { + return nil, fmt.Errorf("google: could not parse key: %v", err) + } + ts := &jwtAccessTokenSource{ + email: cfg.Email, + audience: audience, + pk: pk, + } + tok, err := ts.Token() + if err != nil { + return nil, err + } + return oauth2.ReuseTokenSource(tok, ts), nil +} + +type jwtAccessTokenSource struct { + email, audience string + pk *rsa.PrivateKey +} + +func (ts *jwtAccessTokenSource) Token() (*oauth2.Token, error) { + iat := time.Now() + exp := iat.Add(time.Hour) + cs := &jws.ClaimSet{ + Iss: ts.email, + Sub: ts.email, + Aud: ts.audience, + Iat: iat.Unix(), + Exp: exp.Unix(), + } + hdr := &jws.Header{ + Algorithm: "RS256", + Typ: "JWT", + } + msg, err := jws.Encode(hdr, cs, ts.pk) + if err != nil { + return nil, fmt.Errorf("google: could not encode JWT: %v", err) + } + return &oauth2.Token{AccessToken: msg, TokenType: "Bearer", Expiry: exp}, nil +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/google/sdk.go b/components/engine/vendor/src/golang.org/x/oauth2/google/sdk.go new file mode 100644 index 0000000000..d29a3bb9bb --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/google/sdk.go @@ -0,0 +1,168 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package google + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "os/user" + "path/filepath" + "runtime" + "strings" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/internal" +) + +type sdkCredentials struct { + Data []struct { + Credential struct { + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + TokenExpiry *time.Time `json:"token_expiry"` + } `json:"credential"` + Key struct { + Account string `json:"account"` + Scope string `json:"scope"` + } `json:"key"` + } +} + +// An SDKConfig provides access to tokens from an account already +// authorized via the Google Cloud SDK. +type SDKConfig struct { + conf oauth2.Config + initialToken *oauth2.Token +} + +// NewSDKConfig creates an SDKConfig for the given Google Cloud SDK +// account. If account is empty, the account currently active in +// Google Cloud SDK properties is used. +// Google Cloud SDK credentials must be created by running `gcloud auth` +// before using this function. +// The Google Cloud SDK is available at https://cloud.google.com/sdk/. +func NewSDKConfig(account string) (*SDKConfig, error) { + configPath, err := sdkConfigPath() + if err != nil { + return nil, fmt.Errorf("oauth2/google: error getting SDK config path: %v", err) + } + credentialsPath := filepath.Join(configPath, "credentials") + f, err := os.Open(credentialsPath) + if err != nil { + return nil, fmt.Errorf("oauth2/google: failed to load SDK credentials: %v", err) + } + defer f.Close() + + var c sdkCredentials + if err := json.NewDecoder(f).Decode(&c); err != nil { + return nil, fmt.Errorf("oauth2/google: failed to decode SDK credentials from %q: %v", credentialsPath, err) + } + if len(c.Data) == 0 { + return nil, fmt.Errorf("oauth2/google: no credentials found in %q, run `gcloud auth login` to create one", credentialsPath) + } + if account == "" { + propertiesPath := filepath.Join(configPath, "properties") + f, err := os.Open(propertiesPath) + if err != nil { + return nil, fmt.Errorf("oauth2/google: failed to load SDK properties: %v", err) + } + defer f.Close() + ini, err := internal.ParseINI(f) + if err != nil { + return nil, fmt.Errorf("oauth2/google: failed to parse SDK properties %q: %v", propertiesPath, err) + } + core, ok := ini["core"] + if !ok { + return nil, fmt.Errorf("oauth2/google: failed to find [core] section in %v", ini) + } + active, ok := core["account"] + if !ok { + return nil, fmt.Errorf("oauth2/google: failed to find %q attribute in %v", "account", core) + } + account = active + } + + for _, d := range c.Data { + if account == "" || d.Key.Account == account { + if d.Credential.AccessToken == "" && d.Credential.RefreshToken == "" { + return nil, fmt.Errorf("oauth2/google: no token available for account %q", account) + } + var expiry time.Time + if d.Credential.TokenExpiry != nil { + expiry = *d.Credential.TokenExpiry + } + return &SDKConfig{ + conf: oauth2.Config{ + ClientID: d.Credential.ClientID, + ClientSecret: d.Credential.ClientSecret, + Scopes: strings.Split(d.Key.Scope, " "), + Endpoint: Endpoint, + RedirectURL: "oob", + }, + initialToken: &oauth2.Token{ + AccessToken: d.Credential.AccessToken, + RefreshToken: d.Credential.RefreshToken, + Expiry: expiry, + }, + }, nil + } + } + return nil, fmt.Errorf("oauth2/google: no such credentials for account %q", account) +} + +// Client returns an HTTP client using Google Cloud SDK credentials to +// authorize requests. The token will auto-refresh as necessary. The +// underlying http.RoundTripper will be obtained using the provided +// context. The returned client and its Transport should not be +// modified. +func (c *SDKConfig) Client(ctx context.Context) *http.Client { + return &http.Client{ + Transport: &oauth2.Transport{ + Source: c.TokenSource(ctx), + }, + } +} + +// TokenSource returns an oauth2.TokenSource that retrieve tokens from +// Google Cloud SDK credentials using the provided context. +// It will returns the current access token stored in the credentials, +// and refresh it when it expires, but it won't update the credentials +// with the new access token. +func (c *SDKConfig) TokenSource(ctx context.Context) oauth2.TokenSource { + return c.conf.TokenSource(ctx, c.initialToken) +} + +// Scopes are the OAuth 2.0 scopes the current account is authorized for. +func (c *SDKConfig) Scopes() []string { + return c.conf.Scopes +} + +// sdkConfigPath tries to guess where the gcloud config is located. +// It can be overridden during tests. +var sdkConfigPath = func() (string, error) { + if runtime.GOOS == "windows" { + return filepath.Join(os.Getenv("APPDATA"), "gcloud"), nil + } + homeDir := guessUnixHomeDir() + if homeDir == "" { + return "", errors.New("unable to get current user home directory: os/user lookup failed; $HOME is empty") + } + return filepath.Join(homeDir, ".config", "gcloud"), nil +} + +func guessUnixHomeDir() string { + usr, err := user.Current() + if err == nil { + return usr.HomeDir + } + return os.Getenv("HOME") +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/internal/oauth2.go b/components/engine/vendor/src/golang.org/x/oauth2/internal/oauth2.go new file mode 100644 index 0000000000..fbe1028d64 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/internal/oauth2.go @@ -0,0 +1,76 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package internal contains support packages for oauth2 package. +package internal + +import ( + "bufio" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "io" + "strings" +) + +// ParseKey converts the binary contents of a private key file +// to an *rsa.PrivateKey. It detects whether the private key is in a +// PEM container or not. If so, it extracts the the private key +// from PEM container before conversion. It only supports PEM +// containers with no passphrase. +func ParseKey(key []byte) (*rsa.PrivateKey, error) { + block, _ := pem.Decode(key) + if block != nil { + key = block.Bytes + } + parsedKey, err := x509.ParsePKCS8PrivateKey(key) + if err != nil { + parsedKey, err = x509.ParsePKCS1PrivateKey(key) + if err != nil { + return nil, fmt.Errorf("private key should be a PEM or plain PKSC1 or PKCS8; parse error: %v", err) + } + } + parsed, ok := parsedKey.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("private key is invalid") + } + return parsed, nil +} + +func ParseINI(ini io.Reader) (map[string]map[string]string, error) { + result := map[string]map[string]string{ + "": map[string]string{}, // root section + } + scanner := bufio.NewScanner(ini) + currentSection := "" + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(line, ";") { + // comment. + continue + } + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + currentSection = strings.TrimSpace(line[1 : len(line)-1]) + result[currentSection] = map[string]string{} + continue + } + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 && parts[0] != "" { + result[currentSection][strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1]) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error scanning ini: %v", err) + } + return result, nil +} + +func CondVal(v string) []string { + if v == "" { + return nil + } + return []string{v} +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/internal/token.go b/components/engine/vendor/src/golang.org/x/oauth2/internal/token.go new file mode 100644 index 0000000000..39caf6c617 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/internal/token.go @@ -0,0 +1,221 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package internal contains support packages for oauth2 package. +package internal + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "golang.org/x/net/context" +) + +// Token represents the crendentials used to authorize +// the requests to access protected resources on the OAuth 2.0 +// provider's backend. +// +// This type is a mirror of oauth2.Token and exists to break +// an otherwise-circular dependency. Other internal packages +// should convert this Token into an oauth2.Token before use. +type Token struct { + // AccessToken is the token that authorizes and authenticates + // the requests. + AccessToken string + + // TokenType is the type of token. + // The Type method returns either this or "Bearer", the default. + TokenType string + + // RefreshToken is a token that's used by the application + // (as opposed to the user) to refresh the access token + // if it expires. + RefreshToken string + + // Expiry is the optional expiration time of the access token. + // + // If zero, TokenSource implementations will reuse the same + // token forever and RefreshToken or equivalent + // mechanisms for that TokenSource will not be used. + Expiry time.Time + + // Raw optionally contains extra metadata from the server + // when updating a token. + Raw interface{} +} + +// tokenJSON is the struct representing the HTTP response from OAuth2 +// providers returning a token in JSON form. +type tokenJSON struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + RefreshToken string `json:"refresh_token"` + ExpiresIn expirationTime `json:"expires_in"` // at least PayPal returns string, while most return number + Expires expirationTime `json:"expires"` // broken Facebook spelling of expires_in +} + +func (e *tokenJSON) expiry() (t time.Time) { + if v := e.ExpiresIn; v != 0 { + return time.Now().Add(time.Duration(v) * time.Second) + } + if v := e.Expires; v != 0 { + return time.Now().Add(time.Duration(v) * time.Second) + } + return +} + +type expirationTime int32 + +func (e *expirationTime) UnmarshalJSON(b []byte) error { + var n json.Number + err := json.Unmarshal(b, &n) + if err != nil { + return err + } + i, err := n.Int64() + if err != nil { + return err + } + *e = expirationTime(i) + return nil +} + +var brokenAuthHeaderProviders = []string{ + "https://accounts.google.com/", + "https://api.dropbox.com/", + "https://api.instagram.com/", + "https://api.netatmo.net/", + "https://api.odnoklassniki.ru/", + "https://api.pushbullet.com/", + "https://api.soundcloud.com/", + "https://api.twitch.tv/", + "https://app.box.com/", + "https://connect.stripe.com/", + "https://login.microsoftonline.com/", + "https://login.salesforce.com/", + "https://oauth.sandbox.trainingpeaks.com/", + "https://oauth.trainingpeaks.com/", + "https://oauth.vk.com/", + "https://slack.com/", + "https://test-sandbox.auth.corp.google.com", + "https://test.salesforce.com/", + "https://user.gini.net/", + "https://www.douban.com/", + "https://www.googleapis.com/", + "https://www.linkedin.com/", + "https://www.strava.com/oauth/", +} + +func RegisterBrokenAuthHeaderProvider(tokenURL string) { + brokenAuthHeaderProviders = append(brokenAuthHeaderProviders, tokenURL) +} + +// providerAuthHeaderWorks reports whether the OAuth2 server identified by the tokenURL +// implements the OAuth2 spec correctly +// See https://code.google.com/p/goauth2/issues/detail?id=31 for background. +// In summary: +// - Reddit only accepts client secret in the Authorization header +// - Dropbox accepts either it in URL param or Auth header, but not both. +// - Google only accepts URL param (not spec compliant?), not Auth header +// - Stripe only accepts client secret in Auth header with Bearer method, not Basic +func providerAuthHeaderWorks(tokenURL string) bool { + for _, s := range brokenAuthHeaderProviders { + if strings.HasPrefix(tokenURL, s) { + // Some sites fail to implement the OAuth2 spec fully. + return false + } + } + + // Assume the provider implements the spec properly + // otherwise. We can add more exceptions as they're + // discovered. We will _not_ be adding configurable hooks + // to this package to let users select server bugs. + return true +} + +func RetrieveToken(ctx context.Context, ClientID, ClientSecret, TokenURL string, v url.Values) (*Token, error) { + hc, err := ContextClient(ctx) + if err != nil { + return nil, err + } + v.Set("client_id", ClientID) + bustedAuth := !providerAuthHeaderWorks(TokenURL) + if bustedAuth && ClientSecret != "" { + v.Set("client_secret", ClientSecret) + } + req, err := http.NewRequest("POST", TokenURL, strings.NewReader(v.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + if !bustedAuth { + req.SetBasicAuth(ClientID, ClientSecret) + } + r, err := hc.Do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + return nil, fmt.Errorf("oauth2: cannot fetch token: %v", err) + } + if code := r.StatusCode; code < 200 || code > 299 { + return nil, fmt.Errorf("oauth2: cannot fetch token: %v\nResponse: %s", r.Status, body) + } + + var token *Token + content, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")) + switch content { + case "application/x-www-form-urlencoded", "text/plain": + vals, err := url.ParseQuery(string(body)) + if err != nil { + return nil, err + } + token = &Token{ + AccessToken: vals.Get("access_token"), + TokenType: vals.Get("token_type"), + RefreshToken: vals.Get("refresh_token"), + Raw: vals, + } + e := vals.Get("expires_in") + if e == "" { + // TODO(jbd): Facebook's OAuth2 implementation is broken and + // returns expires_in field in expires. Remove the fallback to expires, + // when Facebook fixes their implementation. + e = vals.Get("expires") + } + expires, _ := strconv.Atoi(e) + if expires != 0 { + token.Expiry = time.Now().Add(time.Duration(expires) * time.Second) + } + default: + var tj tokenJSON + if err = json.Unmarshal(body, &tj); err != nil { + return nil, err + } + token = &Token{ + AccessToken: tj.AccessToken, + TokenType: tj.TokenType, + RefreshToken: tj.RefreshToken, + Expiry: tj.expiry(), + Raw: make(map[string]interface{}), + } + json.Unmarshal(body, &token.Raw) // no error checks for optional fields + } + // Don't overwrite `RefreshToken` with an empty value + // if this was a token refreshing request. + if token.RefreshToken == "" { + token.RefreshToken = v.Get("refresh_token") + } + return token, nil +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/internal/transport.go b/components/engine/vendor/src/golang.org/x/oauth2/internal/transport.go new file mode 100644 index 0000000000..f1f173e345 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/internal/transport.go @@ -0,0 +1,69 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package internal contains support packages for oauth2 package. +package internal + +import ( + "net/http" + + "golang.org/x/net/context" +) + +// HTTPClient is the context key to use with golang.org/x/net/context's +// WithValue function to associate an *http.Client value with a context. +var HTTPClient ContextKey + +// ContextKey is just an empty struct. It exists so HTTPClient can be +// an immutable public variable with a unique type. It's immutable +// because nobody else can create a ContextKey, being unexported. +type ContextKey struct{} + +// ContextClientFunc is a func which tries to return an *http.Client +// given a Context value. If it returns an error, the search stops +// with that error. If it returns (nil, nil), the search continues +// down the list of registered funcs. +type ContextClientFunc func(context.Context) (*http.Client, error) + +var contextClientFuncs []ContextClientFunc + +func RegisterContextClientFunc(fn ContextClientFunc) { + contextClientFuncs = append(contextClientFuncs, fn) +} + +func ContextClient(ctx context.Context) (*http.Client, error) { + if ctx != nil { + if hc, ok := ctx.Value(HTTPClient).(*http.Client); ok { + return hc, nil + } + } + for _, fn := range contextClientFuncs { + c, err := fn(ctx) + if err != nil { + return nil, err + } + if c != nil { + return c, nil + } + } + return http.DefaultClient, nil +} + +func ContextTransport(ctx context.Context) http.RoundTripper { + hc, err := ContextClient(ctx) + // This is a rare error case (somebody using nil on App Engine). + if err != nil { + return ErrorTransport{err} + } + return hc.Transport +} + +// ErrorTransport returns the specified error on RoundTrip. +// This RoundTripper should be used in rare error cases where +// error handling can be postponed to response handling time. +type ErrorTransport struct{ Err error } + +func (t ErrorTransport) RoundTrip(*http.Request) (*http.Response, error) { + return nil, t.Err +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/jws/jws.go b/components/engine/vendor/src/golang.org/x/oauth2/jws/jws.go new file mode 100644 index 0000000000..8ca5978432 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/jws/jws.go @@ -0,0 +1,172 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package jws provides encoding and decoding utilities for +// signed JWS messages. +package jws // import "golang.org/x/oauth2/jws" + +import ( + "bytes" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +// ClaimSet contains information about the JWT signature including the +// permissions being requested (scopes), the target of the token, the issuer, +// the time the token was issued, and the lifetime of the token. +type ClaimSet struct { + Iss string `json:"iss"` // email address of the client_id of the application making the access token request + Scope string `json:"scope,omitempty"` // space-delimited list of the permissions the application requests + Aud string `json:"aud"` // descriptor of the intended target of the assertion (Optional). + Exp int64 `json:"exp"` // the expiration time of the assertion (seconds since Unix epoch) + Iat int64 `json:"iat"` // the time the assertion was issued (seconds since Unix epoch) + Typ string `json:"typ,omitempty"` // token type (Optional). + + // Email for which the application is requesting delegated access (Optional). + Sub string `json:"sub,omitempty"` + + // The old name of Sub. Client keeps setting Prn to be + // complaint with legacy OAuth 2.0 providers. (Optional) + Prn string `json:"prn,omitempty"` + + // See http://tools.ietf.org/html/draft-jones-json-web-token-10#section-4.3 + // This array is marshalled using custom code (see (c *ClaimSet) encode()). + PrivateClaims map[string]interface{} `json:"-"` +} + +func (c *ClaimSet) encode() (string, error) { + // Reverting time back for machines whose time is not perfectly in sync. + // If client machine's time is in the future according + // to Google servers, an access token will not be issued. + now := time.Now().Add(-10 * time.Second) + if c.Iat == 0 { + c.Iat = now.Unix() + } + if c.Exp == 0 { + c.Exp = now.Add(time.Hour).Unix() + } + if c.Exp < c.Iat { + return "", fmt.Errorf("jws: invalid Exp = %v; must be later than Iat = %v", c.Exp, c.Iat) + } + + b, err := json.Marshal(c) + if err != nil { + return "", err + } + + if len(c.PrivateClaims) == 0 { + return base64Encode(b), nil + } + + // Marshal private claim set and then append it to b. + prv, err := json.Marshal(c.PrivateClaims) + if err != nil { + return "", fmt.Errorf("jws: invalid map of private claims %v", c.PrivateClaims) + } + + // Concatenate public and private claim JSON objects. + if !bytes.HasSuffix(b, []byte{'}'}) { + return "", fmt.Errorf("jws: invalid JSON %s", b) + } + if !bytes.HasPrefix(prv, []byte{'{'}) { + return "", fmt.Errorf("jws: invalid JSON %s", prv) + } + b[len(b)-1] = ',' // Replace closing curly brace with a comma. + b = append(b, prv[1:]...) // Append private claims. + return base64Encode(b), nil +} + +// Header represents the header for the signed JWS payloads. +type Header struct { + // The algorithm used for signature. + Algorithm string `json:"alg"` + + // Represents the token type. + Typ string `json:"typ"` +} + +func (h *Header) encode() (string, error) { + b, err := json.Marshal(h) + if err != nil { + return "", err + } + return base64Encode(b), nil +} + +// Decode decodes a claim set from a JWS payload. +func Decode(payload string) (*ClaimSet, error) { + // decode returned id token to get expiry + s := strings.Split(payload, ".") + if len(s) < 2 { + // TODO(jbd): Provide more context about the error. + return nil, errors.New("jws: invalid token received") + } + decoded, err := base64Decode(s[1]) + if err != nil { + return nil, err + } + c := &ClaimSet{} + err = json.NewDecoder(bytes.NewBuffer(decoded)).Decode(c) + return c, err +} + +// Signer returns a signature for the given data. +type Signer func(data []byte) (sig []byte, err error) + +// EncodeWithSigner encodes a header and claim set with the provided signer. +func EncodeWithSigner(header *Header, c *ClaimSet, sg Signer) (string, error) { + head, err := header.encode() + if err != nil { + return "", err + } + cs, err := c.encode() + if err != nil { + return "", err + } + ss := fmt.Sprintf("%s.%s", head, cs) + sig, err := sg([]byte(ss)) + if err != nil { + return "", err + } + return fmt.Sprintf("%s.%s", ss, base64Encode(sig)), nil +} + +// Encode encodes a signed JWS with provided header and claim set. +// This invokes EncodeWithSigner using crypto/rsa.SignPKCS1v15 with the given RSA private key. +func Encode(header *Header, c *ClaimSet, key *rsa.PrivateKey) (string, error) { + sg := func(data []byte) (sig []byte, err error) { + h := sha256.New() + h.Write([]byte(data)) + return rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, h.Sum(nil)) + } + return EncodeWithSigner(header, c, sg) +} + +// base64Encode returns and Base64url encoded version of the input string with any +// trailing "=" stripped. +func base64Encode(b []byte) string { + return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=") +} + +// base64Decode decodes the Base64url encoded string +func base64Decode(s string) ([]byte, error) { + // add back missing padding + switch len(s) % 4 { + case 1: + s += "===" + case 2: + s += "==" + case 3: + s += "=" + } + return base64.URLEncoding.DecodeString(s) +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/jwt/jwt.go b/components/engine/vendor/src/golang.org/x/oauth2/jwt/jwt.go new file mode 100644 index 0000000000..2ffad21a60 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/jwt/jwt.go @@ -0,0 +1,153 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package jwt implements the OAuth 2.0 JSON Web Token flow, commonly +// known as "two-legged OAuth 2.0". +// +// See: https://tools.ietf.org/html/draft-ietf-oauth-jwt-bearer-12 +package jwt + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/internal" + "golang.org/x/oauth2/jws" +) + +var ( + defaultGrantType = "urn:ietf:params:oauth:grant-type:jwt-bearer" + defaultHeader = &jws.Header{Algorithm: "RS256", Typ: "JWT"} +) + +// Config is the configuration for using JWT to fetch tokens, +// commonly known as "two-legged OAuth 2.0". +type Config struct { + // Email is the OAuth client identifier used when communicating with + // the configured OAuth provider. + Email string + + // PrivateKey contains the contents of an RSA private key or the + // contents of a PEM file that contains a private key. The provided + // private key is used to sign JWT payloads. + // PEM containers with a passphrase are not supported. + // Use the following command to convert a PKCS 12 file into a PEM. + // + // $ openssl pkcs12 -in key.p12 -out key.pem -nodes + // + PrivateKey []byte + + // Subject is the optional user to impersonate. + Subject string + + // Scopes optionally specifies a list of requested permission scopes. + Scopes []string + + // TokenURL is the endpoint required to complete the 2-legged JWT flow. + TokenURL string + + // Expires optionally specifies how long the token is valid for. + Expires time.Duration +} + +// TokenSource returns a JWT TokenSource using the configuration +// in c and the HTTP client from the provided context. +func (c *Config) TokenSource(ctx context.Context) oauth2.TokenSource { + return oauth2.ReuseTokenSource(nil, jwtSource{ctx, c}) +} + +// Client returns an HTTP client wrapping the context's +// HTTP transport and adding Authorization headers with tokens +// obtained from c. +// +// The returned client and its Transport should not be modified. +func (c *Config) Client(ctx context.Context) *http.Client { + return oauth2.NewClient(ctx, c.TokenSource(ctx)) +} + +// jwtSource is a source that always does a signed JWT request for a token. +// It should typically be wrapped with a reuseTokenSource. +type jwtSource struct { + ctx context.Context + conf *Config +} + +func (js jwtSource) Token() (*oauth2.Token, error) { + pk, err := internal.ParseKey(js.conf.PrivateKey) + if err != nil { + return nil, err + } + hc := oauth2.NewClient(js.ctx, nil) + claimSet := &jws.ClaimSet{ + Iss: js.conf.Email, + Scope: strings.Join(js.conf.Scopes, " "), + Aud: js.conf.TokenURL, + } + if subject := js.conf.Subject; subject != "" { + claimSet.Sub = subject + // prn is the old name of sub. Keep setting it + // to be compatible with legacy OAuth 2.0 providers. + claimSet.Prn = subject + } + if t := js.conf.Expires; t > 0 { + claimSet.Exp = time.Now().Add(t).Unix() + } + payload, err := jws.Encode(defaultHeader, claimSet, pk) + if err != nil { + return nil, err + } + v := url.Values{} + v.Set("grant_type", defaultGrantType) + v.Set("assertion", payload) + resp, err := hc.PostForm(js.conf.TokenURL, v) + if err != nil { + return nil, fmt.Errorf("oauth2: cannot fetch token: %v", err) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return nil, fmt.Errorf("oauth2: cannot fetch token: %v", err) + } + if c := resp.StatusCode; c < 200 || c > 299 { + return nil, fmt.Errorf("oauth2: cannot fetch token: %v\nResponse: %s", resp.Status, body) + } + // tokenRes is the JSON response body. + var tokenRes struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + IDToken string `json:"id_token"` + ExpiresIn int64 `json:"expires_in"` // relative seconds from now + } + if err := json.Unmarshal(body, &tokenRes); err != nil { + return nil, fmt.Errorf("oauth2: cannot fetch token: %v", err) + } + token := &oauth2.Token{ + AccessToken: tokenRes.AccessToken, + TokenType: tokenRes.TokenType, + } + raw := make(map[string]interface{}) + json.Unmarshal(body, &raw) // no error checks for optional fields + token = token.WithExtra(raw) + + if secs := tokenRes.ExpiresIn; secs > 0 { + token.Expiry = time.Now().Add(time.Duration(secs) * time.Second) + } + if v := tokenRes.IDToken; v != "" { + // decode returned id token to get expiry + claimSet, err := jws.Decode(v) + if err != nil { + return nil, fmt.Errorf("oauth2: error decoding JWT token: %v", err) + } + token.Expiry = time.Unix(claimSet.Exp, 0) + } + return token, nil +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/oauth2.go b/components/engine/vendor/src/golang.org/x/oauth2/oauth2.go new file mode 100644 index 0000000000..a68289607b --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/oauth2.go @@ -0,0 +1,337 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package oauth2 provides support for making +// OAuth2 authorized and authenticated HTTP requests. +// It can additionally grant authorization with Bearer JWT. +package oauth2 // import "golang.org/x/oauth2" + +import ( + "bytes" + "errors" + "net/http" + "net/url" + "strings" + "sync" + + "golang.org/x/net/context" + "golang.org/x/oauth2/internal" +) + +// NoContext is the default context you should supply if not using +// your own context.Context (see https://golang.org/x/net/context). +var NoContext = context.TODO() + +// RegisterBrokenAuthHeaderProvider registers an OAuth2 server +// identified by the tokenURL prefix as an OAuth2 implementation +// which doesn't support the HTTP Basic authentication +// scheme to authenticate with the authorization server. +// Once a server is registered, credentials (client_id and client_secret) +// will be passed as query parameters rather than being present +// in the Authorization header. +// See https://code.google.com/p/goauth2/issues/detail?id=31 for background. +func RegisterBrokenAuthHeaderProvider(tokenURL string) { + internal.RegisterBrokenAuthHeaderProvider(tokenURL) +} + +// Config describes a typical 3-legged OAuth2 flow, with both the +// client application information and the server's endpoint URLs. +type Config struct { + // ClientID is the application's ID. + ClientID string + + // ClientSecret is the application's secret. + ClientSecret string + + // Endpoint contains the resource server's token endpoint + // URLs. These are constants specific to each server and are + // often available via site-specific packages, such as + // google.Endpoint or github.Endpoint. + Endpoint Endpoint + + // RedirectURL is the URL to redirect users going through + // the OAuth flow, after the resource owner's URLs. + RedirectURL string + + // Scope specifies optional requested permissions. + Scopes []string +} + +// A TokenSource is anything that can return a token. +type TokenSource interface { + // Token returns a token or an error. + // Token must be safe for concurrent use by multiple goroutines. + // The returned Token must not be modified. + Token() (*Token, error) +} + +// Endpoint contains the OAuth 2.0 provider's authorization and token +// endpoint URLs. +type Endpoint struct { + AuthURL string + TokenURL string +} + +var ( + // AccessTypeOnline and AccessTypeOffline are options passed + // to the Options.AuthCodeURL method. They modify the + // "access_type" field that gets sent in the URL returned by + // AuthCodeURL. + // + // Online is the default if neither is specified. If your + // application needs to refresh access tokens when the user + // is not present at the browser, then use offline. This will + // result in your application obtaining a refresh token the + // first time your application exchanges an authorization + // code for a user. + AccessTypeOnline AuthCodeOption = SetAuthURLParam("access_type", "online") + AccessTypeOffline AuthCodeOption = SetAuthURLParam("access_type", "offline") + + // ApprovalForce forces the users to view the consent dialog + // and confirm the permissions request at the URL returned + // from AuthCodeURL, even if they've already done so. + ApprovalForce AuthCodeOption = SetAuthURLParam("approval_prompt", "force") +) + +// An AuthCodeOption is passed to Config.AuthCodeURL. +type AuthCodeOption interface { + setValue(url.Values) +} + +type setParam struct{ k, v string } + +func (p setParam) setValue(m url.Values) { m.Set(p.k, p.v) } + +// SetAuthURLParam builds an AuthCodeOption which passes key/value parameters +// to a provider's authorization endpoint. +func SetAuthURLParam(key, value string) AuthCodeOption { + return setParam{key, value} +} + +// AuthCodeURL returns a URL to OAuth 2.0 provider's consent page +// that asks for permissions for the required scopes explicitly. +// +// State is a token to protect the user from CSRF attacks. You must +// always provide a non-zero string and validate that it matches the +// the state query parameter on your redirect callback. +// See http://tools.ietf.org/html/rfc6749#section-10.12 for more info. +// +// Opts may include AccessTypeOnline or AccessTypeOffline, as well +// as ApprovalForce. +func (c *Config) AuthCodeURL(state string, opts ...AuthCodeOption) string { + var buf bytes.Buffer + buf.WriteString(c.Endpoint.AuthURL) + v := url.Values{ + "response_type": {"code"}, + "client_id": {c.ClientID}, + "redirect_uri": internal.CondVal(c.RedirectURL), + "scope": internal.CondVal(strings.Join(c.Scopes, " ")), + "state": internal.CondVal(state), + } + for _, opt := range opts { + opt.setValue(v) + } + if strings.Contains(c.Endpoint.AuthURL, "?") { + buf.WriteByte('&') + } else { + buf.WriteByte('?') + } + buf.WriteString(v.Encode()) + return buf.String() +} + +// PasswordCredentialsToken converts a resource owner username and password +// pair into a token. +// +// Per the RFC, this grant type should only be used "when there is a high +// degree of trust between the resource owner and the client (e.g., the client +// is part of the device operating system or a highly privileged application), +// and when other authorization grant types are not available." +// See https://tools.ietf.org/html/rfc6749#section-4.3 for more info. +// +// The HTTP client to use is derived from the context. +// If nil, http.DefaultClient is used. +func (c *Config) PasswordCredentialsToken(ctx context.Context, username, password string) (*Token, error) { + return retrieveToken(ctx, c, url.Values{ + "grant_type": {"password"}, + "username": {username}, + "password": {password}, + "scope": internal.CondVal(strings.Join(c.Scopes, " ")), + }) +} + +// Exchange converts an authorization code into a token. +// +// It is used after a resource provider redirects the user back +// to the Redirect URI (the URL obtained from AuthCodeURL). +// +// The HTTP client to use is derived from the context. +// If a client is not provided via the context, http.DefaultClient is used. +// +// The code will be in the *http.Request.FormValue("code"). Before +// calling Exchange, be sure to validate FormValue("state"). +func (c *Config) Exchange(ctx context.Context, code string) (*Token, error) { + return retrieveToken(ctx, c, url.Values{ + "grant_type": {"authorization_code"}, + "code": {code}, + "redirect_uri": internal.CondVal(c.RedirectURL), + "scope": internal.CondVal(strings.Join(c.Scopes, " ")), + }) +} + +// Client returns an HTTP client using the provided token. +// The token will auto-refresh as necessary. The underlying +// HTTP transport will be obtained using the provided context. +// The returned client and its Transport should not be modified. +func (c *Config) Client(ctx context.Context, t *Token) *http.Client { + return NewClient(ctx, c.TokenSource(ctx, t)) +} + +// TokenSource returns a TokenSource that returns t until t expires, +// automatically refreshing it as necessary using the provided context. +// +// Most users will use Config.Client instead. +func (c *Config) TokenSource(ctx context.Context, t *Token) TokenSource { + tkr := &tokenRefresher{ + ctx: ctx, + conf: c, + } + if t != nil { + tkr.refreshToken = t.RefreshToken + } + return &reuseTokenSource{ + t: t, + new: tkr, + } +} + +// tokenRefresher is a TokenSource that makes "grant_type"=="refresh_token" +// HTTP requests to renew a token using a RefreshToken. +type tokenRefresher struct { + ctx context.Context // used to get HTTP requests + conf *Config + refreshToken string +} + +// WARNING: Token is not safe for concurrent access, as it +// updates the tokenRefresher's refreshToken field. +// Within this package, it is used by reuseTokenSource which +// synchronizes calls to this method with its own mutex. +func (tf *tokenRefresher) Token() (*Token, error) { + if tf.refreshToken == "" { + return nil, errors.New("oauth2: token expired and refresh token is not set") + } + + tk, err := retrieveToken(tf.ctx, tf.conf, url.Values{ + "grant_type": {"refresh_token"}, + "refresh_token": {tf.refreshToken}, + }) + + if err != nil { + return nil, err + } + if tf.refreshToken != tk.RefreshToken { + tf.refreshToken = tk.RefreshToken + } + return tk, err +} + +// reuseTokenSource is a TokenSource that holds a single token in memory +// and validates its expiry before each call to retrieve it with +// Token. If it's expired, it will be auto-refreshed using the +// new TokenSource. +type reuseTokenSource struct { + new TokenSource // called when t is expired. + + mu sync.Mutex // guards t + t *Token +} + +// Token returns the current token if it's still valid, else will +// refresh the current token (using r.Context for HTTP client +// information) and return the new one. +func (s *reuseTokenSource) Token() (*Token, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.t.Valid() { + return s.t, nil + } + t, err := s.new.Token() + if err != nil { + return nil, err + } + s.t = t + return t, nil +} + +// StaticTokenSource returns a TokenSource that always returns the same token. +// Because the provided token t is never refreshed, StaticTokenSource is only +// useful for tokens that never expire. +func StaticTokenSource(t *Token) TokenSource { + return staticTokenSource{t} +} + +// staticTokenSource is a TokenSource that always returns the same Token. +type staticTokenSource struct { + t *Token +} + +func (s staticTokenSource) Token() (*Token, error) { + return s.t, nil +} + +// HTTPClient is the context key to use with golang.org/x/net/context's +// WithValue function to associate an *http.Client value with a context. +var HTTPClient internal.ContextKey + +// NewClient creates an *http.Client from a Context and TokenSource. +// The returned client is not valid beyond the lifetime of the context. +// +// As a special case, if src is nil, a non-OAuth2 client is returned +// using the provided context. This exists to support related OAuth2 +// packages. +func NewClient(ctx context.Context, src TokenSource) *http.Client { + if src == nil { + c, err := internal.ContextClient(ctx) + if err != nil { + return &http.Client{Transport: internal.ErrorTransport{err}} + } + return c + } + return &http.Client{ + Transport: &Transport{ + Base: internal.ContextTransport(ctx), + Source: ReuseTokenSource(nil, src), + }, + } +} + +// ReuseTokenSource returns a TokenSource which repeatedly returns the +// same token as long as it's valid, starting with t. +// When its cached token is invalid, a new token is obtained from src. +// +// ReuseTokenSource is typically used to reuse tokens from a cache +// (such as a file on disk) between runs of a program, rather than +// obtaining new tokens unnecessarily. +// +// The initial token t may be nil, in which case the TokenSource is +// wrapped in a caching version if it isn't one already. This also +// means it's always safe to wrap ReuseTokenSource around any other +// TokenSource without adverse effects. +func ReuseTokenSource(t *Token, src TokenSource) TokenSource { + // Don't wrap a reuseTokenSource in itself. That would work, + // but cause an unnecessary number of mutex operations. + // Just build the equivalent one. + if rt, ok := src.(*reuseTokenSource); ok { + if t == nil { + // Just use it directly. + return rt + } + src = rt.new + } + return &reuseTokenSource{ + t: t, + new: src, + } +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/token.go b/components/engine/vendor/src/golang.org/x/oauth2/token.go new file mode 100644 index 0000000000..7a3167f15b --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/token.go @@ -0,0 +1,158 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package oauth2 + +import ( + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "golang.org/x/net/context" + "golang.org/x/oauth2/internal" +) + +// expiryDelta determines how earlier a token should be considered +// expired than its actual expiration time. It is used to avoid late +// expirations due to client-server time mismatches. +const expiryDelta = 10 * time.Second + +// Token represents the crendentials used to authorize +// the requests to access protected resources on the OAuth 2.0 +// provider's backend. +// +// Most users of this package should not access fields of Token +// directly. They're exported mostly for use by related packages +// implementing derivative OAuth2 flows. +type Token struct { + // AccessToken is the token that authorizes and authenticates + // the requests. + AccessToken string `json:"access_token"` + + // TokenType is the type of token. + // The Type method returns either this or "Bearer", the default. + TokenType string `json:"token_type,omitempty"` + + // RefreshToken is a token that's used by the application + // (as opposed to the user) to refresh the access token + // if it expires. + RefreshToken string `json:"refresh_token,omitempty"` + + // Expiry is the optional expiration time of the access token. + // + // If zero, TokenSource implementations will reuse the same + // token forever and RefreshToken or equivalent + // mechanisms for that TokenSource will not be used. + Expiry time.Time `json:"expiry,omitempty"` + + // raw optionally contains extra metadata from the server + // when updating a token. + raw interface{} +} + +// Type returns t.TokenType if non-empty, else "Bearer". +func (t *Token) Type() string { + if strings.EqualFold(t.TokenType, "bearer") { + return "Bearer" + } + if strings.EqualFold(t.TokenType, "mac") { + return "MAC" + } + if strings.EqualFold(t.TokenType, "basic") { + return "Basic" + } + if t.TokenType != "" { + return t.TokenType + } + return "Bearer" +} + +// SetAuthHeader sets the Authorization header to r using the access +// token in t. +// +// This method is unnecessary when using Transport or an HTTP Client +// returned by this package. +func (t *Token) SetAuthHeader(r *http.Request) { + r.Header.Set("Authorization", t.Type()+" "+t.AccessToken) +} + +// WithExtra returns a new Token that's a clone of t, but using the +// provided raw extra map. This is only intended for use by packages +// implementing derivative OAuth2 flows. +func (t *Token) WithExtra(extra interface{}) *Token { + t2 := new(Token) + *t2 = *t + t2.raw = extra + return t2 +} + +// Extra returns an extra field. +// Extra fields are key-value pairs returned by the server as a +// part of the token retrieval response. +func (t *Token) Extra(key string) interface{} { + if raw, ok := t.raw.(map[string]interface{}); ok { + return raw[key] + } + + vals, ok := t.raw.(url.Values) + if !ok { + return nil + } + + v := vals.Get(key) + switch s := strings.TrimSpace(v); strings.Count(s, ".") { + case 0: // Contains no "."; try to parse as int + if i, err := strconv.ParseInt(s, 10, 64); err == nil { + return i + } + case 1: // Contains a single "."; try to parse as float + if f, err := strconv.ParseFloat(s, 64); err == nil { + return f + } + } + + return v +} + +// expired reports whether the token is expired. +// t must be non-nil. +func (t *Token) expired() bool { + if t.Expiry.IsZero() { + return false + } + return t.Expiry.Add(-expiryDelta).Before(time.Now()) +} + +// Valid reports whether t is non-nil, has an AccessToken, and is not expired. +func (t *Token) Valid() bool { + return t != nil && t.AccessToken != "" && !t.expired() +} + +// tokenFromInternal maps an *internal.Token struct into +// a *Token struct. +func tokenFromInternal(t *internal.Token) *Token { + if t == nil { + return nil + } + return &Token{ + AccessToken: t.AccessToken, + TokenType: t.TokenType, + RefreshToken: t.RefreshToken, + Expiry: t.Expiry, + raw: t.Raw, + } +} + +// retrieveToken takes a *Config and uses that to retrieve an *internal.Token. +// This token is then mapped from *internal.Token into an *oauth2.Token which is returned along +// with an error.. +func retrieveToken(ctx context.Context, c *Config, v url.Values) (*Token, error) { + tk, err := internal.RetrieveToken(ctx, c.ClientID, c.ClientSecret, c.Endpoint.TokenURL, v) + if err != nil { + return nil, err + } + return tokenFromInternal(tk), nil +} diff --git a/components/engine/vendor/src/golang.org/x/oauth2/transport.go b/components/engine/vendor/src/golang.org/x/oauth2/transport.go new file mode 100644 index 0000000000..92ac7e2531 --- /dev/null +++ b/components/engine/vendor/src/golang.org/x/oauth2/transport.go @@ -0,0 +1,132 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package oauth2 + +import ( + "errors" + "io" + "net/http" + "sync" +) + +// Transport is an http.RoundTripper that makes OAuth 2.0 HTTP requests, +// wrapping a base RoundTripper and adding an Authorization header +// with a token from the supplied Sources. +// +// Transport is a low-level mechanism. Most code will use the +// higher-level Config.Client method instead. +type Transport struct { + // Source supplies the token to add to outgoing requests' + // Authorization headers. + Source TokenSource + + // Base is the base RoundTripper used to make HTTP requests. + // If nil, http.DefaultTransport is used. + Base http.RoundTripper + + mu sync.Mutex // guards modReq + modReq map[*http.Request]*http.Request // original -> modified +} + +// RoundTrip authorizes and authenticates the request with an +// access token. If no token exists or token is expired, +// tries to refresh/fetch a new token. +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + if t.Source == nil { + return nil, errors.New("oauth2: Transport's Source is nil") + } + token, err := t.Source.Token() + if err != nil { + return nil, err + } + + req2 := cloneRequest(req) // per RoundTripper contract + token.SetAuthHeader(req2) + t.setModReq(req, req2) + res, err := t.base().RoundTrip(req2) + if err != nil { + t.setModReq(req, nil) + return nil, err + } + res.Body = &onEOFReader{ + rc: res.Body, + fn: func() { t.setModReq(req, nil) }, + } + return res, nil +} + +// CancelRequest cancels an in-flight request by closing its connection. +func (t *Transport) CancelRequest(req *http.Request) { + type canceler interface { + CancelRequest(*http.Request) + } + if cr, ok := t.base().(canceler); ok { + t.mu.Lock() + modReq := t.modReq[req] + delete(t.modReq, req) + t.mu.Unlock() + cr.CancelRequest(modReq) + } +} + +func (t *Transport) base() http.RoundTripper { + if t.Base != nil { + return t.Base + } + return http.DefaultTransport +} + +func (t *Transport) setModReq(orig, mod *http.Request) { + t.mu.Lock() + defer t.mu.Unlock() + if t.modReq == nil { + t.modReq = make(map[*http.Request]*http.Request) + } + if mod == nil { + delete(t.modReq, orig) + } else { + t.modReq[orig] = mod + } +} + +// cloneRequest returns a clone of the provided *http.Request. +// The clone is a shallow copy of the struct and its Header map. +func cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header, len(r.Header)) + for k, s := range r.Header { + r2.Header[k] = append([]string(nil), s...) + } + return r2 +} + +type onEOFReader struct { + rc io.ReadCloser + fn func() +} + +func (r *onEOFReader) Read(p []byte) (n int, err error) { + n, err = r.rc.Read(p) + if err == io.EOF { + r.runFunc() + } + return +} + +func (r *onEOFReader) Close() error { + err := r.rc.Close() + r.runFunc() + return err +} + +func (r *onEOFReader) runFunc() { + if fn := r.fn; fn != nil { + fn() + r.fn = nil + } +} diff --git a/components/engine/vendor/src/golang.org/x/sys/windows/registry/key.go b/components/engine/vendor/src/golang.org/x/sys/windows/registry/key.go deleted file mode 100644 index f087ce5ada..0000000000 --- a/components/engine/vendor/src/golang.org/x/sys/windows/registry/key.go +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -// Package registry provides access to the Windows registry. -// -// Here is a simple example, opening a registry key and reading a string value from it. -// -// k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE) -// if err != nil { -// log.Fatal(err) -// } -// defer k.Close() -// -// s, _, err := k.GetStringValue("SystemRoot") -// if err != nil { -// log.Fatal(err) -// } -// fmt.Printf("Windows system root is %q\n", s) -// -package registry - -import ( - "io" - "syscall" - "time" -) - -const ( - // Registry key security and access rights. - // See https://msdn.microsoft.com/en-us/library/windows/desktop/ms724878.aspx - // for details. - ALL_ACCESS = 0xf003f - CREATE_LINK = 0x00020 - CREATE_SUB_KEY = 0x00004 - ENUMERATE_SUB_KEYS = 0x00008 - EXECUTE = 0x20019 - NOTIFY = 0x00010 - QUERY_VALUE = 0x00001 - READ = 0x20019 - SET_VALUE = 0x00002 - WOW64_32KEY = 0x00200 - WOW64_64KEY = 0x00100 - WRITE = 0x20006 -) - -// Key is a handle to an open Windows registry key. -// Keys can be obtained by calling OpenKey; there are -// also some predefined root keys such as CURRENT_USER. -// Keys can be used directly in the Windows API. -type Key syscall.Handle - -const ( - // Windows defines some predefined root keys that are always open. - // An application can use these keys as entry points to the registry. - // Normally these keys are used in OpenKey to open new keys, - // but they can also be used anywhere a Key is required. - CLASSES_ROOT = Key(syscall.HKEY_CLASSES_ROOT) - CURRENT_USER = Key(syscall.HKEY_CURRENT_USER) - LOCAL_MACHINE = Key(syscall.HKEY_LOCAL_MACHINE) - USERS = Key(syscall.HKEY_USERS) - CURRENT_CONFIG = Key(syscall.HKEY_CURRENT_CONFIG) -) - -// Close closes open key k. -func (k Key) Close() error { - return syscall.RegCloseKey(syscall.Handle(k)) -} - -// OpenKey opens a new key with path name relative to key k. -// It accepts any open key, including CURRENT_USER and others, -// and returns the new key and an error. -// The access parameter specifies desired access rights to the -// key to be opened. -func OpenKey(k Key, path string, access uint32) (Key, error) { - p, err := syscall.UTF16PtrFromString(path) - if err != nil { - return 0, err - } - var subkey syscall.Handle - err = syscall.RegOpenKeyEx(syscall.Handle(k), p, 0, access, &subkey) - if err != nil { - return 0, err - } - return Key(subkey), nil -} - -// ReadSubKeyNames returns the names of subkeys of key k. -// The parameter n controls the number of returned names, -// analogous to the way os.File.Readdirnames works. -func (k Key) ReadSubKeyNames(n int) ([]string, error) { - ki, err := k.Stat() - if err != nil { - return nil, err - } - names := make([]string, 0, ki.SubKeyCount) - buf := make([]uint16, ki.MaxSubKeyLen+1) // extra room for terminating zero byte -loopItems: - for i := uint32(0); ; i++ { - if n > 0 { - if len(names) == n { - return names, nil - } - } - l := uint32(len(buf)) - for { - err := syscall.RegEnumKeyEx(syscall.Handle(k), i, &buf[0], &l, nil, nil, nil, nil) - if err == nil { - break - } - if err == syscall.ERROR_MORE_DATA { - // Double buffer size and try again. - l = uint32(2 * len(buf)) - buf = make([]uint16, l) - continue - } - if err == _ERROR_NO_MORE_ITEMS { - break loopItems - } - return names, err - } - names = append(names, syscall.UTF16ToString(buf[:l])) - } - if n > len(names) { - return names, io.EOF - } - return names, nil -} - -// CreateKey creates a key named path under open key k. -// CreateKey returns the new key and a boolean flag that reports -// whether the key already existed. -// The access parameter specifies the access rights for the key -// to be created. -func CreateKey(k Key, path string, access uint32) (newk Key, openedExisting bool, err error) { - var h syscall.Handle - var d uint32 - err = regCreateKeyEx(syscall.Handle(k), syscall.StringToUTF16Ptr(path), - 0, nil, _REG_OPTION_NON_VOLATILE, access, nil, &h, &d) - if err != nil { - return 0, false, err - } - return Key(h), d == _REG_OPENED_EXISTING_KEY, nil -} - -// DeleteKey deletes the subkey path of key k and its values. -func DeleteKey(k Key, path string) error { - return regDeleteKey(syscall.Handle(k), syscall.StringToUTF16Ptr(path)) -} - -// A KeyInfo describes the statistics of a key. It is returned by Stat. -type KeyInfo struct { - SubKeyCount uint32 - MaxSubKeyLen uint32 // size of the key's subkey with the longest name, in Unicode characters, not including the terminating zero byte - ValueCount uint32 - MaxValueNameLen uint32 // size of the key's longest value name, in Unicode characters, not including the terminating zero byte - MaxValueLen uint32 // longest data component among the key's values, in bytes - lastWriteTime syscall.Filetime -} - -// ModTime returns the key's last write time. -func (ki *KeyInfo) ModTime() time.Time { - return time.Unix(0, ki.lastWriteTime.Nanoseconds()) -} - -// Stat retrieves information about the open key k. -func (k Key) Stat() (*KeyInfo, error) { - var ki KeyInfo - err := syscall.RegQueryInfoKey(syscall.Handle(k), nil, nil, nil, - &ki.SubKeyCount, &ki.MaxSubKeyLen, nil, &ki.ValueCount, - &ki.MaxValueNameLen, &ki.MaxValueLen, nil, &ki.lastWriteTime) - if err != nil { - return nil, err - } - return &ki, nil -} diff --git a/components/engine/vendor/src/golang.org/x/sys/windows/registry/syscall.go b/components/engine/vendor/src/golang.org/x/sys/windows/registry/syscall.go deleted file mode 100644 index 5426cae909..0000000000 --- a/components/engine/vendor/src/golang.org/x/sys/windows/registry/syscall.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package registry - -import "syscall" - -//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsyscall_windows.go syscall.go - -const ( - _REG_OPTION_NON_VOLATILE = 0 - - _REG_CREATED_NEW_KEY = 1 - _REG_OPENED_EXISTING_KEY = 2 - - _ERROR_NO_MORE_ITEMS syscall.Errno = 259 -) - -func LoadRegLoadMUIString() error { - return procRegLoadMUIStringW.Find() -} - -//sys regCreateKeyEx(key syscall.Handle, subkey *uint16, reserved uint32, class *uint16, options uint32, desired uint32, sa *syscall.SecurityAttributes, result *syscall.Handle, disposition *uint32) (regerrno error) = advapi32.RegCreateKeyExW -//sys regDeleteKey(key syscall.Handle, subkey *uint16) (regerrno error) = advapi32.RegDeleteKeyW -//sys regSetValueEx(key syscall.Handle, valueName *uint16, reserved uint32, vtype uint32, buf *byte, bufsize uint32) (regerrno error) = advapi32.RegSetValueExW -//sys regEnumValue(key syscall.Handle, index uint32, name *uint16, nameLen *uint32, reserved *uint32, valtype *uint32, buf *byte, buflen *uint32) (regerrno error) = advapi32.RegEnumValueW -//sys regDeleteValue(key syscall.Handle, name *uint16) (regerrno error) = advapi32.RegDeleteValueW -//sys regLoadMUIString(key syscall.Handle, name *uint16, buf *uint16, buflen uint32, buflenCopied *uint32, flags uint32, dir *uint16) (regerrno error) = advapi32.RegLoadMUIStringW - -//sys expandEnvironmentStrings(src *uint16, dst *uint16, size uint32) (n uint32, err error) = kernel32.ExpandEnvironmentStringsW diff --git a/components/engine/vendor/src/golang.org/x/sys/windows/registry/value.go b/components/engine/vendor/src/golang.org/x/sys/windows/registry/value.go deleted file mode 100644 index 71d4e15bab..0000000000 --- a/components/engine/vendor/src/golang.org/x/sys/windows/registry/value.go +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package registry - -import ( - "errors" - "io" - "syscall" - "unicode/utf16" - "unsafe" -) - -const ( - // Registry value types. - NONE = 0 - SZ = 1 - EXPAND_SZ = 2 - BINARY = 3 - DWORD = 4 - DWORD_BIG_ENDIAN = 5 - LINK = 6 - MULTI_SZ = 7 - RESOURCE_LIST = 8 - FULL_RESOURCE_DESCRIPTOR = 9 - RESOURCE_REQUIREMENTS_LIST = 10 - QWORD = 11 -) - -var ( - // ErrShortBuffer is returned when the buffer was too short for the operation. - ErrShortBuffer = syscall.ERROR_MORE_DATA - - // ErrNotExist is returned when a registry key or value does not exist. - ErrNotExist = syscall.ERROR_FILE_NOT_FOUND - - // ErrUnexpectedType is returned by Get*Value when the value's type was unexpected. - ErrUnexpectedType = errors.New("unexpected key value type") -) - -// GetValue retrieves the type and data for the specified value associated -// with an open key k. It fills up buffer buf and returns the retrieved -// byte count n. If buf is too small to fit the stored value it returns -// ErrShortBuffer error along with the required buffer size n. -// If no buffer is provided, it returns true and actual buffer size n. -// If no buffer is provided, GetValue returns the value's type only. -// If the value does not exist, the error returned is ErrNotExist. -// -// GetValue is a low level function. If value's type is known, use the appropriate -// Get*Value function instead. -func (k Key) GetValue(name string, buf []byte) (n int, valtype uint32, err error) { - pname, err := syscall.UTF16PtrFromString(name) - if err != nil { - return 0, 0, err - } - var pbuf *byte - if len(buf) > 0 { - pbuf = (*byte)(unsafe.Pointer(&buf[0])) - } - l := uint32(len(buf)) - err = syscall.RegQueryValueEx(syscall.Handle(k), pname, nil, &valtype, pbuf, &l) - if err != nil { - return int(l), valtype, err - } - return int(l), valtype, nil -} - -func (k Key) getValue(name string, buf []byte) (date []byte, valtype uint32, err error) { - p, err := syscall.UTF16PtrFromString(name) - if err != nil { - return nil, 0, err - } - var t uint32 - n := uint32(len(buf)) - for { - err = syscall.RegQueryValueEx(syscall.Handle(k), p, nil, &t, (*byte)(unsafe.Pointer(&buf[0])), &n) - if err == nil { - return buf[:n], t, nil - } - if err != syscall.ERROR_MORE_DATA { - return nil, 0, err - } - if n <= uint32(len(buf)) { - return nil, 0, err - } - buf = make([]byte, n) - } -} - -// GetStringValue retrieves the string value for the specified -// value name associated with an open key k. It also returns the value's type. -// If value does not exist, GetStringValue returns ErrNotExist. -// If value is not SZ or EXPAND_SZ, it will return the correct value -// type and ErrUnexpectedType. -func (k Key) GetStringValue(name string) (val string, valtype uint32, err error) { - data, typ, err2 := k.getValue(name, make([]byte, 64)) - if err2 != nil { - return "", typ, err2 - } - switch typ { - case SZ, EXPAND_SZ: - default: - return "", typ, ErrUnexpectedType - } - if len(data) == 0 { - return "", typ, nil - } - u := (*[1 << 29]uint16)(unsafe.Pointer(&data[0]))[:] - return syscall.UTF16ToString(u), typ, nil -} - -// GetMUIStringValue retrieves the localized string value for -// the specified value name associated with an open key k. -// If the value name doesn't exist or the localized string value -// can't be resolved, GetMUIStringValue returns ErrNotExist. -// GetMUIStringValue panics if the system doesn't support -// regLoadMUIString; use LoadRegLoadMUIString to check if -// regLoadMUIString is supported before calling this function. -func (k Key) GetMUIStringValue(name string) (string, error) { - pname, err := syscall.UTF16PtrFromString(name) - if err != nil { - return "", err - } - - buf := make([]uint16, 1024) - var buflen uint32 - var pdir *uint16 - - err = regLoadMUIString(syscall.Handle(k), pname, &buf[0], uint32(len(buf)), &buflen, 0, pdir) - if err == syscall.ERROR_FILE_NOT_FOUND { // Try fallback path - - // Try to resolve the string value using the system directory as - // a DLL search path; this assumes the string value is of the form - // @[path]\dllname,-strID but with no path given, e.g. @tzres.dll,-320. - - // This approach works with tzres.dll but may have to be revised - // in the future to allow callers to provide custom search paths. - - var s string - s, err = ExpandString("%SystemRoot%\\system32\\") - if err != nil { - return "", err - } - pdir, err = syscall.UTF16PtrFromString(s) - if err != nil { - return "", err - } - - err = regLoadMUIString(syscall.Handle(k), pname, &buf[0], uint32(len(buf)), &buflen, 0, pdir) - } - - for err == syscall.ERROR_MORE_DATA { // Grow buffer if needed - if buflen <= uint32(len(buf)) { - break // Buffer not growing, assume race; break - } - buf = make([]uint16, buflen) - err = regLoadMUIString(syscall.Handle(k), pname, &buf[0], uint32(len(buf)), &buflen, 0, pdir) - } - - if err != nil { - return "", err - } - - return syscall.UTF16ToString(buf), nil -} - -// ExpandString expands environment-variable strings and replaces -// them with the values defined for the current user. -// Use ExpandString to expand EXPAND_SZ strings. -func ExpandString(value string) (string, error) { - if value == "" { - return "", nil - } - p, err := syscall.UTF16PtrFromString(value) - if err != nil { - return "", err - } - r := make([]uint16, 100) - for { - n, err := expandEnvironmentStrings(p, &r[0], uint32(len(r))) - if err != nil { - return "", err - } - if n <= uint32(len(r)) { - u := (*[1 << 29]uint16)(unsafe.Pointer(&r[0]))[:] - return syscall.UTF16ToString(u), nil - } - r = make([]uint16, n) - } -} - -// GetStringsValue retrieves the []string value for the specified -// value name associated with an open key k. It also returns the value's type. -// If value does not exist, GetStringsValue returns ErrNotExist. -// If value is not MULTI_SZ, it will return the correct value -// type and ErrUnexpectedType. -func (k Key) GetStringsValue(name string) (val []string, valtype uint32, err error) { - data, typ, err2 := k.getValue(name, make([]byte, 64)) - if err2 != nil { - return nil, typ, err2 - } - if typ != MULTI_SZ { - return nil, typ, ErrUnexpectedType - } - if len(data) == 0 { - return nil, typ, nil - } - p := (*[1 << 29]uint16)(unsafe.Pointer(&data[0]))[:len(data)/2] - if len(p) == 0 { - return nil, typ, nil - } - if p[len(p)-1] == 0 { - p = p[:len(p)-1] // remove terminating null - } - val = make([]string, 0, 5) - from := 0 - for i, c := range p { - if c == 0 { - val = append(val, string(utf16.Decode(p[from:i]))) - from = i + 1 - } - } - return val, typ, nil -} - -// GetIntegerValue retrieves the integer value for the specified -// value name associated with an open key k. It also returns the value's type. -// If value does not exist, GetIntegerValue returns ErrNotExist. -// If value is not DWORD or QWORD, it will return the correct value -// type and ErrUnexpectedType. -func (k Key) GetIntegerValue(name string) (val uint64, valtype uint32, err error) { - data, typ, err2 := k.getValue(name, make([]byte, 8)) - if err2 != nil { - return 0, typ, err2 - } - switch typ { - case DWORD: - if len(data) != 4 { - return 0, typ, errors.New("DWORD value is not 4 bytes long") - } - return uint64(*(*uint32)(unsafe.Pointer(&data[0]))), DWORD, nil - case QWORD: - if len(data) != 8 { - return 0, typ, errors.New("QWORD value is not 8 bytes long") - } - return uint64(*(*uint64)(unsafe.Pointer(&data[0]))), QWORD, nil - default: - return 0, typ, ErrUnexpectedType - } -} - -// GetBinaryValue retrieves the binary value for the specified -// value name associated with an open key k. It also returns the value's type. -// If value does not exist, GetBinaryValue returns ErrNotExist. -// If value is not BINARY, it will return the correct value -// type and ErrUnexpectedType. -func (k Key) GetBinaryValue(name string) (val []byte, valtype uint32, err error) { - data, typ, err2 := k.getValue(name, make([]byte, 64)) - if err2 != nil { - return nil, typ, err2 - } - if typ != BINARY { - return nil, typ, ErrUnexpectedType - } - return data, typ, nil -} - -func (k Key) setValue(name string, valtype uint32, data []byte) error { - p, err := syscall.UTF16PtrFromString(name) - if err != nil { - return err - } - if len(data) == 0 { - return regSetValueEx(syscall.Handle(k), p, 0, valtype, nil, 0) - } - return regSetValueEx(syscall.Handle(k), p, 0, valtype, &data[0], uint32(len(data))) -} - -// SetDWordValue sets the data and type of a name value -// under key k to value and DWORD. -func (k Key) SetDWordValue(name string, value uint32) error { - return k.setValue(name, DWORD, (*[4]byte)(unsafe.Pointer(&value))[:]) -} - -// SetQWordValue sets the data and type of a name value -// under key k to value and QWORD. -func (k Key) SetQWordValue(name string, value uint64) error { - return k.setValue(name, QWORD, (*[8]byte)(unsafe.Pointer(&value))[:]) -} - -func (k Key) setStringValue(name string, valtype uint32, value string) error { - v, err := syscall.UTF16FromString(value) - if err != nil { - return err - } - buf := (*[1 << 29]byte)(unsafe.Pointer(&v[0]))[:len(v)*2] - return k.setValue(name, valtype, buf) -} - -// SetStringValue sets the data and type of a name value -// under key k to value and SZ. The value must not contain a zero byte. -func (k Key) SetStringValue(name, value string) error { - return k.setStringValue(name, SZ, value) -} - -// SetExpandStringValue sets the data and type of a name value -// under key k to value and EXPAND_SZ. The value must not contain a zero byte. -func (k Key) SetExpandStringValue(name, value string) error { - return k.setStringValue(name, EXPAND_SZ, value) -} - -// SetStringsValue sets the data and type of a name value -// under key k to value and MULTI_SZ. The value strings -// must not contain a zero byte. -func (k Key) SetStringsValue(name string, value []string) error { - ss := "" - for _, s := range value { - for i := 0; i < len(s); i++ { - if s[i] == 0 { - return errors.New("string cannot have 0 inside") - } - } - ss += s + "\x00" - } - v := utf16.Encode([]rune(ss + "\x00")) - buf := (*[1 << 29]byte)(unsafe.Pointer(&v[0]))[:len(v)*2] - return k.setValue(name, MULTI_SZ, buf) -} - -// SetBinaryValue sets the data and type of a name value -// under key k to value and BINARY. -func (k Key) SetBinaryValue(name string, value []byte) error { - return k.setValue(name, BINARY, value) -} - -// DeleteValue removes a named value from the key k. -func (k Key) DeleteValue(name string) error { - return regDeleteValue(syscall.Handle(k), syscall.StringToUTF16Ptr(name)) -} - -// ReadValueNames returns the value names of key k. -// The parameter n controls the number of returned names, -// analogous to the way os.File.Readdirnames works. -func (k Key) ReadValueNames(n int) ([]string, error) { - ki, err := k.Stat() - if err != nil { - return nil, err - } - names := make([]string, 0, ki.ValueCount) - buf := make([]uint16, ki.MaxValueNameLen+1) // extra room for terminating null character -loopItems: - for i := uint32(0); ; i++ { - if n > 0 { - if len(names) == n { - return names, nil - } - } - l := uint32(len(buf)) - for { - err := regEnumValue(syscall.Handle(k), i, &buf[0], &l, nil, nil, nil, nil) - if err == nil { - break - } - if err == syscall.ERROR_MORE_DATA { - // Double buffer size and try again. - l = uint32(2 * len(buf)) - buf = make([]uint16, l) - continue - } - if err == _ERROR_NO_MORE_ITEMS { - break loopItems - } - return names, err - } - names = append(names, syscall.UTF16ToString(buf[:l])) - } - if n > len(names) { - return names, io.EOF - } - return names, nil -} diff --git a/components/engine/vendor/src/golang.org/x/sys/windows/registry/zsyscall_windows.go b/components/engine/vendor/src/golang.org/x/sys/windows/registry/zsyscall_windows.go deleted file mode 100644 index 9c17675a24..0000000000 --- a/components/engine/vendor/src/golang.org/x/sys/windows/registry/zsyscall_windows.go +++ /dev/null @@ -1,82 +0,0 @@ -// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT - -package registry - -import "unsafe" -import "syscall" - -var _ unsafe.Pointer - -var ( - modadvapi32 = syscall.NewLazyDLL("advapi32.dll") - modkernel32 = syscall.NewLazyDLL("kernel32.dll") - - procRegCreateKeyExW = modadvapi32.NewProc("RegCreateKeyExW") - procRegDeleteKeyW = modadvapi32.NewProc("RegDeleteKeyW") - procRegSetValueExW = modadvapi32.NewProc("RegSetValueExW") - procRegEnumValueW = modadvapi32.NewProc("RegEnumValueW") - procRegDeleteValueW = modadvapi32.NewProc("RegDeleteValueW") - procRegLoadMUIStringW = modadvapi32.NewProc("RegLoadMUIStringW") - procExpandEnvironmentStringsW = modkernel32.NewProc("ExpandEnvironmentStringsW") -) - -func regCreateKeyEx(key syscall.Handle, subkey *uint16, reserved uint32, class *uint16, options uint32, desired uint32, sa *syscall.SecurityAttributes, result *syscall.Handle, disposition *uint32) (regerrno error) { - r0, _, _ := syscall.Syscall9(procRegCreateKeyExW.Addr(), 9, uintptr(key), uintptr(unsafe.Pointer(subkey)), uintptr(reserved), uintptr(unsafe.Pointer(class)), uintptr(options), uintptr(desired), uintptr(unsafe.Pointer(sa)), uintptr(unsafe.Pointer(result)), uintptr(unsafe.Pointer(disposition))) - if r0 != 0 { - regerrno = syscall.Errno(r0) - } - return -} - -func regDeleteKey(key syscall.Handle, subkey *uint16) (regerrno error) { - r0, _, _ := syscall.Syscall(procRegDeleteKeyW.Addr(), 2, uintptr(key), uintptr(unsafe.Pointer(subkey)), 0) - if r0 != 0 { - regerrno = syscall.Errno(r0) - } - return -} - -func regSetValueEx(key syscall.Handle, valueName *uint16, reserved uint32, vtype uint32, buf *byte, bufsize uint32) (regerrno error) { - r0, _, _ := syscall.Syscall6(procRegSetValueExW.Addr(), 6, uintptr(key), uintptr(unsafe.Pointer(valueName)), uintptr(reserved), uintptr(vtype), uintptr(unsafe.Pointer(buf)), uintptr(bufsize)) - if r0 != 0 { - regerrno = syscall.Errno(r0) - } - return -} - -func regEnumValue(key syscall.Handle, index uint32, name *uint16, nameLen *uint32, reserved *uint32, valtype *uint32, buf *byte, buflen *uint32) (regerrno error) { - r0, _, _ := syscall.Syscall9(procRegEnumValueW.Addr(), 8, uintptr(key), uintptr(index), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameLen)), uintptr(unsafe.Pointer(reserved)), uintptr(unsafe.Pointer(valtype)), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(buflen)), 0) - if r0 != 0 { - regerrno = syscall.Errno(r0) - } - return -} - -func regDeleteValue(key syscall.Handle, name *uint16) (regerrno error) { - r0, _, _ := syscall.Syscall(procRegDeleteValueW.Addr(), 2, uintptr(key), uintptr(unsafe.Pointer(name)), 0) - if r0 != 0 { - regerrno = syscall.Errno(r0) - } - return -} - -func regLoadMUIString(key syscall.Handle, name *uint16, buf *uint16, buflen uint32, buflenCopied *uint32, flags uint32, dir *uint16) (regerrno error) { - r0, _, _ := syscall.Syscall9(procRegLoadMUIStringW.Addr(), 7, uintptr(key), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buf)), uintptr(buflen), uintptr(unsafe.Pointer(buflenCopied)), uintptr(flags), uintptr(unsafe.Pointer(dir)), 0, 0) - if r0 != 0 { - regerrno = syscall.Errno(r0) - } - return -} - -func expandEnvironmentStrings(src *uint16, dst *uint16, size uint32) (n uint32, err error) { - r0, _, e1 := syscall.Syscall(procExpandEnvironmentStringsW.Addr(), 3, uintptr(unsafe.Pointer(src)), uintptr(unsafe.Pointer(dst)), uintptr(size)) - n = uint32(r0) - if n == 0 { - if e1 != 0 { - err = error(e1) - } else { - err = syscall.EINVAL - } - } - return -} diff --git a/components/engine/vendor/src/google.golang.org/api/LICENSE b/components/engine/vendor/src/google.golang.org/api/LICENSE new file mode 100644 index 0000000000..263aa7a0c1 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/api/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/components/engine/vendor/src/google.golang.org/api/gensupport/json.go b/components/engine/vendor/src/google.golang.org/api/gensupport/json.go new file mode 100644 index 0000000000..193def5938 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/api/gensupport/json.go @@ -0,0 +1,177 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gensupport is an internal implementation detail used by code +// generated by the google-api-go-generator tool. +// +// This package may be modified at any time without regard for backwards +// compatibility. It should not be used directly by API users. +package gensupport + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" +) + +// MarshalJSON returns a JSON encoding of schema containing only selected fields. +// A field is selected if: +// * it has a non-empty value, or +// * its field name is present in forceSendFields, and +// * it is not a nil pointer or nil interface. +// The JSON key for each selected field is taken from the field's json: struct tag. +func MarshalJSON(schema interface{}, forceSendFields []string) ([]byte, error) { + if len(forceSendFields) == 0 { + return json.Marshal(schema) + } + + mustInclude := make(map[string]struct{}) + for _, f := range forceSendFields { + mustInclude[f] = struct{}{} + } + + dataMap, err := schemaToMap(schema, mustInclude) + if err != nil { + return nil, err + } + return json.Marshal(dataMap) +} + +func schemaToMap(schema interface{}, mustInclude map[string]struct{}) (map[string]interface{}, error) { + m := make(map[string]interface{}) + s := reflect.ValueOf(schema) + st := s.Type() + + for i := 0; i < s.NumField(); i++ { + jsonTag := st.Field(i).Tag.Get("json") + if jsonTag == "" { + continue + } + tag, err := parseJSONTag(jsonTag) + if err != nil { + return nil, err + } + if tag.ignore { + continue + } + + v := s.Field(i) + f := st.Field(i) + if !includeField(v, f, mustInclude) { + continue + } + + // nil maps are treated as empty maps. + if f.Type.Kind() == reflect.Map && v.IsNil() { + m[tag.apiName] = map[string]string{} + continue + } + + // nil slices are treated as empty slices. + if f.Type.Kind() == reflect.Slice && v.IsNil() { + m[tag.apiName] = []bool{} + continue + } + + if tag.stringFormat { + m[tag.apiName] = formatAsString(v, f.Type.Kind()) + } else { + m[tag.apiName] = v.Interface() + } + } + return m, nil +} + +// formatAsString returns a string representation of v, dereferencing it first if possible. +func formatAsString(v reflect.Value, kind reflect.Kind) string { + if kind == reflect.Ptr && !v.IsNil() { + v = v.Elem() + } + + return fmt.Sprintf("%v", v.Interface()) +} + +// jsonTag represents a restricted version of the struct tag format used by encoding/json. +// It is used to describe the JSON encoding of fields in a Schema struct. +type jsonTag struct { + apiName string + stringFormat bool + ignore bool +} + +// parseJSONTag parses a restricted version of the struct tag format used by encoding/json. +// The format of the tag must match that generated by the Schema.writeSchemaStruct method +// in the api generator. +func parseJSONTag(val string) (jsonTag, error) { + if val == "-" { + return jsonTag{ignore: true}, nil + } + + var tag jsonTag + + i := strings.Index(val, ",") + if i == -1 || val[:i] == "" { + return tag, fmt.Errorf("malformed json tag: %s", val) + } + + tag = jsonTag{ + apiName: val[:i], + } + + switch val[i+1:] { + case "omitempty": + case "omitempty,string": + tag.stringFormat = true + default: + return tag, fmt.Errorf("malformed json tag: %s", val) + } + + return tag, nil +} + +// Reports whether the struct field "f" with value "v" should be included in JSON output. +func includeField(v reflect.Value, f reflect.StructField, mustInclude map[string]struct{}) bool { + // The regular JSON encoding of a nil pointer is "null", which means "delete this field". + // Therefore, we could enable field deletion by honoring pointer fields' presence in the mustInclude set. + // However, many fields are not pointers, so there would be no way to delete these fields. + // Rather than partially supporting field deletion, we ignore mustInclude for nil pointer fields. + // Deletion will be handled by a separate mechanism. + if f.Type.Kind() == reflect.Ptr && v.IsNil() { + return false + } + + // The "any" type is represented as an interface{}. If this interface + // is nil, there is no reasonable representation to send. We ignore + // these fields, for the same reasons as given above for pointers. + if f.Type.Kind() == reflect.Interface && v.IsNil() { + return false + } + + _, ok := mustInclude[f.Name] + return ok || !isEmptyValue(v) +} + +// isEmptyValue reports whether v is the empty value for its type. This +// implementation is based on that of the encoding/json package, but its +// correctness does not depend on it being identical. What's important is that +// this function return false in situations where v should not be sent as part +// of a PATCH operation. +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + return false +} diff --git a/components/engine/vendor/src/google.golang.org/api/gensupport/params.go b/components/engine/vendor/src/google.golang.org/api/gensupport/params.go new file mode 100644 index 0000000000..dfad3f414d --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/api/gensupport/params.go @@ -0,0 +1,31 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gensupport + +import "net/url" + +// URLParams is a simplified replacement for url.Values +// that safely builds up URL parameters for encoding. +type URLParams map[string][]string + +// Set sets the key to value. +// It replaces any existing values. +func (u URLParams) Set(key, value string) { + u[key] = []string{value} +} + +// SetMulti sets the key to an array of values. +// It replaces any existing values. +// Note that values must not be modified after calling SetMulti +// so the caller is responsible for making a copy if necessary. +func (u URLParams) SetMulti(key string, values []string) { + u[key] = values +} + +// Encode encodes the values into ``URL encoded'' form +// ("bar=baz&foo=quux") sorted by key. +func (u URLParams) Encode() string { + return url.Values(u).Encode() +} diff --git a/components/engine/vendor/src/google.golang.org/api/googleapi/googleapi.go b/components/engine/vendor/src/google.golang.org/api/googleapi/googleapi.go new file mode 100644 index 0000000000..fbae951023 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/api/googleapi/googleapi.go @@ -0,0 +1,588 @@ +// Copyright 2011 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package googleapi contains the common code shared by all Google API +// libraries. +package googleapi // import "google.golang.org/api/googleapi" + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "net/textproto" + "net/url" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/net/context" + "golang.org/x/net/context/ctxhttp" + "google.golang.org/api/googleapi/internal/uritemplates" +) + +// ContentTyper is an interface for Readers which know (or would like +// to override) their Content-Type. If a media body doesn't implement +// ContentTyper, the type is sniffed from the content using +// http.DetectContentType. +type ContentTyper interface { + ContentType() string +} + +// A SizeReaderAt is a ReaderAt with a Size method. +// An io.SectionReader implements SizeReaderAt. +type SizeReaderAt interface { + io.ReaderAt + Size() int64 +} + +// ServerResponse is embedded in each Do response and +// provides the HTTP status code and header sent by the server. +type ServerResponse struct { + // HTTPStatusCode is the server's response status code. + // When using a resource method's Do call, this will always be in the 2xx range. + HTTPStatusCode int + // Header contains the response header fields from the server. + Header http.Header +} + +const ( + Version = "0.5" + + // statusResumeIncomplete is the code returned by the Google uploader when the transfer is not yet complete. + statusResumeIncomplete = 308 + + // UserAgent is the header string used to identify this package. + UserAgent = "google-api-go-client/" + Version + + // uploadPause determines the delay between failed upload attempts + uploadPause = 1 * time.Second +) + +// Error contains an error response from the server. +type Error struct { + // Code is the HTTP response status code and will always be populated. + Code int `json:"code"` + // Message is the server response message and is only populated when + // explicitly referenced by the JSON server response. + Message string `json:"message"` + // Body is the raw response returned by the server. + // It is often but not always JSON, depending on how the request fails. + Body string + // Header contains the response header fields from the server. + Header http.Header + + Errors []ErrorItem +} + +// ErrorItem is a detailed error code & message from the Google API frontend. +type ErrorItem struct { + // Reason is the typed error code. For example: "some_example". + Reason string `json:"reason"` + // Message is the human-readable description of the error. + Message string `json:"message"` +} + +func (e *Error) Error() string { + if len(e.Errors) == 0 && e.Message == "" { + return fmt.Sprintf("googleapi: got HTTP response code %d with body: %v", e.Code, e.Body) + } + var buf bytes.Buffer + fmt.Fprintf(&buf, "googleapi: Error %d: ", e.Code) + if e.Message != "" { + fmt.Fprintf(&buf, "%s", e.Message) + } + if len(e.Errors) == 0 { + return strings.TrimSpace(buf.String()) + } + if len(e.Errors) == 1 && e.Errors[0].Message == e.Message { + fmt.Fprintf(&buf, ", %s", e.Errors[0].Reason) + return buf.String() + } + fmt.Fprintln(&buf, "\nMore details:") + for _, v := range e.Errors { + fmt.Fprintf(&buf, "Reason: %s, Message: %s\n", v.Reason, v.Message) + } + return buf.String() +} + +type errorReply struct { + Error *Error `json:"error"` +} + +// CheckResponse returns an error (of type *Error) if the response +// status code is not 2xx. +func CheckResponse(res *http.Response) error { + if res.StatusCode >= 200 && res.StatusCode <= 299 { + return nil + } + slurp, err := ioutil.ReadAll(res.Body) + if err == nil { + jerr := new(errorReply) + err = json.Unmarshal(slurp, jerr) + if err == nil && jerr.Error != nil { + if jerr.Error.Code == 0 { + jerr.Error.Code = res.StatusCode + } + jerr.Error.Body = string(slurp) + return jerr.Error + } + } + return &Error{ + Code: res.StatusCode, + Body: string(slurp), + Header: res.Header, + } +} + +// IsNotModified reports whether err is the result of the +// server replying with http.StatusNotModified. +// Such error values are sometimes returned by "Do" methods +// on calls when If-None-Match is used. +func IsNotModified(err error) bool { + if err == nil { + return false + } + ae, ok := err.(*Error) + return ok && ae.Code == http.StatusNotModified +} + +// CheckMediaResponse returns an error (of type *Error) if the response +// status code is not 2xx. Unlike CheckResponse it does not assume the +// body is a JSON error document. +func CheckMediaResponse(res *http.Response) error { + if res.StatusCode >= 200 && res.StatusCode <= 299 { + return nil + } + slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<20)) + res.Body.Close() + return &Error{ + Code: res.StatusCode, + Body: string(slurp), + } +} + +type MarshalStyle bool + +var WithDataWrapper = MarshalStyle(true) +var WithoutDataWrapper = MarshalStyle(false) + +func (wrap MarshalStyle) JSONReader(v interface{}) (io.Reader, error) { + buf := new(bytes.Buffer) + if wrap { + buf.Write([]byte(`{"data": `)) + } + err := json.NewEncoder(buf).Encode(v) + if err != nil { + return nil, err + } + if wrap { + buf.Write([]byte(`}`)) + } + return buf, nil +} + +func getMediaType(media io.Reader) (io.Reader, string) { + if typer, ok := media.(ContentTyper); ok { + return media, typer.ContentType() + } + + pr, pw := io.Pipe() + typ := "application/octet-stream" + buf, err := ioutil.ReadAll(io.LimitReader(media, 512)) + if err != nil { + pw.CloseWithError(fmt.Errorf("error reading media: %v", err)) + return pr, typ + } + typ = http.DetectContentType(buf) + mr := io.MultiReader(bytes.NewReader(buf), media) + go func() { + _, err = io.Copy(pw, mr) + if err != nil { + pw.CloseWithError(fmt.Errorf("error reading media: %v", err)) + return + } + pw.Close() + }() + return pr, typ +} + +// DetectMediaType detects and returns the content type of the provided media. +// If the type can not be determined, "application/octet-stream" is returned. +func DetectMediaType(media io.ReaderAt) string { + if typer, ok := media.(ContentTyper); ok { + return typer.ContentType() + } + + typ := "application/octet-stream" + buf := make([]byte, 1024) + n, err := media.ReadAt(buf, 0) + buf = buf[:n] + if err == nil || err == io.EOF { + typ = http.DetectContentType(buf) + } + return typ +} + +type Lengther interface { + Len() int +} + +// endingWithErrorReader from r until it returns an error. If the +// final error from r is io.EOF and e is non-nil, e is used instead. +type endingWithErrorReader struct { + r io.Reader + e error +} + +func (er endingWithErrorReader) Read(p []byte) (n int, err error) { + n, err = er.r.Read(p) + if err == io.EOF && er.e != nil { + err = er.e + } + return +} + +func typeHeader(contentType string) textproto.MIMEHeader { + h := make(textproto.MIMEHeader) + h.Set("Content-Type", contentType) + return h +} + +// countingWriter counts the number of bytes it receives to write, but +// discards them. +type countingWriter struct { + n *int64 +} + +func (w countingWriter) Write(p []byte) (int, error) { + *w.n += int64(len(p)) + return len(p), nil +} + +// ConditionallyIncludeMedia does nothing if media is nil. +// +// bodyp is an in/out parameter. It should initially point to the +// reader of the application/json (or whatever) payload to send in the +// API request. It's updated to point to the multipart body reader. +// +// ctypep is an in/out parameter. It should initially point to the +// content type of the bodyp, usually "application/json". It's updated +// to the "multipart/related" content type, with random boundary. +// +// The return value is the content-length of the entire multpart body. +func ConditionallyIncludeMedia(media io.Reader, bodyp *io.Reader, ctypep *string) (cancel func(), ok bool) { + if media == nil { + return + } + // Get the media type, which might return a different reader instance. + var mediaType string + media, mediaType = getMediaType(media) + + body, bodyType := *bodyp, *ctypep + + pr, pw := io.Pipe() + mpw := multipart.NewWriter(pw) + *bodyp = pr + *ctypep = "multipart/related; boundary=" + mpw.Boundary() + go func() { + w, err := mpw.CreatePart(typeHeader(bodyType)) + if err != nil { + mpw.Close() + pw.CloseWithError(fmt.Errorf("googleapi: body CreatePart failed: %v", err)) + return + } + _, err = io.Copy(w, body) + if err != nil { + mpw.Close() + pw.CloseWithError(fmt.Errorf("googleapi: body Copy failed: %v", err)) + return + } + + w, err = mpw.CreatePart(typeHeader(mediaType)) + if err != nil { + mpw.Close() + pw.CloseWithError(fmt.Errorf("googleapi: media CreatePart failed: %v", err)) + return + } + _, err = io.Copy(w, media) + if err != nil { + mpw.Close() + pw.CloseWithError(fmt.Errorf("googleapi: media Copy failed: %v", err)) + return + } + mpw.Close() + pw.Close() + }() + cancel = func() { pw.CloseWithError(errAborted) } + return cancel, true +} + +var errAborted = errors.New("googleapi: upload aborted") + +// ProgressUpdater is a function that is called upon every progress update of a resumable upload. +// This is the only part of a resumable upload (from googleapi) that is usable by the developer. +// The remaining usable pieces of resumable uploads is exposed in each auto-generated API. +type ProgressUpdater func(current, total int64) + +// ResumableUpload is used by the generated APIs to provide resumable uploads. +// It is not used by developers directly. +type ResumableUpload struct { + Client *http.Client + // URI is the resumable resource destination provided by the server after specifying "&uploadType=resumable". + URI string + UserAgent string // User-Agent for header of the request + // Media is the object being uploaded. + Media io.ReaderAt + // MediaType defines the media type, e.g. "image/jpeg". + MediaType string + // ContentLength is the full size of the object being uploaded. + ContentLength int64 + + mu sync.Mutex // guards progress + progress int64 // number of bytes uploaded so far + + // Callback is an optional function that will be called upon every progress update. + Callback ProgressUpdater +} + +var ( + // rangeRE matches the transfer status response from the server. $1 is the last byte index uploaded. + rangeRE = regexp.MustCompile(`^bytes=0\-(\d+)$`) + // chunkSize is the size of the chunks created during a resumable upload and should be a power of two. + // 1<<18 is the minimum size supported by the Google uploader, and there is no maximum. + chunkSize int64 = 1 << 18 +) + +// Progress returns the number of bytes uploaded at this point. +func (rx *ResumableUpload) Progress() int64 { + rx.mu.Lock() + defer rx.mu.Unlock() + return rx.progress +} + +func (rx *ResumableUpload) transferStatus(ctx context.Context) (int64, *http.Response, error) { + req, _ := http.NewRequest("POST", rx.URI, nil) + req.ContentLength = 0 + req.Header.Set("User-Agent", rx.UserAgent) + req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", rx.ContentLength)) + res, err := ctxhttp.Do(ctx, rx.Client, req) + if err != nil || res.StatusCode != statusResumeIncomplete { + return 0, res, err + } + var start int64 + if m := rangeRE.FindStringSubmatch(res.Header.Get("Range")); len(m) == 2 { + start, err = strconv.ParseInt(m[1], 10, 64) + if err != nil { + return 0, nil, fmt.Errorf("unable to parse range size %v", m[1]) + } + start += 1 // Start at the next byte + } + return start, res, nil +} + +type chunk struct { + body io.Reader + size int64 + err error +} + +func (rx *ResumableUpload) transferChunks(ctx context.Context) (*http.Response, error) { + start, res, err := rx.transferStatus(ctx) + if err != nil || res.StatusCode != statusResumeIncomplete { + if err == context.Canceled { + return &http.Response{StatusCode: http.StatusRequestTimeout}, err + } + return res, err + } + + for { + select { // Check for cancellation + case <-ctx.Done(): + res.StatusCode = http.StatusRequestTimeout + return res, ctx.Err() + default: + } + reqSize := rx.ContentLength - start + if reqSize > chunkSize { + reqSize = chunkSize + } + r := io.NewSectionReader(rx.Media, start, reqSize) + req, _ := http.NewRequest("POST", rx.URI, r) + req.ContentLength = reqSize + req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, rx.ContentLength)) + req.Header.Set("Content-Type", rx.MediaType) + req.Header.Set("User-Agent", rx.UserAgent) + res, err = ctxhttp.Do(ctx, rx.Client, req) + start += reqSize + if err == nil && (res.StatusCode == statusResumeIncomplete || res.StatusCode == http.StatusOK) { + rx.mu.Lock() + rx.progress = start // keep track of number of bytes sent so far + rx.mu.Unlock() + if rx.Callback != nil { + rx.Callback(start, rx.ContentLength) + } + } + if err != nil || res.StatusCode != statusResumeIncomplete { + break + } + } + return res, err +} + +var sleep = time.Sleep // override in unit tests + +// Upload starts the process of a resumable upload with a cancellable context. +// It retries indefinitely (with a pause of uploadPause between attempts) until cancelled. +// It is called from the auto-generated API code and is not visible to the user. +// rx is private to the auto-generated API code. +func (rx *ResumableUpload) Upload(ctx context.Context) (*http.Response, error) { + var res *http.Response + var err error + for { + res, err = rx.transferChunks(ctx) + if err != nil || res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusOK { + return res, err + } + select { // Check for cancellation + case <-ctx.Done(): + res.StatusCode = http.StatusRequestTimeout + return res, ctx.Err() + default: + } + sleep(uploadPause) + } + return res, err +} + +func ResolveRelative(basestr, relstr string) string { + u, _ := url.Parse(basestr) + rel, _ := url.Parse(relstr) + u = u.ResolveReference(rel) + us := u.String() + us = strings.Replace(us, "%7B", "{", -1) + us = strings.Replace(us, "%7D", "}", -1) + return us +} + +// has4860Fix is whether this Go environment contains the fix for +// http://golang.org/issue/4860 +var has4860Fix bool + +// init initializes has4860Fix by checking the behavior of the net/http package. +func init() { + r := http.Request{ + URL: &url.URL{ + Scheme: "http", + Opaque: "//opaque", + }, + } + b := &bytes.Buffer{} + r.Write(b) + has4860Fix = bytes.HasPrefix(b.Bytes(), []byte("GET http")) +} + +// SetOpaque sets u.Opaque from u.Path such that HTTP requests to it +// don't alter any hex-escaped characters in u.Path. +func SetOpaque(u *url.URL) { + u.Opaque = "//" + u.Host + u.Path + if !has4860Fix { + u.Opaque = u.Scheme + ":" + u.Opaque + } +} + +// Expand subsitutes any {encoded} strings in the URL passed in using +// the map supplied. +// +// This calls SetOpaque to avoid encoding of the parameters in the URL path. +func Expand(u *url.URL, expansions map[string]string) { + expanded, err := uritemplates.Expand(u.Path, expansions) + if err == nil { + u.Path = expanded + SetOpaque(u) + } +} + +// CloseBody is used to close res.Body. +// Prior to calling Close, it also tries to Read a small amount to see an EOF. +// Not seeing an EOF can prevent HTTP Transports from reusing connections. +func CloseBody(res *http.Response) { + if res == nil || res.Body == nil { + return + } + // Justification for 3 byte reads: two for up to "\r\n" after + // a JSON/XML document, and then 1 to see EOF if we haven't yet. + // TODO(bradfitz): detect Go 1.3+ and skip these reads. + // See https://codereview.appspot.com/58240043 + // and https://codereview.appspot.com/49570044 + buf := make([]byte, 1) + for i := 0; i < 3; i++ { + _, err := res.Body.Read(buf) + if err != nil { + break + } + } + res.Body.Close() + +} + +// VariantType returns the type name of the given variant. +// If the map doesn't contain the named key or the value is not a []interface{}, "" is returned. +// This is used to support "variant" APIs that can return one of a number of different types. +func VariantType(t map[string]interface{}) string { + s, _ := t["type"].(string) + return s +} + +// ConvertVariant uses the JSON encoder/decoder to fill in the struct 'dst' with the fields found in variant 'v'. +// This is used to support "variant" APIs that can return one of a number of different types. +// It reports whether the conversion was successful. +func ConvertVariant(v map[string]interface{}, dst interface{}) bool { + var buf bytes.Buffer + err := json.NewEncoder(&buf).Encode(v) + if err != nil { + return false + } + return json.Unmarshal(buf.Bytes(), dst) == nil +} + +// A Field names a field to be retrieved with a partial response. +// See https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// +// Partial responses can dramatically reduce the amount of data that must be sent to your application. +// In order to request partial responses, you can specify the full list of fields +// that your application needs by adding the Fields option to your request. +// +// Field strings use camelCase with leading lower-case characters to identify fields within the response. +// +// For example, if your response has a "NextPageToken" and a slice of "Items" with "Id" fields, +// you could request just those fields like this: +// +// svc.Events.List().Fields("nextPageToken", "items/id").Do() +// +// or if you were also interested in each Item's "Updated" field, you can combine them like this: +// +// svc.Events.List().Fields("nextPageToken", "items(id,updated)").Do() +// +// More information about field formatting can be found here: +// https://developers.google.com/+/api/#fields-syntax +// +// Another way to find field names is through the Google API explorer: +// https://developers.google.com/apis-explorer/#p/ +type Field string + +// CombineFields combines fields into a single string. +func CombineFields(s []Field) string { + r := make([]string, len(s)) + for i, v := range s { + r[i] = string(v) + } + return strings.Join(r, ",") +} diff --git a/components/engine/vendor/src/google.golang.org/api/googleapi/internal/uritemplates/LICENSE b/components/engine/vendor/src/google.golang.org/api/googleapi/internal/uritemplates/LICENSE new file mode 100644 index 0000000000..de9c88cb65 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/api/googleapi/internal/uritemplates/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) 2013 Joshua Tacoma + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/components/engine/vendor/src/google.golang.org/api/googleapi/internal/uritemplates/uritemplates.go b/components/engine/vendor/src/google.golang.org/api/googleapi/internal/uritemplates/uritemplates.go new file mode 100644 index 0000000000..8a84813fe5 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/api/googleapi/internal/uritemplates/uritemplates.go @@ -0,0 +1,359 @@ +// Copyright 2013 Joshua Tacoma. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package uritemplates is a level 4 implementation of RFC 6570 (URI +// Template, http://tools.ietf.org/html/rfc6570). +// +// To use uritemplates, parse a template string and expand it with a value +// map: +// +// template, _ := uritemplates.Parse("https://api.github.com/repos{/user,repo}") +// values := make(map[string]interface{}) +// values["user"] = "jtacoma" +// values["repo"] = "uritemplates" +// expanded, _ := template.ExpandString(values) +// fmt.Printf(expanded) +// +package uritemplates + +import ( + "bytes" + "errors" + "fmt" + "reflect" + "regexp" + "strconv" + "strings" +) + +var ( + unreserved = regexp.MustCompile("[^A-Za-z0-9\\-._~]") + reserved = regexp.MustCompile("[^A-Za-z0-9\\-._~:/?#[\\]@!$&'()*+,;=]") + validname = regexp.MustCompile("^([A-Za-z0-9_\\.]|%[0-9A-Fa-f][0-9A-Fa-f])+$") + hex = []byte("0123456789ABCDEF") +) + +func pctEncode(src []byte) []byte { + dst := make([]byte, len(src)*3) + for i, b := range src { + buf := dst[i*3 : i*3+3] + buf[0] = 0x25 + buf[1] = hex[b/16] + buf[2] = hex[b%16] + } + return dst +} + +func escape(s string, allowReserved bool) (escaped string) { + if allowReserved { + escaped = string(reserved.ReplaceAllFunc([]byte(s), pctEncode)) + } else { + escaped = string(unreserved.ReplaceAllFunc([]byte(s), pctEncode)) + } + return escaped +} + +// A UriTemplate is a parsed representation of a URI template. +type UriTemplate struct { + raw string + parts []templatePart +} + +// Parse parses a URI template string into a UriTemplate object. +func Parse(rawtemplate string) (template *UriTemplate, err error) { + template = new(UriTemplate) + template.raw = rawtemplate + split := strings.Split(rawtemplate, "{") + template.parts = make([]templatePart, len(split)*2-1) + for i, s := range split { + if i == 0 { + if strings.Contains(s, "}") { + err = errors.New("unexpected }") + break + } + template.parts[i].raw = s + } else { + subsplit := strings.Split(s, "}") + if len(subsplit) != 2 { + err = errors.New("malformed template") + break + } + expression := subsplit[0] + template.parts[i*2-1], err = parseExpression(expression) + if err != nil { + break + } + template.parts[i*2].raw = subsplit[1] + } + } + if err != nil { + template = nil + } + return template, err +} + +type templatePart struct { + raw string + terms []templateTerm + first string + sep string + named bool + ifemp string + allowReserved bool +} + +type templateTerm struct { + name string + explode bool + truncate int +} + +func parseExpression(expression string) (result templatePart, err error) { + switch expression[0] { + case '+': + result.sep = "," + result.allowReserved = true + expression = expression[1:] + case '.': + result.first = "." + result.sep = "." + expression = expression[1:] + case '/': + result.first = "/" + result.sep = "/" + expression = expression[1:] + case ';': + result.first = ";" + result.sep = ";" + result.named = true + expression = expression[1:] + case '?': + result.first = "?" + result.sep = "&" + result.named = true + result.ifemp = "=" + expression = expression[1:] + case '&': + result.first = "&" + result.sep = "&" + result.named = true + result.ifemp = "=" + expression = expression[1:] + case '#': + result.first = "#" + result.sep = "," + result.allowReserved = true + expression = expression[1:] + default: + result.sep = "," + } + rawterms := strings.Split(expression, ",") + result.terms = make([]templateTerm, len(rawterms)) + for i, raw := range rawterms { + result.terms[i], err = parseTerm(raw) + if err != nil { + break + } + } + return result, err +} + +func parseTerm(term string) (result templateTerm, err error) { + if strings.HasSuffix(term, "*") { + result.explode = true + term = term[:len(term)-1] + } + split := strings.Split(term, ":") + if len(split) == 1 { + result.name = term + } else if len(split) == 2 { + result.name = split[0] + var parsed int64 + parsed, err = strconv.ParseInt(split[1], 10, 0) + result.truncate = int(parsed) + } else { + err = errors.New("multiple colons in same term") + } + if !validname.MatchString(result.name) { + err = errors.New("not a valid name: " + result.name) + } + if result.explode && result.truncate > 0 { + err = errors.New("both explode and prefix modifers on same term") + } + return result, err +} + +// Expand expands a URI template with a set of values to produce a string. +func (self *UriTemplate) Expand(value interface{}) (string, error) { + values, ismap := value.(map[string]interface{}) + if !ismap { + if m, ismap := struct2map(value); !ismap { + return "", errors.New("expected map[string]interface{}, struct, or pointer to struct.") + } else { + return self.Expand(m) + } + } + var buf bytes.Buffer + for _, p := range self.parts { + err := p.expand(&buf, values) + if err != nil { + return "", err + } + } + return buf.String(), nil +} + +func (self *templatePart) expand(buf *bytes.Buffer, values map[string]interface{}) error { + if len(self.raw) > 0 { + buf.WriteString(self.raw) + return nil + } + var zeroLen = buf.Len() + buf.WriteString(self.first) + var firstLen = buf.Len() + for _, term := range self.terms { + value, exists := values[term.name] + if !exists { + continue + } + if buf.Len() != firstLen { + buf.WriteString(self.sep) + } + switch v := value.(type) { + case string: + self.expandString(buf, term, v) + case []interface{}: + self.expandArray(buf, term, v) + case map[string]interface{}: + if term.truncate > 0 { + return errors.New("cannot truncate a map expansion") + } + self.expandMap(buf, term, v) + default: + if m, ismap := struct2map(value); ismap { + if term.truncate > 0 { + return errors.New("cannot truncate a map expansion") + } + self.expandMap(buf, term, m) + } else { + str := fmt.Sprintf("%v", value) + self.expandString(buf, term, str) + } + } + } + if buf.Len() == firstLen { + original := buf.Bytes()[:zeroLen] + buf.Reset() + buf.Write(original) + } + return nil +} + +func (self *templatePart) expandName(buf *bytes.Buffer, name string, empty bool) { + if self.named { + buf.WriteString(name) + if empty { + buf.WriteString(self.ifemp) + } else { + buf.WriteString("=") + } + } +} + +func (self *templatePart) expandString(buf *bytes.Buffer, t templateTerm, s string) { + if len(s) > t.truncate && t.truncate > 0 { + s = s[:t.truncate] + } + self.expandName(buf, t.name, len(s) == 0) + buf.WriteString(escape(s, self.allowReserved)) +} + +func (self *templatePart) expandArray(buf *bytes.Buffer, t templateTerm, a []interface{}) { + if len(a) == 0 { + return + } else if !t.explode { + self.expandName(buf, t.name, false) + } + for i, value := range a { + if t.explode && i > 0 { + buf.WriteString(self.sep) + } else if i > 0 { + buf.WriteString(",") + } + var s string + switch v := value.(type) { + case string: + s = v + default: + s = fmt.Sprintf("%v", v) + } + if len(s) > t.truncate && t.truncate > 0 { + s = s[:t.truncate] + } + if self.named && t.explode { + self.expandName(buf, t.name, len(s) == 0) + } + buf.WriteString(escape(s, self.allowReserved)) + } +} + +func (self *templatePart) expandMap(buf *bytes.Buffer, t templateTerm, m map[string]interface{}) { + if len(m) == 0 { + return + } + if !t.explode { + self.expandName(buf, t.name, len(m) == 0) + } + var firstLen = buf.Len() + for k, value := range m { + if firstLen != buf.Len() { + if t.explode { + buf.WriteString(self.sep) + } else { + buf.WriteString(",") + } + } + var s string + switch v := value.(type) { + case string: + s = v + default: + s = fmt.Sprintf("%v", v) + } + if t.explode { + buf.WriteString(escape(k, self.allowReserved)) + buf.WriteRune('=') + buf.WriteString(escape(s, self.allowReserved)) + } else { + buf.WriteString(escape(k, self.allowReserved)) + buf.WriteRune(',') + buf.WriteString(escape(s, self.allowReserved)) + } + } +} + +func struct2map(v interface{}) (map[string]interface{}, bool) { + value := reflect.ValueOf(v) + switch value.Type().Kind() { + case reflect.Ptr: + return struct2map(value.Elem().Interface()) + case reflect.Struct: + m := make(map[string]interface{}) + for i := 0; i < value.NumField(); i++ { + tag := value.Type().Field(i).Tag + var name string + if strings.Contains(string(tag), ":") { + name = tag.Get("uri") + } else { + name = strings.TrimSpace(string(tag)) + } + if len(name) == 0 { + name = value.Type().Field(i).Name + } + m[name] = value.Field(i).Interface() + } + return m, true + } + return nil, false +} diff --git a/components/engine/vendor/src/google.golang.org/api/googleapi/internal/uritemplates/utils.go b/components/engine/vendor/src/google.golang.org/api/googleapi/internal/uritemplates/utils.go new file mode 100644 index 0000000000..399ef46236 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/api/googleapi/internal/uritemplates/utils.go @@ -0,0 +1,13 @@ +package uritemplates + +func Expand(path string, expansions map[string]string) (string, error) { + template, err := Parse(path) + if err != nil { + return "", err + } + values := make(map[string]interface{}) + for k, v := range expansions { + values[k] = v + } + return template.Expand(values) +} diff --git a/components/engine/vendor/src/google.golang.org/api/googleapi/types.go b/components/engine/vendor/src/google.golang.org/api/googleapi/types.go new file mode 100644 index 0000000000..a02b4b0716 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/api/googleapi/types.go @@ -0,0 +1,182 @@ +// Copyright 2013 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package googleapi + +import ( + "encoding/json" + "strconv" +) + +// Int64s is a slice of int64s that marshal as quoted strings in JSON. +type Int64s []int64 + +func (q *Int64s) UnmarshalJSON(raw []byte) error { + *q = (*q)[:0] + var ss []string + if err := json.Unmarshal(raw, &ss); err != nil { + return err + } + for _, s := range ss { + v, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return err + } + *q = append(*q, int64(v)) + } + return nil +} + +// Int32s is a slice of int32s that marshal as quoted strings in JSON. +type Int32s []int32 + +func (q *Int32s) UnmarshalJSON(raw []byte) error { + *q = (*q)[:0] + var ss []string + if err := json.Unmarshal(raw, &ss); err != nil { + return err + } + for _, s := range ss { + v, err := strconv.ParseInt(s, 10, 32) + if err != nil { + return err + } + *q = append(*q, int32(v)) + } + return nil +} + +// Uint64s is a slice of uint64s that marshal as quoted strings in JSON. +type Uint64s []uint64 + +func (q *Uint64s) UnmarshalJSON(raw []byte) error { + *q = (*q)[:0] + var ss []string + if err := json.Unmarshal(raw, &ss); err != nil { + return err + } + for _, s := range ss { + v, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return err + } + *q = append(*q, uint64(v)) + } + return nil +} + +// Uint32s is a slice of uint32s that marshal as quoted strings in JSON. +type Uint32s []uint32 + +func (q *Uint32s) UnmarshalJSON(raw []byte) error { + *q = (*q)[:0] + var ss []string + if err := json.Unmarshal(raw, &ss); err != nil { + return err + } + for _, s := range ss { + v, err := strconv.ParseUint(s, 10, 32) + if err != nil { + return err + } + *q = append(*q, uint32(v)) + } + return nil +} + +// Float64s is a slice of float64s that marshal as quoted strings in JSON. +type Float64s []float64 + +func (q *Float64s) UnmarshalJSON(raw []byte) error { + *q = (*q)[:0] + var ss []string + if err := json.Unmarshal(raw, &ss); err != nil { + return err + } + for _, s := range ss { + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return err + } + *q = append(*q, float64(v)) + } + return nil +} + +func quotedList(n int, fn func(dst []byte, i int) []byte) ([]byte, error) { + dst := make([]byte, 0, 2+n*10) // somewhat arbitrary + dst = append(dst, '[') + for i := 0; i < n; i++ { + if i > 0 { + dst = append(dst, ',') + } + dst = append(dst, '"') + dst = fn(dst, i) + dst = append(dst, '"') + } + dst = append(dst, ']') + return dst, nil +} + +func (s Int64s) MarshalJSON() ([]byte, error) { + return quotedList(len(s), func(dst []byte, i int) []byte { + return strconv.AppendInt(dst, s[i], 10) + }) +} + +func (s Int32s) MarshalJSON() ([]byte, error) { + return quotedList(len(s), func(dst []byte, i int) []byte { + return strconv.AppendInt(dst, int64(s[i]), 10) + }) +} + +func (s Uint64s) MarshalJSON() ([]byte, error) { + return quotedList(len(s), func(dst []byte, i int) []byte { + return strconv.AppendUint(dst, s[i], 10) + }) +} + +func (s Uint32s) MarshalJSON() ([]byte, error) { + return quotedList(len(s), func(dst []byte, i int) []byte { + return strconv.AppendUint(dst, uint64(s[i]), 10) + }) +} + +func (s Float64s) MarshalJSON() ([]byte, error) { + return quotedList(len(s), func(dst []byte, i int) []byte { + return strconv.AppendFloat(dst, s[i], 'g', -1, 64) + }) +} + +/* + * Helper routines for simplifying the creation of optional fields of basic type. + */ + +// Bool is a helper routine that allocates a new bool value +// to store v and returns a pointer to it. +func Bool(v bool) *bool { return &v } + +// Int32 is a helper routine that allocates a new int32 value +// to store v and returns a pointer to it. +func Int32(v int32) *int32 { return &v } + +// Int64 is a helper routine that allocates a new int64 value +// to store v and returns a pointer to it. +func Int64(v int64) *int64 { return &v } + +// Float64 is a helper routine that allocates a new float64 value +// to store v and returns a pointer to it. +func Float64(v float64) *float64 { return &v } + +// Uint32 is a helper routine that allocates a new uint32 value +// to store v and returns a pointer to it. +func Uint32(v uint32) *uint32 { return &v } + +// Uint64 is a helper routine that allocates a new uint64 value +// to store v and returns a pointer to it. +func Uint64(v uint64) *uint64 { return &v } + +// String is a helper routine that allocates a new string value +// to store v and returns a pointer to it. +func String(v string) *string { return &v } diff --git a/components/engine/vendor/src/google.golang.org/api/logging/v1beta3/logging-api.json b/components/engine/vendor/src/google.golang.org/api/logging/v1beta3/logging-api.json new file mode 100644 index 0000000000..48a141e712 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/api/logging/v1beta3/logging-api.json @@ -0,0 +1,1692 @@ +{ + "kind": "discovery#restDescription", + "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/JyzTrhH0rHlFw8M4zq31tTXViEA\"", + "discoveryVersion": "v1", + "id": "logging:v1beta3", + "name": "logging", + "version": "v1beta3", + "revision": "20151109", + "title": "Google Cloud Logging API", + "description": "The Google Cloud Logging API lets you write log entries and manage your logs, log sinks and logs-based metrics.", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://cloud.google.com/logging/docs/", + "protocol": "rest", + "baseUrl": "https://logging.googleapis.com/", + "basePath": "/", + "rootUrl": "https://logging.googleapis.com/", + "servicePath": "", + "batchPath": "batch", + "parameters": { + "access_token": { + "type": "string", + "description": "OAuth access token.", + "location": "query" + }, + "alt": { + "type": "string", + "description": "Data format for response.", + "default": "json", + "enumDescriptions": [ + "Responses with Content-Type of application/json", + "Media download with context-dependent Content-Type", + "Responses with Content-Type of application/x-protobuf" + ], + "location": "query" + }, + "bearer_token": { + "type": "string", + "description": "OAuth bearer token.", + "location": "query" + }, + "callback": { + "type": "string", + "description": "JSONP", + "location": "query" + }, + "fields": { + "type": "string", + "description": "Selector specifying which fields to include in a partial response.", + "location": "query" + }, + "key": { + "type": "string", + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "location": "query" + }, + "oauth_token": { + "type": "string", + "description": "OAuth 2.0 token for the current user.", + "location": "query" + }, + "pp": { + "type": "boolean", + "description": "Pretty-print response.", + "default": "true", + "location": "query" + }, + "prettyPrint": { + "type": "boolean", + "description": "Returns response with indentations and line breaks.", + "default": "true", + "location": "query" + }, + "quotaUser": { + "type": "string", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters.", + "location": "query" + }, + "upload_protocol": { + "type": "string", + "description": "Upload protocol for media (e.g. \"raw\", \"multipart\").", + "location": "query" + }, + "uploadType": { + "type": "string", + "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\").", + "location": "query" + }, + "$.xgafv": { + "type": "string", + "description": "V1 error format.", + "enumDescriptions": [ + "v1 error format", + "v2 error format" + ], + "location": "query" + } + }, + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/cloud-platform": { + "description": "View and manage your data across Google Cloud Platform services" + }, + "https://www.googleapis.com/auth/cloud-platform.read-only": { + "description": "View your data across Google Cloud Platform services" + }, + "https://www.googleapis.com/auth/logging.admin": { + "description": "Administrate log data for your projects" + }, + "https://www.googleapis.com/auth/logging.read": { + "description": "View log data for your projects" + }, + "https://www.googleapis.com/auth/logging.write": { + "description": "Submit log data for your projects" + } + } + } + }, + "schemas": { + "ListLogsResponse": { + "id": "ListLogsResponse", + "type": "object", + "description": "Result returned from ListLogs.", + "properties": { + "logs": { + "type": "array", + "description": "A list of log descriptions matching the criteria.", + "items": { + "$ref": "Log" + } + }, + "nextPageToken": { + "type": "string", + "description": "If there are more results, then `nextPageToken` is returned in the response. To get the next batch of logs, use the value of `nextPageToken` as `pageToken` in the next call of `ListLogs`. If `nextPageToken` is empty, then there are no more results." + } + } + }, + "Log": { + "id": "Log", + "type": "object", + "description": "_Output only._ Describes a log, which is a named stream of log entries.", + "properties": { + "name": { + "type": "string", + "description": "The resource name of the log. Example: `\"/projects/my-gcp-project-id/logs/LOG_NAME\"`, where `LOG_NAME` is the URL-encoded given name of the log. The log includes those log entries whose `LogEntry.log` field contains this given name. To avoid name collisions, it is a best practice to prefix the given log name with the service name, but this is not required. Examples of log given names: `\"appengine.googleapis.com/request_log\"`, `\"apache-access\"`." + }, + "displayName": { + "type": "string", + "description": "_Optional._ The common name of the log. Example: `\"request_log\"`." + }, + "payloadType": { + "type": "string", + "description": "_Optional_. A URI representing the expected payload type for log entries." + } + } + }, + "Empty": { + "id": "Empty", + "type": "object", + "description": "A generic empty message that you can re-use to avoid defining duplicated empty messages in your APIs. A typical example is to use it as the request or the response type of an API method. For instance: service Foo { rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); } The JSON representation for `Empty` is empty JSON object `{}`." + }, + "WriteLogEntriesRequest": { + "id": "WriteLogEntriesRequest", + "type": "object", + "description": "The parameters to WriteLogEntries.", + "properties": { + "commonLabels": { + "type": "object", + "description": "Metadata labels that apply to all log entries in this request, so that you don't have to repeat them in each log entry's `metadata.labels` field. If any of the log entries contains a (key, value) with the same key that is in `commonLabels`, then the entry's (key, value) overrides the one in `commonLabels`.", + "additionalProperties": { + "type": "string" + } + }, + "entries": { + "type": "array", + "description": "Log entries to insert.", + "items": { + "$ref": "LogEntry" + } + } + } + }, + "LogEntry": { + "id": "LogEntry", + "type": "object", + "description": "An individual entry in a log.", + "properties": { + "metadata": { + "$ref": "LogEntryMetadata", + "description": "Information about the log entry." + }, + "protoPayload": { + "type": "object", + "description": "The log entry payload, represented as a protocol buffer that is expressed as a JSON object. You can only pass `protoPayload` values that belong to a set of approved types.", + "additionalProperties": { + "type": "any", + "description": "Properties of the object. Contains field @ype with type URL." + } + }, + "textPayload": { + "type": "string", + "description": "The log entry payload, represented as a Unicode string (UTF-8)." + }, + "structPayload": { + "type": "object", + "description": "The log entry payload, represented as a structure that is expressed as a JSON object.", + "additionalProperties": { + "type": "any", + "description": "Properties of the object." + } + }, + "insertId": { + "type": "string", + "description": "A unique ID for the log entry. If you provide this field, the logging service considers other log entries in the same log with the same ID as duplicates which can be removed." + }, + "log": { + "type": "string", + "description": "The log to which this entry belongs. When a log entry is ingested, the value of this field is set by the logging system." + }, + "httpRequest": { + "$ref": "HttpRequest", + "description": "Information about the HTTP request associated with this log entry, if applicable." + } + } + }, + "LogEntryMetadata": { + "id": "LogEntryMetadata", + "type": "object", + "description": "Additional data that is associated with a log entry, set by the service creating the log entry.", + "properties": { + "timestamp": { + "type": "string", + "description": "The time the event described by the log entry occurred. Timestamps must be later than January 1, 1970." + }, + "severity": { + "type": "string", + "description": "The severity of the log entry.", + "enum": [ + "DEFAULT", + "DEBUG", + "INFO", + "NOTICE", + "WARNING", + "ERROR", + "CRITICAL", + "ALERT", + "EMERGENCY" + ] + }, + "projectId": { + "type": "string", + "description": "The project ID of the Google Cloud Platform service that created the log entry." + }, + "serviceName": { + "type": "string", + "description": "The API name of the Google Cloud Platform service that created the log entry. For example, `\"compute.googleapis.com\"`." + }, + "region": { + "type": "string", + "description": "The region name of the Google Cloud Platform service that created the log entry. For example, `\"us-central1\"`." + }, + "zone": { + "type": "string", + "description": "The zone of the Google Cloud Platform service that created the log entry. For example, `\"us-central1-a\"`." + }, + "userId": { + "type": "string", + "description": "The fully-qualified email address of the authenticated user that performed or requested the action represented by the log entry. If the log entry does not apply to an action taken by an authenticated user, then the field should be empty." + }, + "labels": { + "type": "object", + "description": "A set of (key, value) data that provides additional information about the log entry. If the log entry is from one of the Google Cloud Platform sources listed below, the indicated (key, value) information must be provided: Google App Engine, service_name `appengine.googleapis.com`: \"appengine.googleapis.com/module_id\", \"appengine.googleapis.com/version_id\", and one of: \"appengine.googleapis.com/replica_index\", \"appengine.googleapis.com/clone_id\", or else provide the following Compute Engine labels: Google Compute Engine, service_name `compute.googleapis.com`: \"compute.googleapis.com/resource_type\", \"instance\" \"compute.googleapis.com/resource_id\",", + "additionalProperties": { + "type": "string" + } + } + } + }, + "HttpRequest": { + "id": "HttpRequest", + "type": "object", + "description": "A common proto for logging HTTP requests.", + "properties": { + "requestMethod": { + "type": "string", + "description": "Request method, such as `GET`, `HEAD`, `PUT` or `POST`." + }, + "requestUrl": { + "type": "string", + "description": "Contains the scheme (http|https), the host name, the path and the query portion of the URL that was requested." + }, + "requestSize": { + "type": "string", + "description": "Size of the HTTP request message in bytes, including request headers and the request body.", + "format": "int64" + }, + "status": { + "type": "integer", + "description": "A response code indicates the status of response, e.g., 200.", + "format": "int32" + }, + "responseSize": { + "type": "string", + "description": "Size of the HTTP response message in bytes sent back to the client, including response headers and response body.", + "format": "int64" + }, + "userAgent": { + "type": "string", + "description": "User agent sent by the client, e.g., \"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Q312461; .NET CLR 1.0.3705)\"." + }, + "remoteIp": { + "type": "string", + "description": "IP address of the client who issues the HTTP request. Could be either IPv4 or IPv6." + }, + "referer": { + "type": "string", + "description": "Referer (a.k.a. referrer) URL of request, as defined in http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html." + }, + "cacheHit": { + "type": "boolean", + "description": "Whether or not an entity was served from cache (with or without validation)." + }, + "validatedWithOriginServer": { + "type": "boolean", + "description": "Whether or not the response was validated with the origin server before being served from cache. This field is only meaningful if cache_hit is True." + } + } + }, + "WriteLogEntriesResponse": { + "id": "WriteLogEntriesResponse", + "type": "object", + "description": "Result returned from WriteLogEntries. empty" + }, + "ListLogServicesResponse": { + "id": "ListLogServicesResponse", + "type": "object", + "description": "Result returned from `ListLogServicesRequest`.", + "properties": { + "logServices": { + "type": "array", + "description": "A list of log services.", + "items": { + "$ref": "LogService" + } + }, + "nextPageToken": { + "type": "string", + "description": "If there are more results, then `nextPageToken` is returned in the response. To get the next batch of services, use the value of `nextPageToken` as `pageToken` in the next call of `ListLogServices`. If `nextPageToken` is empty, then there are no more results." + } + } + }, + "LogService": { + "id": "LogService", + "type": "object", + "description": "_Output only._ Describes a service that writes log entries.", + "properties": { + "name": { + "type": "string", + "description": "The service's name. Example: `\"appengine.googleapis.com\"`. Log names beginning with this string are reserved for this service. This value can appear in the `LogEntry.metadata.serviceName` field of log entries associated with this log service." + }, + "indexKeys": { + "type": "array", + "description": "A list of the names of the keys used to index and label individual log entries from this service. The first two keys are used as the primary and secondary index, respectively. Additional keys may be used to label the entries. For example, App Engine indexes its entries by module and by version, so its `indexKeys` field is the following: [ \"appengine.googleapis.com/module_id\", \"appengine.googleapis.com/version_id\" ]", + "items": { + "type": "string" + } + } + } + }, + "ListLogServiceIndexesResponse": { + "id": "ListLogServiceIndexesResponse", + "type": "object", + "description": "Result returned from ListLogServiceIndexesRequest.", + "properties": { + "serviceIndexPrefixes": { + "type": "array", + "description": "A list of log service index values. Each index value has the form `\"/value1/value2/...\"`, where `value1` is a value in the primary index, `value2` is a value in the secondary index, and so forth.", + "items": { + "type": "string" + } + }, + "nextPageToken": { + "type": "string", + "description": "If there are more results, then `nextPageToken` is returned in the response. To get the next batch of indexes, use the value of `nextPageToken` as `pageToken` in the next call of `ListLogServiceIndexes`. If `nextPageToken` is empty, then there are no more results." + } + } + }, + "ListLogSinksResponse": { + "id": "ListLogSinksResponse", + "type": "object", + "description": "Result returned from `ListLogSinks`.", + "properties": { + "sinks": { + "type": "array", + "description": "The requested log sinks. If a returned `LogSink` object has an empty `destination` field, the client can retrieve the complete `LogSink` object by calling `log.sinks.get`.", + "items": { + "$ref": "LogSink" + } + } + } + }, + "LogSink": { + "id": "LogSink", + "type": "object", + "description": "Describes where log entries are written outside of Cloud Logging.", + "properties": { + "name": { + "type": "string", + "description": "The client-assigned name of this sink. For example, `\"my-syslog-sink\"`. The name must be unique among the sinks of a similar kind in the project." + }, + "destination": { + "type": "string", + "description": "The resource name of the destination. Cloud Logging writes designated log entries to this destination. For example, `\"storage.googleapis.com/my-output-bucket\"`." + }, + "filter": { + "type": "string", + "description": "An advanced logs filter. If present, only log entries matching the filter are written. Only project sinks use this field; log sinks and log service sinks must not include a filter." + }, + "errors": { + "type": "array", + "description": "_Output only._ If any errors occur when invoking a sink method, then this field contains descriptions of the errors.", + "items": { + "$ref": "LogError" + } + } + } + }, + "LogError": { + "id": "LogError", + "type": "object", + "description": "Describes a problem with a logging resource or operation.", + "properties": { + "resource": { + "type": "string", + "description": "A resource name associated with this error. For example, the name of a Cloud Storage bucket that has insufficient permissions to be a destination for log entries." + }, + "status": { + "$ref": "Status", + "description": "The error description, including a classification code, an error message, and other details." + }, + "timeNanos": { + "type": "string", + "description": "The time the error was observed, in nanoseconds since the Unix epoch.", + "format": "int64" + } + } + }, + "Status": { + "id": "Status", + "type": "object", + "description": "The `Status` type defines a logical error model that is suitable for different programming environments, including REST APIs and RPC APIs. It is used by [gRPC](https://github.com/grpc). The error model is designed to be: - Simple to use and understand for most users - Flexible enough to meet unexpected needs # Overview The `Status` message contains three pieces of data: error code, error message, and error details. The error code should be an enum value of google.rpc.Code, but it may accept additional error codes if needed. The error message should be a developer-facing English message that helps developers *understand* and *resolve* the error. If a localized user-facing error message is needed, put the localized message in the error details or localize it in the client. The optional error details may contain arbitrary information about the error. There is a predefined set of error detail types in the package `google.rpc` which can be used for common error conditions. # Language mapping The `Status` message is the logical representation of the error model, but it is not necessarily the actual wire format. When the `Status` message is exposed in different client libraries and different wire protocols, it can be mapped differently. For example, it will likely be mapped to some exceptions in Java, but more likely mapped to some error codes in C. # Other uses The error model and the `Status` message can be used in a variety of environments, either with or without APIs, to provide a consistent developer experience across different environments. Example uses of this error model include: - Partial errors. If a service needs to return partial errors to the client, it may embed the `Status` in the normal response to indicate the partial errors. - Workflow errors. A typical workflow has multiple steps. Each step may have a `Status` message for error reporting purpose. - Batch operations. If a client uses batch request and batch response, the `Status` message should be used directly inside batch response, one for each error sub-response. - Asynchronous operations. If an API call embeds asynchronous operation results in its response, the status of those operations should be represented directly using the `Status` message. - Logging. If some API errors are stored in logs, the message `Status` could be used directly after any stripping needed for security/privacy reasons.", + "properties": { + "code": { + "type": "integer", + "description": "The status code, which should be an enum value of google.rpc.Code.", + "format": "int32" + }, + "message": { + "type": "string", + "description": "A developer-facing error message, which should be in English. Any user-facing error message should be localized and sent in the google.rpc.Status.details field, or localized by the client." + }, + "details": { + "type": "array", + "description": "A list of messages that carry the error details. There will be a common set of message types for APIs to use.", + "items": { + "type": "object", + "additionalProperties": { + "type": "any", + "description": "Properties of the object. Contains field @ype with type URL." + } + } + } + } + }, + "ListLogServiceSinksResponse": { + "id": "ListLogServiceSinksResponse", + "type": "object", + "description": "Result returned from `ListLogServiceSinks`.", + "properties": { + "sinks": { + "type": "array", + "description": "The requested log service sinks. If a returned `LogSink` object has an empty `destination` field, the client can retrieve the complete `LogSink` object by calling `logServices.sinks.get`.", + "items": { + "$ref": "LogSink" + } + } + } + }, + "ListSinksResponse": { + "id": "ListSinksResponse", + "type": "object", + "description": "Result returned from `ListSinks`.", + "properties": { + "sinks": { + "type": "array", + "description": "The requested sinks. If a returned `LogSink` object has an empty `destination` field, the client can retrieve the complete `LogSink` object by calling `projects.sinks.get`.", + "items": { + "$ref": "LogSink" + } + } + } + }, + "ListLogMetricsResponse": { + "id": "ListLogMetricsResponse", + "type": "object", + "description": "Result returned from ListLogMetrics.", + "properties": { + "metrics": { + "type": "array", + "description": "The list of metrics that was requested.", + "items": { + "$ref": "LogMetric" + } + }, + "nextPageToken": { + "type": "string", + "description": "If there are more results, then `nextPageToken` is returned in the response. To get the next batch of entries, use the value of `nextPageToken` as `pageToken` in the next call of `ListLogMetrics`. If `nextPageToken` is empty, then there are no more results." + } + } + }, + "LogMetric": { + "id": "LogMetric", + "type": "object", + "description": "Describes a logs-based metric. The value of the metric is the number of log entries in your project that match a logs filter.", + "properties": { + "name": { + "type": "string", + "description": "The client-assigned name for this metric, such as `\"severe_errors\"`. Metric names are limited to 1000 characters and can include only the following characters: `A-Z`, `a-z`, `0-9`, and the special characters `_-.,+!*',()%/\\`. The slash character (`/`) denotes a hierarchy of name pieces, and it cannot be the first character of the name." + }, + "description": { + "type": "string", + "description": "A description of this metric." + }, + "filter": { + "type": "string", + "description": "An [advanced logs filter](/logging/docs/view/advanced_filters). Example: `\"log:syslog AND metadata.severity\u003e=ERROR\"`." + } + } + }, + "RequestLog": { + "id": "RequestLog", + "type": "object", + "description": "Complete log information about a single request to an application.", + "properties": { + "appId": { + "type": "string", + "description": "Identifies the application that handled this request." + }, + "moduleId": { + "type": "string", + "description": "Identifies the module of the application that handled this request." + }, + "versionId": { + "type": "string", + "description": "Version of the application that handled this request." + }, + "requestId": { + "type": "string", + "description": "Globally unique identifier for a request, based on request start time. Request IDs for requests which started later will compare greater as strings than those for requests which started earlier." + }, + "ip": { + "type": "string", + "description": "Origin IP address." + }, + "startTime": { + "type": "string", + "description": "Time at which request was known to have begun processing." + }, + "endTime": { + "type": "string", + "description": "Time at which request was known to end processing." + }, + "latency": { + "type": "string", + "description": "Latency of the request." + }, + "megaCycles": { + "type": "string", + "description": "Number of CPU megacycles used to process request.", + "format": "int64" + }, + "method": { + "type": "string", + "description": "Request method, such as `GET`, `HEAD`, `PUT`, `POST`, or `DELETE`." + }, + "resource": { + "type": "string", + "description": "Contains the path and query portion of the URL that was requested. For example, if the URL was \"http://example.com/app?name=val\", the resource would be \"/app?name=val\". Any trailing fragment (separated by a '#' character) will not be included." + }, + "httpVersion": { + "type": "string", + "description": "HTTP version of request." + }, + "status": { + "type": "integer", + "description": "Response status of request.", + "format": "int32" + }, + "responseSize": { + "type": "string", + "description": "Size in bytes sent back to client by request.", + "format": "int64" + }, + "referrer": { + "type": "string", + "description": "Referrer URL of request." + }, + "userAgent": { + "type": "string", + "description": "User agent used for making request." + }, + "nickname": { + "type": "string", + "description": "A string that identifies a logged-in user who made this request, or empty if the user is not logged in. Most likely, this is the part of the user's email before the '@' sign. The field value is the same for different requests from the same user, but different users may have a similar name. This information is also available to the application via Users API. This field will be populated starting with App Engine 1.9.21." + }, + "urlMapEntry": { + "type": "string", + "description": "File or class within URL mapping used for request. Useful for tracking down the source code which was responsible for managing request. Especially for multiply mapped handlers." + }, + "host": { + "type": "string", + "description": "The Internet host and port number of the resource being requested." + }, + "cost": { + "type": "number", + "description": "An indication of the relative cost of serving this request.", + "format": "double" + }, + "taskQueueName": { + "type": "string", + "description": "Queue name of the request (for an offline request)." + }, + "taskName": { + "type": "string", + "description": "Task name of the request (for an offline request)." + }, + "wasLoadingRequest": { + "type": "boolean", + "description": "Was this request a loading request for this instance?" + }, + "pendingTime": { + "type": "string", + "description": "Time this request spent in the pending request queue, if it was pending at all." + }, + "instanceIndex": { + "type": "integer", + "description": "If the instance that processed this request was individually addressable (i.e. belongs to a manually scaled module), this is the index of the instance.", + "format": "int32" + }, + "finished": { + "type": "boolean", + "description": "If true, represents a finished request. Otherwise, the request is active." + }, + "instanceId": { + "type": "string", + "description": "An opaque identifier for the instance that handled the request." + }, + "line": { + "type": "array", + "description": "List of log lines emitted by the application while serving this request, if requested.", + "items": { + "$ref": "LogLine" + } + }, + "appEngineRelease": { + "type": "string", + "description": "App Engine release version string." + }, + "traceId": { + "type": "string", + "description": "Cloud Trace identifier of the trace for this request." + }, + "sourceReference": { + "type": "array", + "description": "Source code for the application that handled this request. There can be more than one source reference per deployed application if source code is distributed among multiple repositories.", + "items": { + "$ref": "SourceReference" + } + } + } + }, + "LogLine": { + "id": "LogLine", + "type": "object", + "description": "Application log line emitted while processing a request.", + "properties": { + "time": { + "type": "string", + "description": "Time when log entry was made. May be inaccurate." + }, + "severity": { + "type": "string", + "description": "Severity of log.", + "enum": [ + "DEFAULT", + "DEBUG", + "INFO", + "NOTICE", + "WARNING", + "ERROR", + "CRITICAL", + "ALERT", + "EMERGENCY" + ] + }, + "logMessage": { + "type": "string", + "description": "App provided log message." + }, + "sourceLocation": { + "$ref": "SourceLocation", + "description": "Line of code that generated this log message." + } + } + }, + "SourceLocation": { + "id": "SourceLocation", + "type": "object", + "description": "Specifies a location in a source file.", + "properties": { + "file": { + "type": "string", + "description": "Source file name. May or may not be a fully qualified name, depending on the runtime environment." + }, + "line": { + "type": "string", + "description": "Line within the source file.", + "format": "int64" + }, + "functionName": { + "type": "string", + "description": "Human-readable name of the function or method being invoked, with optional context such as the class or package name, for use in contexts such as the logs viewer where file:line number is less meaningful. This may vary by language, for example: in Java: qual.if.ied.Class.method in Go: dir/package.func in Python: function ..." + } + } + }, + "SourceReference": { + "id": "SourceReference", + "type": "object", + "description": "A reference to a particular snapshot of the source tree used to build and deploy an application.", + "properties": { + "repository": { + "type": "string", + "description": "Optional. A URI string identifying the repository. Example: \"https://github.com/GoogleCloudPlatform/kubernetes.git\"" + }, + "revisionId": { + "type": "string", + "description": "The canonical (and persistent) identifier of the deployed revision. Example (git): \"0035781c50ec7aa23385dc841529ce8a4b70db1b\"" + } + } + } + }, + "resources": { + "projects": { + "resources": { + "logs": { + "methods": { + "list": { + "id": "logging.projects.logs.list", + "path": "v1beta3/projects/{projectsId}/logs", + "httpMethod": "GET", + "description": "Lists the logs in the project. Only logs that have entries are listed.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `projectName`. The resource name of the project whose logs are requested. If both `serviceName` and `serviceIndexPrefix` are empty, then all logs with entries in this project are listed.", + "required": true, + "location": "path" + }, + "serviceName": { + "type": "string", + "description": "If not empty, this field must be a log service name such as `\"compute.googleapis.com\"`. Only logs associated with that that log service are listed.", + "location": "query" + }, + "serviceIndexPrefix": { + "type": "string", + "description": "The purpose of this field is to restrict the listed logs to those with entries of a certain kind. If `serviceName` is the name of a log service, then this field may contain values for the log service's indexes. Only logs that have entries whose indexes include the values are listed. The format for this field is `\"/val1/val2.../valN\"`, where `val1` is a value for the first index, `val2` for the second index, etc. An empty value (a single slash) for an index matches all values, and you can omit values for later indexes entirely.", + "location": "query" + }, + "pageSize": { + "type": "integer", + "description": "The maximum number of results to return.", + "format": "int32", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "An opaque token, returned as `nextPageToken` by a prior `ListLogs` operation. If `pageToken` is supplied, then the other fields of this request are ignored, and instead the previous `ListLogs` operation is continued.", + "location": "query" + } + }, + "parameterOrder": [ + "projectsId" + ], + "response": { + "$ref": "ListLogsResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.read" + ] + }, + "delete": { + "id": "logging.projects.logs.delete", + "path": "v1beta3/projects/{projectsId}/logs/{logsId}", + "httpMethod": "DELETE", + "description": "Deletes a log and all its log entries. The log will reappear if it receives new entries.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `logName`. The resource name of the log to be deleted.", + "required": true, + "location": "path" + }, + "logsId": { + "type": "string", + "description": "Part of `logName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "logsId" + ], + "response": { + "$ref": "Empty" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin" + ] + } + }, + "resources": { + "entries": { + "methods": { + "write": { + "id": "logging.projects.logs.entries.write", + "path": "v1beta3/projects/{projectsId}/logs/{logsId}/entries:write", + "httpMethod": "POST", + "description": "Writes log entries to Cloud Logging. Each entry consists of a `LogEntry` object. You must fill in all the fields of the object, including one of the payload fields. You may supply a map, `commonLabels`, that holds default (key, value) data for the `entries[].metadata.labels` map in each entry, saving you the trouble of creating identical copies for each entry.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `logName`. The resource name of the log that will receive the log entries.", + "required": true, + "location": "path" + }, + "logsId": { + "type": "string", + "description": "Part of `logName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "logsId" + ], + "request": { + "$ref": "WriteLogEntriesRequest" + }, + "response": { + "$ref": "WriteLogEntriesResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.write" + ] + } + } + }, + "sinks": { + "methods": { + "list": { + "id": "logging.projects.logs.sinks.list", + "path": "v1beta3/projects/{projectsId}/logs/{logsId}/sinks", + "httpMethod": "GET", + "description": "Lists log sinks associated with a log.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `logName`. The log whose sinks are wanted. For example, `\"compute.google.com/syslog\"`.", + "required": true, + "location": "path" + }, + "logsId": { + "type": "string", + "description": "Part of `logName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "logsId" + ], + "response": { + "$ref": "ListLogSinksResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.read" + ] + }, + "get": { + "id": "logging.projects.logs.sinks.get", + "path": "v1beta3/projects/{projectsId}/logs/{logsId}/sinks/{sinksId}", + "httpMethod": "GET", + "description": "Gets a log sink.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `sinkName`. The resource name of the log sink to return.", + "required": true, + "location": "path" + }, + "logsId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + }, + "sinksId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "logsId", + "sinksId" + ], + "response": { + "$ref": "LogSink" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.read" + ] + }, + "create": { + "id": "logging.projects.logs.sinks.create", + "path": "v1beta3/projects/{projectsId}/logs/{logsId}/sinks", + "httpMethod": "POST", + "description": "Creates a log sink. All log entries for a specified log are written to the destination.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `logName`. The resource name of the log to which to the sink is bound.", + "required": true, + "location": "path" + }, + "logsId": { + "type": "string", + "description": "Part of `logName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "logsId" + ], + "request": { + "$ref": "LogSink" + }, + "response": { + "$ref": "LogSink" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin" + ] + }, + "update": { + "id": "logging.projects.logs.sinks.update", + "path": "v1beta3/projects/{projectsId}/logs/{logsId}/sinks/{sinksId}", + "httpMethod": "PUT", + "description": "Updates a log sink. If the sink does not exist, it is created.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `sinkName`. The resource name of the sink to update.", + "required": true, + "location": "path" + }, + "logsId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + }, + "sinksId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "logsId", + "sinksId" + ], + "request": { + "$ref": "LogSink" + }, + "response": { + "$ref": "LogSink" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin" + ] + }, + "delete": { + "id": "logging.projects.logs.sinks.delete", + "path": "v1beta3/projects/{projectsId}/logs/{logsId}/sinks/{sinksId}", + "httpMethod": "DELETE", + "description": "Deletes a log sink. After deletion, no new log entries are written to the destination.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `sinkName`. The resource name of the log sink to delete.", + "required": true, + "location": "path" + }, + "logsId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + }, + "sinksId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "logsId", + "sinksId" + ], + "response": { + "$ref": "Empty" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin" + ] + } + } + } + } + }, + "logServices": { + "methods": { + "list": { + "id": "logging.projects.logServices.list", + "path": "v1beta3/projects/{projectsId}/logServices", + "httpMethod": "GET", + "description": "Lists the log services that have log entries in this project.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `projectName`. The resource name of the project whose services are to be listed.", + "required": true, + "location": "path" + }, + "pageSize": { + "type": "integer", + "description": "The maximum number of `LogService` objects to return in one operation.", + "format": "int32", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "An opaque token, returned as `nextPageToken` by a prior `ListLogServices` operation. If `pageToken` is supplied, then the other fields of this request are ignored, and instead the previous `ListLogServices` operation is continued.", + "location": "query" + } + }, + "parameterOrder": [ + "projectsId" + ], + "response": { + "$ref": "ListLogServicesResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.read" + ] + } + }, + "resources": { + "indexes": { + "methods": { + "list": { + "id": "logging.projects.logServices.indexes.list", + "path": "v1beta3/projects/{projectsId}/logServices/{logServicesId}/indexes", + "httpMethod": "GET", + "description": "Lists the current index values for a log service.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `serviceName`. The resource name of a log service whose service indexes are requested. Example: `\"projects/my-project-id/logServices/appengine.googleapis.com\"`.", + "required": true, + "location": "path" + }, + "logServicesId": { + "type": "string", + "description": "Part of `serviceName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + }, + "indexPrefix": { + "type": "string", + "description": "Restricts the index values returned to be those with a specified prefix for each index key. This field has the form `\"/prefix1/prefix2/...\"`, in order corresponding to the `LogService indexKeys`. Non-empty prefixes must begin with `/`. For example, App Engine's two keys are the module ID and the version ID. Following is the effect of using various values for `indexPrefix`: + `\"/Mod/\"` retrieves `/Mod/10` and `/Mod/11` but not `/ModA/10`. + `\"/Mod` retrieves `/Mod/10`, `/Mod/11` and `/ModA/10` but not `/XXX/33`. + `\"/Mod/1\"` retrieves `/Mod/10` and `/Mod/11` but not `/ModA/10`. + `\"/Mod/10/\"` retrieves `/Mod/10` only. + An empty prefix or `\"/\"` retrieves all values.", + "location": "query" + }, + "depth": { + "type": "integer", + "description": "A non-negative integer that limits the number of levels of the index hierarchy that are returned. If `depth` is 1 (default), only the first index key value is returned. If `depth` is 2, both primary and secondary key values are returned. If `depth` is 0, the depth is the number of slash-separators in the `indexPrefix` field, not counting a slash appearing as the last character of the prefix. If the `indexPrefix` field is empty, the default depth is 1. It is an error for `depth` to be any positive value less than the number of components in `indexPrefix`.", + "format": "int32", + "location": "query" + }, + "pageSize": { + "type": "integer", + "description": "The maximum number of log service index resources to return in one operation.", + "format": "int32", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "An opaque token, returned as `nextPageToken` by a prior `ListLogServiceIndexes` operation. If `pageToken` is supplied, then the other fields of this request are ignored, and instead the previous `ListLogServiceIndexes` operation is continued.", + "location": "query" + } + }, + "parameterOrder": [ + "projectsId", + "logServicesId" + ], + "response": { + "$ref": "ListLogServiceIndexesResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.read" + ] + } + } + }, + "sinks": { + "methods": { + "list": { + "id": "logging.projects.logServices.sinks.list", + "path": "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks", + "httpMethod": "GET", + "description": "Lists log service sinks associated with a log service.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `serviceName`. The log service whose sinks are wanted.", + "required": true, + "location": "path" + }, + "logServicesId": { + "type": "string", + "description": "Part of `serviceName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "logServicesId" + ], + "response": { + "$ref": "ListLogServiceSinksResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.read" + ] + }, + "get": { + "id": "logging.projects.logServices.sinks.get", + "path": "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks/{sinksId}", + "httpMethod": "GET", + "description": "Gets a log service sink.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `sinkName`. The resource name of the log service sink to return.", + "required": true, + "location": "path" + }, + "logServicesId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + }, + "sinksId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "logServicesId", + "sinksId" + ], + "response": { + "$ref": "LogSink" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.read" + ] + }, + "create": { + "id": "logging.projects.logServices.sinks.create", + "path": "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks", + "httpMethod": "POST", + "description": "Creates a log service sink. All log entries from a specified log service are written to the destination.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `serviceName`. The resource name of the log service to which the sink is bound.", + "required": true, + "location": "path" + }, + "logServicesId": { + "type": "string", + "description": "Part of `serviceName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "logServicesId" + ], + "request": { + "$ref": "LogSink" + }, + "response": { + "$ref": "LogSink" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin" + ] + }, + "update": { + "id": "logging.projects.logServices.sinks.update", + "path": "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks/{sinksId}", + "httpMethod": "PUT", + "description": "Updates a log service sink. If the sink does not exist, it is created.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `sinkName`. The resource name of the log service sink to update.", + "required": true, + "location": "path" + }, + "logServicesId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + }, + "sinksId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "logServicesId", + "sinksId" + ], + "request": { + "$ref": "LogSink" + }, + "response": { + "$ref": "LogSink" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin" + ] + }, + "delete": { + "id": "logging.projects.logServices.sinks.delete", + "path": "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks/{sinksId}", + "httpMethod": "DELETE", + "description": "Deletes a log service sink. After deletion, no new log entries are written to the destination.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `sinkName`. The resource name of the log service sink to delete.", + "required": true, + "location": "path" + }, + "logServicesId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + }, + "sinksId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "logServicesId", + "sinksId" + ], + "response": { + "$ref": "Empty" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin" + ] + } + } + } + } + }, + "sinks": { + "methods": { + "list": { + "id": "logging.projects.sinks.list", + "path": "v1beta3/projects/{projectsId}/sinks", + "httpMethod": "GET", + "description": "Lists project sinks associated with a project.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `projectName`. The project whose sinks are wanted.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId" + ], + "response": { + "$ref": "ListSinksResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.read" + ] + }, + "get": { + "id": "logging.projects.sinks.get", + "path": "v1beta3/projects/{projectsId}/sinks/{sinksId}", + "httpMethod": "GET", + "description": "Gets a project sink.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `sinkName`. The resource name of the project sink to return.", + "required": true, + "location": "path" + }, + "sinksId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "sinksId" + ], + "response": { + "$ref": "LogSink" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.read" + ] + }, + "create": { + "id": "logging.projects.sinks.create", + "path": "v1beta3/projects/{projectsId}/sinks", + "httpMethod": "POST", + "description": "Creates a project sink. A logs filter determines which log entries are written to the destination.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `projectName`. The resource name of the project to which the sink is bound.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId" + ], + "request": { + "$ref": "LogSink" + }, + "response": { + "$ref": "LogSink" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin" + ] + }, + "update": { + "id": "logging.projects.sinks.update", + "path": "v1beta3/projects/{projectsId}/sinks/{sinksId}", + "httpMethod": "PUT", + "description": "Updates a project sink. If the sink does not exist, it is created. The destination, filter, or both may be updated.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `sinkName`. The resource name of the project sink to update.", + "required": true, + "location": "path" + }, + "sinksId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "sinksId" + ], + "request": { + "$ref": "LogSink" + }, + "response": { + "$ref": "LogSink" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin" + ] + }, + "delete": { + "id": "logging.projects.sinks.delete", + "path": "v1beta3/projects/{projectsId}/sinks/{sinksId}", + "httpMethod": "DELETE", + "description": "Deletes a project sink. After deletion, no new log entries are written to the destination.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `sinkName`. The resource name of the project sink to delete.", + "required": true, + "location": "path" + }, + "sinksId": { + "type": "string", + "description": "Part of `sinkName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "sinksId" + ], + "response": { + "$ref": "Empty" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin" + ] + } + } + }, + "metrics": { + "methods": { + "list": { + "id": "logging.projects.metrics.list", + "path": "v1beta3/projects/{projectsId}/metrics", + "httpMethod": "GET", + "description": "Lists the logs-based metrics associated with a project.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `projectName`. The resource name for the project whose metrics are wanted.", + "required": true, + "location": "path" + }, + "pageToken": { + "type": "string", + "description": "An opaque token, returned as `nextPageToken` by a prior `ListLogMetrics` operation. If `pageToken` is supplied, then the other fields of this request are ignored, and instead the previous `ListLogMetrics` operation is continued.", + "location": "query" + }, + "pageSize": { + "type": "integer", + "description": "The maximum number of `LogMetric` objects to return in one operation.", + "format": "int32", + "location": "query" + } + }, + "parameterOrder": [ + "projectsId" + ], + "response": { + "$ref": "ListLogMetricsResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.read" + ] + }, + "get": { + "id": "logging.projects.metrics.get", + "path": "v1beta3/projects/{projectsId}/metrics/{metricsId}", + "httpMethod": "GET", + "description": "Gets a logs-based metric.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `metricName`. The resource name of the desired metric.", + "required": true, + "location": "path" + }, + "metricsId": { + "type": "string", + "description": "Part of `metricName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "metricsId" + ], + "response": { + "$ref": "LogMetric" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.read" + ] + }, + "create": { + "id": "logging.projects.metrics.create", + "path": "v1beta3/projects/{projectsId}/metrics", + "httpMethod": "POST", + "description": "Creates a logs-based metric.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `projectName`. The resource name of the project in which to create the metric.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId" + ], + "request": { + "$ref": "LogMetric" + }, + "response": { + "$ref": "LogMetric" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.write" + ] + }, + "update": { + "id": "logging.projects.metrics.update", + "path": "v1beta3/projects/{projectsId}/metrics/{metricsId}", + "httpMethod": "PUT", + "description": "Creates or updates a logs-based metric.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `metricName`. The resource name of the metric to update.", + "required": true, + "location": "path" + }, + "metricsId": { + "type": "string", + "description": "Part of `metricName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "metricsId" + ], + "request": { + "$ref": "LogMetric" + }, + "response": { + "$ref": "LogMetric" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.write" + ] + }, + "delete": { + "id": "logging.projects.metrics.delete", + "path": "v1beta3/projects/{projectsId}/metrics/{metricsId}", + "httpMethod": "DELETE", + "description": "Deletes a logs-based metric.", + "parameters": { + "projectsId": { + "type": "string", + "description": "Part of `metricName`. The resource name of the metric to delete.", + "required": true, + "location": "path" + }, + "metricsId": { + "type": "string", + "description": "Part of `metricName`. See documentation of `projectsId`.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectsId", + "metricsId" + ], + "response": { + "$ref": "Empty" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.admin", + "https://www.googleapis.com/auth/logging.write" + ] + } + } + } + } + } + } +} diff --git a/components/engine/vendor/src/google.golang.org/api/logging/v1beta3/logging-gen.go b/components/engine/vendor/src/google.golang.org/api/logging/v1beta3/logging-gen.go new file mode 100644 index 0000000000..362338c180 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/api/logging/v1beta3/logging-gen.go @@ -0,0 +1,4787 @@ +// Package logging provides access to the Google Cloud Logging API. +// +// See https://cloud.google.com/logging/docs/ +// +// Usage example: +// +// import "google.golang.org/api/logging/v1beta3" +// ... +// loggingService, err := logging.New(oauthHttpClient) +package logging // import "google.golang.org/api/logging/v1beta3" + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + context "golang.org/x/net/context" + ctxhttp "golang.org/x/net/context/ctxhttp" + gensupport "google.golang.org/api/gensupport" + googleapi "google.golang.org/api/googleapi" + "io" + "net/http" + "net/url" + "strconv" + "strings" +) + +// Always reference these packages, just in case the auto-generated code +// below doesn't. +var _ = bytes.NewBuffer +var _ = strconv.Itoa +var _ = fmt.Sprintf +var _ = json.NewDecoder +var _ = io.Copy +var _ = url.Parse +var _ = gensupport.MarshalJSON +var _ = googleapi.Version +var _ = errors.New +var _ = strings.Replace +var _ = context.Canceled +var _ = ctxhttp.Do + +const apiId = "logging:v1beta3" +const apiName = "logging" +const apiVersion = "v1beta3" +const basePath = "https://logging.googleapis.com/" + +// OAuth2 scopes used by this API. +const ( + // View and manage your data across Google Cloud Platform services + CloudPlatformScope = "https://www.googleapis.com/auth/cloud-platform" + + // View your data across Google Cloud Platform services + CloudPlatformReadOnlyScope = "https://www.googleapis.com/auth/cloud-platform.read-only" + + // Administrate log data for your projects + LoggingAdminScope = "https://www.googleapis.com/auth/logging.admin" + + // View log data for your projects + LoggingReadScope = "https://www.googleapis.com/auth/logging.read" + + // Submit log data for your projects + LoggingWriteScope = "https://www.googleapis.com/auth/logging.write" +) + +func New(client *http.Client) (*Service, error) { + if client == nil { + return nil, errors.New("client is nil") + } + s := &Service{client: client, BasePath: basePath} + s.Projects = NewProjectsService(s) + return s, nil +} + +type Service struct { + client *http.Client + BasePath string // API endpoint base URL + UserAgent string // optional additional User-Agent fragment + + Projects *ProjectsService +} + +func (s *Service) userAgent() string { + if s.UserAgent == "" { + return googleapi.UserAgent + } + return googleapi.UserAgent + " " + s.UserAgent +} + +func NewProjectsService(s *Service) *ProjectsService { + rs := &ProjectsService{s: s} + rs.LogServices = NewProjectsLogServicesService(s) + rs.Logs = NewProjectsLogsService(s) + rs.Metrics = NewProjectsMetricsService(s) + rs.Sinks = NewProjectsSinksService(s) + return rs +} + +type ProjectsService struct { + s *Service + + LogServices *ProjectsLogServicesService + + Logs *ProjectsLogsService + + Metrics *ProjectsMetricsService + + Sinks *ProjectsSinksService +} + +func NewProjectsLogServicesService(s *Service) *ProjectsLogServicesService { + rs := &ProjectsLogServicesService{s: s} + rs.Indexes = NewProjectsLogServicesIndexesService(s) + rs.Sinks = NewProjectsLogServicesSinksService(s) + return rs +} + +type ProjectsLogServicesService struct { + s *Service + + Indexes *ProjectsLogServicesIndexesService + + Sinks *ProjectsLogServicesSinksService +} + +func NewProjectsLogServicesIndexesService(s *Service) *ProjectsLogServicesIndexesService { + rs := &ProjectsLogServicesIndexesService{s: s} + return rs +} + +type ProjectsLogServicesIndexesService struct { + s *Service +} + +func NewProjectsLogServicesSinksService(s *Service) *ProjectsLogServicesSinksService { + rs := &ProjectsLogServicesSinksService{s: s} + return rs +} + +type ProjectsLogServicesSinksService struct { + s *Service +} + +func NewProjectsLogsService(s *Service) *ProjectsLogsService { + rs := &ProjectsLogsService{s: s} + rs.Entries = NewProjectsLogsEntriesService(s) + rs.Sinks = NewProjectsLogsSinksService(s) + return rs +} + +type ProjectsLogsService struct { + s *Service + + Entries *ProjectsLogsEntriesService + + Sinks *ProjectsLogsSinksService +} + +func NewProjectsLogsEntriesService(s *Service) *ProjectsLogsEntriesService { + rs := &ProjectsLogsEntriesService{s: s} + return rs +} + +type ProjectsLogsEntriesService struct { + s *Service +} + +func NewProjectsLogsSinksService(s *Service) *ProjectsLogsSinksService { + rs := &ProjectsLogsSinksService{s: s} + return rs +} + +type ProjectsLogsSinksService struct { + s *Service +} + +func NewProjectsMetricsService(s *Service) *ProjectsMetricsService { + rs := &ProjectsMetricsService{s: s} + return rs +} + +type ProjectsMetricsService struct { + s *Service +} + +func NewProjectsSinksService(s *Service) *ProjectsSinksService { + rs := &ProjectsSinksService{s: s} + return rs +} + +type ProjectsSinksService struct { + s *Service +} + +// Empty: A generic empty message that you can re-use to avoid defining +// duplicated empty messages in your APIs. A typical example is to use +// it as the request or the response type of an API method. For +// instance: service Foo { rpc Bar(google.protobuf.Empty) returns +// (google.protobuf.Empty); } The JSON representation for `Empty` is +// empty JSON object `{}`. +type Empty struct { + // ServerResponse contains the HTTP response code and headers from the + // server. + googleapi.ServerResponse `json:"-"` +} + +// HttpRequest: A common proto for logging HTTP requests. +type HttpRequest struct { + // CacheHit: Whether or not an entity was served from cache (with or + // without validation). + CacheHit bool `json:"cacheHit,omitempty"` + + // Referer: Referer (a.k.a. referrer) URL of request, as defined in + // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html. + Referer string `json:"referer,omitempty"` + + // RemoteIp: IP address of the client who issues the HTTP request. Could + // be either IPv4 or IPv6. + RemoteIp string `json:"remoteIp,omitempty"` + + // RequestMethod: Request method, such as `GET`, `HEAD`, `PUT` or + // `POST`. + RequestMethod string `json:"requestMethod,omitempty"` + + // RequestSize: Size of the HTTP request message in bytes, including + // request headers and the request body. + RequestSize int64 `json:"requestSize,omitempty,string"` + + // RequestUrl: Contains the scheme (http|https), the host name, the path + // and the query portion of the URL that was requested. + RequestUrl string `json:"requestUrl,omitempty"` + + // ResponseSize: Size of the HTTP response message in bytes sent back to + // the client, including response headers and response body. + ResponseSize int64 `json:"responseSize,omitempty,string"` + + // Status: A response code indicates the status of response, e.g., 200. + Status int64 `json:"status,omitempty"` + + // UserAgent: User agent sent by the client, e.g., "Mozilla/4.0 + // (compatible; MSIE 6.0; Windows 98; Q312461; .NET CLR 1.0.3705)". + UserAgent string `json:"userAgent,omitempty"` + + // ValidatedWithOriginServer: Whether or not the response was validated + // with the origin server before being served from cache. This field is + // only meaningful if cache_hit is True. + ValidatedWithOriginServer bool `json:"validatedWithOriginServer,omitempty"` + + // ForceSendFields is a list of field names (e.g. "CacheHit") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *HttpRequest) MarshalJSON() ([]byte, error) { + type noMethod HttpRequest + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// ListLogMetricsResponse: Result returned from ListLogMetrics. +type ListLogMetricsResponse struct { + // Metrics: The list of metrics that was requested. + Metrics []*LogMetric `json:"metrics,omitempty"` + + // NextPageToken: If there are more results, then `nextPageToken` is + // returned in the response. To get the next batch of entries, use the + // value of `nextPageToken` as `pageToken` in the next call of + // `ListLogMetrics`. If `nextPageToken` is empty, then there are no more + // results. + NextPageToken string `json:"nextPageToken,omitempty"` + + // ServerResponse contains the HTTP response code and headers from the + // server. + googleapi.ServerResponse `json:"-"` + + // ForceSendFields is a list of field names (e.g. "Metrics") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *ListLogMetricsResponse) MarshalJSON() ([]byte, error) { + type noMethod ListLogMetricsResponse + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// ListLogServiceIndexesResponse: Result returned from +// ListLogServiceIndexesRequest. +type ListLogServiceIndexesResponse struct { + // NextPageToken: If there are more results, then `nextPageToken` is + // returned in the response. To get the next batch of indexes, use the + // value of `nextPageToken` as `pageToken` in the next call of + // `ListLogServiceIndexes`. If `nextPageToken` is empty, then there are + // no more results. + NextPageToken string `json:"nextPageToken,omitempty"` + + // ServiceIndexPrefixes: A list of log service index values. Each index + // value has the form "/value1/value2/...", where `value1` is a value + // in the primary index, `value2` is a value in the secondary index, and + // so forth. + ServiceIndexPrefixes []string `json:"serviceIndexPrefixes,omitempty"` + + // ServerResponse contains the HTTP response code and headers from the + // server. + googleapi.ServerResponse `json:"-"` + + // ForceSendFields is a list of field names (e.g. "NextPageToken") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *ListLogServiceIndexesResponse) MarshalJSON() ([]byte, error) { + type noMethod ListLogServiceIndexesResponse + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// ListLogServiceSinksResponse: Result returned from +// `ListLogServiceSinks`. +type ListLogServiceSinksResponse struct { + // Sinks: The requested log service sinks. If a returned `LogSink` + // object has an empty `destination` field, the client can retrieve the + // complete `LogSink` object by calling `logServices.sinks.get`. + Sinks []*LogSink `json:"sinks,omitempty"` + + // ServerResponse contains the HTTP response code and headers from the + // server. + googleapi.ServerResponse `json:"-"` + + // ForceSendFields is a list of field names (e.g. "Sinks") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *ListLogServiceSinksResponse) MarshalJSON() ([]byte, error) { + type noMethod ListLogServiceSinksResponse + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// ListLogServicesResponse: Result returned from +// `ListLogServicesRequest`. +type ListLogServicesResponse struct { + // LogServices: A list of log services. + LogServices []*LogService `json:"logServices,omitempty"` + + // NextPageToken: If there are more results, then `nextPageToken` is + // returned in the response. To get the next batch of services, use the + // value of `nextPageToken` as `pageToken` in the next call of + // `ListLogServices`. If `nextPageToken` is empty, then there are no + // more results. + NextPageToken string `json:"nextPageToken,omitempty"` + + // ServerResponse contains the HTTP response code and headers from the + // server. + googleapi.ServerResponse `json:"-"` + + // ForceSendFields is a list of field names (e.g. "LogServices") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *ListLogServicesResponse) MarshalJSON() ([]byte, error) { + type noMethod ListLogServicesResponse + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// ListLogSinksResponse: Result returned from `ListLogSinks`. +type ListLogSinksResponse struct { + // Sinks: The requested log sinks. If a returned `LogSink` object has an + // empty `destination` field, the client can retrieve the complete + // `LogSink` object by calling `log.sinks.get`. + Sinks []*LogSink `json:"sinks,omitempty"` + + // ServerResponse contains the HTTP response code and headers from the + // server. + googleapi.ServerResponse `json:"-"` + + // ForceSendFields is a list of field names (e.g. "Sinks") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *ListLogSinksResponse) MarshalJSON() ([]byte, error) { + type noMethod ListLogSinksResponse + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// ListLogsResponse: Result returned from ListLogs. +type ListLogsResponse struct { + // Logs: A list of log descriptions matching the criteria. + Logs []*Log `json:"logs,omitempty"` + + // NextPageToken: If there are more results, then `nextPageToken` is + // returned in the response. To get the next batch of logs, use the + // value of `nextPageToken` as `pageToken` in the next call of + // `ListLogs`. If `nextPageToken` is empty, then there are no more + // results. + NextPageToken string `json:"nextPageToken,omitempty"` + + // ServerResponse contains the HTTP response code and headers from the + // server. + googleapi.ServerResponse `json:"-"` + + // ForceSendFields is a list of field names (e.g. "Logs") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *ListLogsResponse) MarshalJSON() ([]byte, error) { + type noMethod ListLogsResponse + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// ListSinksResponse: Result returned from `ListSinks`. +type ListSinksResponse struct { + // Sinks: The requested sinks. If a returned `LogSink` object has an + // empty `destination` field, the client can retrieve the complete + // `LogSink` object by calling `projects.sinks.get`. + Sinks []*LogSink `json:"sinks,omitempty"` + + // ServerResponse contains the HTTP response code and headers from the + // server. + googleapi.ServerResponse `json:"-"` + + // ForceSendFields is a list of field names (e.g. "Sinks") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *ListSinksResponse) MarshalJSON() ([]byte, error) { + type noMethod ListSinksResponse + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// Log: _Output only._ Describes a log, which is a named stream of log +// entries. +type Log struct { + // DisplayName: _Optional._ The common name of the log. Example: + // "request_log". + DisplayName string `json:"displayName,omitempty"` + + // Name: The resource name of the log. Example: + // "/projects/my-gcp-project-id/logs/LOG_NAME", where `LOG_NAME` is + // the URL-encoded given name of the log. The log includes those log + // entries whose `LogEntry.log` field contains this given name. To avoid + // name collisions, it is a best practice to prefix the given log name + // with the service name, but this is not required. Examples of log + // given names: "appengine.googleapis.com/request_log", + // "apache-access". + Name string `json:"name,omitempty"` + + // PayloadType: _Optional_. A URI representing the expected payload type + // for log entries. + PayloadType string `json:"payloadType,omitempty"` + + // ForceSendFields is a list of field names (e.g. "DisplayName") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *Log) MarshalJSON() ([]byte, error) { + type noMethod Log + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// LogEntry: An individual entry in a log. +type LogEntry struct { + // HttpRequest: Information about the HTTP request associated with this + // log entry, if applicable. + HttpRequest *HttpRequest `json:"httpRequest,omitempty"` + + // InsertId: A unique ID for the log entry. If you provide this field, + // the logging service considers other log entries in the same log with + // the same ID as duplicates which can be removed. + InsertId string `json:"insertId,omitempty"` + + // Log: The log to which this entry belongs. When a log entry is + // ingested, the value of this field is set by the logging system. + Log string `json:"log,omitempty"` + + // Metadata: Information about the log entry. + Metadata *LogEntryMetadata `json:"metadata,omitempty"` + + // ProtoPayload: The log entry payload, represented as a protocol buffer + // that is expressed as a JSON object. You can only pass `protoPayload` + // values that belong to a set of approved types. + ProtoPayload LogEntryProtoPayload `json:"protoPayload,omitempty"` + + // StructPayload: The log entry payload, represented as a structure that + // is expressed as a JSON object. + StructPayload LogEntryStructPayload `json:"structPayload,omitempty"` + + // TextPayload: The log entry payload, represented as a Unicode string + // (UTF-8). + TextPayload string `json:"textPayload,omitempty"` + + // ForceSendFields is a list of field names (e.g. "HttpRequest") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *LogEntry) MarshalJSON() ([]byte, error) { + type noMethod LogEntry + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +type LogEntryProtoPayload interface{} + +type LogEntryStructPayload interface{} + +// LogEntryMetadata: Additional data that is associated with a log +// entry, set by the service creating the log entry. +type LogEntryMetadata struct { + // Labels: A set of (key, value) data that provides additional + // information about the log entry. If the log entry is from one of the + // Google Cloud Platform sources listed below, the indicated (key, + // value) information must be provided: Google App Engine, service_name + // `appengine.googleapis.com`: "appengine.googleapis.com/module_id", + // "appengine.googleapis.com/version_id", and one of: + // "appengine.googleapis.com/replica_index", + // "appengine.googleapis.com/clone_id", or else provide the following + // Compute Engine labels: Google Compute Engine, service_name + // `compute.googleapis.com`: "compute.googleapis.com/resource_type", + // "instance" "compute.googleapis.com/resource_id", + Labels map[string]string `json:"labels,omitempty"` + + // ProjectId: The project ID of the Google Cloud Platform service that + // created the log entry. + ProjectId string `json:"projectId,omitempty"` + + // Region: The region name of the Google Cloud Platform service that + // created the log entry. For example, "us-central1". + Region string `json:"region,omitempty"` + + // ServiceName: The API name of the Google Cloud Platform service that + // created the log entry. For example, "compute.googleapis.com". + ServiceName string `json:"serviceName,omitempty"` + + // Severity: The severity of the log entry. + // + // Possible values: + // "DEFAULT" + // "DEBUG" + // "INFO" + // "NOTICE" + // "WARNING" + // "ERROR" + // "CRITICAL" + // "ALERT" + // "EMERGENCY" + Severity string `json:"severity,omitempty"` + + // Timestamp: The time the event described by the log entry occurred. + // Timestamps must be later than January 1, 1970. + Timestamp string `json:"timestamp,omitempty"` + + // UserId: The fully-qualified email address of the authenticated user + // that performed or requested the action represented by the log entry. + // If the log entry does not apply to an action taken by an + // authenticated user, then the field should be empty. + UserId string `json:"userId,omitempty"` + + // Zone: The zone of the Google Cloud Platform service that created the + // log entry. For example, "us-central1-a". + Zone string `json:"zone,omitempty"` + + // ForceSendFields is a list of field names (e.g. "Labels") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *LogEntryMetadata) MarshalJSON() ([]byte, error) { + type noMethod LogEntryMetadata + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// LogError: Describes a problem with a logging resource or operation. +type LogError struct { + // Resource: A resource name associated with this error. For example, + // the name of a Cloud Storage bucket that has insufficient permissions + // to be a destination for log entries. + Resource string `json:"resource,omitempty"` + + // Status: The error description, including a classification code, an + // error message, and other details. + Status *Status `json:"status,omitempty"` + + // TimeNanos: The time the error was observed, in nanoseconds since the + // Unix epoch. + TimeNanos int64 `json:"timeNanos,omitempty,string"` + + // ForceSendFields is a list of field names (e.g. "Resource") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *LogError) MarshalJSON() ([]byte, error) { + type noMethod LogError + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// LogLine: Application log line emitted while processing a request. +type LogLine struct { + // LogMessage: App provided log message. + LogMessage string `json:"logMessage,omitempty"` + + // Severity: Severity of log. + // + // Possible values: + // "DEFAULT" + // "DEBUG" + // "INFO" + // "NOTICE" + // "WARNING" + // "ERROR" + // "CRITICAL" + // "ALERT" + // "EMERGENCY" + Severity string `json:"severity,omitempty"` + + // SourceLocation: Line of code that generated this log message. + SourceLocation *SourceLocation `json:"sourceLocation,omitempty"` + + // Time: Time when log entry was made. May be inaccurate. + Time string `json:"time,omitempty"` + + // ForceSendFields is a list of field names (e.g. "LogMessage") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *LogLine) MarshalJSON() ([]byte, error) { + type noMethod LogLine + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// LogMetric: Describes a logs-based metric. The value of the metric is +// the number of log entries in your project that match a logs filter. +type LogMetric struct { + // Description: A description of this metric. + Description string `json:"description,omitempty"` + + // Filter: An [advanced logs + // filter](/logging/docs/view/advanced_filters). Example: "log:syslog + // AND metadata.severity>=ERROR". + Filter string `json:"filter,omitempty"` + + // Name: The client-assigned name for this metric, such as + // "severe_errors". Metric names are limited to 1000 characters and + // can include only the following characters: `A-Z`, `a-z`, `0-9`, and + // the special characters `_-.,+!*',()%/\`. The slash character (`/`) + // denotes a hierarchy of name pieces, and it cannot be the first + // character of the name. + Name string `json:"name,omitempty"` + + // ServerResponse contains the HTTP response code and headers from the + // server. + googleapi.ServerResponse `json:"-"` + + // ForceSendFields is a list of field names (e.g. "Description") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *LogMetric) MarshalJSON() ([]byte, error) { + type noMethod LogMetric + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// LogService: _Output only._ Describes a service that writes log +// entries. +type LogService struct { + // IndexKeys: A list of the names of the keys used to index and label + // individual log entries from this service. The first two keys are used + // as the primary and secondary index, respectively. Additional keys may + // be used to label the entries. For example, App Engine indexes its + // entries by module and by version, so its `indexKeys` field is the + // following: [ "appengine.googleapis.com/module_id", + // "appengine.googleapis.com/version_id" ] + IndexKeys []string `json:"indexKeys,omitempty"` + + // Name: The service's name. Example: "appengine.googleapis.com". Log + // names beginning with this string are reserved for this service. This + // value can appear in the `LogEntry.metadata.serviceName` field of log + // entries associated with this log service. + Name string `json:"name,omitempty"` + + // ForceSendFields is a list of field names (e.g. "IndexKeys") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *LogService) MarshalJSON() ([]byte, error) { + type noMethod LogService + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// LogSink: Describes where log entries are written outside of Cloud +// Logging. +type LogSink struct { + // Destination: The resource name of the destination. Cloud Logging + // writes designated log entries to this destination. For example, + // "storage.googleapis.com/my-output-bucket". + Destination string `json:"destination,omitempty"` + + // Errors: _Output only._ If any errors occur when invoking a sink + // method, then this field contains descriptions of the errors. + Errors []*LogError `json:"errors,omitempty"` + + // Filter: An advanced logs filter. If present, only log entries + // matching the filter are written. Only project sinks use this field; + // log sinks and log service sinks must not include a filter. + Filter string `json:"filter,omitempty"` + + // Name: The client-assigned name of this sink. For example, + // "my-syslog-sink". The name must be unique among the sinks of a + // similar kind in the project. + Name string `json:"name,omitempty"` + + // ServerResponse contains the HTTP response code and headers from the + // server. + googleapi.ServerResponse `json:"-"` + + // ForceSendFields is a list of field names (e.g. "Destination") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *LogSink) MarshalJSON() ([]byte, error) { + type noMethod LogSink + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// RequestLog: Complete log information about a single request to an +// application. +type RequestLog struct { + // AppEngineRelease: App Engine release version string. + AppEngineRelease string `json:"appEngineRelease,omitempty"` + + // AppId: Identifies the application that handled this request. + AppId string `json:"appId,omitempty"` + + // Cost: An indication of the relative cost of serving this request. + Cost float64 `json:"cost,omitempty"` + + // EndTime: Time at which request was known to end processing. + EndTime string `json:"endTime,omitempty"` + + // Finished: If true, represents a finished request. Otherwise, the + // request is active. + Finished bool `json:"finished,omitempty"` + + // Host: The Internet host and port number of the resource being + // requested. + Host string `json:"host,omitempty"` + + // HttpVersion: HTTP version of request. + HttpVersion string `json:"httpVersion,omitempty"` + + // InstanceId: An opaque identifier for the instance that handled the + // request. + InstanceId string `json:"instanceId,omitempty"` + + // InstanceIndex: If the instance that processed this request was + // individually addressable (i.e. belongs to a manually scaled module), + // this is the index of the instance. + InstanceIndex int64 `json:"instanceIndex,omitempty"` + + // Ip: Origin IP address. + Ip string `json:"ip,omitempty"` + + // Latency: Latency of the request. + Latency string `json:"latency,omitempty"` + + // Line: List of log lines emitted by the application while serving this + // request, if requested. + Line []*LogLine `json:"line,omitempty"` + + // MegaCycles: Number of CPU megacycles used to process request. + MegaCycles int64 `json:"megaCycles,omitempty,string"` + + // Method: Request method, such as `GET`, `HEAD`, `PUT`, `POST`, or + // `DELETE`. + Method string `json:"method,omitempty"` + + // ModuleId: Identifies the module of the application that handled this + // request. + ModuleId string `json:"moduleId,omitempty"` + + // Nickname: A string that identifies a logged-in user who made this + // request, or empty if the user is not logged in. Most likely, this is + // the part of the user's email before the '@' sign. The field value is + // the same for different requests from the same user, but different + // users may have a similar name. This information is also available to + // the application via Users API. This field will be populated starting + // with App Engine 1.9.21. + Nickname string `json:"nickname,omitempty"` + + // PendingTime: Time this request spent in the pending request queue, if + // it was pending at all. + PendingTime string `json:"pendingTime,omitempty"` + + // Referrer: Referrer URL of request. + Referrer string `json:"referrer,omitempty"` + + // RequestId: Globally unique identifier for a request, based on request + // start time. Request IDs for requests which started later will compare + // greater as strings than those for requests which started earlier. + RequestId string `json:"requestId,omitempty"` + + // Resource: Contains the path and query portion of the URL that was + // requested. For example, if the URL was + // "http://example.com/app?name=val", the resource would be + // "/app?name=val". Any trailing fragment (separated by a '#' character) + // will not be included. + Resource string `json:"resource,omitempty"` + + // ResponseSize: Size in bytes sent back to client by request. + ResponseSize int64 `json:"responseSize,omitempty,string"` + + // SourceReference: Source code for the application that handled this + // request. There can be more than one source reference per deployed + // application if source code is distributed among multiple + // repositories. + SourceReference []*SourceReference `json:"sourceReference,omitempty"` + + // StartTime: Time at which request was known to have begun processing. + StartTime string `json:"startTime,omitempty"` + + // Status: Response status of request. + Status int64 `json:"status,omitempty"` + + // TaskName: Task name of the request (for an offline request). + TaskName string `json:"taskName,omitempty"` + + // TaskQueueName: Queue name of the request (for an offline request). + TaskQueueName string `json:"taskQueueName,omitempty"` + + // TraceId: Cloud Trace identifier of the trace for this request. + TraceId string `json:"traceId,omitempty"` + + // UrlMapEntry: File or class within URL mapping used for request. + // Useful for tracking down the source code which was responsible for + // managing request. Especially for multiply mapped handlers. + UrlMapEntry string `json:"urlMapEntry,omitempty"` + + // UserAgent: User agent used for making request. + UserAgent string `json:"userAgent,omitempty"` + + // VersionId: Version of the application that handled this request. + VersionId string `json:"versionId,omitempty"` + + // WasLoadingRequest: Was this request a loading request for this + // instance? + WasLoadingRequest bool `json:"wasLoadingRequest,omitempty"` + + // ForceSendFields is a list of field names (e.g. "AppEngineRelease") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *RequestLog) MarshalJSON() ([]byte, error) { + type noMethod RequestLog + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// SourceLocation: Specifies a location in a source file. +type SourceLocation struct { + // File: Source file name. May or may not be a fully qualified name, + // depending on the runtime environment. + File string `json:"file,omitempty"` + + // FunctionName: Human-readable name of the function or method being + // invoked, with optional context such as the class or package name, for + // use in contexts such as the logs viewer where file:line number is + // less meaningful. This may vary by language, for example: in Java: + // qual.if.ied.Class.method in Go: dir/package.func in Python: function + // ... + FunctionName string `json:"functionName,omitempty"` + + // Line: Line within the source file. + Line int64 `json:"line,omitempty,string"` + + // ForceSendFields is a list of field names (e.g. "File") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *SourceLocation) MarshalJSON() ([]byte, error) { + type noMethod SourceLocation + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// SourceReference: A reference to a particular snapshot of the source +// tree used to build and deploy an application. +type SourceReference struct { + // Repository: Optional. A URI string identifying the repository. + // Example: "https://github.com/GoogleCloudPlatform/kubernetes.git" + Repository string `json:"repository,omitempty"` + + // RevisionId: The canonical (and persistent) identifier of the deployed + // revision. Example (git): "0035781c50ec7aa23385dc841529ce8a4b70db1b" + RevisionId string `json:"revisionId,omitempty"` + + // ForceSendFields is a list of field names (e.g. "Repository") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *SourceReference) MarshalJSON() ([]byte, error) { + type noMethod SourceReference + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// Status: The `Status` type defines a logical error model that is +// suitable for different programming environments, including REST APIs +// and RPC APIs. It is used by [gRPC](https://github.com/grpc). The +// error model is designed to be: - Simple to use and understand for +// most users - Flexible enough to meet unexpected needs # Overview The +// `Status` message contains three pieces of data: error code, error +// message, and error details. The error code should be an enum value of +// google.rpc.Code, but it may accept additional error codes if needed. +// The error message should be a developer-facing English message that +// helps developers *understand* and *resolve* the error. If a localized +// user-facing error message is needed, put the localized message in the +// error details or localize it in the client. The optional error +// details may contain arbitrary information about the error. There is a +// predefined set of error detail types in the package `google.rpc` +// which can be used for common error conditions. # Language mapping The +// `Status` message is the logical representation of the error model, +// but it is not necessarily the actual wire format. When the `Status` +// message is exposed in different client libraries and different wire +// protocols, it can be mapped differently. For example, it will likely +// be mapped to some exceptions in Java, but more likely mapped to some +// error codes in C. # Other uses The error model and the `Status` +// message can be used in a variety of environments, either with or +// without APIs, to provide a consistent developer experience across +// different environments. Example uses of this error model include: - +// Partial errors. If a service needs to return partial errors to the +// client, it may embed the `Status` in the normal response to indicate +// the partial errors. - Workflow errors. A typical workflow has +// multiple steps. Each step may have a `Status` message for error +// reporting purpose. - Batch operations. If a client uses batch request +// and batch response, the `Status` message should be used directly +// inside batch response, one for each error sub-response. - +// Asynchronous operations. If an API call embeds asynchronous operation +// results in its response, the status of those operations should be +// represented directly using the `Status` message. - Logging. If some +// API errors are stored in logs, the message `Status` could be used +// directly after any stripping needed for security/privacy reasons. +type Status struct { + // Code: The status code, which should be an enum value of + // google.rpc.Code. + Code int64 `json:"code,omitempty"` + + // Details: A list of messages that carry the error details. There will + // be a common set of message types for APIs to use. + Details []StatusDetails `json:"details,omitempty"` + + // Message: A developer-facing error message, which should be in + // English. Any user-facing error message should be localized and sent + // in the google.rpc.Status.details field, or localized by the client. + Message string `json:"message,omitempty"` + + // ForceSendFields is a list of field names (e.g. "Code") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *Status) MarshalJSON() ([]byte, error) { + type noMethod Status + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +type StatusDetails interface{} + +// WriteLogEntriesRequest: The parameters to WriteLogEntries. +type WriteLogEntriesRequest struct { + // CommonLabels: Metadata labels that apply to all log entries in this + // request, so that you don't have to repeat them in each log entry's + // `metadata.labels` field. If any of the log entries contains a (key, + // value) with the same key that is in `commonLabels`, then the entry's + // (key, value) overrides the one in `commonLabels`. + CommonLabels map[string]string `json:"commonLabels,omitempty"` + + // Entries: Log entries to insert. + Entries []*LogEntry `json:"entries,omitempty"` + + // ForceSendFields is a list of field names (e.g. "CommonLabels") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` +} + +func (s *WriteLogEntriesRequest) MarshalJSON() ([]byte, error) { + type noMethod WriteLogEntriesRequest + raw := noMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields) +} + +// WriteLogEntriesResponse: Result returned from WriteLogEntries. empty +type WriteLogEntriesResponse struct { + // ServerResponse contains the HTTP response code and headers from the + // server. + googleapi.ServerResponse `json:"-"` +} + +// method id "logging.projects.logServices.list": + +type ProjectsLogServicesListCall struct { + s *Service + projectsId string + urlParams_ gensupport.URLParams + ifNoneMatch_ string + ctx_ context.Context +} + +// List: Lists the log services that have log entries in this project. +func (r *ProjectsLogServicesService) List(projectsId string) *ProjectsLogServicesListCall { + c := &ProjectsLogServicesListCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + return c +} + +// PageSize sets the optional parameter "pageSize": The maximum number +// of `LogService` objects to return in one operation. +func (c *ProjectsLogServicesListCall) PageSize(pageSize int64) *ProjectsLogServicesListCall { + c.urlParams_.Set("pageSize", fmt.Sprint(pageSize)) + return c +} + +// PageToken sets the optional parameter "pageToken": An opaque token, +// returned as `nextPageToken` by a prior `ListLogServices` operation. +// If `pageToken` is supplied, then the other fields of this request are +// ignored, and instead the previous `ListLogServices` operation is +// continued. +func (c *ProjectsLogServicesListCall) PageToken(pageToken string) *ProjectsLogServicesListCall { + c.urlParams_.Set("pageToken", pageToken) + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogServicesListCall) QuotaUser(quotaUser string) *ProjectsLogServicesListCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogServicesListCall) Fields(s ...googleapi.Field) *ProjectsLogServicesListCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// IfNoneMatch sets the optional parameter which makes the operation +// fail if the object's ETag matches the given value. This is useful for +// getting updates only after the object has changed since the last +// request. Use googleapi.IsNotModified to check whether the response +// error from Do is the result of In-None-Match. +func (c *ProjectsLogServicesListCall) IfNoneMatch(entityTag string) *ProjectsLogServicesListCall { + c.ifNoneMatch_ = entityTag + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogServicesListCall) Context(ctx context.Context) *ProjectsLogServicesListCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogServicesListCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logServices") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("GET", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ifNoneMatch_ != "" { + req.Header.Set("If-None-Match", c.ifNoneMatch_) + } + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logServices.list" call. +// Exactly one of *ListLogServicesResponse or error will be non-nil. Any +// non-2xx status code is an error. Response headers are in either +// *ListLogServicesResponse.ServerResponse.Header or (if a response was +// returned at all) in error.(*googleapi.Error).Header. Use +// googleapi.IsNotModified to check whether the returned error was +// because http.StatusNotModified was returned. +func (c *ProjectsLogServicesListCall) Do() (*ListLogServicesResponse, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &ListLogServicesResponse{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Lists the log services that have log entries in this project.", + // "httpMethod": "GET", + // "id": "logging.projects.logServices.list", + // "parameterOrder": [ + // "projectsId" + // ], + // "parameters": { + // "pageSize": { + // "description": "The maximum number of `LogService` objects to return in one operation.", + // "format": "int32", + // "location": "query", + // "type": "integer" + // }, + // "pageToken": { + // "description": "An opaque token, returned as `nextPageToken` by a prior `ListLogServices` operation. If `pageToken` is supplied, then the other fields of this request are ignored, and instead the previous `ListLogServices` operation is continued.", + // "location": "query", + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `projectName`. The resource name of the project whose services are to be listed.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logServices", + // "response": { + // "$ref": "ListLogServicesResponse" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/cloud-platform.read-only", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.read" + // ] + // } + +} + +// method id "logging.projects.logServices.indexes.list": + +type ProjectsLogServicesIndexesListCall struct { + s *Service + projectsId string + logServicesId string + urlParams_ gensupport.URLParams + ifNoneMatch_ string + ctx_ context.Context +} + +// List: Lists the current index values for a log service. +func (r *ProjectsLogServicesIndexesService) List(projectsId string, logServicesId string) *ProjectsLogServicesIndexesListCall { + c := &ProjectsLogServicesIndexesListCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logServicesId = logServicesId + return c +} + +// Depth sets the optional parameter "depth": A non-negative integer +// that limits the number of levels of the index hierarchy that are +// returned. If `depth` is 1 (default), only the first index key value +// is returned. If `depth` is 2, both primary and secondary key values +// are returned. If `depth` is 0, the depth is the number of +// slash-separators in the `indexPrefix` field, not counting a slash +// appearing as the last character of the prefix. If the `indexPrefix` +// field is empty, the default depth is 1. It is an error for `depth` to +// be any positive value less than the number of components in +// `indexPrefix`. +func (c *ProjectsLogServicesIndexesListCall) Depth(depth int64) *ProjectsLogServicesIndexesListCall { + c.urlParams_.Set("depth", fmt.Sprint(depth)) + return c +} + +// IndexPrefix sets the optional parameter "indexPrefix": Restricts the +// index values returned to be those with a specified prefix for each +// index key. This field has the form "/prefix1/prefix2/...", in order +// corresponding to the `LogService indexKeys`. Non-empty prefixes must +// begin with `/`. For example, App Engine's two keys are the module ID +// and the version ID. Following is the effect of using various values +// for `indexPrefix`: + "/Mod/" retrieves `/Mod/10` and `/Mod/11` but +// not `/ModA/10`. + "/Mod` retrieves `/Mod/10`, `/Mod/11` and +// `/ModA/10` but not `/XXX/33`. + "/Mod/1" retrieves `/Mod/10` and +// `/Mod/11` but not `/ModA/10`. + "/Mod/10/" retrieves `/Mod/10` +// only. + An empty prefix or "/" retrieves all values. +func (c *ProjectsLogServicesIndexesListCall) IndexPrefix(indexPrefix string) *ProjectsLogServicesIndexesListCall { + c.urlParams_.Set("indexPrefix", indexPrefix) + return c +} + +// PageSize sets the optional parameter "pageSize": The maximum number +// of log service index resources to return in one operation. +func (c *ProjectsLogServicesIndexesListCall) PageSize(pageSize int64) *ProjectsLogServicesIndexesListCall { + c.urlParams_.Set("pageSize", fmt.Sprint(pageSize)) + return c +} + +// PageToken sets the optional parameter "pageToken": An opaque token, +// returned as `nextPageToken` by a prior `ListLogServiceIndexes` +// operation. If `pageToken` is supplied, then the other fields of this +// request are ignored, and instead the previous `ListLogServiceIndexes` +// operation is continued. +func (c *ProjectsLogServicesIndexesListCall) PageToken(pageToken string) *ProjectsLogServicesIndexesListCall { + c.urlParams_.Set("pageToken", pageToken) + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogServicesIndexesListCall) QuotaUser(quotaUser string) *ProjectsLogServicesIndexesListCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogServicesIndexesListCall) Fields(s ...googleapi.Field) *ProjectsLogServicesIndexesListCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// IfNoneMatch sets the optional parameter which makes the operation +// fail if the object's ETag matches the given value. This is useful for +// getting updates only after the object has changed since the last +// request. Use googleapi.IsNotModified to check whether the response +// error from Do is the result of In-None-Match. +func (c *ProjectsLogServicesIndexesListCall) IfNoneMatch(entityTag string) *ProjectsLogServicesIndexesListCall { + c.ifNoneMatch_ = entityTag + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogServicesIndexesListCall) Context(ctx context.Context) *ProjectsLogServicesIndexesListCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogServicesIndexesListCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logServices/{logServicesId}/indexes") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("GET", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logServicesId": c.logServicesId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ifNoneMatch_ != "" { + req.Header.Set("If-None-Match", c.ifNoneMatch_) + } + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logServices.indexes.list" call. +// Exactly one of *ListLogServiceIndexesResponse or error will be +// non-nil. Any non-2xx status code is an error. Response headers are in +// either *ListLogServiceIndexesResponse.ServerResponse.Header or (if a +// response was returned at all) in error.(*googleapi.Error).Header. Use +// googleapi.IsNotModified to check whether the returned error was +// because http.StatusNotModified was returned. +func (c *ProjectsLogServicesIndexesListCall) Do() (*ListLogServiceIndexesResponse, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &ListLogServiceIndexesResponse{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Lists the current index values for a log service.", + // "httpMethod": "GET", + // "id": "logging.projects.logServices.indexes.list", + // "parameterOrder": [ + // "projectsId", + // "logServicesId" + // ], + // "parameters": { + // "depth": { + // "description": "A non-negative integer that limits the number of levels of the index hierarchy that are returned. If `depth` is 1 (default), only the first index key value is returned. If `depth` is 2, both primary and secondary key values are returned. If `depth` is 0, the depth is the number of slash-separators in the `indexPrefix` field, not counting a slash appearing as the last character of the prefix. If the `indexPrefix` field is empty, the default depth is 1. It is an error for `depth` to be any positive value less than the number of components in `indexPrefix`.", + // "format": "int32", + // "location": "query", + // "type": "integer" + // }, + // "indexPrefix": { + // "description": "Restricts the index values returned to be those with a specified prefix for each index key. This field has the form `\"/prefix1/prefix2/...\"`, in order corresponding to the `LogService indexKeys`. Non-empty prefixes must begin with `/`. For example, App Engine's two keys are the module ID and the version ID. Following is the effect of using various values for `indexPrefix`: + `\"/Mod/\"` retrieves `/Mod/10` and `/Mod/11` but not `/ModA/10`. + `\"/Mod` retrieves `/Mod/10`, `/Mod/11` and `/ModA/10` but not `/XXX/33`. + `\"/Mod/1\"` retrieves `/Mod/10` and `/Mod/11` but not `/ModA/10`. + `\"/Mod/10/\"` retrieves `/Mod/10` only. + An empty prefix or `\"/\"` retrieves all values.", + // "location": "query", + // "type": "string" + // }, + // "logServicesId": { + // "description": "Part of `serviceName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "pageSize": { + // "description": "The maximum number of log service index resources to return in one operation.", + // "format": "int32", + // "location": "query", + // "type": "integer" + // }, + // "pageToken": { + // "description": "An opaque token, returned as `nextPageToken` by a prior `ListLogServiceIndexes` operation. If `pageToken` is supplied, then the other fields of this request are ignored, and instead the previous `ListLogServiceIndexes` operation is continued.", + // "location": "query", + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `serviceName`. The resource name of a log service whose service indexes are requested. Example: `\"projects/my-project-id/logServices/appengine.googleapis.com\"`.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logServices/{logServicesId}/indexes", + // "response": { + // "$ref": "ListLogServiceIndexesResponse" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/cloud-platform.read-only", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.read" + // ] + // } + +} + +// method id "logging.projects.logServices.sinks.create": + +type ProjectsLogServicesSinksCreateCall struct { + s *Service + projectsId string + logServicesId string + logsink *LogSink + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Create: Creates a log service sink. All log entries from a specified +// log service are written to the destination. +func (r *ProjectsLogServicesSinksService) Create(projectsId string, logServicesId string, logsink *LogSink) *ProjectsLogServicesSinksCreateCall { + c := &ProjectsLogServicesSinksCreateCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logServicesId = logServicesId + c.logsink = logsink + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogServicesSinksCreateCall) QuotaUser(quotaUser string) *ProjectsLogServicesSinksCreateCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogServicesSinksCreateCall) Fields(s ...googleapi.Field) *ProjectsLogServicesSinksCreateCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogServicesSinksCreateCall) Context(ctx context.Context) *ProjectsLogServicesSinksCreateCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogServicesSinksCreateCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + body, err := googleapi.WithoutDataWrapper.JSONReader(c.logsink) + if err != nil { + return nil, err + } + ctype := "application/json" + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("POST", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logServicesId": c.logServicesId, + }) + req.Header.Set("Content-Type", ctype) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logServices.sinks.create" call. +// Exactly one of *LogSink or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *LogSink.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsLogServicesSinksCreateCall) Do() (*LogSink, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &LogSink{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Creates a log service sink. All log entries from a specified log service are written to the destination.", + // "httpMethod": "POST", + // "id": "logging.projects.logServices.sinks.create", + // "parameterOrder": [ + // "projectsId", + // "logServicesId" + // ], + // "parameters": { + // "logServicesId": { + // "description": "Part of `serviceName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `serviceName`. The resource name of the log service to which the sink is bound.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks", + // "request": { + // "$ref": "LogSink" + // }, + // "response": { + // "$ref": "LogSink" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin" + // ] + // } + +} + +// method id "logging.projects.logServices.sinks.delete": + +type ProjectsLogServicesSinksDeleteCall struct { + s *Service + projectsId string + logServicesId string + sinksId string + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Delete: Deletes a log service sink. After deletion, no new log +// entries are written to the destination. +func (r *ProjectsLogServicesSinksService) Delete(projectsId string, logServicesId string, sinksId string) *ProjectsLogServicesSinksDeleteCall { + c := &ProjectsLogServicesSinksDeleteCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logServicesId = logServicesId + c.sinksId = sinksId + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogServicesSinksDeleteCall) QuotaUser(quotaUser string) *ProjectsLogServicesSinksDeleteCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogServicesSinksDeleteCall) Fields(s ...googleapi.Field) *ProjectsLogServicesSinksDeleteCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogServicesSinksDeleteCall) Context(ctx context.Context) *ProjectsLogServicesSinksDeleteCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogServicesSinksDeleteCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks/{sinksId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("DELETE", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logServicesId": c.logServicesId, + "sinksId": c.sinksId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logServices.sinks.delete" call. +// Exactly one of *Empty or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *Empty.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsLogServicesSinksDeleteCall) Do() (*Empty, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &Empty{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Deletes a log service sink. After deletion, no new log entries are written to the destination.", + // "httpMethod": "DELETE", + // "id": "logging.projects.logServices.sinks.delete", + // "parameterOrder": [ + // "projectsId", + // "logServicesId", + // "sinksId" + // ], + // "parameters": { + // "logServicesId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `sinkName`. The resource name of the log service sink to delete.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "sinksId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks/{sinksId}", + // "response": { + // "$ref": "Empty" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin" + // ] + // } + +} + +// method id "logging.projects.logServices.sinks.get": + +type ProjectsLogServicesSinksGetCall struct { + s *Service + projectsId string + logServicesId string + sinksId string + urlParams_ gensupport.URLParams + ifNoneMatch_ string + ctx_ context.Context +} + +// Get: Gets a log service sink. +func (r *ProjectsLogServicesSinksService) Get(projectsId string, logServicesId string, sinksId string) *ProjectsLogServicesSinksGetCall { + c := &ProjectsLogServicesSinksGetCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logServicesId = logServicesId + c.sinksId = sinksId + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogServicesSinksGetCall) QuotaUser(quotaUser string) *ProjectsLogServicesSinksGetCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogServicesSinksGetCall) Fields(s ...googleapi.Field) *ProjectsLogServicesSinksGetCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// IfNoneMatch sets the optional parameter which makes the operation +// fail if the object's ETag matches the given value. This is useful for +// getting updates only after the object has changed since the last +// request. Use googleapi.IsNotModified to check whether the response +// error from Do is the result of In-None-Match. +func (c *ProjectsLogServicesSinksGetCall) IfNoneMatch(entityTag string) *ProjectsLogServicesSinksGetCall { + c.ifNoneMatch_ = entityTag + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogServicesSinksGetCall) Context(ctx context.Context) *ProjectsLogServicesSinksGetCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogServicesSinksGetCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks/{sinksId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("GET", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logServicesId": c.logServicesId, + "sinksId": c.sinksId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ifNoneMatch_ != "" { + req.Header.Set("If-None-Match", c.ifNoneMatch_) + } + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logServices.sinks.get" call. +// Exactly one of *LogSink or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *LogSink.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsLogServicesSinksGetCall) Do() (*LogSink, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &LogSink{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Gets a log service sink.", + // "httpMethod": "GET", + // "id": "logging.projects.logServices.sinks.get", + // "parameterOrder": [ + // "projectsId", + // "logServicesId", + // "sinksId" + // ], + // "parameters": { + // "logServicesId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `sinkName`. The resource name of the log service sink to return.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "sinksId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks/{sinksId}", + // "response": { + // "$ref": "LogSink" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/cloud-platform.read-only", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.read" + // ] + // } + +} + +// method id "logging.projects.logServices.sinks.list": + +type ProjectsLogServicesSinksListCall struct { + s *Service + projectsId string + logServicesId string + urlParams_ gensupport.URLParams + ifNoneMatch_ string + ctx_ context.Context +} + +// List: Lists log service sinks associated with a log service. +func (r *ProjectsLogServicesSinksService) List(projectsId string, logServicesId string) *ProjectsLogServicesSinksListCall { + c := &ProjectsLogServicesSinksListCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logServicesId = logServicesId + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogServicesSinksListCall) QuotaUser(quotaUser string) *ProjectsLogServicesSinksListCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogServicesSinksListCall) Fields(s ...googleapi.Field) *ProjectsLogServicesSinksListCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// IfNoneMatch sets the optional parameter which makes the operation +// fail if the object's ETag matches the given value. This is useful for +// getting updates only after the object has changed since the last +// request. Use googleapi.IsNotModified to check whether the response +// error from Do is the result of In-None-Match. +func (c *ProjectsLogServicesSinksListCall) IfNoneMatch(entityTag string) *ProjectsLogServicesSinksListCall { + c.ifNoneMatch_ = entityTag + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogServicesSinksListCall) Context(ctx context.Context) *ProjectsLogServicesSinksListCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogServicesSinksListCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("GET", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logServicesId": c.logServicesId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ifNoneMatch_ != "" { + req.Header.Set("If-None-Match", c.ifNoneMatch_) + } + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logServices.sinks.list" call. +// Exactly one of *ListLogServiceSinksResponse or error will be non-nil. +// Any non-2xx status code is an error. Response headers are in either +// *ListLogServiceSinksResponse.ServerResponse.Header or (if a response +// was returned at all) in error.(*googleapi.Error).Header. Use +// googleapi.IsNotModified to check whether the returned error was +// because http.StatusNotModified was returned. +func (c *ProjectsLogServicesSinksListCall) Do() (*ListLogServiceSinksResponse, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &ListLogServiceSinksResponse{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Lists log service sinks associated with a log service.", + // "httpMethod": "GET", + // "id": "logging.projects.logServices.sinks.list", + // "parameterOrder": [ + // "projectsId", + // "logServicesId" + // ], + // "parameters": { + // "logServicesId": { + // "description": "Part of `serviceName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `serviceName`. The log service whose sinks are wanted.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks", + // "response": { + // "$ref": "ListLogServiceSinksResponse" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/cloud-platform.read-only", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.read" + // ] + // } + +} + +// method id "logging.projects.logServices.sinks.update": + +type ProjectsLogServicesSinksUpdateCall struct { + s *Service + projectsId string + logServicesId string + sinksId string + logsink *LogSink + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Update: Updates a log service sink. If the sink does not exist, it is +// created. +func (r *ProjectsLogServicesSinksService) Update(projectsId string, logServicesId string, sinksId string, logsink *LogSink) *ProjectsLogServicesSinksUpdateCall { + c := &ProjectsLogServicesSinksUpdateCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logServicesId = logServicesId + c.sinksId = sinksId + c.logsink = logsink + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogServicesSinksUpdateCall) QuotaUser(quotaUser string) *ProjectsLogServicesSinksUpdateCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogServicesSinksUpdateCall) Fields(s ...googleapi.Field) *ProjectsLogServicesSinksUpdateCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogServicesSinksUpdateCall) Context(ctx context.Context) *ProjectsLogServicesSinksUpdateCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogServicesSinksUpdateCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + body, err := googleapi.WithoutDataWrapper.JSONReader(c.logsink) + if err != nil { + return nil, err + } + ctype := "application/json" + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks/{sinksId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("PUT", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logServicesId": c.logServicesId, + "sinksId": c.sinksId, + }) + req.Header.Set("Content-Type", ctype) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logServices.sinks.update" call. +// Exactly one of *LogSink or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *LogSink.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsLogServicesSinksUpdateCall) Do() (*LogSink, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &LogSink{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Updates a log service sink. If the sink does not exist, it is created.", + // "httpMethod": "PUT", + // "id": "logging.projects.logServices.sinks.update", + // "parameterOrder": [ + // "projectsId", + // "logServicesId", + // "sinksId" + // ], + // "parameters": { + // "logServicesId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `sinkName`. The resource name of the log service sink to update.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "sinksId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logServices/{logServicesId}/sinks/{sinksId}", + // "request": { + // "$ref": "LogSink" + // }, + // "response": { + // "$ref": "LogSink" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin" + // ] + // } + +} + +// method id "logging.projects.logs.delete": + +type ProjectsLogsDeleteCall struct { + s *Service + projectsId string + logsId string + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Delete: Deletes a log and all its log entries. The log will reappear +// if it receives new entries. +func (r *ProjectsLogsService) Delete(projectsId string, logsId string) *ProjectsLogsDeleteCall { + c := &ProjectsLogsDeleteCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logsId = logsId + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogsDeleteCall) QuotaUser(quotaUser string) *ProjectsLogsDeleteCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogsDeleteCall) Fields(s ...googleapi.Field) *ProjectsLogsDeleteCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogsDeleteCall) Context(ctx context.Context) *ProjectsLogsDeleteCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogsDeleteCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logs/{logsId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("DELETE", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logsId": c.logsId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logs.delete" call. +// Exactly one of *Empty or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *Empty.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsLogsDeleteCall) Do() (*Empty, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &Empty{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Deletes a log and all its log entries. The log will reappear if it receives new entries.", + // "httpMethod": "DELETE", + // "id": "logging.projects.logs.delete", + // "parameterOrder": [ + // "projectsId", + // "logsId" + // ], + // "parameters": { + // "logsId": { + // "description": "Part of `logName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `logName`. The resource name of the log to be deleted.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logs/{logsId}", + // "response": { + // "$ref": "Empty" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin" + // ] + // } + +} + +// method id "logging.projects.logs.list": + +type ProjectsLogsListCall struct { + s *Service + projectsId string + urlParams_ gensupport.URLParams + ifNoneMatch_ string + ctx_ context.Context +} + +// List: Lists the logs in the project. Only logs that have entries are +// listed. +func (r *ProjectsLogsService) List(projectsId string) *ProjectsLogsListCall { + c := &ProjectsLogsListCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + return c +} + +// PageSize sets the optional parameter "pageSize": The maximum number +// of results to return. +func (c *ProjectsLogsListCall) PageSize(pageSize int64) *ProjectsLogsListCall { + c.urlParams_.Set("pageSize", fmt.Sprint(pageSize)) + return c +} + +// PageToken sets the optional parameter "pageToken": An opaque token, +// returned as `nextPageToken` by a prior `ListLogs` operation. If +// `pageToken` is supplied, then the other fields of this request are +// ignored, and instead the previous `ListLogs` operation is continued. +func (c *ProjectsLogsListCall) PageToken(pageToken string) *ProjectsLogsListCall { + c.urlParams_.Set("pageToken", pageToken) + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogsListCall) QuotaUser(quotaUser string) *ProjectsLogsListCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// ServiceIndexPrefix sets the optional parameter "serviceIndexPrefix": +// The purpose of this field is to restrict the listed logs to those +// with entries of a certain kind. If `serviceName` is the name of a log +// service, then this field may contain values for the log service's +// indexes. Only logs that have entries whose indexes include the values +// are listed. The format for this field is "/val1/val2.../valN", +// where `val1` is a value for the first index, `val2` for the second +// index, etc. An empty value (a single slash) for an index matches all +// values, and you can omit values for later indexes entirely. +func (c *ProjectsLogsListCall) ServiceIndexPrefix(serviceIndexPrefix string) *ProjectsLogsListCall { + c.urlParams_.Set("serviceIndexPrefix", serviceIndexPrefix) + return c +} + +// ServiceName sets the optional parameter "serviceName": If not empty, +// this field must be a log service name such as +// "compute.googleapis.com". Only logs associated with that that log +// service are listed. +func (c *ProjectsLogsListCall) ServiceName(serviceName string) *ProjectsLogsListCall { + c.urlParams_.Set("serviceName", serviceName) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogsListCall) Fields(s ...googleapi.Field) *ProjectsLogsListCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// IfNoneMatch sets the optional parameter which makes the operation +// fail if the object's ETag matches the given value. This is useful for +// getting updates only after the object has changed since the last +// request. Use googleapi.IsNotModified to check whether the response +// error from Do is the result of In-None-Match. +func (c *ProjectsLogsListCall) IfNoneMatch(entityTag string) *ProjectsLogsListCall { + c.ifNoneMatch_ = entityTag + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogsListCall) Context(ctx context.Context) *ProjectsLogsListCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogsListCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logs") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("GET", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ifNoneMatch_ != "" { + req.Header.Set("If-None-Match", c.ifNoneMatch_) + } + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logs.list" call. +// Exactly one of *ListLogsResponse or error will be non-nil. Any +// non-2xx status code is an error. Response headers are in either +// *ListLogsResponse.ServerResponse.Header or (if a response was +// returned at all) in error.(*googleapi.Error).Header. Use +// googleapi.IsNotModified to check whether the returned error was +// because http.StatusNotModified was returned. +func (c *ProjectsLogsListCall) Do() (*ListLogsResponse, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &ListLogsResponse{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Lists the logs in the project. Only logs that have entries are listed.", + // "httpMethod": "GET", + // "id": "logging.projects.logs.list", + // "parameterOrder": [ + // "projectsId" + // ], + // "parameters": { + // "pageSize": { + // "description": "The maximum number of results to return.", + // "format": "int32", + // "location": "query", + // "type": "integer" + // }, + // "pageToken": { + // "description": "An opaque token, returned as `nextPageToken` by a prior `ListLogs` operation. If `pageToken` is supplied, then the other fields of this request are ignored, and instead the previous `ListLogs` operation is continued.", + // "location": "query", + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `projectName`. The resource name of the project whose logs are requested. If both `serviceName` and `serviceIndexPrefix` are empty, then all logs with entries in this project are listed.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "serviceIndexPrefix": { + // "description": "The purpose of this field is to restrict the listed logs to those with entries of a certain kind. If `serviceName` is the name of a log service, then this field may contain values for the log service's indexes. Only logs that have entries whose indexes include the values are listed. The format for this field is `\"/val1/val2.../valN\"`, where `val1` is a value for the first index, `val2` for the second index, etc. An empty value (a single slash) for an index matches all values, and you can omit values for later indexes entirely.", + // "location": "query", + // "type": "string" + // }, + // "serviceName": { + // "description": "If not empty, this field must be a log service name such as `\"compute.googleapis.com\"`. Only logs associated with that that log service are listed.", + // "location": "query", + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logs", + // "response": { + // "$ref": "ListLogsResponse" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/cloud-platform.read-only", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.read" + // ] + // } + +} + +// method id "logging.projects.logs.entries.write": + +type ProjectsLogsEntriesWriteCall struct { + s *Service + projectsId string + logsId string + writelogentriesrequest *WriteLogEntriesRequest + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Write: Writes log entries to Cloud Logging. Each entry consists of a +// `LogEntry` object. You must fill in all the fields of the object, +// including one of the payload fields. You may supply a map, +// `commonLabels`, that holds default (key, value) data for the +// `entries[].metadata.labels` map in each entry, saving you the trouble +// of creating identical copies for each entry. +func (r *ProjectsLogsEntriesService) Write(projectsId string, logsId string, writelogentriesrequest *WriteLogEntriesRequest) *ProjectsLogsEntriesWriteCall { + c := &ProjectsLogsEntriesWriteCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logsId = logsId + c.writelogentriesrequest = writelogentriesrequest + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogsEntriesWriteCall) QuotaUser(quotaUser string) *ProjectsLogsEntriesWriteCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogsEntriesWriteCall) Fields(s ...googleapi.Field) *ProjectsLogsEntriesWriteCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogsEntriesWriteCall) Context(ctx context.Context) *ProjectsLogsEntriesWriteCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogsEntriesWriteCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + body, err := googleapi.WithoutDataWrapper.JSONReader(c.writelogentriesrequest) + if err != nil { + return nil, err + } + ctype := "application/json" + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logs/{logsId}/entries:write") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("POST", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logsId": c.logsId, + }) + req.Header.Set("Content-Type", ctype) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logs.entries.write" call. +// Exactly one of *WriteLogEntriesResponse or error will be non-nil. Any +// non-2xx status code is an error. Response headers are in either +// *WriteLogEntriesResponse.ServerResponse.Header or (if a response was +// returned at all) in error.(*googleapi.Error).Header. Use +// googleapi.IsNotModified to check whether the returned error was +// because http.StatusNotModified was returned. +func (c *ProjectsLogsEntriesWriteCall) Do() (*WriteLogEntriesResponse, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &WriteLogEntriesResponse{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Writes log entries to Cloud Logging. Each entry consists of a `LogEntry` object. You must fill in all the fields of the object, including one of the payload fields. You may supply a map, `commonLabels`, that holds default (key, value) data for the `entries[].metadata.labels` map in each entry, saving you the trouble of creating identical copies for each entry.", + // "httpMethod": "POST", + // "id": "logging.projects.logs.entries.write", + // "parameterOrder": [ + // "projectsId", + // "logsId" + // ], + // "parameters": { + // "logsId": { + // "description": "Part of `logName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `logName`. The resource name of the log that will receive the log entries.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logs/{logsId}/entries:write", + // "request": { + // "$ref": "WriteLogEntriesRequest" + // }, + // "response": { + // "$ref": "WriteLogEntriesResponse" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.write" + // ] + // } + +} + +// method id "logging.projects.logs.sinks.create": + +type ProjectsLogsSinksCreateCall struct { + s *Service + projectsId string + logsId string + logsink *LogSink + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Create: Creates a log sink. All log entries for a specified log are +// written to the destination. +func (r *ProjectsLogsSinksService) Create(projectsId string, logsId string, logsink *LogSink) *ProjectsLogsSinksCreateCall { + c := &ProjectsLogsSinksCreateCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logsId = logsId + c.logsink = logsink + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogsSinksCreateCall) QuotaUser(quotaUser string) *ProjectsLogsSinksCreateCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogsSinksCreateCall) Fields(s ...googleapi.Field) *ProjectsLogsSinksCreateCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogsSinksCreateCall) Context(ctx context.Context) *ProjectsLogsSinksCreateCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogsSinksCreateCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + body, err := googleapi.WithoutDataWrapper.JSONReader(c.logsink) + if err != nil { + return nil, err + } + ctype := "application/json" + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logs/{logsId}/sinks") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("POST", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logsId": c.logsId, + }) + req.Header.Set("Content-Type", ctype) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logs.sinks.create" call. +// Exactly one of *LogSink or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *LogSink.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsLogsSinksCreateCall) Do() (*LogSink, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &LogSink{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Creates a log sink. All log entries for a specified log are written to the destination.", + // "httpMethod": "POST", + // "id": "logging.projects.logs.sinks.create", + // "parameterOrder": [ + // "projectsId", + // "logsId" + // ], + // "parameters": { + // "logsId": { + // "description": "Part of `logName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `logName`. The resource name of the log to which to the sink is bound.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logs/{logsId}/sinks", + // "request": { + // "$ref": "LogSink" + // }, + // "response": { + // "$ref": "LogSink" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin" + // ] + // } + +} + +// method id "logging.projects.logs.sinks.delete": + +type ProjectsLogsSinksDeleteCall struct { + s *Service + projectsId string + logsId string + sinksId string + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Delete: Deletes a log sink. After deletion, no new log entries are +// written to the destination. +func (r *ProjectsLogsSinksService) Delete(projectsId string, logsId string, sinksId string) *ProjectsLogsSinksDeleteCall { + c := &ProjectsLogsSinksDeleteCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logsId = logsId + c.sinksId = sinksId + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogsSinksDeleteCall) QuotaUser(quotaUser string) *ProjectsLogsSinksDeleteCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogsSinksDeleteCall) Fields(s ...googleapi.Field) *ProjectsLogsSinksDeleteCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogsSinksDeleteCall) Context(ctx context.Context) *ProjectsLogsSinksDeleteCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogsSinksDeleteCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logs/{logsId}/sinks/{sinksId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("DELETE", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logsId": c.logsId, + "sinksId": c.sinksId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logs.sinks.delete" call. +// Exactly one of *Empty or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *Empty.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsLogsSinksDeleteCall) Do() (*Empty, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &Empty{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Deletes a log sink. After deletion, no new log entries are written to the destination.", + // "httpMethod": "DELETE", + // "id": "logging.projects.logs.sinks.delete", + // "parameterOrder": [ + // "projectsId", + // "logsId", + // "sinksId" + // ], + // "parameters": { + // "logsId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `sinkName`. The resource name of the log sink to delete.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "sinksId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logs/{logsId}/sinks/{sinksId}", + // "response": { + // "$ref": "Empty" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin" + // ] + // } + +} + +// method id "logging.projects.logs.sinks.get": + +type ProjectsLogsSinksGetCall struct { + s *Service + projectsId string + logsId string + sinksId string + urlParams_ gensupport.URLParams + ifNoneMatch_ string + ctx_ context.Context +} + +// Get: Gets a log sink. +func (r *ProjectsLogsSinksService) Get(projectsId string, logsId string, sinksId string) *ProjectsLogsSinksGetCall { + c := &ProjectsLogsSinksGetCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logsId = logsId + c.sinksId = sinksId + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogsSinksGetCall) QuotaUser(quotaUser string) *ProjectsLogsSinksGetCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogsSinksGetCall) Fields(s ...googleapi.Field) *ProjectsLogsSinksGetCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// IfNoneMatch sets the optional parameter which makes the operation +// fail if the object's ETag matches the given value. This is useful for +// getting updates only after the object has changed since the last +// request. Use googleapi.IsNotModified to check whether the response +// error from Do is the result of In-None-Match. +func (c *ProjectsLogsSinksGetCall) IfNoneMatch(entityTag string) *ProjectsLogsSinksGetCall { + c.ifNoneMatch_ = entityTag + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogsSinksGetCall) Context(ctx context.Context) *ProjectsLogsSinksGetCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogsSinksGetCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logs/{logsId}/sinks/{sinksId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("GET", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logsId": c.logsId, + "sinksId": c.sinksId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ifNoneMatch_ != "" { + req.Header.Set("If-None-Match", c.ifNoneMatch_) + } + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logs.sinks.get" call. +// Exactly one of *LogSink or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *LogSink.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsLogsSinksGetCall) Do() (*LogSink, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &LogSink{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Gets a log sink.", + // "httpMethod": "GET", + // "id": "logging.projects.logs.sinks.get", + // "parameterOrder": [ + // "projectsId", + // "logsId", + // "sinksId" + // ], + // "parameters": { + // "logsId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `sinkName`. The resource name of the log sink to return.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "sinksId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logs/{logsId}/sinks/{sinksId}", + // "response": { + // "$ref": "LogSink" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/cloud-platform.read-only", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.read" + // ] + // } + +} + +// method id "logging.projects.logs.sinks.list": + +type ProjectsLogsSinksListCall struct { + s *Service + projectsId string + logsId string + urlParams_ gensupport.URLParams + ifNoneMatch_ string + ctx_ context.Context +} + +// List: Lists log sinks associated with a log. +func (r *ProjectsLogsSinksService) List(projectsId string, logsId string) *ProjectsLogsSinksListCall { + c := &ProjectsLogsSinksListCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logsId = logsId + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogsSinksListCall) QuotaUser(quotaUser string) *ProjectsLogsSinksListCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogsSinksListCall) Fields(s ...googleapi.Field) *ProjectsLogsSinksListCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// IfNoneMatch sets the optional parameter which makes the operation +// fail if the object's ETag matches the given value. This is useful for +// getting updates only after the object has changed since the last +// request. Use googleapi.IsNotModified to check whether the response +// error from Do is the result of In-None-Match. +func (c *ProjectsLogsSinksListCall) IfNoneMatch(entityTag string) *ProjectsLogsSinksListCall { + c.ifNoneMatch_ = entityTag + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogsSinksListCall) Context(ctx context.Context) *ProjectsLogsSinksListCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogsSinksListCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logs/{logsId}/sinks") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("GET", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logsId": c.logsId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ifNoneMatch_ != "" { + req.Header.Set("If-None-Match", c.ifNoneMatch_) + } + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logs.sinks.list" call. +// Exactly one of *ListLogSinksResponse or error will be non-nil. Any +// non-2xx status code is an error. Response headers are in either +// *ListLogSinksResponse.ServerResponse.Header or (if a response was +// returned at all) in error.(*googleapi.Error).Header. Use +// googleapi.IsNotModified to check whether the returned error was +// because http.StatusNotModified was returned. +func (c *ProjectsLogsSinksListCall) Do() (*ListLogSinksResponse, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &ListLogSinksResponse{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Lists log sinks associated with a log.", + // "httpMethod": "GET", + // "id": "logging.projects.logs.sinks.list", + // "parameterOrder": [ + // "projectsId", + // "logsId" + // ], + // "parameters": { + // "logsId": { + // "description": "Part of `logName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `logName`. The log whose sinks are wanted. For example, `\"compute.google.com/syslog\"`.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logs/{logsId}/sinks", + // "response": { + // "$ref": "ListLogSinksResponse" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/cloud-platform.read-only", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.read" + // ] + // } + +} + +// method id "logging.projects.logs.sinks.update": + +type ProjectsLogsSinksUpdateCall struct { + s *Service + projectsId string + logsId string + sinksId string + logsink *LogSink + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Update: Updates a log sink. If the sink does not exist, it is +// created. +func (r *ProjectsLogsSinksService) Update(projectsId string, logsId string, sinksId string, logsink *LogSink) *ProjectsLogsSinksUpdateCall { + c := &ProjectsLogsSinksUpdateCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logsId = logsId + c.sinksId = sinksId + c.logsink = logsink + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsLogsSinksUpdateCall) QuotaUser(quotaUser string) *ProjectsLogsSinksUpdateCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsLogsSinksUpdateCall) Fields(s ...googleapi.Field) *ProjectsLogsSinksUpdateCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsLogsSinksUpdateCall) Context(ctx context.Context) *ProjectsLogsSinksUpdateCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsLogsSinksUpdateCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + body, err := googleapi.WithoutDataWrapper.JSONReader(c.logsink) + if err != nil { + return nil, err + } + ctype := "application/json" + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/logs/{logsId}/sinks/{sinksId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("PUT", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "logsId": c.logsId, + "sinksId": c.sinksId, + }) + req.Header.Set("Content-Type", ctype) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.logs.sinks.update" call. +// Exactly one of *LogSink or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *LogSink.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsLogsSinksUpdateCall) Do() (*LogSink, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &LogSink{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Updates a log sink. If the sink does not exist, it is created.", + // "httpMethod": "PUT", + // "id": "logging.projects.logs.sinks.update", + // "parameterOrder": [ + // "projectsId", + // "logsId", + // "sinksId" + // ], + // "parameters": { + // "logsId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `sinkName`. The resource name of the sink to update.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "sinksId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/logs/{logsId}/sinks/{sinksId}", + // "request": { + // "$ref": "LogSink" + // }, + // "response": { + // "$ref": "LogSink" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin" + // ] + // } + +} + +// method id "logging.projects.metrics.create": + +type ProjectsMetricsCreateCall struct { + s *Service + projectsId string + logmetric *LogMetric + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Create: Creates a logs-based metric. +func (r *ProjectsMetricsService) Create(projectsId string, logmetric *LogMetric) *ProjectsMetricsCreateCall { + c := &ProjectsMetricsCreateCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logmetric = logmetric + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsMetricsCreateCall) QuotaUser(quotaUser string) *ProjectsMetricsCreateCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsMetricsCreateCall) Fields(s ...googleapi.Field) *ProjectsMetricsCreateCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsMetricsCreateCall) Context(ctx context.Context) *ProjectsMetricsCreateCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsMetricsCreateCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + body, err := googleapi.WithoutDataWrapper.JSONReader(c.logmetric) + if err != nil { + return nil, err + } + ctype := "application/json" + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/metrics") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("POST", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + }) + req.Header.Set("Content-Type", ctype) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.metrics.create" call. +// Exactly one of *LogMetric or error will be non-nil. Any non-2xx +// status code is an error. Response headers are in either +// *LogMetric.ServerResponse.Header or (if a response was returned at +// all) in error.(*googleapi.Error).Header. Use googleapi.IsNotModified +// to check whether the returned error was because +// http.StatusNotModified was returned. +func (c *ProjectsMetricsCreateCall) Do() (*LogMetric, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &LogMetric{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Creates a logs-based metric.", + // "httpMethod": "POST", + // "id": "logging.projects.metrics.create", + // "parameterOrder": [ + // "projectsId" + // ], + // "parameters": { + // "projectsId": { + // "description": "Part of `projectName`. The resource name of the project in which to create the metric.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/metrics", + // "request": { + // "$ref": "LogMetric" + // }, + // "response": { + // "$ref": "LogMetric" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.write" + // ] + // } + +} + +// method id "logging.projects.metrics.delete": + +type ProjectsMetricsDeleteCall struct { + s *Service + projectsId string + metricsId string + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Delete: Deletes a logs-based metric. +func (r *ProjectsMetricsService) Delete(projectsId string, metricsId string) *ProjectsMetricsDeleteCall { + c := &ProjectsMetricsDeleteCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.metricsId = metricsId + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsMetricsDeleteCall) QuotaUser(quotaUser string) *ProjectsMetricsDeleteCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsMetricsDeleteCall) Fields(s ...googleapi.Field) *ProjectsMetricsDeleteCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsMetricsDeleteCall) Context(ctx context.Context) *ProjectsMetricsDeleteCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsMetricsDeleteCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/metrics/{metricsId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("DELETE", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "metricsId": c.metricsId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.metrics.delete" call. +// Exactly one of *Empty or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *Empty.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsMetricsDeleteCall) Do() (*Empty, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &Empty{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Deletes a logs-based metric.", + // "httpMethod": "DELETE", + // "id": "logging.projects.metrics.delete", + // "parameterOrder": [ + // "projectsId", + // "metricsId" + // ], + // "parameters": { + // "metricsId": { + // "description": "Part of `metricName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `metricName`. The resource name of the metric to delete.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/metrics/{metricsId}", + // "response": { + // "$ref": "Empty" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.write" + // ] + // } + +} + +// method id "logging.projects.metrics.get": + +type ProjectsMetricsGetCall struct { + s *Service + projectsId string + metricsId string + urlParams_ gensupport.URLParams + ifNoneMatch_ string + ctx_ context.Context +} + +// Get: Gets a logs-based metric. +func (r *ProjectsMetricsService) Get(projectsId string, metricsId string) *ProjectsMetricsGetCall { + c := &ProjectsMetricsGetCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.metricsId = metricsId + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsMetricsGetCall) QuotaUser(quotaUser string) *ProjectsMetricsGetCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsMetricsGetCall) Fields(s ...googleapi.Field) *ProjectsMetricsGetCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// IfNoneMatch sets the optional parameter which makes the operation +// fail if the object's ETag matches the given value. This is useful for +// getting updates only after the object has changed since the last +// request. Use googleapi.IsNotModified to check whether the response +// error from Do is the result of In-None-Match. +func (c *ProjectsMetricsGetCall) IfNoneMatch(entityTag string) *ProjectsMetricsGetCall { + c.ifNoneMatch_ = entityTag + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsMetricsGetCall) Context(ctx context.Context) *ProjectsMetricsGetCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsMetricsGetCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/metrics/{metricsId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("GET", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "metricsId": c.metricsId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ifNoneMatch_ != "" { + req.Header.Set("If-None-Match", c.ifNoneMatch_) + } + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.metrics.get" call. +// Exactly one of *LogMetric or error will be non-nil. Any non-2xx +// status code is an error. Response headers are in either +// *LogMetric.ServerResponse.Header or (if a response was returned at +// all) in error.(*googleapi.Error).Header. Use googleapi.IsNotModified +// to check whether the returned error was because +// http.StatusNotModified was returned. +func (c *ProjectsMetricsGetCall) Do() (*LogMetric, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &LogMetric{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Gets a logs-based metric.", + // "httpMethod": "GET", + // "id": "logging.projects.metrics.get", + // "parameterOrder": [ + // "projectsId", + // "metricsId" + // ], + // "parameters": { + // "metricsId": { + // "description": "Part of `metricName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `metricName`. The resource name of the desired metric.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/metrics/{metricsId}", + // "response": { + // "$ref": "LogMetric" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/cloud-platform.read-only", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.read" + // ] + // } + +} + +// method id "logging.projects.metrics.list": + +type ProjectsMetricsListCall struct { + s *Service + projectsId string + urlParams_ gensupport.URLParams + ifNoneMatch_ string + ctx_ context.Context +} + +// List: Lists the logs-based metrics associated with a project. +func (r *ProjectsMetricsService) List(projectsId string) *ProjectsMetricsListCall { + c := &ProjectsMetricsListCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + return c +} + +// PageSize sets the optional parameter "pageSize": The maximum number +// of `LogMetric` objects to return in one operation. +func (c *ProjectsMetricsListCall) PageSize(pageSize int64) *ProjectsMetricsListCall { + c.urlParams_.Set("pageSize", fmt.Sprint(pageSize)) + return c +} + +// PageToken sets the optional parameter "pageToken": An opaque token, +// returned as `nextPageToken` by a prior `ListLogMetrics` operation. If +// `pageToken` is supplied, then the other fields of this request are +// ignored, and instead the previous `ListLogMetrics` operation is +// continued. +func (c *ProjectsMetricsListCall) PageToken(pageToken string) *ProjectsMetricsListCall { + c.urlParams_.Set("pageToken", pageToken) + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsMetricsListCall) QuotaUser(quotaUser string) *ProjectsMetricsListCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsMetricsListCall) Fields(s ...googleapi.Field) *ProjectsMetricsListCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// IfNoneMatch sets the optional parameter which makes the operation +// fail if the object's ETag matches the given value. This is useful for +// getting updates only after the object has changed since the last +// request. Use googleapi.IsNotModified to check whether the response +// error from Do is the result of In-None-Match. +func (c *ProjectsMetricsListCall) IfNoneMatch(entityTag string) *ProjectsMetricsListCall { + c.ifNoneMatch_ = entityTag + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsMetricsListCall) Context(ctx context.Context) *ProjectsMetricsListCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsMetricsListCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/metrics") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("GET", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ifNoneMatch_ != "" { + req.Header.Set("If-None-Match", c.ifNoneMatch_) + } + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.metrics.list" call. +// Exactly one of *ListLogMetricsResponse or error will be non-nil. Any +// non-2xx status code is an error. Response headers are in either +// *ListLogMetricsResponse.ServerResponse.Header or (if a response was +// returned at all) in error.(*googleapi.Error).Header. Use +// googleapi.IsNotModified to check whether the returned error was +// because http.StatusNotModified was returned. +func (c *ProjectsMetricsListCall) Do() (*ListLogMetricsResponse, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &ListLogMetricsResponse{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Lists the logs-based metrics associated with a project.", + // "httpMethod": "GET", + // "id": "logging.projects.metrics.list", + // "parameterOrder": [ + // "projectsId" + // ], + // "parameters": { + // "pageSize": { + // "description": "The maximum number of `LogMetric` objects to return in one operation.", + // "format": "int32", + // "location": "query", + // "type": "integer" + // }, + // "pageToken": { + // "description": "An opaque token, returned as `nextPageToken` by a prior `ListLogMetrics` operation. If `pageToken` is supplied, then the other fields of this request are ignored, and instead the previous `ListLogMetrics` operation is continued.", + // "location": "query", + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `projectName`. The resource name for the project whose metrics are wanted.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/metrics", + // "response": { + // "$ref": "ListLogMetricsResponse" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/cloud-platform.read-only", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.read" + // ] + // } + +} + +// method id "logging.projects.metrics.update": + +type ProjectsMetricsUpdateCall struct { + s *Service + projectsId string + metricsId string + logmetric *LogMetric + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Update: Creates or updates a logs-based metric. +func (r *ProjectsMetricsService) Update(projectsId string, metricsId string, logmetric *LogMetric) *ProjectsMetricsUpdateCall { + c := &ProjectsMetricsUpdateCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.metricsId = metricsId + c.logmetric = logmetric + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsMetricsUpdateCall) QuotaUser(quotaUser string) *ProjectsMetricsUpdateCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsMetricsUpdateCall) Fields(s ...googleapi.Field) *ProjectsMetricsUpdateCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsMetricsUpdateCall) Context(ctx context.Context) *ProjectsMetricsUpdateCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsMetricsUpdateCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + body, err := googleapi.WithoutDataWrapper.JSONReader(c.logmetric) + if err != nil { + return nil, err + } + ctype := "application/json" + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/metrics/{metricsId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("PUT", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "metricsId": c.metricsId, + }) + req.Header.Set("Content-Type", ctype) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.metrics.update" call. +// Exactly one of *LogMetric or error will be non-nil. Any non-2xx +// status code is an error. Response headers are in either +// *LogMetric.ServerResponse.Header or (if a response was returned at +// all) in error.(*googleapi.Error).Header. Use googleapi.IsNotModified +// to check whether the returned error was because +// http.StatusNotModified was returned. +func (c *ProjectsMetricsUpdateCall) Do() (*LogMetric, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &LogMetric{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Creates or updates a logs-based metric.", + // "httpMethod": "PUT", + // "id": "logging.projects.metrics.update", + // "parameterOrder": [ + // "projectsId", + // "metricsId" + // ], + // "parameters": { + // "metricsId": { + // "description": "Part of `metricName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "projectsId": { + // "description": "Part of `metricName`. The resource name of the metric to update.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/metrics/{metricsId}", + // "request": { + // "$ref": "LogMetric" + // }, + // "response": { + // "$ref": "LogMetric" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.write" + // ] + // } + +} + +// method id "logging.projects.sinks.create": + +type ProjectsSinksCreateCall struct { + s *Service + projectsId string + logsink *LogSink + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Create: Creates a project sink. A logs filter determines which log +// entries are written to the destination. +func (r *ProjectsSinksService) Create(projectsId string, logsink *LogSink) *ProjectsSinksCreateCall { + c := &ProjectsSinksCreateCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.logsink = logsink + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsSinksCreateCall) QuotaUser(quotaUser string) *ProjectsSinksCreateCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsSinksCreateCall) Fields(s ...googleapi.Field) *ProjectsSinksCreateCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsSinksCreateCall) Context(ctx context.Context) *ProjectsSinksCreateCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsSinksCreateCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + body, err := googleapi.WithoutDataWrapper.JSONReader(c.logsink) + if err != nil { + return nil, err + } + ctype := "application/json" + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/sinks") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("POST", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + }) + req.Header.Set("Content-Type", ctype) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.sinks.create" call. +// Exactly one of *LogSink or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *LogSink.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsSinksCreateCall) Do() (*LogSink, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &LogSink{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Creates a project sink. A logs filter determines which log entries are written to the destination.", + // "httpMethod": "POST", + // "id": "logging.projects.sinks.create", + // "parameterOrder": [ + // "projectsId" + // ], + // "parameters": { + // "projectsId": { + // "description": "Part of `projectName`. The resource name of the project to which the sink is bound.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/sinks", + // "request": { + // "$ref": "LogSink" + // }, + // "response": { + // "$ref": "LogSink" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin" + // ] + // } + +} + +// method id "logging.projects.sinks.delete": + +type ProjectsSinksDeleteCall struct { + s *Service + projectsId string + sinksId string + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Delete: Deletes a project sink. After deletion, no new log entries +// are written to the destination. +func (r *ProjectsSinksService) Delete(projectsId string, sinksId string) *ProjectsSinksDeleteCall { + c := &ProjectsSinksDeleteCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.sinksId = sinksId + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsSinksDeleteCall) QuotaUser(quotaUser string) *ProjectsSinksDeleteCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsSinksDeleteCall) Fields(s ...googleapi.Field) *ProjectsSinksDeleteCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsSinksDeleteCall) Context(ctx context.Context) *ProjectsSinksDeleteCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsSinksDeleteCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/sinks/{sinksId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("DELETE", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "sinksId": c.sinksId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.sinks.delete" call. +// Exactly one of *Empty or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *Empty.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsSinksDeleteCall) Do() (*Empty, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &Empty{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Deletes a project sink. After deletion, no new log entries are written to the destination.", + // "httpMethod": "DELETE", + // "id": "logging.projects.sinks.delete", + // "parameterOrder": [ + // "projectsId", + // "sinksId" + // ], + // "parameters": { + // "projectsId": { + // "description": "Part of `sinkName`. The resource name of the project sink to delete.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "sinksId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/sinks/{sinksId}", + // "response": { + // "$ref": "Empty" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin" + // ] + // } + +} + +// method id "logging.projects.sinks.get": + +type ProjectsSinksGetCall struct { + s *Service + projectsId string + sinksId string + urlParams_ gensupport.URLParams + ifNoneMatch_ string + ctx_ context.Context +} + +// Get: Gets a project sink. +func (r *ProjectsSinksService) Get(projectsId string, sinksId string) *ProjectsSinksGetCall { + c := &ProjectsSinksGetCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.sinksId = sinksId + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsSinksGetCall) QuotaUser(quotaUser string) *ProjectsSinksGetCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsSinksGetCall) Fields(s ...googleapi.Field) *ProjectsSinksGetCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// IfNoneMatch sets the optional parameter which makes the operation +// fail if the object's ETag matches the given value. This is useful for +// getting updates only after the object has changed since the last +// request. Use googleapi.IsNotModified to check whether the response +// error from Do is the result of In-None-Match. +func (c *ProjectsSinksGetCall) IfNoneMatch(entityTag string) *ProjectsSinksGetCall { + c.ifNoneMatch_ = entityTag + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsSinksGetCall) Context(ctx context.Context) *ProjectsSinksGetCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsSinksGetCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/sinks/{sinksId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("GET", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "sinksId": c.sinksId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ifNoneMatch_ != "" { + req.Header.Set("If-None-Match", c.ifNoneMatch_) + } + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.sinks.get" call. +// Exactly one of *LogSink or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *LogSink.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsSinksGetCall) Do() (*LogSink, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &LogSink{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Gets a project sink.", + // "httpMethod": "GET", + // "id": "logging.projects.sinks.get", + // "parameterOrder": [ + // "projectsId", + // "sinksId" + // ], + // "parameters": { + // "projectsId": { + // "description": "Part of `sinkName`. The resource name of the project sink to return.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "sinksId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/sinks/{sinksId}", + // "response": { + // "$ref": "LogSink" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/cloud-platform.read-only", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.read" + // ] + // } + +} + +// method id "logging.projects.sinks.list": + +type ProjectsSinksListCall struct { + s *Service + projectsId string + urlParams_ gensupport.URLParams + ifNoneMatch_ string + ctx_ context.Context +} + +// List: Lists project sinks associated with a project. +func (r *ProjectsSinksService) List(projectsId string) *ProjectsSinksListCall { + c := &ProjectsSinksListCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsSinksListCall) QuotaUser(quotaUser string) *ProjectsSinksListCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsSinksListCall) Fields(s ...googleapi.Field) *ProjectsSinksListCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// IfNoneMatch sets the optional parameter which makes the operation +// fail if the object's ETag matches the given value. This is useful for +// getting updates only after the object has changed since the last +// request. Use googleapi.IsNotModified to check whether the response +// error from Do is the result of In-None-Match. +func (c *ProjectsSinksListCall) IfNoneMatch(entityTag string) *ProjectsSinksListCall { + c.ifNoneMatch_ = entityTag + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsSinksListCall) Context(ctx context.Context) *ProjectsSinksListCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsSinksListCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/sinks") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("GET", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + }) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ifNoneMatch_ != "" { + req.Header.Set("If-None-Match", c.ifNoneMatch_) + } + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.sinks.list" call. +// Exactly one of *ListSinksResponse or error will be non-nil. Any +// non-2xx status code is an error. Response headers are in either +// *ListSinksResponse.ServerResponse.Header or (if a response was +// returned at all) in error.(*googleapi.Error).Header. Use +// googleapi.IsNotModified to check whether the returned error was +// because http.StatusNotModified was returned. +func (c *ProjectsSinksListCall) Do() (*ListSinksResponse, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &ListSinksResponse{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Lists project sinks associated with a project.", + // "httpMethod": "GET", + // "id": "logging.projects.sinks.list", + // "parameterOrder": [ + // "projectsId" + // ], + // "parameters": { + // "projectsId": { + // "description": "Part of `projectName`. The project whose sinks are wanted.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/sinks", + // "response": { + // "$ref": "ListSinksResponse" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/cloud-platform.read-only", + // "https://www.googleapis.com/auth/logging.admin", + // "https://www.googleapis.com/auth/logging.read" + // ] + // } + +} + +// method id "logging.projects.sinks.update": + +type ProjectsSinksUpdateCall struct { + s *Service + projectsId string + sinksId string + logsink *LogSink + urlParams_ gensupport.URLParams + ctx_ context.Context +} + +// Update: Updates a project sink. If the sink does not exist, it is +// created. The destination, filter, or both may be updated. +func (r *ProjectsSinksService) Update(projectsId string, sinksId string, logsink *LogSink) *ProjectsSinksUpdateCall { + c := &ProjectsSinksUpdateCall{s: r.s, urlParams_: make(gensupport.URLParams)} + c.projectsId = projectsId + c.sinksId = sinksId + c.logsink = logsink + return c +} + +// QuotaUser sets the optional parameter "quotaUser": Available to use +// for quota purposes for server-side applications. Can be any arbitrary +// string assigned to a user, but should not exceed 40 characters. +func (c *ProjectsSinksUpdateCall) QuotaUser(quotaUser string) *ProjectsSinksUpdateCall { + c.urlParams_.Set("quotaUser", quotaUser) + return c +} + +// Fields allows partial responses to be retrieved. See +// https://developers.google.com/gdata/docs/2.0/basics#PartialResponse +// for more information. +func (c *ProjectsSinksUpdateCall) Fields(s ...googleapi.Field) *ProjectsSinksUpdateCall { + c.urlParams_.Set("fields", googleapi.CombineFields(s)) + return c +} + +// Context sets the context to be used in this call's Do method. Any +// pending HTTP request will be aborted if the provided context is +// canceled. +func (c *ProjectsSinksUpdateCall) Context(ctx context.Context) *ProjectsSinksUpdateCall { + c.ctx_ = ctx + return c +} + +func (c *ProjectsSinksUpdateCall) doRequest(alt string) (*http.Response, error) { + var body io.Reader = nil + body, err := googleapi.WithoutDataWrapper.JSONReader(c.logsink) + if err != nil { + return nil, err + } + ctype := "application/json" + c.urlParams_.Set("alt", alt) + urls := googleapi.ResolveRelative(c.s.BasePath, "v1beta3/projects/{projectsId}/sinks/{sinksId}") + urls += "?" + c.urlParams_.Encode() + req, _ := http.NewRequest("PUT", urls, body) + googleapi.Expand(req.URL, map[string]string{ + "projectsId": c.projectsId, + "sinksId": c.sinksId, + }) + req.Header.Set("Content-Type", ctype) + req.Header.Set("User-Agent", c.s.userAgent()) + if c.ctx_ != nil { + return ctxhttp.Do(c.ctx_, c.s.client, req) + } + return c.s.client.Do(req) +} + +// Do executes the "logging.projects.sinks.update" call. +// Exactly one of *LogSink or error will be non-nil. Any non-2xx status +// code is an error. Response headers are in either +// *LogSink.ServerResponse.Header or (if a response was returned at all) +// in error.(*googleapi.Error).Header. Use googleapi.IsNotModified to +// check whether the returned error was because http.StatusNotModified +// was returned. +func (c *ProjectsSinksUpdateCall) Do() (*LogSink, error) { + res, err := c.doRequest("json") + if res != nil && res.StatusCode == http.StatusNotModified { + if res.Body != nil { + res.Body.Close() + } + return nil, &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + } + } + if err != nil { + return nil, err + } + defer googleapi.CloseBody(res) + if err := googleapi.CheckResponse(res); err != nil { + return nil, err + } + ret := &LogSink{ + ServerResponse: googleapi.ServerResponse{ + Header: res.Header, + HTTPStatusCode: res.StatusCode, + }, + } + if err := json.NewDecoder(res.Body).Decode(&ret); err != nil { + return nil, err + } + return ret, nil + // { + // "description": "Updates a project sink. If the sink does not exist, it is created. The destination, filter, or both may be updated.", + // "httpMethod": "PUT", + // "id": "logging.projects.sinks.update", + // "parameterOrder": [ + // "projectsId", + // "sinksId" + // ], + // "parameters": { + // "projectsId": { + // "description": "Part of `sinkName`. The resource name of the project sink to update.", + // "location": "path", + // "required": true, + // "type": "string" + // }, + // "sinksId": { + // "description": "Part of `sinkName`. See documentation of `projectsId`.", + // "location": "path", + // "required": true, + // "type": "string" + // } + // }, + // "path": "v1beta3/projects/{projectsId}/sinks/{sinksId}", + // "request": { + // "$ref": "LogSink" + // }, + // "response": { + // "$ref": "LogSink" + // }, + // "scopes": [ + // "https://www.googleapis.com/auth/cloud-platform", + // "https://www.googleapis.com/auth/logging.admin" + // ] + // } + +} diff --git a/components/engine/vendor/src/google.golang.org/cloud/.travis.yml b/components/engine/vendor/src/google.golang.org/cloud/.travis.yml new file mode 100644 index 0000000000..c037df0de0 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/.travis.yml @@ -0,0 +1,11 @@ +sudo: false +language: go +go: +- 1.4 +- 1.5 +install: +- go get -v google.golang.org/cloud/... +script: +- openssl aes-256-cbc -K $encrypted_912ff8fa81ad_key -iv $encrypted_912ff8fa81ad_iv -in key.json.enc -out key.json -d +- GCLOUD_TESTS_GOLANG_PROJECT_ID="dulcet-port-762" GCLOUD_TESTS_GOLANG_KEY="$(pwd)/key.json" + go test -v -tags=integration google.golang.org/cloud/... diff --git a/components/engine/vendor/src/google.golang.org/cloud/AUTHORS b/components/engine/vendor/src/google.golang.org/cloud/AUTHORS new file mode 100644 index 0000000000..3da443dc9f --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/AUTHORS @@ -0,0 +1,12 @@ +# This is the official list of cloud authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as: +# Name or Organization +# The email address is not required for organizations. + +Google Inc. +Palm Stone Games, Inc. +Péter Szilágyi +Tyler Treat diff --git a/components/engine/vendor/src/google.golang.org/cloud/CONTRIBUTING.md b/components/engine/vendor/src/google.golang.org/cloud/CONTRIBUTING.md new file mode 100644 index 0000000000..9a1cab2878 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/CONTRIBUTING.md @@ -0,0 +1,114 @@ +# Contributing + +1. Sign one of the contributor license agreements below. +1. `go get golang.org/x/review/git-codereview` to install the code reviewing tool. +1. Get the cloud package by running `go get -d google.golang.org/cloud`. + 1. If you have already checked out the source, make sure that the remote git + origin is https://code.googlesource.com/gocloud: + + git remote set-url origin https://code.googlesource.com/gocloud +1. Make changes and create a change by running `git codereview change `, +provide a command message, and use `git codereview mail` to create a Gerrit CL. +1. Keep amending to the change and mail as your recieve feedback. + +## Integration Tests + +Additional to the unit tests, you may run the integration test suite. + +To run the integrations tests, creating and configuration of a project in the +Google Developers Console is required. Once you create a project, set the +following environment variables to be able to run the against the actual APIs. + +- **GCLOUD_TESTS_GOLANG_PROJECT_ID**: Developers Console project's ID (e.g. bamboo-shift-455) +- **GCLOUD_TESTS_GOLANG_KEY**: The path to the JSON key file. + +Create a storage bucket with the same name as the project id set in **GCLOUD_TESTS_GOLANG_PROJECT_ID**. +The storage integration test will create and delete some objects in this bucket. + +Install the [gcloud command-line tool][gcloudcli] to your machine and use it +to create the indexes used in the datastore integration tests with indexes +found in `datastore/testdata/index.yaml`: + +From the project's root directory: + +``` sh +# Install the app component +$ gcloud components update app + +# Set the default project in your env +$ gcloud config set project $GCLOUD_TESTS_GOLANG_PROJECT_ID + +# Authenticate the gcloud tool with your account +$ gcloud auth login + +# Create the indexes +$ gcloud preview datastore create-indexes datastore/testdata/index.yaml + +``` + +You can run the integration tests by running: + +``` sh +$ go test -v -tags=integration google.golang.org/cloud/... +``` + +## Contributor License Agreements + +Before we can accept your pull requests you'll need to sign a Contributor +License Agreement (CLA): + +- **If you are an individual writing original source code** and **you own the +- intellectual property**, then you'll need to sign an [individual CLA][indvcla]. +- **If you work for a company that wants to allow you to contribute your work**, +then you'll need to sign a [corporate CLA][corpcla]. + +You can sign these electronically (just scroll to the bottom). After that, +we'll be able to accept your pull requests. + +## Contributor Code of Conduct + +As contributors and maintainers of this project, +and in the interest of fostering an open and welcoming community, +we pledge to respect all people who contribute through reporting issues, +posting feature requests, updating documentation, +submitting pull requests or patches, and other activities. + +We are committed to making participation in this project +a harassment-free experience for everyone, +regardless of level of experience, gender, gender identity and expression, +sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, +such as physical or electronic +addresses, without explicit permission +* Other unethical or unprofessional conduct. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct. +By adopting this Code of Conduct, +project maintainers commit themselves to fairly and consistently +applying these principles to every aspect of managing this project. +Project maintainers who do not follow or enforce the Code of Conduct +may be permanently removed from the project team. + +This code of conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior +may be reported by opening an issue +or contacting one or more of the project maintainers. + +This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0, +available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/) + +[gcloudcli]: https://developers.google.com/cloud/sdk/gcloud/ +[indvcla]: https://developers.google.com/open-source/cla/individual +[corpcla]: https://developers.google.com/open-source/cla/corporate diff --git a/components/engine/vendor/src/google.golang.org/cloud/CONTRIBUTORS b/components/engine/vendor/src/google.golang.org/cloud/CONTRIBUTORS new file mode 100644 index 0000000000..475ac6a667 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/CONTRIBUTORS @@ -0,0 +1,24 @@ +# People who have agreed to one of the CLAs and can contribute patches. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# https://developers.google.com/open-source/cla/individual +# https://developers.google.com/open-source/cla/corporate +# +# Names should be added to this file as: +# Name + +# Keep the list alphabetically sorted. + +Andrew Gerrand +Brad Fitzpatrick +Burcu Dogan +Dave Day +David Symonds +Glenn Lewis +Johan Euphrosine +Luna Duclos +Michael McGreevy +Péter Szilágyi +Tyler Treat diff --git a/components/engine/vendor/src/google.golang.org/cloud/LICENSE b/components/engine/vendor/src/google.golang.org/cloud/LICENSE new file mode 100644 index 0000000000..a4c5efd822 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2014 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/components/engine/vendor/src/google.golang.org/cloud/README.md b/components/engine/vendor/src/google.golang.org/cloud/README.md new file mode 100644 index 0000000000..10d3995d58 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/README.md @@ -0,0 +1,135 @@ +# Google Cloud for Go + +[![Build Status](https://travis-ci.org/GoogleCloudPlatform/gcloud-golang.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/gcloud-golang) + +**NOTE:** These packages are experimental, and may occasionally make +backwards-incompatible changes. + +**NOTE:** Github repo is a mirror of [https://code.googlesource.com/gocloud](https://code.googlesource.com/gocloud). + +Go packages for Google Cloud Platform services. Supported APIs include: + + * Google Cloud Datastore + * Google Cloud Storage + * Google Cloud Pub/Sub + * Google Cloud Container Engine + +``` go +import "google.golang.org/cloud" +``` + +Documentation and examples are available at +[https://godoc.org/google.golang.org/cloud](https://godoc.org/google.golang.org/cloud). + +## Authorization + +Authorization, throughout the package, is delegated to the godoc.org/golang.org/x/oauth2. +Refer to the [godoc documentation](https://godoc.org/golang.org/x/oauth2) +for examples on using oauth2 with the Cloud package. + +## Google Cloud Datastore + +[Google Cloud Datastore][cloud-datastore] ([docs][cloud-datastore-docs]) is a fully +managed, schemaless database for storing non-relational data. Cloud Datastore +automatically scales with your users and supports ACID transactions, high availability +of reads and writes, strong consistency for reads and ancestor queries, and eventual +consistency for all other queries. + +Follow the [activation instructions][cloud-datastore-activation] to use the Google +Cloud Datastore API with your project. + +[https://godoc.org/google.golang.org/cloud/datastore](https://godoc.org/google.golang.org/cloud/datastore) + + +```go +type Post struct { + Title string + Body string `datastore:",noindex"` + PublishedAt time.Time +} +keys := []*datastore.Key{ + datastore.NewKey(ctx, "Post", "post1", 0, nil), + datastore.NewKey(ctx, "Post", "post2", 0, nil), +} +posts := []*Post{ + {Title: "Post 1", Body: "...", PublishedAt: time.Now()}, + {Title: "Post 2", Body: "...", PublishedAt: time.Now()}, +} +if _, err := datastore.PutMulti(ctx, keys, posts); err != nil { + log.Println(err) +} +``` + +## Google Cloud Storage + +[Google Cloud Storage][cloud-storage] ([docs][cloud-storage-docs]) allows you to store +data on Google infrastructure with very high reliability, performance and availability, +and can be used to distribute large data objects to users via direct download. + +[https://godoc.org/google.golang.org/cloud/storage](https://godoc.org/google.golang.org/cloud/storage) + + +```go +// Read the object1 from bucket. +rc, err := storage.NewReader(ctx, "bucket", "object1") +if err != nil { + log.Fatal(err) +} +slurp, err := ioutil.ReadAll(rc) +rc.Close() +if err != nil { + log.Fatal(err) +} +``` + +## Google Cloud Pub/Sub (Alpha) + +> Google Cloud Pub/Sub is in **Alpha status**. As a result, it might change in +> backward-incompatible ways and is not recommended for production use. It is not +> subject to any SLA or deprecation policy. + +[Google Cloud Pub/Sub][cloud-pubsub] ([docs][cloud-pubsub-docs]) allows you to connect +your services with reliable, many-to-many, asynchronous messaging hosted on Google's +infrastructure. Cloud Pub/Sub automatically scales as you need it and provides a foundation +for building your own robust, global services. + +[https://godoc.org/google.golang.org/cloud/pubsub](https://godoc.org/google.golang.org/cloud/pubsub) + + +```go +// Publish "hello world" on topic1. +msgIDs, err := pubsub.Publish(ctx, "topic1", &pubsub.Message{ + Data: []byte("hello world"), +}) +if err != nil { + log.Println(err) +} +// Pull messages via subscription1. +msgs, err := pubsub.Pull(ctx, "subscription1", 1) +if err != nil { + log.Println(err) +} +``` + +## Contributing + +Contributions are welcome. Please, see the +[CONTRIBUTING](https://github.com/GoogleCloudPlatform/gcloud-golang/blob/master/CONTRIBUTING.md) +document for details. We're using Gerrit for our code reviews. Please don't open pull +requests against this repo, new pull requests will be automatically closed. + +Please note that this project is released with a Contributor Code of Conduct. +By participating in this project you agree to abide by its terms. +See [Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/gcloud-golang/blob/master/CONTRIBUTING.md#contributor-code-of-conduct) +for more information. + +[cloud-datastore]: https://cloud.google.com/datastore/ +[cloud-datastore-docs]: https://cloud.google.com/datastore/docs +[cloud-datastore-activation]: https://cloud.google.com/datastore/docs/activate + +[cloud-pubsub]: https://cloud.google.com/pubsub/ +[cloud-pubsub-docs]: https://cloud.google.com/pubsub/docs + +[cloud-storage]: https://cloud.google.com/storage/ +[cloud-storage-docs]: https://cloud.google.com/storage/docs/overview +[cloud-storage-create-bucket]: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets diff --git a/components/engine/vendor/src/google.golang.org/cloud/cloud.go b/components/engine/vendor/src/google.golang.org/cloud/cloud.go new file mode 100644 index 0000000000..96d36baf2c --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/cloud.go @@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cloud contains Google Cloud Platform APIs related types +// and common functions. +package cloud // import "google.golang.org/cloud" + +import ( + "net/http" + + "golang.org/x/net/context" + "google.golang.org/cloud/internal" +) + +// NewContext returns a new context that uses the provided http.Client. +// Provided http.Client is responsible to authorize and authenticate +// the requests made to the Google Cloud APIs. +// It mutates the client's original Transport to append the cloud +// package's user-agent to the outgoing requests. +// You can obtain the project ID from the Google Developers Console, +// https://console.developers.google.com. +func NewContext(projID string, c *http.Client) context.Context { + if c == nil { + panic("invalid nil *http.Client passed to NewContext") + } + return WithContext(context.Background(), projID, c) +} + +// WithContext returns a new context in a similar way NewContext does, +// but initiates the new context with the specified parent. +func WithContext(parent context.Context, projID string, c *http.Client) context.Context { + // TODO(bradfitz): delete internal.Transport. It's too wrappy for what it does. + // Do User-Agent some other way. + if _, ok := c.Transport.(*internal.Transport); !ok { + c.Transport = &internal.Transport{Base: c.Transport} + } + return internal.WithContext(parent, projID, c) +} diff --git a/components/engine/vendor/src/google.golang.org/cloud/compute/metadata/metadata.go b/components/engine/vendor/src/google.golang.org/cloud/compute/metadata/metadata.go new file mode 100644 index 0000000000..972972dd76 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/compute/metadata/metadata.go @@ -0,0 +1,327 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package metadata provides access to Google Compute Engine (GCE) +// metadata and API service accounts. +// +// This package is a wrapper around the GCE metadata service, +// as documented at https://developers.google.com/compute/docs/metadata. +package metadata // import "google.golang.org/cloud/compute/metadata" + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" + + "google.golang.org/cloud/internal" +) + +type cachedValue struct { + k string + trim bool + mu sync.Mutex + v string +} + +var ( + projID = &cachedValue{k: "project/project-id", trim: true} + projNum = &cachedValue{k: "project/numeric-project-id", trim: true} + instID = &cachedValue{k: "instance/id", trim: true} +) + +var metaClient = &http.Client{ + Transport: &internal.Transport{ + Base: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 750 * time.Millisecond, + KeepAlive: 30 * time.Second, + }).Dial, + ResponseHeaderTimeout: 750 * time.Millisecond, + }, + }, +} + +// NotDefinedError is returned when requested metadata is not defined. +// +// The underlying string is the suffix after "/computeMetadata/v1/". +// +// This error is not returned if the value is defined to be the empty +// string. +type NotDefinedError string + +func (suffix NotDefinedError) Error() string { + return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix)) +} + +// Get returns a value from the metadata service. +// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/". +// +// If the GCE_METADATA_HOST environment variable is not defined, a default of +// 169.254.169.254 will be used instead. +// +// If the requested metadata is not defined, the returned error will +// be of type NotDefinedError. +func Get(suffix string) (string, error) { + val, _, err := getETag(suffix) + return val, err +} + +// getETag returns a value from the metadata service as well as the associated +// ETag. This func is otherwise equivalent to Get. +func getETag(suffix string) (value, etag string, err error) { + // Using a fixed IP makes it very difficult to spoof the metadata service in + // a container, which is an important use-case for local testing of cloud + // deployments. To enable spoofing of the metadata service, the environment + // variable GCE_METADATA_HOST is first inspected to decide where metadata + // requests shall go. + host := os.Getenv("GCE_METADATA_HOST") + if host == "" { + // Using 169.254.169.254 instead of "metadata" here because Go + // binaries built with the "netgo" tag and without cgo won't + // know the search suffix for "metadata" is + // ".google.internal", and this IP address is documented as + // being stable anyway. + host = "169.254.169.254" + } + url := "http://" + host + "/computeMetadata/v1/" + suffix + req, _ := http.NewRequest("GET", url, nil) + req.Header.Set("Metadata-Flavor", "Google") + res, err := metaClient.Do(req) + if err != nil { + return "", "", err + } + defer res.Body.Close() + if res.StatusCode == http.StatusNotFound { + return "", "", NotDefinedError(suffix) + } + if res.StatusCode != 200 { + return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url) + } + all, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", "", err + } + return string(all), res.Header.Get("Etag"), nil +} + +func getTrimmed(suffix string) (s string, err error) { + s, err = Get(suffix) + s = strings.TrimSpace(s) + return +} + +func (c *cachedValue) get() (v string, err error) { + defer c.mu.Unlock() + c.mu.Lock() + if c.v != "" { + return c.v, nil + } + if c.trim { + v, err = getTrimmed(c.k) + } else { + v, err = Get(c.k) + } + if err == nil { + c.v = v + } + return +} + +var onGCE struct { + sync.Mutex + set bool + v bool +} + +// OnGCE reports whether this process is running on Google Compute Engine. +func OnGCE() bool { + defer onGCE.Unlock() + onGCE.Lock() + if onGCE.set { + return onGCE.v + } + onGCE.set = true + + // We use the DNS name of the metadata service here instead of the IP address + // because we expect that to fail faster in the not-on-GCE case. + res, err := metaClient.Get("http://metadata.google.internal") + if err != nil { + return false + } + onGCE.v = res.Header.Get("Metadata-Flavor") == "Google" + return onGCE.v +} + +// Subscribe subscribes to a value from the metadata service. +// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/". +// +// Subscribe calls fn with the latest metadata value indicated by the provided +// suffix. If the metadata value is deleted, fn is called with the empty string +// and ok false. Subscribe blocks until fn returns a non-nil error or the value +// is deleted. Subscribe returns the error value returned from the last call to +// fn, which may be nil when ok == false. +func Subscribe(suffix string, fn func(v string, ok bool) error) error { + const failedSubscribeSleep = time.Second * 5 + + // First check to see if the metadata value exists at all. + val, lastETag, err := getETag(suffix) + if err != nil { + return err + } + + if err := fn(val, true); err != nil { + return err + } + + ok := true + suffix += "?wait_for_change=true&last_etag=" + for { + val, etag, err := getETag(suffix + url.QueryEscape(lastETag)) + if err != nil { + if _, deleted := err.(NotDefinedError); !deleted { + time.Sleep(failedSubscribeSleep) + continue // Retry on other errors. + } + ok = false + } + lastETag = etag + + if err := fn(val, ok); err != nil || !ok { + return err + } + } +} + +// ProjectID returns the current instance's project ID string. +func ProjectID() (string, error) { return projID.get() } + +// NumericProjectID returns the current instance's numeric project ID. +func NumericProjectID() (string, error) { return projNum.get() } + +// InternalIP returns the instance's primary internal IP address. +func InternalIP() (string, error) { + return getTrimmed("instance/network-interfaces/0/ip") +} + +// ExternalIP returns the instance's primary external (public) IP address. +func ExternalIP() (string, error) { + return getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip") +} + +// Hostname returns the instance's hostname. This will be of the form +// ".c..internal". +func Hostname() (string, error) { + return getTrimmed("instance/hostname") +} + +// InstanceTags returns the list of user-defined instance tags, +// assigned when initially creating a GCE instance. +func InstanceTags() ([]string, error) { + var s []string + j, err := Get("instance/tags") + if err != nil { + return nil, err + } + if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil { + return nil, err + } + return s, nil +} + +// InstanceID returns the current VM's numeric instance ID. +func InstanceID() (string, error) { + return instID.get() +} + +// InstanceName returns the current VM's instance ID string. +func InstanceName() (string, error) { + host, err := Hostname() + if err != nil { + return "", err + } + return strings.Split(host, ".")[0], nil +} + +// Zone returns the current VM's zone, such as "us-central1-b". +func Zone() (string, error) { + zone, err := getTrimmed("instance/zone") + // zone is of the form "projects//zones/". + if err != nil { + return "", err + } + return zone[strings.LastIndex(zone, "/")+1:], nil +} + +// InstanceAttributes returns the list of user-defined attributes, +// assigned when initially creating a GCE VM instance. The value of an +// attribute can be obtained with InstanceAttributeValue. +func InstanceAttributes() ([]string, error) { return lines("instance/attributes/") } + +// ProjectAttributes returns the list of user-defined attributes +// applying to the project as a whole, not just this VM. The value of +// an attribute can be obtained with ProjectAttributeValue. +func ProjectAttributes() ([]string, error) { return lines("project/attributes/") } + +func lines(suffix string) ([]string, error) { + j, err := Get(suffix) + if err != nil { + return nil, err + } + s := strings.Split(strings.TrimSpace(j), "\n") + for i := range s { + s[i] = strings.TrimSpace(s[i]) + } + return s, nil +} + +// InstanceAttributeValue returns the value of the provided VM +// instance attribute. +// +// If the requested attribute is not defined, the returned error will +// be of type NotDefinedError. +// +// InstanceAttributeValue may return ("", nil) if the attribute was +// defined to be the empty string. +func InstanceAttributeValue(attr string) (string, error) { + return Get("instance/attributes/" + attr) +} + +// ProjectAttributeValue returns the value of the provided +// project attribute. +// +// If the requested attribute is not defined, the returned error will +// be of type NotDefinedError. +// +// ProjectAttributeValue may return ("", nil) if the attribute was +// defined to be the empty string. +func ProjectAttributeValue(attr string) (string, error) { + return Get("project/attributes/" + attr) +} + +// Scopes returns the service account scopes for the given account. +// The account may be empty or the string "default" to use the instance's +// main account. +func Scopes(serviceAccount string) ([]string, error) { + if serviceAccount == "" { + serviceAccount = "default" + } + return lines("instance/service-accounts/" + serviceAccount + "/scopes") +} diff --git a/components/engine/vendor/src/google.golang.org/cloud/internal/cloud.go b/components/engine/vendor/src/google.golang.org/cloud/internal/cloud.go new file mode 100644 index 0000000000..8b0db1b5da --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/internal/cloud.go @@ -0,0 +1,128 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package internal provides support for the cloud packages. +// +// Users should not import this package directly. +package internal + +import ( + "fmt" + "net/http" + "sync" + + "golang.org/x/net/context" +) + +type contextKey struct{} + +func WithContext(parent context.Context, projID string, c *http.Client) context.Context { + if c == nil { + panic("nil *http.Client passed to WithContext") + } + if projID == "" { + panic("empty project ID passed to WithContext") + } + return context.WithValue(parent, contextKey{}, &cloudContext{ + ProjectID: projID, + HTTPClient: c, + }) +} + +const userAgent = "gcloud-golang/0.1" + +type cloudContext struct { + ProjectID string + HTTPClient *http.Client + + mu sync.Mutex // guards svc + svc map[string]interface{} // e.g. "storage" => *rawStorage.Service +} + +// Service returns the result of the fill function if it's never been +// called before for the given name (which is assumed to be an API +// service name, like "datastore"). If it has already been cached, the fill +// func is not run. +// It's safe for concurrent use by multiple goroutines. +func Service(ctx context.Context, name string, fill func(*http.Client) interface{}) interface{} { + return cc(ctx).service(name, fill) +} + +func (c *cloudContext) service(name string, fill func(*http.Client) interface{}) interface{} { + c.mu.Lock() + defer c.mu.Unlock() + + if c.svc == nil { + c.svc = make(map[string]interface{}) + } else if v, ok := c.svc[name]; ok { + return v + } + v := fill(c.HTTPClient) + c.svc[name] = v + return v +} + +// Transport is an http.RoundTripper that appends +// Google Cloud client's user-agent to the original +// request's user-agent header. +type Transport struct { + // Base represents the actual http.RoundTripper + // the requests will be delegated to. + Base http.RoundTripper +} + +// RoundTrip appends a user-agent to the existing user-agent +// header and delegates the request to the base http.RoundTripper. +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + req = cloneRequest(req) + ua := req.Header.Get("User-Agent") + if ua == "" { + ua = userAgent + } else { + ua = fmt.Sprintf("%s %s", ua, userAgent) + } + req.Header.Set("User-Agent", ua) + return t.Base.RoundTrip(req) +} + +// cloneRequest returns a clone of the provided *http.Request. +// The clone is a shallow copy of the struct and its Header map. +func cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header) + for k, s := range r.Header { + r2.Header[k] = s + } + return r2 +} + +func ProjID(ctx context.Context) string { + return cc(ctx).ProjectID +} + +func HTTPClient(ctx context.Context) *http.Client { + return cc(ctx).HTTPClient +} + +// cc returns the internal *cloudContext (cc) state for a context.Context. +// It panics if the user did it wrong. +func cc(ctx context.Context) *cloudContext { + if c, ok := ctx.Value(contextKey{}).(*cloudContext); ok { + return c + } + panic("invalid context.Context type; it should be created with cloud.NewContext") +} diff --git a/components/engine/vendor/src/google.golang.org/cloud/internal/opts/option.go b/components/engine/vendor/src/google.golang.org/cloud/internal/opts/option.go new file mode 100644 index 0000000000..c5ccf4f56d --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/internal/opts/option.go @@ -0,0 +1,24 @@ +// Package opts holds the DialOpts struct, configurable by +// cloud.ClientOptions to set up transports for cloud packages. +// +// This is a separate page to prevent cycles between the core +// cloud packages. +package opts + +import ( + "net/http" + + "golang.org/x/oauth2" + "google.golang.org/grpc" +) + +type DialOpt struct { + Endpoint string + Scopes []string + UserAgent string + + TokenSource oauth2.TokenSource + + HTTPClient *http.Client + GRPCClient *grpc.ClientConn +} diff --git a/components/engine/vendor/src/google.golang.org/cloud/internal/transport/cancelreq.go b/components/engine/vendor/src/google.golang.org/cloud/internal/transport/cancelreq.go new file mode 100644 index 0000000000..ddae71ccef --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/internal/transport/cancelreq.go @@ -0,0 +1,29 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.5 + +package transport + +import "net/http" + +// makeReqCancel returns a closure that cancels the given http.Request +// when called. +func makeReqCancel(req *http.Request) func(http.RoundTripper) { + c := make(chan struct{}) + req.Cancel = c + return func(http.RoundTripper) { + close(c) + } +} diff --git a/components/engine/vendor/src/google.golang.org/cloud/internal/transport/cancelreq_legacy.go b/components/engine/vendor/src/google.golang.org/cloud/internal/transport/cancelreq_legacy.go new file mode 100644 index 0000000000..c11a4ddebc --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/internal/transport/cancelreq_legacy.go @@ -0,0 +1,31 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !go1.5 + +package transport + +import "net/http" + +// makeReqCancel returns a closure that cancels the given http.Request +// when called. +func makeReqCancel(req *http.Request) func(http.RoundTripper) { + // Go 1.4 and prior do not have a reliable way of cancelling a request. + // Transport.CancelRequest will only work if the request is already in-flight. + return func(r http.RoundTripper) { + if t, ok := r.(*http.Transport); ok { + t.CancelRequest(req) + } + } +} diff --git a/components/engine/vendor/src/google.golang.org/cloud/internal/transport/dial.go b/components/engine/vendor/src/google.golang.org/cloud/internal/transport/dial.go new file mode 100644 index 0000000000..29624410d3 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/internal/transport/dial.go @@ -0,0 +1,134 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "errors" + "fmt" + "net/http" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/cloud" + "google.golang.org/cloud/internal/opts" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/oauth" +) + +// ErrHTTP is returned when on a non-200 HTTP response. +type ErrHTTP struct { + StatusCode int + Body []byte + err error +} + +func (e *ErrHTTP) Error() string { + if e.err == nil { + return fmt.Sprintf("error during call, http status code: %v %s", e.StatusCode, e.Body) + } + return e.err.Error() +} + +// NewHTTPClient returns an HTTP client for use communicating with a Google cloud +// service, configured with the given ClientOptions. It also returns the endpoint +// for the service as specified in the options. +func NewHTTPClient(ctx context.Context, opt ...cloud.ClientOption) (*http.Client, string, error) { + var o opts.DialOpt + for _, opt := range opt { + opt.Resolve(&o) + } + if o.GRPCClient != nil { + return nil, "", errors.New("unsupported GRPC base transport specified") + } + // TODO(djd): Wrap all http.Clients with appropriate internal version to add + // UserAgent header and prepend correct endpoint. + if o.HTTPClient != nil { + return o.HTTPClient, o.Endpoint, nil + } + if o.TokenSource == nil { + var err error + o.TokenSource, err = google.DefaultTokenSource(ctx, o.Scopes...) + if err != nil { + return nil, "", fmt.Errorf("google.DefaultTokenSource: %v", err) + } + } + return oauth2.NewClient(ctx, o.TokenSource), o.Endpoint, nil +} + +// NewProtoClient returns a ProtoClient for communicating with a Google cloud service, +// configured with the given ClientOptions. +func NewProtoClient(ctx context.Context, opt ...cloud.ClientOption) (*ProtoClient, error) { + var o opts.DialOpt + for _, opt := range opt { + opt.Resolve(&o) + } + if o.GRPCClient != nil { + return nil, errors.New("unsupported GRPC base transport specified") + } + var client *http.Client + switch { + case o.HTTPClient != nil: + if o.TokenSource != nil { + return nil, errors.New("at most one of WithTokenSource or WithBaseHTTP may be provided") + } + client = o.HTTPClient + case o.TokenSource != nil: + client = oauth2.NewClient(ctx, o.TokenSource) + default: + var err error + client, err = google.DefaultClient(ctx, o.Scopes...) + if err != nil { + return nil, err + } + } + + return &ProtoClient{ + client: client, + endpoint: o.Endpoint, + userAgent: o.UserAgent, + }, nil +} + +// DialGRPC returns a GRPC connection for use communicating with a Google cloud +// service, configured with the given ClientOptions. +func DialGRPC(ctx context.Context, opt ...cloud.ClientOption) (*grpc.ClientConn, error) { + var o opts.DialOpt + for _, opt := range opt { + opt.Resolve(&o) + } + if o.HTTPClient != nil { + return nil, errors.New("unsupported HTTP base transport specified") + } + if o.GRPCClient != nil { + return o.GRPCClient, nil + } + if o.TokenSource == nil { + var err error + o.TokenSource, err = google.DefaultTokenSource(ctx, o.Scopes...) + if err != nil { + return nil, fmt.Errorf("google.DefaultTokenSource: %v", err) + } + } + grpcOpts := []grpc.DialOption{ + grpc.WithPerRPCCredentials(oauth.TokenSource{o.TokenSource}), + grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")), + } + if o.UserAgent != "" { + grpcOpts = append(grpcOpts, grpc.WithUserAgent(o.UserAgent)) + } + return grpc.Dial(o.Endpoint, grpcOpts...) +} diff --git a/components/engine/vendor/src/google.golang.org/cloud/internal/transport/proto.go b/components/engine/vendor/src/google.golang.org/cloud/internal/transport/proto.go new file mode 100644 index 0000000000..05b11cde1e --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/internal/transport/proto.go @@ -0,0 +1,80 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "bytes" + "io/ioutil" + "net/http" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" +) + +type ProtoClient struct { + client *http.Client + endpoint string + userAgent string +} + +func (c *ProtoClient) Call(ctx context.Context, method string, req, resp proto.Message) error { + payload, err := proto.Marshal(req) + if err != nil { + return err + } + + httpReq, err := http.NewRequest("POST", c.endpoint+method, bytes.NewReader(payload)) + if err != nil { + return err + } + httpReq.Header.Set("Content-Type", "application/x-protobuf") + if ua := c.userAgent; ua != "" { + httpReq.Header.Set("User-Agent", ua) + } + + errc := make(chan error, 1) + cancel := makeReqCancel(httpReq) + + go func() { + r, err := c.client.Do(httpReq) + if err != nil { + errc <- err + return + } + defer r.Body.Close() + + body, err := ioutil.ReadAll(r.Body) + if r.StatusCode != http.StatusOK { + err = &ErrHTTP{ + StatusCode: r.StatusCode, + Body: body, + err: err, + } + } + if err != nil { + errc <- err + return + } + errc <- proto.Unmarshal(body, resp) + }() + + select { + case <-ctx.Done(): + cancel(c.client.Transport) // Cancel the HTTP request. + return ctx.Err() + case err := <-errc: + return err + } +} diff --git a/components/engine/vendor/src/google.golang.org/cloud/key.json.enc b/components/engine/vendor/src/google.golang.org/cloud/key.json.enc new file mode 100644 index 0000000000..2f673a84b1 Binary files /dev/null and b/components/engine/vendor/src/google.golang.org/cloud/key.json.enc differ diff --git a/components/engine/vendor/src/google.golang.org/cloud/logging/logging.go b/components/engine/vendor/src/google.golang.org/cloud/logging/logging.go new file mode 100644 index 0000000000..bd33e26ecf --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/logging/logging.go @@ -0,0 +1,468 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package logging contains a Google Cloud Logging client. +// +// This package is experimental and subject to API changes. +package logging // import "google.golang.org/cloud/logging" + +import ( + "errors" + "io" + "log" + "sync" + "time" + + "golang.org/x/net/context" + api "google.golang.org/api/logging/v1beta3" + "google.golang.org/cloud" + "google.golang.org/cloud/internal/transport" +) + +// Scope is the OAuth2 scope necessary to use Google Cloud Logging. +const Scope = api.LoggingWriteScope + +// Level is the log level. +type Level int + +const ( + // Default means no assigned severity level. + Default Level = iota + Debug + Info + Warning + Error + Critical + Alert + Emergency + nLevel +) + +var levelName = [nLevel]string{ + Default: "", + Debug: "DEBUG", + Info: "INFO", + Warning: "WARNING", + Error: "ERROR", + Critical: "CRITICAL", + Alert: "ALERT", + Emergency: "EMERGENCY", +} + +func (v Level) String() string { + return levelName[v] +} + +// Client is a Google Cloud Logging client. +// It must be constructed via NewClient. +type Client struct { + svc *api.Service + logs *api.ProjectsLogsEntriesService + projID string + logName string + writer [nLevel]io.Writer + logger [nLevel]*log.Logger + + mu sync.Mutex + queued []*api.LogEntry + curFlush *flushCall // currently in-flight flush + flushTimer *time.Timer // nil before first use + timerActive bool // whether flushTimer is armed + inFlight int // number of log entries sent to API service but not yet ACKed + + // For testing: + timeNow func() time.Time // optional + + // ServiceName may be "appengine.googleapis.com", + // "compute.googleapis.com" or "custom.googleapis.com". + // + // The default is "custom.googleapis.com". + // + // The service name is only used by the API server to + // determine which of the labels are used to index the logs. + ServiceName string + + // CommonLabels are metadata labels that apply to all log + // entries in this request, so that you don't have to repeat + // them in each log entry's metadata.labels field. If any of + // the log entries contains a (key, value) with the same key + // that is in CommonLabels, then the entry's (key, value) + // overrides the one in CommonLabels. + CommonLabels map[string]string + + // BufferLimit is the maximum number of items to keep in memory + // before flushing. Zero means automatic. A value of 1 means to + // flush after each log entry. + // The default is currently 10,000. + BufferLimit int + + // FlushAfter optionally specifies a threshold count at which buffered + // log entries are flushed, even if the BufferInterval has not yet + // been reached. + // The default is currently 10. + FlushAfter int + + // BufferInterval is the maximum amount of time that an item + // should remain buffered in memory before being flushed to + // the logging service. + // The default is currently 1 second. + BufferInterval time.Duration + + // Overflow is a function which runs when the Log function + // overflows its configured buffer limit. If nil, the log + // entry is dropped. The return value from Overflow is + // returned by Log. + Overflow func(*Client, Entry) error +} + +func (c *Client) flushAfter() int { + if v := c.FlushAfter; v > 0 { + return v + } + return 10 +} + +func (c *Client) bufferInterval() time.Duration { + if v := c.BufferInterval; v > 0 { + return v + } + return time.Second +} + +func (c *Client) bufferLimit() int { + if v := c.BufferLimit; v > 0 { + return v + } + return 10000 +} + +func (c *Client) serviceName() string { + if v := c.ServiceName; v != "" { + return v + } + return "custom.googleapis.com" +} + +func (c *Client) now() time.Time { + if now := c.timeNow; now != nil { + return now() + } + return time.Now() +} + +// Writer returns an io.Writer for the provided log level. +// +// Each Write call on the returned Writer generates a log entry. +// +// This Writer accessor does not allocate, so callers do not need to +// cache. +func (c *Client) Writer(v Level) io.Writer { return c.writer[v] } + +// Logger returns a *log.Logger for the provided log level. +// +// A Logger for each Level is pre-allocated by NewClient with an empty +// prefix and no flags. This Logger accessor does not allocate. +// Callers wishing to use alternate flags (such as log.Lshortfile) may +// mutate the returned Logger with SetFlags. Such mutations affect all +// callers in the program. +func (c *Client) Logger(v Level) *log.Logger { return c.logger[v] } + +type levelWriter struct { + level Level + c *Client +} + +func (w levelWriter) Write(p []byte) (n int, err error) { + return len(p), w.c.Log(Entry{ + Level: w.level, + Payload: string(p), + }) +} + +// Entry is a log entry. +type Entry struct { + // Time is the time of the entry. If the zero value, the current time is used. + Time time.Time + + // Level is log entry's severity level. + // The zero value means no assigned severity level. + Level Level + + // Payload must be either a string, []byte, or something that + // marshals via the encoding/json package to a JSON object + // (and not any other type of JSON value). + Payload interface{} + + // Labels optionally specifies key/value labels for the log entry. + // Depending on the Client's ServiceName, these are indexed differently + // by the Cloud Logging Service. + // See https://cloud.google.com/logging/docs/logs_index + // The Client.Log method takes ownership of this map. + Labels map[string]string + + // TODO: de-duping id +} + +func (c *Client) apiEntry(e Entry) (*api.LogEntry, error) { + t := e.Time + if t.IsZero() { + t = c.now() + } + + ent := &api.LogEntry{ + Metadata: &api.LogEntryMetadata{ + Timestamp: t.UTC().Format(time.RFC3339Nano), + ServiceName: c.serviceName(), + Severity: e.Level.String(), + Labels: e.Labels, + }, + } + switch p := e.Payload.(type) { + case string: + ent.TextPayload = p + case []byte: + ent.TextPayload = string(p) + default: + ent.StructPayload = api.LogEntryStructPayload(p) + } + return ent, nil +} + +// LogSync logs e synchronously without any buffering. +// This is mostly intended for debugging or critical errors. +func (c *Client) LogSync(e Entry) error { + ent, err := c.apiEntry(e) + if err != nil { + return err + } + _, err = c.logs.Write(c.projID, c.logName, &api.WriteLogEntriesRequest{ + CommonLabels: c.CommonLabels, + Entries: []*api.LogEntry{ent}, + }).Do() + return err +} + +var ErrOverflow = errors.New("logging: log entry overflowed buffer limits") + +// Log queues an entry to be sent to the logging service, subject to the +// Client's parameters. By default, the log will be flushed within +// one second. +// Log only returns an error if the entry is invalid or the queue is at +// capacity. If the queue is at capacity and the entry can't be added, +// Log returns either ErrOverflow when c.Overflow is nil, or the +// value returned by c.Overflow. +func (c *Client) Log(e Entry) error { + ent, err := c.apiEntry(e) + if err != nil { + return err + } + + c.mu.Lock() + buffered := len(c.queued) + c.inFlight + + if buffered >= c.bufferLimit() { + c.mu.Unlock() + if fn := c.Overflow; fn != nil { + return fn(c, e) + } + return ErrOverflow + } + defer c.mu.Unlock() + + c.queued = append(c.queued, ent) + if len(c.queued) >= c.flushAfter() { + c.scheduleFlushLocked(0) + return nil + } + c.scheduleFlushLocked(c.bufferInterval()) + return nil +} + +// c.mu must be held. +// +// d will be one of two values: either c.BufferInterval (or its +// default value) or 0. +func (c *Client) scheduleFlushLocked(d time.Duration) { + if c.inFlight > 0 { + // For now to keep things simple, only allow one HTTP + // request in flight at a time. + return + } + switch { + case c.flushTimer == nil: + // First flush. + c.timerActive = true + c.flushTimer = time.AfterFunc(d, c.timeoutFlush) + case c.timerActive && d == 0: + // Make it happen sooner. For example, this is the + // case of transitioning from a 1 second flush after + // the 1st item to an immediate flush after the 10th + // item. + c.flushTimer.Reset(0) + case !c.timerActive: + c.timerActive = true + c.flushTimer.Reset(d) + default: + // else timer was already active, also at d > 0, + // so we don't touch it and let it fire as previously + // scheduled. + } +} + +// timeoutFlush runs in its own goroutine (from time.AfterFunc) and +// flushes c.queued. +func (c *Client) timeoutFlush() { + c.mu.Lock() + c.timerActive = false + c.mu.Unlock() + if err := c.Flush(); err != nil { + // schedule another try + // TODO: smarter back-off? + c.mu.Lock() + c.scheduleFlushLocked(5 * time.Second) + c.mu.Unlock() + } +} + +// Ping reports whether the client's connection to Google Cloud +// Logging and the authentication configuration are valid. +func (c *Client) Ping() error { + _, err := c.logs.Write(c.projID, c.logName, &api.WriteLogEntriesRequest{ + Entries: []*api.LogEntry{}, + }).Do() + return err +} + +// Flush flushes any buffered log entries. +func (c *Client) Flush() error { + var numFlush int + c.mu.Lock() + for { + // We're already flushing (or we just started flushing + // ourselves), so wait for it to finish. + if f := c.curFlush; f != nil { + wasEmpty := len(c.queued) == 0 + c.mu.Unlock() + <-f.donec // wait for it + numFlush++ + // Terminate whenever there's an error, we've + // already flushed twice (one that was already + // in-flight when flush was called, and then + // one we instigated), or the queue was empty + // when we released the locked (meaning this + // in-flight flush removes everything present + // when Flush was called, and we don't need to + // kick off a new flush for things arriving + // afterward) + if f.err != nil || numFlush == 2 || wasEmpty { + return f.err + } + // Otherwise, re-obtain the lock and loop, + // starting over with seeing if a flush is in + // progress, which might've been started by a + // different goroutine before aquiring this + // lock again. + c.mu.Lock() + continue + } + + // Terminal case: + if len(c.queued) == 0 { + c.mu.Unlock() + return nil + } + + c.startFlushLocked() + } +} + +// requires c.mu be held. +func (c *Client) startFlushLocked() { + if c.curFlush != nil { + panic("internal error: flush already in flight") + } + if len(c.queued) == 0 { + panic("internal error: no items queued") + } + logEntries := c.queued + c.inFlight = len(logEntries) + c.queued = nil + + flush := &flushCall{ + donec: make(chan struct{}), + } + c.curFlush = flush + go func() { + defer close(flush.donec) + _, err := c.logs.Write(c.projID, c.logName, &api.WriteLogEntriesRequest{ + CommonLabels: c.CommonLabels, + Entries: logEntries, + }).Do() + flush.err = err + c.mu.Lock() + defer c.mu.Unlock() + c.inFlight = 0 + c.curFlush = nil + if err != nil { + c.queued = append(c.queued, logEntries...) + } else if len(c.queued) > 0 { + c.scheduleFlushLocked(c.bufferInterval()) + } + }() + +} + +const prodAddr = "https://logging.googleapis.com/" + +const userAgent = "gcloud-golang-logging/20150922" + +// NewClient returns a new log client, logging to the named log in the +// provided project. +// +// The exported fields on the returned client may be modified before +// the client is used for logging. Once log entries are in flight, +// the fields must not be modified. +func NewClient(ctx context.Context, projectID, logName string, opts ...cloud.ClientOption) (*Client, error) { + httpClient, endpoint, err := transport.NewHTTPClient(ctx, append([]cloud.ClientOption{ + cloud.WithEndpoint(prodAddr), + cloud.WithScopes(api.CloudPlatformScope), + cloud.WithUserAgent(userAgent), + }, opts...)...) + if err != nil { + return nil, err + } + svc, err := api.New(httpClient) + if err != nil { + return nil, err + } + svc.BasePath = endpoint + c := &Client{ + svc: svc, + logs: api.NewProjectsLogsEntriesService(svc), + logName: logName, + projID: projectID, + } + for i := range c.writer { + level := Level(i) + c.writer[level] = levelWriter{level, c} + c.logger[level] = log.New(c.writer[level], "", 0) + } + return c, nil +} + +// flushCall is an in-flight or completed flush. +type flushCall struct { + donec chan struct{} // closed when response is in + err error // error is valid after wg is Done +} diff --git a/components/engine/vendor/src/google.golang.org/cloud/option.go b/components/engine/vendor/src/google.golang.org/cloud/option.go new file mode 100644 index 0000000000..d8614eb206 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/cloud/option.go @@ -0,0 +1,102 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloud + +import ( + "net/http" + + "golang.org/x/oauth2" + "google.golang.org/cloud/internal/opts" + "google.golang.org/grpc" +) + +// ClientOption is used when construct clients for each cloud service. +type ClientOption interface { + // Resolve configures the given DialOpts for this option. + Resolve(*opts.DialOpt) +} + +// WithTokenSource returns a ClientOption that specifies an OAuth2 token +// source to be used as the basis for authentication. +func WithTokenSource(s oauth2.TokenSource) ClientOption { + return withTokenSource{s} +} + +type withTokenSource struct{ ts oauth2.TokenSource } + +func (w withTokenSource) Resolve(o *opts.DialOpt) { + o.TokenSource = w.ts +} + +// WithEndpoint returns a ClientOption that overrides the default endpoint +// to be used for a service. +func WithEndpoint(url string) ClientOption { + return withEndpoint(url) +} + +type withEndpoint string + +func (w withEndpoint) Resolve(o *opts.DialOpt) { + o.Endpoint = string(w) +} + +// WithScopes returns a ClientOption that overrides the default OAuth2 scopes +// to be used for a service. +func WithScopes(scope ...string) ClientOption { + return withScopes(scope) +} + +type withScopes []string + +func (w withScopes) Resolve(o *opts.DialOpt) { + s := make([]string, len(w)) + copy(s, w) + o.Scopes = s +} + +// WithUserAgent returns a ClientOption that sets the User-Agent. +func WithUserAgent(ua string) ClientOption { + return withUA(ua) +} + +type withUA string + +func (w withUA) Resolve(o *opts.DialOpt) { o.UserAgent = string(w) } + +// WithBaseHTTP returns a ClientOption that specifies the HTTP client to +// use as the basis of communications. This option may only be used with +// services that support HTTP as their communication transport. +func WithBaseHTTP(client *http.Client) ClientOption { + return withBaseHTTP{client} +} + +type withBaseHTTP struct{ client *http.Client } + +func (w withBaseHTTP) Resolve(o *opts.DialOpt) { + o.HTTPClient = w.client +} + +// WithBaseGRPC returns a ClientOption that specifies the GRPC client +// connection to use as the basis of communications. This option many only be +// used with services that support HRPC as their communication transport. +func WithBaseGRPC(client *grpc.ClientConn) ClientOption { + return withBaseGRPC{client} +} + +type withBaseGRPC struct{ client *grpc.ClientConn } + +func (w withBaseGRPC) Resolve(o *opts.DialOpt) { + o.GRPCClient = w.client +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/.travis.yml b/components/engine/vendor/src/google.golang.org/grpc/.travis.yml new file mode 100644 index 0000000000..3f83776ec5 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/.travis.yml @@ -0,0 +1,14 @@ +language: go + +before_install: + - go get github.com/axw/gocov/gocov + - go get github.com/mattn/goveralls + - go get golang.org/x/tools/cmd/cover + +install: + - mkdir -p "$GOPATH/src/google.golang.org" + - mv "$TRAVIS_BUILD_DIR" "$GOPATH/src/google.golang.org/grpc" + +script: + - make test testrace + - make coverage diff --git a/components/engine/vendor/src/google.golang.org/grpc/CONTRIBUTING.md b/components/engine/vendor/src/google.golang.org/grpc/CONTRIBUTING.md new file mode 100644 index 0000000000..407d384a7c --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/CONTRIBUTING.md @@ -0,0 +1,23 @@ +# How to contribute + +We definitely welcome patches and contribution to grpc! Here is some guideline +and information about how to do so. + +## Getting started + +### Legal requirements + +In order to protect both you and ourselves, you will need to sign the +[Contributor License Agreement](https://cla.developers.google.com/clas). + +### Filing Issues +When filing an issue, make sure to answer these five questions: + +1. What version of Go are you using (`go version`)? +2. What operating system and processor architecture are you using? +3. What did you do? +4. What did you expect to see? +5. What did you see instead? + +### Contributing code +Unless otherwise noted, the Go source files are distributed under the BSD-style license found in the LICENSE file. diff --git a/components/engine/vendor/src/google.golang.org/grpc/Makefile b/components/engine/vendor/src/google.golang.org/grpc/Makefile new file mode 100644 index 0000000000..5bc38be209 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/Makefile @@ -0,0 +1,50 @@ +.PHONY: \ + all \ + deps \ + updatedeps \ + testdeps \ + updatetestdeps \ + build \ + proto \ + test \ + testrace \ + clean \ + +all: test testrace + +deps: + go get -d -v google.golang.org/grpc/... + +updatedeps: + go get -d -v -u -f google.golang.org/grpc/... + +testdeps: + go get -d -v -t google.golang.org/grpc/... + +updatetestdeps: + go get -d -v -t -u -f google.golang.org/grpc/... + +build: deps + go build google.golang.org/grpc/... + +proto: + @ if ! which protoc > /dev/null; then \ + echo "error: protoc not installed" >&2; \ + exit 1; \ + fi + go get -v github.com/golang/protobuf/protoc-gen-go + for file in $$(git ls-files '*.proto'); do \ + protoc -I $$(dirname $$file) --go_out=plugins=grpc:$$(dirname $$file) $$file; \ + done + +test: testdeps + go test -v -cpu 1,4 google.golang.org/grpc/... + +testrace: testdeps + go test -v -race -cpu 1,4 google.golang.org/grpc/... + +clean: + go clean google.golang.org/grpc/... + +coverage: testdeps + goveralls -v google.golang.org/grpc/... diff --git a/components/engine/vendor/src/google.golang.org/grpc/PATENTS b/components/engine/vendor/src/google.golang.org/grpc/PATENTS new file mode 100644 index 0000000000..619f9dbfe6 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the GRPC project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of GRPC, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of GRPC. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of GRPC or any code incorporated within this +implementation of GRPC constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of GRPC +shall terminate as of the date such litigation is filed. diff --git a/components/engine/vendor/src/google.golang.org/grpc/README.md b/components/engine/vendor/src/google.golang.org/grpc/README.md new file mode 100644 index 0000000000..37b05f0953 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/README.md @@ -0,0 +1,32 @@ +#gRPC-Go + +[![Build Status](https://travis-ci.org/grpc/grpc-go.svg)](https://travis-ci.org/grpc/grpc-go) [![GoDoc](https://godoc.org/google.golang.org/grpc?status.svg)](https://godoc.org/google.golang.org/grpc) + +The Go implementation of [gRPC](http://www.grpc.io/): A high performance, open source, general RPC framework that puts mobile and HTTP/2 first. For more information see the [gRPC Quick Start](http://www.grpc.io/docs/) guide. + +Installation +------------ + +To install this package, you need to install Go 1.4 or above and setup your Go workspace on your computer. The simplest way to install the library is to run: + +``` +$ go get google.golang.org/grpc +``` + +Prerequisites +------------- + +This requires Go 1.4 or above. + +Constraints +----------- +The grpc package should only depend on standard Go packages and a small number of exceptions. If your contribution introduces new dependencies which are NOT in the [list](http://godoc.org/google.golang.org/grpc?imports), you need a discussion with gRPC-Go authors and consultants. + +Documentation +------------- +See [API documentation](https://godoc.org/google.golang.org/grpc) for package and API descriptions and find examples in the [examples directory](examples/). + +Status +------ +Beta release + diff --git a/components/engine/vendor/src/google.golang.org/grpc/call.go b/components/engine/vendor/src/google.golang.org/grpc/call.go new file mode 100644 index 0000000000..9d815af39d --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/call.go @@ -0,0 +1,192 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +package grpc + +import ( + "io" + "time" + + "golang.org/x/net/context" + "golang.org/x/net/trace" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/transport" +) + +// recvResponse receives and parses an RPC response. +// On error, it returns the error and indicates whether the call should be retried. +// +// TODO(zhaoq): Check whether the received message sequence is valid. +func recvResponse(codec Codec, t transport.ClientTransport, c *callInfo, stream *transport.Stream, reply interface{}) error { + // Try to acquire header metadata from the server if there is any. + var err error + c.headerMD, err = stream.Header() + if err != nil { + return err + } + p := &parser{s: stream} + for { + if err = recv(p, codec, reply); err != nil { + if err == io.EOF { + break + } + return err + } + } + c.trailerMD = stream.Trailer() + return nil +} + +// sendRequest writes out various information of an RPC such as Context and Message. +func sendRequest(ctx context.Context, codec Codec, callHdr *transport.CallHdr, t transport.ClientTransport, args interface{}, opts *transport.Options) (_ *transport.Stream, err error) { + stream, err := t.NewStream(ctx, callHdr) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + if _, ok := err.(transport.ConnectionError); !ok { + t.CloseStream(stream, err) + } + } + }() + // TODO(zhaoq): Support compression. + outBuf, err := encode(codec, args, compressionNone) + if err != nil { + return nil, transport.StreamErrorf(codes.Internal, "grpc: %v", err) + } + err = t.Write(stream, outBuf, opts) + if err != nil { + return nil, err + } + // Sent successfully. + return stream, nil +} + +// callInfo contains all related configuration and information about an RPC. +type callInfo struct { + failFast bool + headerMD metadata.MD + trailerMD metadata.MD + traceInfo traceInfo // in trace.go +} + +// Invoke is called by the generated code. It sends the RPC request on the +// wire and returns after response is received. +func Invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) (err error) { + var c callInfo + for _, o := range opts { + if err := o.before(&c); err != nil { + return toRPCErr(err) + } + } + defer func() { + for _, o := range opts { + o.after(&c) + } + }() + if EnableTracing { + c.traceInfo.tr = trace.New("grpc.Sent."+methodFamily(method), method) + defer c.traceInfo.tr.Finish() + c.traceInfo.firstLine.client = true + if deadline, ok := ctx.Deadline(); ok { + c.traceInfo.firstLine.deadline = deadline.Sub(time.Now()) + } + c.traceInfo.tr.LazyLog(&c.traceInfo.firstLine, false) + // TODO(dsymonds): Arrange for c.traceInfo.firstLine.remoteAddr to be set. + defer func() { + if err != nil { + c.traceInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + c.traceInfo.tr.SetError() + } + }() + } + topts := &transport.Options{ + Last: true, + Delay: false, + } + var ( + lastErr error // record the error that happened + ) + for { + var ( + err error + t transport.ClientTransport + stream *transport.Stream + ) + // TODO(zhaoq): Need a formal spec of retry strategy for non-failfast rpcs. + if lastErr != nil && c.failFast { + return toRPCErr(lastErr) + } + callHdr := &transport.CallHdr{ + Host: cc.authority, + Method: method, + } + t, err = cc.dopts.picker.Pick(ctx) + if err != nil { + if lastErr != nil { + // This was a retry; return the error from the last attempt. + return toRPCErr(lastErr) + } + return toRPCErr(err) + } + if c.traceInfo.tr != nil { + c.traceInfo.tr.LazyLog(&payload{sent: true, msg: args}, true) + } + stream, err = sendRequest(ctx, cc.dopts.codec, callHdr, t, args, topts) + if err != nil { + if _, ok := err.(transport.ConnectionError); ok { + lastErr = err + continue + } + if lastErr != nil { + return toRPCErr(lastErr) + } + return toRPCErr(err) + } + // Receive the response + lastErr = recvResponse(cc.dopts.codec, t, &c, stream, reply) + if _, ok := lastErr.(transport.ConnectionError); ok { + continue + } + if c.traceInfo.tr != nil { + c.traceInfo.tr.LazyLog(&payload{sent: false, msg: reply}, true) + } + t.CloseStream(stream, lastErr) + if lastErr != nil { + return toRPCErr(lastErr) + } + return Errorf(stream.StatusCode(), stream.StatusDesc()) + } +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/clientconn.go b/components/engine/vendor/src/google.golang.org/grpc/clientconn.go new file mode 100644 index 0000000000..4729bbd6ab --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/clientconn.go @@ -0,0 +1,525 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +package grpc + +import ( + "errors" + "fmt" + "net" + "strings" + "sync" + "time" + + "golang.org/x/net/context" + "golang.org/x/net/trace" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/transport" +) + +var ( + // ErrUnspecTarget indicates that the target address is unspecified. + ErrUnspecTarget = errors.New("grpc: target is unspecified") + // ErrNoTransportSecurity indicates that there is no transport security + // being set for ClientConn. Users should either set one or explicityly + // call WithInsecure DialOption to disable security. + ErrNoTransportSecurity = errors.New("grpc: no transport security set (use grpc.WithInsecure() explicitly or set credentials)") + // ErrCredentialsMisuse indicates that users want to transmit security infomation + // (e.g., oauth2 token) which requires secure connection on an insecure + // connection. + ErrCredentialsMisuse = errors.New("grpc: the credentials require transport level security (use grpc.WithTransportAuthenticator() to set)") + // ErrClientConnClosing indicates that the operation is illegal because + // the session is closing. + ErrClientConnClosing = errors.New("grpc: the client connection is closing") + // ErrClientConnTimeout indicates that the connection could not be + // established or re-established within the specified timeout. + ErrClientConnTimeout = errors.New("grpc: timed out trying to connect") + // minimum time to give a connection to complete + minConnectTimeout = 20 * time.Second +) + +// dialOptions configure a Dial call. dialOptions are set by the DialOption +// values passed to Dial. +type dialOptions struct { + codec Codec + picker Picker + block bool + insecure bool + copts transport.ConnectOptions +} + +// DialOption configures how we set up the connection. +type DialOption func(*dialOptions) + +// WithCodec returns a DialOption which sets a codec for message marshaling and unmarshaling. +func WithCodec(c Codec) DialOption { + return func(o *dialOptions) { + o.codec = c + } +} + +// WithBlock returns a DialOption which makes caller of Dial blocks until the underlying +// connection is up. Without this, Dial returns immediately and connecting the server +// happens in background. +func WithBlock() DialOption { + return func(o *dialOptions) { + o.block = true + } +} + +func WithInsecure() DialOption { + return func(o *dialOptions) { + o.insecure = true + } +} + +// WithTransportCredentials returns a DialOption which configures a +// connection level security credentials (e.g., TLS/SSL). +func WithTransportCredentials(creds credentials.TransportAuthenticator) DialOption { + return func(o *dialOptions) { + o.copts.AuthOptions = append(o.copts.AuthOptions, creds) + } +} + +// WithPerRPCCredentials returns a DialOption which sets +// credentials which will place auth state on each outbound RPC. +func WithPerRPCCredentials(creds credentials.Credentials) DialOption { + return func(o *dialOptions) { + o.copts.AuthOptions = append(o.copts.AuthOptions, creds) + } +} + +// WithTimeout returns a DialOption that configures a timeout for dialing a client connection. +func WithTimeout(d time.Duration) DialOption { + return func(o *dialOptions) { + o.copts.Timeout = d + } +} + +// WithDialer returns a DialOption that specifies a function to use for dialing network addresses. +func WithDialer(f func(addr string, timeout time.Duration) (net.Conn, error)) DialOption { + return func(o *dialOptions) { + o.copts.Dialer = f + } +} + +// WithUserAgent returns a DialOption that specifies a user agent string for all the RPCs. +func WithUserAgent(s string) DialOption { + return func(o *dialOptions) { + o.copts.UserAgent = s + } +} + +// Dial creates a client connection the given target. +func Dial(target string, opts ...DialOption) (*ClientConn, error) { + cc := &ClientConn{ + target: target, + } + for _, opt := range opts { + opt(&cc.dopts) + } + if cc.dopts.codec == nil { + // Set the default codec. + cc.dopts.codec = protoCodec{} + } + if cc.dopts.picker == nil { + cc.dopts.picker = &unicastPicker{} + } + if err := cc.dopts.picker.Init(cc); err != nil { + return nil, err + } + colonPos := strings.LastIndex(target, ":") + if colonPos == -1 { + colonPos = len(target) + } + cc.authority = target[:colonPos] + return cc, nil +} + +// ConnectivityState indicates the state of a client connection. +type ConnectivityState int + +const ( + // Idle indicates the ClientConn is idle. + Idle ConnectivityState = iota + // Connecting indicates the ClienConn is connecting. + Connecting + // Ready indicates the ClientConn is ready for work. + Ready + // TransientFailure indicates the ClientConn has seen a failure but expects to recover. + TransientFailure + // Shutdown indicates the ClientConn has started shutting down. + Shutdown +) + +func (s ConnectivityState) String() string { + switch s { + case Idle: + return "IDLE" + case Connecting: + return "CONNECTING" + case Ready: + return "READY" + case TransientFailure: + return "TRANSIENT_FAILURE" + case Shutdown: + return "SHUTDOWN" + default: + panic(fmt.Sprintf("unknown connectivity state: %d", s)) + } +} + +// ClientConn represents a client connection to an RPC service. +type ClientConn struct { + target string + authority string + dopts dialOptions +} + +// State returns the connectivity state of cc. +// This is EXPERIMENTAL API. +func (cc *ClientConn) State() ConnectivityState { + return cc.dopts.picker.State() +} + +// WaitForStateChange blocks until the state changes to something other than the sourceState +// or timeout fires on cc. It returns false if timeout fires, and true otherwise. +// This is EXPERIMENTAL API. +func (cc *ClientConn) WaitForStateChange(timeout time.Duration, sourceState ConnectivityState) bool { + return cc.dopts.picker.WaitForStateChange(timeout, sourceState) +} + +// Close starts to tear down the ClientConn. +func (cc *ClientConn) Close() error { + return cc.dopts.picker.Close() +} + +// Conn is a client connection to a single destination. +type Conn struct { + target string + dopts dialOptions + shutdownChan chan struct{} + events trace.EventLog + + mu sync.Mutex + state ConnectivityState + stateCV *sync.Cond + // ready is closed and becomes nil when a new transport is up or failed + // due to timeout. + ready chan struct{} + transport transport.ClientTransport +} + +// NewConn creates a Conn. +func NewConn(cc *ClientConn) (*Conn, error) { + if cc.target == "" { + return nil, ErrUnspecTarget + } + c := &Conn{ + target: cc.target, + dopts: cc.dopts, + shutdownChan: make(chan struct{}), + } + if EnableTracing { + c.events = trace.NewEventLog("grpc.ClientConn", c.target) + } + if !c.dopts.insecure { + var ok bool + for _, cd := range c.dopts.copts.AuthOptions { + if _, ok := cd.(credentials.TransportAuthenticator); !ok { + continue + } + ok = true + } + if !ok { + return nil, ErrNoTransportSecurity + } + } else { + for _, cd := range c.dopts.copts.AuthOptions { + if cd.RequireTransportSecurity() { + return nil, ErrCredentialsMisuse + } + } + } + c.stateCV = sync.NewCond(&c.mu) + if c.dopts.block { + if err := c.resetTransport(false); err != nil { + c.Close() + return nil, err + } + // Start to monitor the error status of transport. + go c.transportMonitor() + } else { + // Start a goroutine connecting to the server asynchronously. + go func() { + if err := c.resetTransport(false); err != nil { + grpclog.Printf("Failed to dial %s: %v; please retry.", c.target, err) + c.Close() + return + } + c.transportMonitor() + }() + } + return c, nil +} + +// printf records an event in cc's event log, unless cc has been closed. +// REQUIRES cc.mu is held. +func (cc *Conn) printf(format string, a ...interface{}) { + if cc.events != nil { + cc.events.Printf(format, a...) + } +} + +// errorf records an error in cc's event log, unless cc has been closed. +// REQUIRES cc.mu is held. +func (cc *Conn) errorf(format string, a ...interface{}) { + if cc.events != nil { + cc.events.Errorf(format, a...) + } +} + +// State returns the connectivity state of the Conn +func (cc *Conn) State() ConnectivityState { + cc.mu.Lock() + defer cc.mu.Unlock() + return cc.state +} + +// WaitForStateChange blocks until the state changes to something other than the sourceState +// or timeout fires. It returns false if timeout fires and true otherwise. +// TODO(zhaoq): Rewrite for complex Picker. +func (cc *Conn) WaitForStateChange(timeout time.Duration, sourceState ConnectivityState) bool { + start := time.Now() + cc.mu.Lock() + defer cc.mu.Unlock() + if sourceState != cc.state { + return true + } + expired := timeout <= time.Since(start) + if expired { + return false + } + done := make(chan struct{}) + go func() { + select { + case <-time.After(timeout - time.Since(start)): + cc.mu.Lock() + expired = true + cc.stateCV.Broadcast() + cc.mu.Unlock() + case <-done: + } + }() + defer close(done) + for sourceState == cc.state { + cc.stateCV.Wait() + if expired { + return false + } + } + return true +} + +func (cc *Conn) resetTransport(closeTransport bool) error { + var retries int + start := time.Now() + for { + cc.mu.Lock() + cc.printf("connecting") + if cc.state == Shutdown { + cc.mu.Unlock() + return ErrClientConnClosing + } + cc.state = Connecting + cc.stateCV.Broadcast() + cc.mu.Unlock() + if closeTransport { + cc.transport.Close() + } + // Adjust timeout for the current try. + copts := cc.dopts.copts + if copts.Timeout < 0 { + cc.Close() + return ErrClientConnTimeout + } + if copts.Timeout > 0 { + copts.Timeout -= time.Since(start) + if copts.Timeout <= 0 { + cc.Close() + return ErrClientConnTimeout + } + } + sleepTime := backoff(retries) + timeout := sleepTime + if timeout < minConnectTimeout { + timeout = minConnectTimeout + } + if copts.Timeout == 0 || copts.Timeout > timeout { + copts.Timeout = timeout + } + connectTime := time.Now() + newTransport, err := transport.NewClientTransport(cc.target, &copts) + if err != nil { + cc.mu.Lock() + cc.errorf("transient failure: %v", err) + cc.state = TransientFailure + cc.stateCV.Broadcast() + if cc.ready != nil { + close(cc.ready) + cc.ready = nil + } + cc.mu.Unlock() + sleepTime -= time.Since(connectTime) + if sleepTime < 0 { + sleepTime = 0 + } + // Fail early before falling into sleep. + if cc.dopts.copts.Timeout > 0 && cc.dopts.copts.Timeout < sleepTime+time.Since(start) { + cc.mu.Lock() + cc.errorf("connection timeout") + cc.mu.Unlock() + cc.Close() + return ErrClientConnTimeout + } + closeTransport = false + time.Sleep(sleepTime) + retries++ + grpclog.Printf("grpc: ClientConn.resetTransport failed to create client transport: %v; Reconnecting to %q", err, cc.target) + continue + } + cc.mu.Lock() + cc.printf("ready") + if cc.state == Shutdown { + // cc.Close() has been invoked. + cc.mu.Unlock() + newTransport.Close() + return ErrClientConnClosing + } + cc.state = Ready + cc.stateCV.Broadcast() + cc.transport = newTransport + if cc.ready != nil { + close(cc.ready) + cc.ready = nil + } + cc.mu.Unlock() + return nil + } +} + +// Run in a goroutine to track the error in transport and create the +// new transport if an error happens. It returns when the channel is closing. +func (cc *Conn) transportMonitor() { + for { + select { + // shutdownChan is needed to detect the teardown when + // the ClientConn is idle (i.e., no RPC in flight). + case <-cc.shutdownChan: + return + case <-cc.transport.Error(): + cc.mu.Lock() + cc.state = TransientFailure + cc.stateCV.Broadcast() + cc.mu.Unlock() + if err := cc.resetTransport(true); err != nil { + // The ClientConn is closing. + cc.mu.Lock() + cc.printf("transport exiting: %v", err) + cc.mu.Unlock() + grpclog.Printf("grpc: ClientConn.transportMonitor exits due to: %v", err) + return + } + continue + } + } +} + +// Wait blocks until i) the new transport is up or ii) ctx is done or iii) cc is closed. +func (cc *Conn) Wait(ctx context.Context) (transport.ClientTransport, error) { + for { + cc.mu.Lock() + switch { + case cc.state == Shutdown: + cc.mu.Unlock() + return nil, ErrClientConnClosing + case cc.state == Ready: + cc.mu.Unlock() + return cc.transport, nil + default: + ready := cc.ready + if ready == nil { + ready = make(chan struct{}) + cc.ready = ready + } + cc.mu.Unlock() + select { + case <-ctx.Done(): + return nil, transport.ContextErr(ctx.Err()) + // Wait until the new transport is ready or failed. + case <-ready: + } + } + } +} + +// Close starts to tear down the Conn. Returns ErrClientConnClosing if +// it has been closed (mostly due to dial time-out). +// TODO(zhaoq): Make this synchronous to avoid unbounded memory consumption in +// some edge cases (e.g., the caller opens and closes many ClientConn's in a +// tight loop. +func (cc *Conn) Close() error { + cc.mu.Lock() + defer cc.mu.Unlock() + if cc.state == Shutdown { + return ErrClientConnClosing + } + cc.state = Shutdown + cc.stateCV.Broadcast() + if cc.events != nil { + cc.events.Finish() + cc.events = nil + } + if cc.ready != nil { + close(cc.ready) + cc.ready = nil + } + if cc.transport != nil { + cc.transport.Close() + } + if cc.shutdownChan != nil { + close(cc.shutdownChan) + } + return nil +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/codegen.sh b/components/engine/vendor/src/google.golang.org/grpc/codegen.sh new file mode 100755 index 0000000000..b009488842 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/codegen.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# This script serves as an example to demonstrate how to generate the gRPC-Go +# interface and the related messages from .proto file. +# +# It assumes the installation of i) Google proto buffer compiler at +# https://github.com/google/protobuf (after v2.6.1) and ii) the Go codegen +# plugin at https://github.com/golang/protobuf (after 2015-02-20). If you have +# not, please install them first. +# +# We recommend running this script at $GOPATH/src. +# +# If this is not what you need, feel free to make your own scripts. Again, this +# script is for demonstration purpose. +# +proto=$1 +protoc --go_out=plugins=grpc:. $proto diff --git a/components/engine/vendor/src/google.golang.org/grpc/codes/code_string.go b/components/engine/vendor/src/google.golang.org/grpc/codes/code_string.go new file mode 100644 index 0000000000..e6762d0845 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/codes/code_string.go @@ -0,0 +1,16 @@ +// generated by stringer -type=Code; DO NOT EDIT + +package codes + +import "fmt" + +const _Code_name = "OKCanceledUnknownInvalidArgumentDeadlineExceededNotFoundAlreadyExistsPermissionDeniedResourceExhaustedFailedPreconditionAbortedOutOfRangeUnimplementedInternalUnavailableDataLossUnauthenticated" + +var _Code_index = [...]uint8{0, 2, 10, 17, 32, 48, 56, 69, 85, 102, 120, 127, 137, 150, 158, 169, 177, 192} + +func (i Code) String() string { + if i+1 >= Code(len(_Code_index)) { + return fmt.Sprintf("Code(%d)", i) + } + return _Code_name[_Code_index[i]:_Code_index[i+1]] +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/codes/codes.go b/components/engine/vendor/src/google.golang.org/grpc/codes/codes.go new file mode 100644 index 0000000000..e14b464acf --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/codes/codes.go @@ -0,0 +1,159 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +// Package codes defines the canonical error codes used by gRPC. It is +// consistent across various languages. +package codes // import "google.golang.org/grpc/codes" + +// A Code is an unsigned 32-bit error code as defined in the gRPC spec. +type Code uint32 + +//go:generate stringer -type=Code + +const ( + // OK is returned on success. + OK Code = 0 + + // Canceled indicates the operation was cancelled (typically by the caller). + Canceled Code = 1 + + // Unknown error. An example of where this error may be returned is + // if a Status value received from another address space belongs to + // an error-space that is not known in this address space. Also + // errors raised by APIs that do not return enough error information + // may be converted to this error. + Unknown Code = 2 + + // InvalidArgument indicates client specified an invalid argument. + // Note that this differs from FailedPrecondition. It indicates arguments + // that are problematic regardless of the state of the system + // (e.g., a malformed file name). + InvalidArgument Code = 3 + + // DeadlineExceeded means operation expired before completion. + // For operations that change the state of the system, this error may be + // returned even if the operation has completed successfully. For + // example, a successful response from a server could have been delayed + // long enough for the deadline to expire. + DeadlineExceeded Code = 4 + + // NotFound means some requested entity (e.g., file or directory) was + // not found. + NotFound Code = 5 + + // AlreadyExists means an attempt to create an entity failed because one + // already exists. + AlreadyExists Code = 6 + + // PermissionDenied indicates the caller does not have permission to + // execute the specified operation. It must not be used for rejections + // caused by exhausting some resource (use ResourceExhausted + // instead for those errors). It must not be + // used if the caller cannot be identified (use Unauthenticated + // instead for those errors). + PermissionDenied Code = 7 + + // Unauthenticated indicates the request does not have valid + // authentication credentials for the operation. + Unauthenticated Code = 16 + + // ResourceExhausted indicates some resource has been exhausted, perhaps + // a per-user quota, or perhaps the entire file system is out of space. + ResourceExhausted Code = 8 + + // FailedPrecondition indicates operation was rejected because the + // system is not in a state required for the operation's execution. + // For example, directory to be deleted may be non-empty, an rmdir + // operation is applied to a non-directory, etc. + // + // A litmus test that may help a service implementor in deciding + // between FailedPrecondition, Aborted, and Unavailable: + // (a) Use Unavailable if the client can retry just the failing call. + // (b) Use Aborted if the client should retry at a higher-level + // (e.g., restarting a read-modify-write sequence). + // (c) Use FailedPrecondition if the client should not retry until + // the system state has been explicitly fixed. E.g., if an "rmdir" + // fails because the directory is non-empty, FailedPrecondition + // should be returned since the client should not retry unless + // they have first fixed up the directory by deleting files from it. + // (d) Use FailedPrecondition if the client performs conditional + // REST Get/Update/Delete on a resource and the resource on the + // server does not match the condition. E.g., conflicting + // read-modify-write on the same resource. + FailedPrecondition Code = 9 + + // Aborted indicates the operation was aborted, typically due to a + // concurrency issue like sequencer check failures, transaction aborts, + // etc. + // + // See litmus test above for deciding between FailedPrecondition, + // Aborted, and Unavailable. + Aborted Code = 10 + + // OutOfRange means operation was attempted past the valid range. + // E.g., seeking or reading past end of file. + // + // Unlike InvalidArgument, this error indicates a problem that may + // be fixed if the system state changes. For example, a 32-bit file + // system will generate InvalidArgument if asked to read at an + // offset that is not in the range [0,2^32-1], but it will generate + // OutOfRange if asked to read from an offset past the current + // file size. + // + // There is a fair bit of overlap between FailedPrecondition and + // OutOfRange. We recommend using OutOfRange (the more specific + // error) when it applies so that callers who are iterating through + // a space can easily look for an OutOfRange error to detect when + // they are done. + OutOfRange Code = 11 + + // Unimplemented indicates operation is not implemented or not + // supported/enabled in this service. + Unimplemented Code = 12 + + // Internal errors. Means some invariants expected by underlying + // system has been broken. If you see one of these errors, + // something is very broken. + Internal Code = 13 + + // Unavailable indicates the service is currently unavailable. + // This is a most likely a transient condition and may be corrected + // by retrying with a backoff. + // + // See litmus test above for deciding between FailedPrecondition, + // Aborted, and Unavailable. + Unavailable Code = 14 + + // DataLoss indicates unrecoverable data loss or corruption. + DataLoss Code = 15 +) diff --git a/components/engine/vendor/src/google.golang.org/grpc/credentials/credentials.go b/components/engine/vendor/src/google.golang.org/grpc/credentials/credentials.go new file mode 100644 index 0000000000..cde38dc4e4 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/credentials/credentials.go @@ -0,0 +1,239 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +// Package credentials implements various credentials supported by gRPC library, +// which encapsulate all the state needed by a client to authenticate with a +// server and make various assertions, e.g., about the client's identity, role, +// or whether it is authorized to make a particular call. +package credentials // import "google.golang.org/grpc/credentials" + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "net" + "strings" + "time" + + "golang.org/x/net/context" +) + +var ( + // alpnProtoStr are the specified application level protocols for gRPC. + alpnProtoStr = []string{"h2"} +) + +// Credentials defines the common interface all supported credentials must +// implement. +type Credentials interface { + // GetRequestMetadata gets the current request metadata, refreshing + // tokens if required. This should be called by the transport layer on + // each request, and the data should be populated in headers or other + // context. uri is the URI of the entry point for the request. When + // supported by the underlying implementation, ctx can be used for + // timeout and cancellation. + // TODO(zhaoq): Define the set of the qualified keys instead of leaving + // it as an arbitrary string. + GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) + // RequireTransportSecurity indicates whether the credentails requires + // transport security. + RequireTransportSecurity() bool +} + +// ProtocolInfo provides information regarding the gRPC wire protocol version, +// security protocol, security protocol version in use, etc. +type ProtocolInfo struct { + // ProtocolVersion is the gRPC wire protocol version. + ProtocolVersion string + // SecurityProtocol is the security protocol in use. + SecurityProtocol string + // SecurityVersion is the security protocol version. + SecurityVersion string +} + +// AuthInfo defines the common interface for the auth information the users are interested in. +type AuthInfo interface { + AuthType() string +} + +type authInfoKey struct{} + +// NewContext creates a new context with authInfo attached. +func NewContext(ctx context.Context, authInfo AuthInfo) context.Context { + return context.WithValue(ctx, authInfoKey{}, authInfo) +} + +// FromContext returns the authInfo in ctx if it exists. +func FromContext(ctx context.Context) (authInfo AuthInfo, ok bool) { + authInfo, ok = ctx.Value(authInfoKey{}).(AuthInfo) + return +} + +// TransportAuthenticator defines the common interface for all the live gRPC wire +// protocols and supported transport security protocols (e.g., TLS, SSL). +type TransportAuthenticator interface { + // ClientHandshake does the authentication handshake specified by the corresponding + // authentication protocol on rawConn for clients. It returns the authenticated + // connection and the corresponding auth information about the connection. + ClientHandshake(addr string, rawConn net.Conn, timeout time.Duration) (net.Conn, AuthInfo, error) + // ServerHandshake does the authentication handshake for servers. It returns + // the authenticated connection and the corresponding auth information about + // the connection. + ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) + // Info provides the ProtocolInfo of this TransportAuthenticator. + Info() ProtocolInfo + Credentials +} + +// TLSInfo contains the auth information for a TLS authenticated connection. +// It implements the AuthInfo interface. +type TLSInfo struct { + State tls.ConnectionState +} + +func (t TLSInfo) AuthType() string { + return "tls" +} + +// tlsCreds is the credentials required for authenticating a connection using TLS. +type tlsCreds struct { + // TLS configuration + config tls.Config +} + +func (c tlsCreds) Info() ProtocolInfo { + return ProtocolInfo{ + SecurityProtocol: "tls", + SecurityVersion: "1.2", + } +} + +// GetRequestMetadata returns nil, nil since TLS credentials does not have +// metadata. +func (c *tlsCreds) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + return nil, nil +} + +func (c *tlsCreds) RequireTransportSecurity() bool { + return true +} + +type timeoutError struct{} + +func (timeoutError) Error() string { return "credentials: Dial timed out" } +func (timeoutError) Timeout() bool { return true } +func (timeoutError) Temporary() bool { return true } + +func (c *tlsCreds) ClientHandshake(addr string, rawConn net.Conn, timeout time.Duration) (_ net.Conn, _ AuthInfo, err error) { + // borrow some code from tls.DialWithDialer + var errChannel chan error + if timeout != 0 { + errChannel = make(chan error, 2) + time.AfterFunc(timeout, func() { + errChannel <- timeoutError{} + }) + } + if c.config.ServerName == "" { + colonPos := strings.LastIndex(addr, ":") + if colonPos == -1 { + colonPos = len(addr) + } + c.config.ServerName = addr[:colonPos] + } + conn := tls.Client(rawConn, &c.config) + if timeout == 0 { + err = conn.Handshake() + } else { + go func() { + errChannel <- conn.Handshake() + }() + err = <-errChannel + } + if err != nil { + rawConn.Close() + return nil, nil, err + } + // TODO(zhaoq): Omit the auth info for client now. It is more for + // information than anything else. + return conn, nil, nil +} + +func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) { + conn := tls.Server(rawConn, &c.config) + if err := conn.Handshake(); err != nil { + rawConn.Close() + return nil, nil, err + } + return conn, TLSInfo{conn.ConnectionState()}, nil +} + +// NewTLS uses c to construct a TransportAuthenticator based on TLS. +func NewTLS(c *tls.Config) TransportAuthenticator { + tc := &tlsCreds{*c} + tc.config.NextProtos = alpnProtoStr + return tc +} + +// NewClientTLSFromCert constructs a TLS from the input certificate for client. +func NewClientTLSFromCert(cp *x509.CertPool, serverName string) TransportAuthenticator { + return NewTLS(&tls.Config{ServerName: serverName, RootCAs: cp}) +} + +// NewClientTLSFromFile constructs a TLS from the input certificate file for client. +func NewClientTLSFromFile(certFile, serverName string) (TransportAuthenticator, error) { + b, err := ioutil.ReadFile(certFile) + if err != nil { + return nil, err + } + cp := x509.NewCertPool() + if !cp.AppendCertsFromPEM(b) { + return nil, fmt.Errorf("credentials: failed to append certificates") + } + return NewTLS(&tls.Config{ServerName: serverName, RootCAs: cp}), nil +} + +// NewServerTLSFromCert constructs a TLS from the input certificate for server. +func NewServerTLSFromCert(cert *tls.Certificate) TransportAuthenticator { + return NewTLS(&tls.Config{Certificates: []tls.Certificate{*cert}}) +} + +// NewServerTLSFromFile constructs a TLS from the input certificate file and key +// file for server. +func NewServerTLSFromFile(certFile, keyFile string) (TransportAuthenticator, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + return NewTLS(&tls.Config{Certificates: []tls.Certificate{cert}}), nil +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/credentials/oauth/oauth.go b/components/engine/vendor/src/google.golang.org/grpc/credentials/oauth/oauth.go new file mode 100644 index 0000000000..04943fdf03 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/credentials/oauth/oauth.go @@ -0,0 +1,177 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +// Package oauth implements gRPC credentials using OAuth. +package oauth + +import ( + "fmt" + "io/ioutil" + + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" + "google.golang.org/grpc/credentials" +) + +// TokenSource supplies credentials from an oauth2.TokenSource. +type TokenSource struct { + oauth2.TokenSource +} + +// GetRequestMetadata gets the request metadata as a map from a TokenSource. +func (ts TokenSource) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + token, err := ts.Token() + if err != nil { + return nil, err + } + return map[string]string{ + "authorization": token.TokenType + " " + token.AccessToken, + }, nil +} + +func (ts TokenSource) RequireTransportSecurity() bool { + return true +} + +type jwtAccess struct { + jsonKey []byte +} + +func NewJWTAccessFromFile(keyFile string) (credentials.Credentials, error) { + jsonKey, err := ioutil.ReadFile(keyFile) + if err != nil { + return nil, fmt.Errorf("credentials: failed to read the service account key file: %v", err) + } + return NewJWTAccessFromKey(jsonKey) +} + +func NewJWTAccessFromKey(jsonKey []byte) (credentials.Credentials, error) { + return jwtAccess{jsonKey}, nil +} + +func (j jwtAccess) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + ts, err := google.JWTAccessTokenSourceFromJSON(j.jsonKey, uri[0]) + if err != nil { + return nil, err + } + token, err := ts.Token() + if err != nil { + return nil, err + } + return map[string]string{ + "authorization": token.TokenType + " " + token.AccessToken, + }, nil +} + +func (j jwtAccess) RequireTransportSecurity() bool { + return true +} + +// oauthAccess supplies credentials from a given token. +type oauthAccess struct { + token oauth2.Token +} + +// NewOauthAccess constructs the credentials using a given token. +func NewOauthAccess(token *oauth2.Token) credentials.Credentials { + return oauthAccess{token: *token} +} + +func (oa oauthAccess) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + return map[string]string{ + "authorization": oa.token.TokenType + " " + oa.token.AccessToken, + }, nil +} + +func (oa oauthAccess) RequireTransportSecurity() bool { + return true +} + +// NewComputeEngine constructs the credentials that fetches access tokens from +// Google Compute Engine (GCE)'s metadata server. It is only valid to use this +// if your program is running on a GCE instance. +// TODO(dsymonds): Deprecate and remove this. +func NewComputeEngine() credentials.Credentials { + return TokenSource{google.ComputeTokenSource("")} +} + +// serviceAccount represents credentials via JWT signing key. +type serviceAccount struct { + config *jwt.Config +} + +func (s serviceAccount) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + token, err := s.config.TokenSource(ctx).Token() + if err != nil { + return nil, err + } + return map[string]string{ + "authorization": token.TokenType + " " + token.AccessToken, + }, nil +} + +func (s serviceAccount) RequireTransportSecurity() bool { + return true +} + +// NewServiceAccountFromKey constructs the credentials using the JSON key slice +// from a Google Developers service account. +func NewServiceAccountFromKey(jsonKey []byte, scope ...string) (credentials.Credentials, error) { + config, err := google.JWTConfigFromJSON(jsonKey, scope...) + if err != nil { + return nil, err + } + return serviceAccount{config: config}, nil +} + +// NewServiceAccountFromFile constructs the credentials using the JSON key file +// of a Google Developers service account. +func NewServiceAccountFromFile(keyFile string, scope ...string) (credentials.Credentials, error) { + jsonKey, err := ioutil.ReadFile(keyFile) + if err != nil { + return nil, fmt.Errorf("credentials: failed to read the service account key file: %v", err) + } + return NewServiceAccountFromKey(jsonKey, scope...) +} + +// NewApplicationDefault returns "Application Default Credentials". For more +// detail, see https://developers.google.com/accounts/docs/application-default-credentials. +func NewApplicationDefault(ctx context.Context, scope ...string) (credentials.Credentials, error) { + t, err := google.DefaultTokenSource(ctx, scope...) + if err != nil { + return nil, err + } + return TokenSource{t}, nil +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/doc.go b/components/engine/vendor/src/google.golang.org/grpc/doc.go new file mode 100644 index 0000000000..c63847745d --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/doc.go @@ -0,0 +1,6 @@ +/* +Package grpc implements an RPC system called gRPC. + +See https://github.com/grpc/grpc for more information about gRPC. +*/ +package grpc // import "google.golang.org/grpc" diff --git a/components/engine/vendor/src/google.golang.org/grpc/grpclog/logger.go b/components/engine/vendor/src/google.golang.org/grpc/grpclog/logger.go new file mode 100644 index 0000000000..ec089f70f8 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/grpclog/logger.go @@ -0,0 +1,90 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* +Package grpclog defines logging for grpc. +*/ +package grpclog // import "google.golang.org/grpc/grpclog" + +import ( + "log" + "os" +) + +// Use golang's standard logger by default. +var logger Logger = log.New(os.Stderr, "", log.LstdFlags) + +// Logger mimics golang's standard Logger as an interface. +type Logger interface { + Fatal(args ...interface{}) + Fatalf(format string, args ...interface{}) + Fatalln(args ...interface{}) + Print(args ...interface{}) + Printf(format string, args ...interface{}) + Println(args ...interface{}) +} + +// SetLogger sets the logger that is used in grpc. +func SetLogger(l Logger) { + logger = l +} + +// Fatal is equivalent to Print() followed by a call to os.Exit() with a non-zero exit code. +func Fatal(args ...interface{}) { + logger.Fatal(args...) +} + +// Fatalf is equivalent to Printf() followed by a call to os.Exit() with a non-zero exit code. +func Fatalf(format string, args ...interface{}) { + logger.Fatalf(format, args...) +} + +// Fatalln is equivalent to Println() followed by a call to os.Exit()) with a non-zero exit code. +func Fatalln(args ...interface{}) { + logger.Fatalln(args...) +} + +// Print prints to the logger. Arguments are handled in the manner of fmt.Print. +func Print(args ...interface{}) { + logger.Print(args...) +} + +// Printf prints to the logger. Arguments are handled in the manner of fmt.Printf. +func Printf(format string, args ...interface{}) { + logger.Printf(format, args...) +} + +// Println prints to the logger. Arguments are handled in the manner of fmt.Println. +func Println(args ...interface{}) { + logger.Println(args...) +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/metadata/metadata.go b/components/engine/vendor/src/google.golang.org/grpc/metadata/metadata.go new file mode 100644 index 0000000000..adebc38f8e --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/metadata/metadata.go @@ -0,0 +1,146 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +// Package metadata define the structure of the metadata supported by gRPC library. +package metadata // import "google.golang.org/grpc/metadata" + +import ( + "encoding/base64" + "fmt" + "strings" + + "golang.org/x/net/context" +) + +const ( + binHdrSuffix = "-bin" +) + +// grpc-http2 requires ASCII header key and value (more detail can be found in +// "Requests" subsection in go/grpc-http2). +func isASCII(s string) bool { + for _, c := range s { + if c > 127 { + return false + } + } + return true +} + +// encodeKeyValue encodes key and value qualified for transmission via gRPC. +// Transmitting binary headers violates HTTP/2 spec. +// TODO(zhaoq): Maybe check if k is ASCII also. +func encodeKeyValue(k, v string) (string, string) { + if isASCII(v) { + return k, v + } + key := strings.ToLower(k + binHdrSuffix) + val := base64.StdEncoding.EncodeToString([]byte(v)) + return key, string(val) +} + +// DecodeKeyValue returns the original key and value corresponding to the +// encoded data in k, v. +func DecodeKeyValue(k, v string) (string, string, error) { + if !strings.HasSuffix(k, binHdrSuffix) { + return k, v, nil + } + key := k[:len(k)-len(binHdrSuffix)] + val, err := base64.StdEncoding.DecodeString(v) + if err != nil { + return "", "", err + } + return key, string(val), nil +} + +// MD is a mapping from metadata keys to values. Users should use the following +// two convenience functions New and Pairs to generate MD. +type MD map[string][]string + +// New creates a MD from given key-value map. +func New(m map[string]string) MD { + md := MD{} + for k, v := range m { + key, val := encodeKeyValue(k, v) + md[key] = append(md[key], val) + } + return md +} + +// Pairs returns an MD formed by the mapping of key, value ... +// Pairs panics if len(kv) is odd. +func Pairs(kv ...string) MD { + if len(kv)%2 == 1 { + panic(fmt.Sprintf("metadata: Pairs got the odd number of input pairs for metadata: %d", len(kv))) + } + md := MD{} + var k string + for i, s := range kv { + if i%2 == 0 { + k = s + continue + } + key, val := encodeKeyValue(k, s) + md[key] = append(md[key], val) + } + return md +} + +// Len returns the number of items in md. +func (md MD) Len() int { + return len(md) +} + +// Copy returns a copy of md. +func (md MD) Copy() MD { + out := MD{} + for k, v := range md { + for _, i := range v { + out[k] = append(out[k], i) + } + } + return out +} + +type mdKey struct{} + +// NewContext creates a new context with md attached. +func NewContext(ctx context.Context, md MD) context.Context { + return context.WithValue(ctx, mdKey{}, md) +} + +// FromContext returns the MD in ctx if it exists. +func FromContext(ctx context.Context) (md MD, ok bool) { + md, ok = ctx.Value(mdKey{}).(MD) + return +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/picker.go b/components/engine/vendor/src/google.golang.org/grpc/picker.go new file mode 100644 index 0000000000..bc48573a41 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/picker.go @@ -0,0 +1,93 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +package grpc + +import ( + "time" + + "golang.org/x/net/context" + "google.golang.org/grpc/transport" +) + +// Picker picks a Conn for RPC requests. +// This is EXPERIMENTAL and please do not implement your own Picker for now. +type Picker interface { + // Init does initial processing for the Picker, e.g., initiate some connections. + Init(cc *ClientConn) error + // Pick blocks until either a transport.ClientTransport is ready for the upcoming RPC + // or some error happens. + Pick(ctx context.Context) (transport.ClientTransport, error) + // State returns the connectivity state of the underlying connections. + State() ConnectivityState + // WaitForStateChange blocks until the state changes to something other than + // the sourceState or timeout fires on cc. It returns false if timeout fires, + // and true otherwise. + WaitForStateChange(timeout time.Duration, sourceState ConnectivityState) bool + // Close closes all the Conn's owned by this Picker. + Close() error +} + +// unicastPicker is the default Picker which is used when there is no custom Picker +// specified by users. It always picks the same Conn. +type unicastPicker struct { + conn *Conn +} + +func (p *unicastPicker) Init(cc *ClientConn) error { + c, err := NewConn(cc) + if err != nil { + return err + } + p.conn = c + return nil +} + +func (p *unicastPicker) Pick(ctx context.Context) (transport.ClientTransport, error) { + return p.conn.Wait(ctx) +} + +func (p *unicastPicker) State() ConnectivityState { + return p.conn.State() +} + +func (p *unicastPicker) WaitForStateChange(timeout time.Duration, sourceState ConnectivityState) bool { + return p.conn.WaitForStateChange(timeout, sourceState) +} + +func (p *unicastPicker) Close() error { + if p.conn != nil { + return p.conn.Close() + } + return nil +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/rpc_util.go b/components/engine/vendor/src/google.golang.org/grpc/rpc_util.go new file mode 100644 index 0000000000..46a6801b07 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/rpc_util.go @@ -0,0 +1,337 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +package grpc + +import ( + "encoding/binary" + "fmt" + "io" + "math" + "math/rand" + "os" + "time" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/transport" +) + +// Codec defines the interface gRPC uses to encode and decode messages. +type Codec interface { + // Marshal returns the wire format of v. + Marshal(v interface{}) ([]byte, error) + // Unmarshal parses the wire format into v. + Unmarshal(data []byte, v interface{}) error + // String returns the name of the Codec implementation. The returned + // string will be used as part of content type in transmission. + String() string +} + +// protoCodec is a Codec implemetation with protobuf. It is the default codec for gRPC. +type protoCodec struct{} + +func (protoCodec) Marshal(v interface{}) ([]byte, error) { + return proto.Marshal(v.(proto.Message)) +} + +func (protoCodec) Unmarshal(data []byte, v interface{}) error { + return proto.Unmarshal(data, v.(proto.Message)) +} + +func (protoCodec) String() string { + return "proto" +} + +// CallOption configures a Call before it starts or extracts information from +// a Call after it completes. +type CallOption interface { + // before is called before the call is sent to any server. If before + // returns a non-nil error, the RPC fails with that error. + before(*callInfo) error + + // after is called after the call has completed. after cannot return an + // error, so any failures should be reported via output parameters. + after(*callInfo) +} + +type beforeCall func(c *callInfo) error + +func (o beforeCall) before(c *callInfo) error { return o(c) } +func (o beforeCall) after(c *callInfo) {} + +type afterCall func(c *callInfo) + +func (o afterCall) before(c *callInfo) error { return nil } +func (o afterCall) after(c *callInfo) { o(c) } + +// Header returns a CallOptions that retrieves the header metadata +// for a unary RPC. +func Header(md *metadata.MD) CallOption { + return afterCall(func(c *callInfo) { + *md = c.headerMD + }) +} + +// Trailer returns a CallOptions that retrieves the trailer metadata +// for a unary RPC. +func Trailer(md *metadata.MD) CallOption { + return afterCall(func(c *callInfo) { + *md = c.trailerMD + }) +} + +// The format of the payload: compressed or not? +type payloadFormat uint8 + +const ( + compressionNone payloadFormat = iota // no compression + compressionFlate + // More formats +) + +// parser reads complelete gRPC messages from the underlying reader. +type parser struct { + s io.Reader +} + +// recvMsg is to read a complete gRPC message from the stream. It is blocking if +// the message has not been complete yet. It returns the message and its type, +// EOF is returned with nil msg and 0 pf if the entire stream is done. Other +// non-nil error is returned if something is wrong on reading. +func (p *parser) recvMsg() (pf payloadFormat, msg []byte, err error) { + // The header of a gRPC message. Find more detail + // at http://www.grpc.io/docs/guides/wire.html. + var buf [5]byte + + if _, err := io.ReadFull(p.s, buf[:]); err != nil { + return 0, nil, err + } + + pf = payloadFormat(buf[0]) + length := binary.BigEndian.Uint32(buf[1:]) + + if length == 0 { + return pf, nil, nil + } + msg = make([]byte, int(length)) + if _, err := io.ReadFull(p.s, msg); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return 0, nil, err + } + return pf, msg, nil +} + +// encode serializes msg and prepends the message header. If msg is nil, it +// generates the message header of 0 message length. +func encode(c Codec, msg interface{}, pf payloadFormat) ([]byte, error) { + var b []byte + var length uint + if msg != nil { + var err error + // TODO(zhaoq): optimize to reduce memory alloc and copying. + b, err = c.Marshal(msg) + if err != nil { + return nil, err + } + length = uint(len(b)) + } + if length > math.MaxUint32 { + return nil, Errorf(codes.InvalidArgument, "grpc: message too large (%d bytes)", length) + } + + const ( + payloadLen = 1 + sizeLen = 4 + ) + + var buf = make([]byte, payloadLen+sizeLen+len(b)) + + // Write payload format + buf[0] = byte(pf) + // Write length of b into buf + binary.BigEndian.PutUint32(buf[1:], uint32(length)) + // Copy encoded msg to buf + copy(buf[5:], b) + + return buf, nil +} + +func recv(p *parser, c Codec, m interface{}) error { + pf, d, err := p.recvMsg() + if err != nil { + return err + } + switch pf { + case compressionNone: + if err := c.Unmarshal(d, m); err != nil { + if rErr, ok := err.(rpcError); ok { + return rErr + } else { + return Errorf(codes.Internal, "grpc: %v", err) + } + } + default: + return Errorf(codes.Internal, "gprc: compression is not supported yet.") + } + return nil +} + +// rpcError defines the status from an RPC. +type rpcError struct { + code codes.Code + desc string +} + +func (e rpcError) Error() string { + return fmt.Sprintf("rpc error: code = %d desc = %q", e.code, e.desc) +} + +// Code returns the error code for err if it was produced by the rpc system. +// Otherwise, it returns codes.Unknown. +func Code(err error) codes.Code { + if err == nil { + return codes.OK + } + if e, ok := err.(rpcError); ok { + return e.code + } + return codes.Unknown +} + +// ErrorDesc returns the error description of err if it was produced by the rpc system. +// Otherwise, it returns err.Error() or empty string when err is nil. +func ErrorDesc(err error) string { + if err == nil { + return "" + } + if e, ok := err.(rpcError); ok { + return e.desc + } + return err.Error() +} + +// Errorf returns an error containing an error code and a description; +// Errorf returns nil if c is OK. +func Errorf(c codes.Code, format string, a ...interface{}) error { + if c == codes.OK { + return nil + } + return rpcError{ + code: c, + desc: fmt.Sprintf(format, a...), + } +} + +// toRPCErr converts an error into a rpcError. +func toRPCErr(err error) error { + switch e := err.(type) { + case rpcError: + return err + case transport.StreamError: + return rpcError{ + code: e.Code, + desc: e.Desc, + } + case transport.ConnectionError: + return rpcError{ + code: codes.Internal, + desc: e.Desc, + } + } + return Errorf(codes.Unknown, "%v", err) +} + +// convertCode converts a standard Go error into its canonical code. Note that +// this is only used to translate the error returned by the server applications. +func convertCode(err error) codes.Code { + switch err { + case nil: + return codes.OK + case io.EOF: + return codes.OutOfRange + case io.ErrClosedPipe, io.ErrNoProgress, io.ErrShortBuffer, io.ErrShortWrite, io.ErrUnexpectedEOF: + return codes.FailedPrecondition + case os.ErrInvalid: + return codes.InvalidArgument + case context.Canceled: + return codes.Canceled + case context.DeadlineExceeded: + return codes.DeadlineExceeded + } + switch { + case os.IsExist(err): + return codes.AlreadyExists + case os.IsNotExist(err): + return codes.NotFound + case os.IsPermission(err): + return codes.PermissionDenied + } + return codes.Unknown +} + +const ( + // how long to wait after the first failure before retrying + baseDelay = 1.0 * time.Second + // upper bound of backoff delay + maxDelay = 120 * time.Second + // backoff increases by this factor on each retry + backoffFactor = 1.6 + // backoff is randomized downwards by this factor + backoffJitter = 0.2 +) + +func backoff(retries int) (t time.Duration) { + if retries == 0 { + return baseDelay + } + backoff, max := float64(baseDelay), float64(maxDelay) + for backoff < max && retries > 0 { + backoff *= backoffFactor + retries-- + } + if backoff > max { + backoff = max + } + // Randomize backoff delays so that if a cluster of requests start at + // the same time, they won't operate in lockstep. + backoff *= 1 + backoffJitter*(rand.Float64()*2-1) + if backoff < 0 { + return 0 + } + return time.Duration(backoff) +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/server.go b/components/engine/vendor/src/google.golang.org/grpc/server.go new file mode 100644 index 0000000000..487a75c5ee --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/server.go @@ -0,0 +1,542 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +package grpc + +import ( + "errors" + "fmt" + "io" + "net" + "reflect" + "runtime" + "strings" + "sync" + "time" + + "golang.org/x/net/context" + "golang.org/x/net/trace" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/transport" +) + +type methodHandler func(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) + +// MethodDesc represents an RPC service's method specification. +type MethodDesc struct { + MethodName string + Handler methodHandler +} + +// ServiceDesc represents an RPC service's specification. +type ServiceDesc struct { + ServiceName string + // The pointer to the service interface. Used to check whether the user + // provided implementation satisfies the interface requirements. + HandlerType interface{} + Methods []MethodDesc + Streams []StreamDesc +} + +// service consists of the information of the server serving this service and +// the methods in this service. +type service struct { + server interface{} // the server for service methods + md map[string]*MethodDesc + sd map[string]*StreamDesc +} + +// Server is a gRPC server to serve RPC requests. +type Server struct { + opts options + mu sync.Mutex + lis map[net.Listener]bool + conns map[transport.ServerTransport]bool + m map[string]*service // service name -> service info + events trace.EventLog +} + +type options struct { + creds credentials.Credentials + codec Codec + maxConcurrentStreams uint32 +} + +// A ServerOption sets options. +type ServerOption func(*options) + +// CustomCodec returns a ServerOption that sets a codec for message marshaling and unmarshaling. +func CustomCodec(codec Codec) ServerOption { + return func(o *options) { + o.codec = codec + } +} + +// MaxConcurrentStreams returns a ServerOption that will apply a limit on the number +// of concurrent streams to each ServerTransport. +func MaxConcurrentStreams(n uint32) ServerOption { + return func(o *options) { + o.maxConcurrentStreams = n + } +} + +// Creds returns a ServerOption that sets credentials for server connections. +func Creds(c credentials.Credentials) ServerOption { + return func(o *options) { + o.creds = c + } +} + +// NewServer creates a gRPC server which has no service registered and has not +// started to accept requests yet. +func NewServer(opt ...ServerOption) *Server { + var opts options + for _, o := range opt { + o(&opts) + } + if opts.codec == nil { + // Set the default codec. + opts.codec = protoCodec{} + } + s := &Server{ + lis: make(map[net.Listener]bool), + opts: opts, + conns: make(map[transport.ServerTransport]bool), + m: make(map[string]*service), + } + if EnableTracing { + _, file, line, _ := runtime.Caller(1) + s.events = trace.NewEventLog("grpc.Server", fmt.Sprintf("%s:%d", file, line)) + } + return s +} + +// printf records an event in s's event log, unless s has been stopped. +// REQUIRES s.mu is held. +func (s *Server) printf(format string, a ...interface{}) { + if s.events != nil { + s.events.Printf(format, a...) + } +} + +// errorf records an error in s's event log, unless s has been stopped. +// REQUIRES s.mu is held. +func (s *Server) errorf(format string, a ...interface{}) { + if s.events != nil { + s.events.Errorf(format, a...) + } +} + +// RegisterService register a service and its implementation to the gRPC +// server. Called from the IDL generated code. This must be called before +// invoking Serve. +func (s *Server) RegisterService(sd *ServiceDesc, ss interface{}) { + ht := reflect.TypeOf(sd.HandlerType).Elem() + st := reflect.TypeOf(ss) + if !st.Implements(ht) { + grpclog.Fatalf("grpc: Server.RegisterService found the handler of type %v that does not satisfy %v", st, ht) + } + s.register(sd, ss) +} + +func (s *Server) register(sd *ServiceDesc, ss interface{}) { + s.mu.Lock() + defer s.mu.Unlock() + s.printf("RegisterService(%q)", sd.ServiceName) + if _, ok := s.m[sd.ServiceName]; ok { + grpclog.Fatalf("grpc: Server.RegisterService found duplicate service registration for %q", sd.ServiceName) + } + srv := &service{ + server: ss, + md: make(map[string]*MethodDesc), + sd: make(map[string]*StreamDesc), + } + for i := range sd.Methods { + d := &sd.Methods[i] + srv.md[d.MethodName] = d + } + for i := range sd.Streams { + d := &sd.Streams[i] + srv.sd[d.StreamName] = d + } + s.m[sd.ServiceName] = srv +} + +var ( + // ErrServerStopped indicates that the operation is now illegal because of + // the server being stopped. + ErrServerStopped = errors.New("grpc: the server has been stopped") +) + +// Serve accepts incoming connections on the listener lis, creating a new +// ServerTransport and service goroutine for each. The service goroutines +// read gRPC request and then call the registered handlers to reply to them. +// Service returns when lis.Accept fails. +func (s *Server) Serve(lis net.Listener) error { + s.mu.Lock() + s.printf("serving") + if s.lis == nil { + s.mu.Unlock() + return ErrServerStopped + } + s.lis[lis] = true + s.mu.Unlock() + defer func() { + lis.Close() + s.mu.Lock() + delete(s.lis, lis) + s.mu.Unlock() + }() + for { + c, err := lis.Accept() + if err != nil { + s.mu.Lock() + s.printf("done serving; Accept = %v", err) + s.mu.Unlock() + return err + } + var authInfo credentials.AuthInfo + if creds, ok := s.opts.creds.(credentials.TransportAuthenticator); ok { + var conn net.Conn + conn, authInfo, err = creds.ServerHandshake(c) + if err != nil { + s.mu.Lock() + s.errorf("ServerHandshake(%q) failed: %v", c.RemoteAddr(), err) + s.mu.Unlock() + grpclog.Println("grpc: Server.Serve failed to complete security handshake.") + continue + } + c = conn + } + s.mu.Lock() + if s.conns == nil { + s.mu.Unlock() + c.Close() + return nil + } + st, err := transport.NewServerTransport("http2", c, s.opts.maxConcurrentStreams, authInfo) + if err != nil { + s.errorf("NewServerTransport(%q) failed: %v", c.RemoteAddr(), err) + s.mu.Unlock() + c.Close() + grpclog.Println("grpc: Server.Serve failed to create ServerTransport: ", err) + continue + } + s.conns[st] = true + s.mu.Unlock() + + go func() { + var wg sync.WaitGroup + st.HandleStreams(func(stream *transport.Stream) { + var trInfo *traceInfo + if EnableTracing { + trInfo = &traceInfo{ + tr: trace.New("grpc.Recv."+methodFamily(stream.Method()), stream.Method()), + } + trInfo.firstLine.client = false + trInfo.firstLine.remoteAddr = st.RemoteAddr() + stream.TraceContext(trInfo.tr) + if dl, ok := stream.Context().Deadline(); ok { + trInfo.firstLine.deadline = dl.Sub(time.Now()) + } + } + wg.Add(1) + go func() { + s.handleStream(st, stream, trInfo) + wg.Done() + }() + }) + wg.Wait() + s.mu.Lock() + delete(s.conns, st) + s.mu.Unlock() + }() + } +} + +func (s *Server) sendResponse(t transport.ServerTransport, stream *transport.Stream, msg interface{}, pf payloadFormat, opts *transport.Options) error { + p, err := encode(s.opts.codec, msg, pf) + if err != nil { + // This typically indicates a fatal issue (e.g., memory + // corruption or hardware faults) the application program + // cannot handle. + // + // TODO(zhaoq): There exist other options also such as only closing the + // faulty stream locally and remotely (Other streams can keep going). Find + // the optimal option. + grpclog.Fatalf("grpc: Server failed to encode response %v", err) + } + return t.Write(stream, p, opts) +} + +func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.Stream, srv *service, md *MethodDesc, trInfo *traceInfo) (err error) { + if trInfo != nil { + defer trInfo.tr.Finish() + trInfo.firstLine.client = false + trInfo.tr.LazyLog(&trInfo.firstLine, false) + defer func() { + if err != nil && err != io.EOF { + trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + trInfo.tr.SetError() + } + }() + } + p := &parser{s: stream} + for { + pf, req, err := p.recvMsg() + if err == io.EOF { + // The entire stream is done (for unary RPC only). + return err + } + if err != nil { + switch err := err.(type) { + case transport.ConnectionError: + // Nothing to do here. + case transport.StreamError: + if err := t.WriteStatus(stream, err.Code, err.Desc); err != nil { + grpclog.Printf("grpc: Server.processUnaryRPC failed to write status: %v", err) + } + default: + panic(fmt.Sprintf("grpc: Unexpected error (%T) from recvMsg: %v", err, err)) + } + return err + } + switch pf { + case compressionNone: + statusCode := codes.OK + statusDesc := "" + df := func(v interface{}) error { + if err := s.opts.codec.Unmarshal(req, v); err != nil { + return err + } + if trInfo != nil { + trInfo.tr.LazyLog(&payload{sent: false, msg: v}, true) + } + return nil + } + reply, appErr := md.Handler(srv.server, stream.Context(), df) + if appErr != nil { + if err, ok := appErr.(rpcError); ok { + statusCode = err.code + statusDesc = err.desc + } else { + statusCode = convertCode(appErr) + statusDesc = appErr.Error() + } + if trInfo != nil && statusCode != codes.OK { + trInfo.tr.LazyLog(stringer(statusDesc), true) + trInfo.tr.SetError() + } + + if err := t.WriteStatus(stream, statusCode, statusDesc); err != nil { + grpclog.Printf("grpc: Server.processUnaryRPC failed to write status: %v", err) + return err + } + return nil + } + if trInfo != nil { + trInfo.tr.LazyLog(stringer("OK"), false) + } + opts := &transport.Options{ + Last: true, + Delay: false, + } + if err := s.sendResponse(t, stream, reply, compressionNone, opts); err != nil { + switch err := err.(type) { + case transport.ConnectionError: + // Nothing to do here. + case transport.StreamError: + statusCode = err.Code + statusDesc = err.Desc + default: + statusCode = codes.Unknown + statusDesc = err.Error() + } + return err + } + if trInfo != nil { + trInfo.tr.LazyLog(&payload{sent: true, msg: reply}, true) + } + return t.WriteStatus(stream, statusCode, statusDesc) + default: + panic(fmt.Sprintf("payload format to be supported: %d", pf)) + } + } +} + +func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transport.Stream, srv *service, sd *StreamDesc, trInfo *traceInfo) (err error) { + ss := &serverStream{ + t: t, + s: stream, + p: &parser{s: stream}, + codec: s.opts.codec, + trInfo: trInfo, + } + if trInfo != nil { + trInfo.tr.LazyLog(&trInfo.firstLine, false) + defer func() { + ss.mu.Lock() + if err != nil && err != io.EOF { + trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + trInfo.tr.SetError() + } + trInfo.tr.Finish() + trInfo.tr = nil + ss.mu.Unlock() + }() + } + if appErr := sd.Handler(srv.server, ss); appErr != nil { + if err, ok := appErr.(rpcError); ok { + ss.statusCode = err.code + ss.statusDesc = err.desc + } else { + ss.statusCode = convertCode(appErr) + ss.statusDesc = appErr.Error() + } + } + if trInfo != nil { + ss.mu.Lock() + if ss.statusCode != codes.OK { + trInfo.tr.LazyLog(stringer(ss.statusDesc), true) + trInfo.tr.SetError() + } else { + trInfo.tr.LazyLog(stringer("OK"), false) + } + ss.mu.Unlock() + } + return t.WriteStatus(ss.s, ss.statusCode, ss.statusDesc) + +} + +func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Stream, trInfo *traceInfo) { + sm := stream.Method() + if sm != "" && sm[0] == '/' { + sm = sm[1:] + } + pos := strings.LastIndex(sm, "/") + if pos == -1 { + if err := t.WriteStatus(stream, codes.InvalidArgument, fmt.Sprintf("malformed method name: %q", stream.Method())); err != nil { + grpclog.Printf("grpc: Server.handleStream failed to write status: %v", err) + } + return + } + service := sm[:pos] + method := sm[pos+1:] + srv, ok := s.m[service] + if !ok { + if err := t.WriteStatus(stream, codes.Unimplemented, fmt.Sprintf("unknown service %v", service)); err != nil { + grpclog.Printf("grpc: Server.handleStream failed to write status: %v", err) + } + return + } + // Unary RPC or Streaming RPC? + if md, ok := srv.md[method]; ok { + s.processUnaryRPC(t, stream, srv, md, trInfo) + return + } + if sd, ok := srv.sd[method]; ok { + s.processStreamingRPC(t, stream, srv, sd, trInfo) + return + } + if err := t.WriteStatus(stream, codes.Unimplemented, fmt.Sprintf("unknown method %v", method)); err != nil { + grpclog.Printf("grpc: Server.handleStream failed to write status: %v", err) + } +} + +// Stop stops the gRPC server. Once Stop returns, the server stops accepting +// connection requests and closes all the connected connections. +func (s *Server) Stop() { + s.mu.Lock() + listeners := s.lis + s.lis = nil + cs := s.conns + s.conns = nil + s.mu.Unlock() + for lis := range listeners { + lis.Close() + } + for c := range cs { + c.Close() + } + s.mu.Lock() + if s.events != nil { + s.events.Finish() + s.events = nil + } + s.mu.Unlock() +} + +// TestingCloseConns closes all exiting transports but keeps s.lis accepting new +// connections. This is for test only now. +func (s *Server) TestingCloseConns() { + s.mu.Lock() + for c := range s.conns { + c.Close() + } + s.conns = make(map[transport.ServerTransport]bool) + s.mu.Unlock() +} + +// SendHeader sends header metadata. It may be called at most once from a unary +// RPC handler. The ctx is the RPC handler's Context or one derived from it. +func SendHeader(ctx context.Context, md metadata.MD) error { + if md.Len() == 0 { + return nil + } + stream, ok := transport.StreamFromContext(ctx) + if !ok { + return fmt.Errorf("grpc: failed to fetch the stream from the context %v", ctx) + } + t := stream.ServerTransport() + if t == nil { + grpclog.Fatalf("grpc: SendHeader: %v has no ServerTransport to send header metadata.", stream) + } + return t.WriteHeader(stream, md) +} + +// SetTrailer sets the trailer metadata that will be sent when an RPC returns. +// It may be called at most once from a unary RPC handler. The ctx is the RPC +// handler's Context or one derived from it. +func SetTrailer(ctx context.Context, md metadata.MD) error { + if md.Len() == 0 { + return nil + } + stream, ok := transport.StreamFromContext(ctx) + if !ok { + return fmt.Errorf("grpc: failed to fetch the stream from the context %v", ctx) + } + return stream.SetTrailer(md) +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/stream.go b/components/engine/vendor/src/google.golang.org/grpc/stream.go new file mode 100644 index 0000000000..2370dd0e9d --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/stream.go @@ -0,0 +1,368 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +package grpc + +import ( + "errors" + "io" + "sync" + "time" + + "golang.org/x/net/context" + "golang.org/x/net/trace" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/transport" +) + +type streamHandler func(srv interface{}, stream ServerStream) error + +// StreamDesc represents a streaming RPC service's method specification. +type StreamDesc struct { + StreamName string + Handler streamHandler + + // At least one of these is true. + ServerStreams bool + ClientStreams bool +} + +// Stream defines the common interface a client or server stream has to satisfy. +type Stream interface { + // Context returns the context for this stream. + Context() context.Context + // SendMsg blocks until it sends m, the stream is done or the stream + // breaks. + // On error, it aborts the stream and returns an RPC status on client + // side. On server side, it simply returns the error to the caller. + // SendMsg is called by generated code. + SendMsg(m interface{}) error + // RecvMsg blocks until it receives a message or the stream is + // done. On client side, it returns io.EOF when the stream is done. On + // any other error, it aborts the streama nd returns an RPC status. On + // server side, it simply returns the error to the caller. + RecvMsg(m interface{}) error +} + +// ClientStream defines the interface a client stream has to satify. +type ClientStream interface { + // Header returns the header metedata received from the server if there + // is any. It blocks if the metadata is not ready to read. + Header() (metadata.MD, error) + // Trailer returns the trailer metadata from the server. It must be called + // after stream.Recv() returns non-nil error (including io.EOF) for + // bi-directional streaming and server streaming or stream.CloseAndRecv() + // returns for client streaming in order to receive trailer metadata if + // present. Otherwise, it could returns an empty MD even though trailer + // is present. + Trailer() metadata.MD + // CloseSend closes the send direction of the stream. It closes the stream + // when non-nil error is met. + CloseSend() error + Stream +} + +// NewClientStream creates a new Stream for the client side. This is called +// by generated code. +func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) { + var ( + t transport.ClientTransport + err error + ) + t, err = cc.dopts.picker.Pick(ctx) + if err != nil { + return nil, toRPCErr(err) + } + // TODO(zhaoq): CallOption is omitted. Add support when it is needed. + callHdr := &transport.CallHdr{ + Host: cc.authority, + Method: method, + } + cs := &clientStream{ + desc: desc, + codec: cc.dopts.codec, + tracing: EnableTracing, + } + if cs.tracing { + cs.trInfo.tr = trace.New("grpc.Sent."+methodFamily(method), method) + cs.trInfo.firstLine.client = true + if deadline, ok := ctx.Deadline(); ok { + cs.trInfo.firstLine.deadline = deadline.Sub(time.Now()) + } + cs.trInfo.tr.LazyLog(&cs.trInfo.firstLine, false) + ctx = trace.NewContext(ctx, cs.trInfo.tr) + } + s, err := t.NewStream(ctx, callHdr) + if err != nil { + return nil, toRPCErr(err) + } + cs.t = t + cs.s = s + cs.p = &parser{s: s} + // Listen on ctx.Done() to detect cancellation when there is no pending + // I/O operations on this stream. + go func() { + <-s.Context().Done() + cs.closeTransportStream(transport.ContextErr(s.Context().Err())) + }() + return cs, nil +} + +// clientStream implements a client side Stream. +type clientStream struct { + t transport.ClientTransport + s *transport.Stream + p *parser + desc *StreamDesc + codec Codec + + tracing bool // set to EnableTracing when the clientStream is created. + + mu sync.Mutex + closed bool + // trInfo.tr is set when the clientStream is created (if EnableTracing is true), + // and is set to nil when the clientStream's finish method is called. + trInfo traceInfo +} + +func (cs *clientStream) Context() context.Context { + return cs.s.Context() +} + +func (cs *clientStream) Header() (metadata.MD, error) { + m, err := cs.s.Header() + if err != nil { + if _, ok := err.(transport.ConnectionError); !ok { + cs.closeTransportStream(err) + } + } + return m, err +} + +func (cs *clientStream) Trailer() metadata.MD { + return cs.s.Trailer() +} + +func (cs *clientStream) SendMsg(m interface{}) (err error) { + if cs.tracing { + cs.mu.Lock() + if cs.trInfo.tr != nil { + cs.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true) + } + cs.mu.Unlock() + } + defer func() { + if err == nil || err == io.EOF { + return + } + if _, ok := err.(transport.ConnectionError); !ok { + cs.closeTransportStream(err) + } + err = toRPCErr(err) + }() + out, err := encode(cs.codec, m, compressionNone) + if err != nil { + return transport.StreamErrorf(codes.Internal, "grpc: %v", err) + } + return cs.t.Write(cs.s, out, &transport.Options{Last: false}) +} + +func (cs *clientStream) RecvMsg(m interface{}) (err error) { + err = recv(cs.p, cs.codec, m) + defer func() { + // err != nil indicates the termination of the stream. + if err != nil { + cs.finish(err) + } + }() + if err == nil { + if cs.tracing { + cs.mu.Lock() + if cs.trInfo.tr != nil { + cs.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true) + } + cs.mu.Unlock() + } + if !cs.desc.ClientStreams || cs.desc.ServerStreams { + return + } + // Special handling for client streaming rpc. + err = recv(cs.p, cs.codec, m) + cs.closeTransportStream(err) + if err == nil { + return toRPCErr(errors.New("grpc: client streaming protocol violation: get , want ")) + } + if err == io.EOF { + if cs.s.StatusCode() == codes.OK { + return nil + } + return Errorf(cs.s.StatusCode(), cs.s.StatusDesc()) + } + return toRPCErr(err) + } + if _, ok := err.(transport.ConnectionError); !ok { + cs.closeTransportStream(err) + } + if err == io.EOF { + if cs.s.StatusCode() == codes.OK { + // Returns io.EOF to indicate the end of the stream. + return + } + return Errorf(cs.s.StatusCode(), cs.s.StatusDesc()) + } + return toRPCErr(err) +} + +func (cs *clientStream) CloseSend() (err error) { + err = cs.t.Write(cs.s, nil, &transport.Options{Last: true}) + if err == nil || err == io.EOF { + return + } + if _, ok := err.(transport.ConnectionError); !ok { + cs.closeTransportStream(err) + } + err = toRPCErr(err) + return +} + +func (cs *clientStream) closeTransportStream(err error) { + cs.mu.Lock() + if cs.closed { + cs.mu.Unlock() + return + } + cs.closed = true + cs.mu.Unlock() + cs.t.CloseStream(cs.s, err) +} + +func (cs *clientStream) finish(err error) { + if !cs.tracing { + return + } + cs.mu.Lock() + defer cs.mu.Unlock() + if cs.trInfo.tr != nil { + if err == nil || err == io.EOF { + cs.trInfo.tr.LazyPrintf("RPC: [OK]") + } else { + cs.trInfo.tr.LazyPrintf("RPC: [%v]", err) + cs.trInfo.tr.SetError() + } + cs.trInfo.tr.Finish() + cs.trInfo.tr = nil + } +} + +// ServerStream defines the interface a server stream has to satisfy. +type ServerStream interface { + // SendHeader sends the header metadata. It should not be called + // after SendProto. It fails if called multiple times or if + // called after SendProto. + SendHeader(metadata.MD) error + // SetTrailer sets the trailer metadata which will be sent with the + // RPC status. + SetTrailer(metadata.MD) + Stream +} + +// serverStream implements a server side Stream. +type serverStream struct { + t transport.ServerTransport + s *transport.Stream + p *parser + codec Codec + statusCode codes.Code + statusDesc string + trInfo *traceInfo + + mu sync.Mutex // protects trInfo.tr after the service handler runs. +} + +func (ss *serverStream) Context() context.Context { + return ss.s.Context() +} + +func (ss *serverStream) SendHeader(md metadata.MD) error { + return ss.t.WriteHeader(ss.s, md) +} + +func (ss *serverStream) SetTrailer(md metadata.MD) { + if md.Len() == 0 { + return + } + ss.s.SetTrailer(md) + return +} + +func (ss *serverStream) SendMsg(m interface{}) (err error) { + defer func() { + if ss.trInfo != nil { + ss.mu.Lock() + if ss.trInfo.tr != nil { + if err == nil { + ss.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true) + } else { + ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + ss.trInfo.tr.SetError() + } + } + ss.mu.Unlock() + } + }() + out, err := encode(ss.codec, m, compressionNone) + if err != nil { + err = transport.StreamErrorf(codes.Internal, "grpc: %v", err) + return err + } + return ss.t.Write(ss.s, out, &transport.Options{Last: false}) +} + +func (ss *serverStream) RecvMsg(m interface{}) (err error) { + defer func() { + if ss.trInfo != nil { + ss.mu.Lock() + if ss.trInfo.tr != nil { + if err == nil { + ss.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true) + } else if err != io.EOF { + ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + ss.trInfo.tr.SetError() + } + } + ss.mu.Unlock() + } + }() + return recv(ss.p, ss.codec, m) +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/trace.go b/components/engine/vendor/src/google.golang.org/grpc/trace.go new file mode 100644 index 0000000000..cde04fbfc9 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/trace.go @@ -0,0 +1,120 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +package grpc + +import ( + "bytes" + "fmt" + "io" + "net" + "strings" + "time" + + "golang.org/x/net/trace" +) + +// EnableTracing controls whether to trace RPCs using the golang.org/x/net/trace package. +// This should only be set before any RPCs are sent or received by this program. +var EnableTracing = true + +// methodFamily returns the trace family for the given method. +// It turns "/pkg.Service/GetFoo" into "pkg.Service". +func methodFamily(m string) string { + m = strings.TrimPrefix(m, "/") // remove leading slash + if i := strings.Index(m, "/"); i >= 0 { + m = m[:i] // remove everything from second slash + } + if i := strings.LastIndex(m, "."); i >= 0 { + m = m[i+1:] // cut down to last dotted component + } + return m +} + +// traceInfo contains tracing information for an RPC. +type traceInfo struct { + tr trace.Trace + firstLine firstLine +} + +// firstLine is the first line of an RPC trace. +type firstLine struct { + client bool // whether this is a client (outgoing) RPC + remoteAddr net.Addr + deadline time.Duration // may be zero +} + +func (f *firstLine) String() string { + var line bytes.Buffer + io.WriteString(&line, "RPC: ") + if f.client { + io.WriteString(&line, "to") + } else { + io.WriteString(&line, "from") + } + fmt.Fprintf(&line, " %v deadline:", f.remoteAddr) + if f.deadline != 0 { + fmt.Fprint(&line, f.deadline) + } else { + io.WriteString(&line, "none") + } + return line.String() +} + +// payload represents an RPC request or response payload. +type payload struct { + sent bool // whether this is an outgoing payload + msg interface{} // e.g. a proto.Message + // TODO(dsymonds): add stringifying info to codec, and limit how much we hold here? +} + +func (p payload) String() string { + if p.sent { + return fmt.Sprintf("sent: %v", p.msg) + } else { + return fmt.Sprintf("recv: %v", p.msg) + } +} + +type fmtStringer struct { + format string + a []interface{} +} + +func (f *fmtStringer) String() string { + return fmt.Sprintf(f.format, f.a...) +} + +type stringer string + +func (s stringer) String() string { return string(s) } diff --git a/components/engine/vendor/src/google.golang.org/grpc/transport/control.go b/components/engine/vendor/src/google.golang.org/grpc/transport/control.go new file mode 100644 index 0000000000..6b5201e7a6 --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/transport/control.go @@ -0,0 +1,259 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +package transport + +import ( + "fmt" + "sync" + + "golang.org/x/net/http2" +) + +const ( + // The default value of flow control window size in HTTP2 spec. + defaultWindowSize = 65535 + // The initial window size for flow control. + initialWindowSize = defaultWindowSize // for an RPC + initialConnWindowSize = defaultWindowSize * 16 // for a connection +) + +// The following defines various control items which could flow through +// the control buffer of transport. They represent different aspects of +// control tasks, e.g., flow control, settings, streaming resetting, etc. +type windowUpdate struct { + streamID uint32 + increment uint32 +} + +func (windowUpdate) isItem() bool { + return true +} + +type settings struct { + ack bool + ss []http2.Setting +} + +func (settings) isItem() bool { + return true +} + +type resetStream struct { + streamID uint32 + code http2.ErrCode +} + +func (resetStream) isItem() bool { + return true +} + +type flushIO struct { +} + +func (flushIO) isItem() bool { + return true +} + +type ping struct { + ack bool +} + +func (ping) isItem() bool { + return true +} + +// quotaPool is a pool which accumulates the quota and sends it to acquire() +// when it is available. +type quotaPool struct { + c chan int + + mu sync.Mutex + quota int +} + +// newQuotaPool creates a quotaPool which has quota q available to consume. +func newQuotaPool(q int) *quotaPool { + qb := "aPool{ + c: make(chan int, 1), + } + if q > 0 { + qb.c <- q + } else { + qb.quota = q + } + return qb +} + +// add adds n to the available quota and tries to send it on acquire. +func (qb *quotaPool) add(n int) { + qb.mu.Lock() + defer qb.mu.Unlock() + qb.quota += n + if qb.quota <= 0 { + return + } + select { + case qb.c <- qb.quota: + qb.quota = 0 + default: + } +} + +// cancel cancels the pending quota sent on acquire, if any. +func (qb *quotaPool) cancel() { + qb.mu.Lock() + defer qb.mu.Unlock() + select { + case n := <-qb.c: + qb.quota += n + default: + } +} + +// reset cancels the pending quota sent on acquired, incremented by v and sends +// it back on acquire. +func (qb *quotaPool) reset(v int) { + qb.mu.Lock() + defer qb.mu.Unlock() + select { + case n := <-qb.c: + qb.quota += n + default: + } + qb.quota += v + if qb.quota <= 0 { + return + } + select { + case qb.c <- qb.quota: + qb.quota = 0 + default: + } +} + +// acquire returns the channel on which available quota amounts are sent. +func (qb *quotaPool) acquire() <-chan int { + return qb.c +} + +// inFlow deals with inbound flow control +type inFlow struct { + // The inbound flow control limit for pending data. + limit uint32 + // conn points to the shared connection-level inFlow that is shared + // by all streams on that conn. It is nil for the inFlow on the conn + // directly. + conn *inFlow + + mu sync.Mutex + // pendingData is the overall data which have been received but not been + // consumed by applications. + pendingData uint32 + // The amount of data the application has consumed but grpc has not sent + // window update for them. Used to reduce window update frequency. + pendingUpdate uint32 +} + +// onData is invoked when some data frame is received. It increments not only its +// own pendingData but also that of the associated connection-level flow. +func (f *inFlow) onData(n uint32) error { + if n == 0 { + return nil + } + f.mu.Lock() + defer f.mu.Unlock() + if f.pendingData+f.pendingUpdate+n > f.limit { + return fmt.Errorf("recieved %d-bytes data exceeding the limit %d bytes", f.pendingData+f.pendingUpdate+n, f.limit) + } + if f.conn != nil { + if err := f.conn.onData(n); err != nil { + return ConnectionErrorf("%v", err) + } + } + f.pendingData += n + return nil +} + +// connOnRead updates the connection level states when the application consumes data. +func (f *inFlow) connOnRead(n uint32) uint32 { + if n == 0 || f.conn != nil { + return 0 + } + f.mu.Lock() + defer f.mu.Unlock() + f.pendingData -= n + f.pendingUpdate += n + if f.pendingUpdate >= f.limit/4 { + ret := f.pendingUpdate + f.pendingUpdate = 0 + return ret + } + return 0 +} + +// onRead is invoked when the application reads the data. It returns the window updates +// for both stream and connection level. +func (f *inFlow) onRead(n uint32) (swu, cwu uint32) { + if n == 0 { + return + } + f.mu.Lock() + defer f.mu.Unlock() + if f.pendingData == 0 { + // pendingData has been adjusted by restoreConn. + return + } + f.pendingData -= n + f.pendingUpdate += n + if f.pendingUpdate >= f.limit/4 { + swu = f.pendingUpdate + f.pendingUpdate = 0 + } + cwu = f.conn.connOnRead(n) + return +} + +// restoreConn is invoked when a stream is terminated. It removes its stake in +// the connection-level flow and resets its own state. +func (f *inFlow) restoreConn() uint32 { + if f.conn == nil { + return 0 + } + f.mu.Lock() + defer f.mu.Unlock() + n := f.pendingData + f.pendingData = 0 + f.pendingUpdate = 0 + return f.conn.connOnRead(n) +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/transport/http2_client.go b/components/engine/vendor/src/google.golang.org/grpc/transport/http2_client.go new file mode 100644 index 0000000000..b13fb78c4c --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/transport/http2_client.go @@ -0,0 +1,860 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +package transport + +import ( + "bytes" + "errors" + "io" + "math" + "net" + "strings" + "sync" + "time" + + "golang.org/x/net/context" + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" +) + +// http2Client implements the ClientTransport interface with HTTP2. +type http2Client struct { + target string // server name/addr + userAgent string + conn net.Conn // underlying communication channel + authInfo credentials.AuthInfo // auth info about the connection + nextID uint32 // the next stream ID to be used + + // writableChan synchronizes write access to the transport. + // A writer acquires the write lock by sending a value on writableChan + // and releases it by receiving from writableChan. + writableChan chan int + // shutdownChan is closed when Close is called. + // Blocking operations should select on shutdownChan to avoid + // blocking forever after Close. + // TODO(zhaoq): Maybe have a channel context? + shutdownChan chan struct{} + // errorChan is closed to notify the I/O error to the caller. + errorChan chan struct{} + + framer *framer + hBuf *bytes.Buffer // the buffer for HPACK encoding + hEnc *hpack.Encoder // HPACK encoder + + // controlBuf delivers all the control related tasks (e.g., window + // updates, reset streams, and various settings) to the controller. + controlBuf *recvBuffer + fc *inFlow + // sendQuotaPool provides flow control to outbound message. + sendQuotaPool *quotaPool + // streamsQuota limits the max number of concurrent streams. + streamsQuota *quotaPool + + // The scheme used: https if TLS is on, http otherwise. + scheme string + + authCreds []credentials.Credentials + + mu sync.Mutex // guard the following variables + state transportState // the state of underlying connection + activeStreams map[uint32]*Stream + // The max number of concurrent streams + maxStreams int + // the per-stream outbound flow control window size set by the peer. + streamSendQuota uint32 +} + +// newHTTP2Client constructs a connected ClientTransport to addr based on HTTP2 +// and starts to receive messages on it. Non-nil error returns if construction +// fails. +func newHTTP2Client(addr string, opts *ConnectOptions) (_ ClientTransport, err error) { + if opts.Dialer == nil { + // Set the default Dialer. + opts.Dialer = func(addr string, timeout time.Duration) (net.Conn, error) { + return net.DialTimeout("tcp", addr, timeout) + } + } + scheme := "http" + startT := time.Now() + timeout := opts.Timeout + conn, connErr := opts.Dialer(addr, timeout) + if connErr != nil { + return nil, ConnectionErrorf("transport: %v", connErr) + } + var authInfo credentials.AuthInfo + for _, c := range opts.AuthOptions { + if ccreds, ok := c.(credentials.TransportAuthenticator); ok { + scheme = "https" + // TODO(zhaoq): Now the first TransportAuthenticator is used if there are + // multiple ones provided. Revisit this if it is not appropriate. Probably + // place the ClientTransport construction into a separate function to make + // things clear. + if timeout > 0 { + timeout -= time.Since(startT) + } + conn, authInfo, connErr = ccreds.ClientHandshake(addr, conn, timeout) + break + } + } + if connErr != nil { + return nil, ConnectionErrorf("transport: %v", connErr) + } + defer func() { + if err != nil { + conn.Close() + } + }() + // Send connection preface to server. + n, err := conn.Write(clientPreface) + if err != nil { + return nil, ConnectionErrorf("transport: %v", err) + } + if n != len(clientPreface) { + return nil, ConnectionErrorf("transport: preface mismatch, wrote %d bytes; want %d", n, len(clientPreface)) + } + framer := newFramer(conn) + if initialWindowSize != defaultWindowSize { + err = framer.writeSettings(true, http2.Setting{http2.SettingInitialWindowSize, uint32(initialWindowSize)}) + } else { + err = framer.writeSettings(true) + } + if err != nil { + return nil, ConnectionErrorf("transport: %v", err) + } + // Adjust the connection flow control window if needed. + if delta := uint32(initialConnWindowSize - defaultWindowSize); delta > 0 { + if err := framer.writeWindowUpdate(true, 0, delta); err != nil { + return nil, ConnectionErrorf("transport: %v", err) + } + } + ua := primaryUA + if opts.UserAgent != "" { + ua = opts.UserAgent + " " + ua + } + var buf bytes.Buffer + t := &http2Client{ + target: addr, + userAgent: ua, + conn: conn, + authInfo: authInfo, + // The client initiated stream id is odd starting from 1. + nextID: 1, + writableChan: make(chan int, 1), + shutdownChan: make(chan struct{}), + errorChan: make(chan struct{}), + framer: framer, + hBuf: &buf, + hEnc: hpack.NewEncoder(&buf), + controlBuf: newRecvBuffer(), + fc: &inFlow{limit: initialConnWindowSize}, + sendQuotaPool: newQuotaPool(defaultWindowSize), + scheme: scheme, + state: reachable, + activeStreams: make(map[uint32]*Stream), + authCreds: opts.AuthOptions, + maxStreams: math.MaxInt32, + streamSendQuota: defaultWindowSize, + } + go t.controller() + t.writableChan <- 0 + // Start the reader goroutine for incoming message. The threading model + // on receiving is that each transport has a dedicated goroutine which + // reads HTTP2 frame from network. Then it dispatches the frame to the + // corresponding stream entity. + go t.reader() + return t, nil +} + +func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream { + fc := &inFlow{ + limit: initialWindowSize, + conn: t.fc, + } + // TODO(zhaoq): Handle uint32 overflow of Stream.id. + s := &Stream{ + id: t.nextID, + method: callHdr.Method, + buf: newRecvBuffer(), + fc: fc, + sendQuotaPool: newQuotaPool(int(t.streamSendQuota)), + headerChan: make(chan struct{}), + } + t.nextID += 2 + s.windowHandler = func(n int) { + t.updateWindow(s, uint32(n)) + } + // Make a stream be able to cancel the pending operations by itself. + s.ctx, s.cancel = context.WithCancel(ctx) + s.dec = &recvBufferReader{ + ctx: s.ctx, + recv: s.buf, + } + return s +} + +// NewStream creates a stream and register it into the transport as "active" +// streams. +func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Stream, err error) { + // Record the timeout value on the context. + var timeout time.Duration + if dl, ok := ctx.Deadline(); ok { + timeout = dl.Sub(time.Now()) + if timeout <= 0 { + return nil, ContextErr(context.DeadlineExceeded) + } + } + // Attach Auth info if there is any. + if t.authInfo != nil { + ctx = credentials.NewContext(ctx, t.authInfo) + } + authData := make(map[string]string) + for _, c := range t.authCreds { + // Construct URI required to get auth request metadata. + var port string + if pos := strings.LastIndex(t.target, ":"); pos != -1 { + // Omit port if it is the default one. + if t.target[pos+1:] != "443" { + port = ":" + t.target[pos+1:] + } + } + pos := strings.LastIndex(callHdr.Method, "/") + if pos == -1 { + return nil, StreamErrorf(codes.InvalidArgument, "transport: malformed method name: %q", callHdr.Method) + } + audience := "https://" + callHdr.Host + port + callHdr.Method[:pos] + data, err := c.GetRequestMetadata(ctx, audience) + if err != nil { + return nil, StreamErrorf(codes.InvalidArgument, "transport: %v", err) + } + for k, v := range data { + authData[k] = v + } + } + t.mu.Lock() + if t.state != reachable { + t.mu.Unlock() + return nil, ErrConnClosing + } + checkStreamsQuota := t.streamsQuota != nil + t.mu.Unlock() + if checkStreamsQuota { + sq, err := wait(ctx, t.shutdownChan, t.streamsQuota.acquire()) + if err != nil { + return nil, err + } + // Returns the quota balance back. + if sq > 1 { + t.streamsQuota.add(sq - 1) + } + } + if _, err := wait(ctx, t.shutdownChan, t.writableChan); err != nil { + // t.streamsQuota will be updated when t.CloseStream is invoked. + return nil, err + } + t.mu.Lock() + if t.state != reachable { + t.mu.Unlock() + return nil, ErrConnClosing + } + s := t.newStream(ctx, callHdr) + t.activeStreams[s.id] = s + + // This stream is not counted when applySetings(...) initialize t.streamsQuota. + // Reset t.streamsQuota to the right value. + var reset bool + if !checkStreamsQuota && t.streamsQuota != nil { + reset = true + } + t.mu.Unlock() + if reset { + t.streamsQuota.reset(-1) + } + + // HPACK encodes various headers. Note that once WriteField(...) is + // called, the corresponding headers/continuation frame has to be sent + // because hpack.Encoder is stateful. + t.hBuf.Reset() + t.hEnc.WriteField(hpack.HeaderField{Name: ":method", Value: "POST"}) + t.hEnc.WriteField(hpack.HeaderField{Name: ":scheme", Value: t.scheme}) + t.hEnc.WriteField(hpack.HeaderField{Name: ":path", Value: callHdr.Method}) + t.hEnc.WriteField(hpack.HeaderField{Name: ":authority", Value: callHdr.Host}) + t.hEnc.WriteField(hpack.HeaderField{Name: "content-type", Value: "application/grpc"}) + t.hEnc.WriteField(hpack.HeaderField{Name: "user-agent", Value: t.userAgent}) + t.hEnc.WriteField(hpack.HeaderField{Name: "te", Value: "trailers"}) + + if timeout > 0 { + t.hEnc.WriteField(hpack.HeaderField{Name: "grpc-timeout", Value: timeoutEncode(timeout)}) + } + for k, v := range authData { + t.hEnc.WriteField(hpack.HeaderField{Name: k, Value: v}) + } + var ( + hasMD bool + endHeaders bool + ) + if md, ok := metadata.FromContext(ctx); ok { + hasMD = true + for k, v := range md { + for _, entry := range v { + t.hEnc.WriteField(hpack.HeaderField{Name: k, Value: entry}) + } + } + } + first := true + // Sends the headers in a single batch even when they span multiple frames. + for !endHeaders { + size := t.hBuf.Len() + if size > http2MaxFrameLen { + size = http2MaxFrameLen + } else { + endHeaders = true + } + if first { + // Sends a HeadersFrame to server to start a new stream. + p := http2.HeadersFrameParam{ + StreamID: s.id, + BlockFragment: t.hBuf.Next(size), + EndStream: false, + EndHeaders: endHeaders, + } + // Do a force flush for the buffered frames iff it is the last headers frame + // and there is header metadata to be sent. Otherwise, there is flushing until + // the corresponding data frame is written. + err = t.framer.writeHeaders(hasMD && endHeaders, p) + first = false + } else { + // Sends Continuation frames for the leftover headers. + err = t.framer.writeContinuation(hasMD && endHeaders, s.id, endHeaders, t.hBuf.Next(size)) + } + if err != nil { + t.notifyError(err) + return nil, ConnectionErrorf("transport: %v", err) + } + } + t.writableChan <- 0 + return s, nil +} + +// CloseStream clears the footprint of a stream when the stream is not needed any more. +// This must not be executed in reader's goroutine. +func (t *http2Client) CloseStream(s *Stream, err error) { + var updateStreams bool + t.mu.Lock() + if t.streamsQuota != nil { + updateStreams = true + } + delete(t.activeStreams, s.id) + t.mu.Unlock() + if updateStreams { + t.streamsQuota.add(1) + } + // In case stream sending and receiving are invoked in separate + // goroutines (e.g., bi-directional streaming), the caller needs + // to call cancel on the stream to interrupt the blocking on + // other goroutines. + s.cancel() + s.mu.Lock() + if q := s.fc.restoreConn(); q > 0 { + t.controlBuf.put(&windowUpdate{0, q}) + } + if s.state == streamDone { + s.mu.Unlock() + return + } + if !s.headerDone { + close(s.headerChan) + s.headerDone = true + } + s.state = streamDone + s.mu.Unlock() + if _, ok := err.(StreamError); ok { + t.controlBuf.put(&resetStream{s.id, http2.ErrCodeCancel}) + } +} + +// Close kicks off the shutdown process of the transport. This should be called +// only once on a transport. Once it is called, the transport should not be +// accessed any more. +func (t *http2Client) Close() (err error) { + t.mu.Lock() + if t.state == closing { + t.mu.Unlock() + return errors.New("transport: Close() was already called") + } + t.state = closing + t.mu.Unlock() + close(t.shutdownChan) + err = t.conn.Close() + t.mu.Lock() + streams := t.activeStreams + t.activeStreams = nil + t.mu.Unlock() + // Notify all active streams. + for _, s := range streams { + s.mu.Lock() + if !s.headerDone { + close(s.headerChan) + s.headerDone = true + } + s.mu.Unlock() + s.write(recvMsg{err: ErrConnClosing}) + } + return +} + +// Write formats the data into HTTP2 data frame(s) and sends it out. The caller +// should proceed only if Write returns nil. +// TODO(zhaoq): opts.Delay is ignored in this implementation. Support it later +// if it improves the performance. +func (t *http2Client) Write(s *Stream, data []byte, opts *Options) error { + r := bytes.NewBuffer(data) + for { + var p []byte + if r.Len() > 0 { + size := http2MaxFrameLen + s.sendQuotaPool.add(0) + // Wait until the stream has some quota to send the data. + sq, err := wait(s.ctx, t.shutdownChan, s.sendQuotaPool.acquire()) + if err != nil { + return err + } + t.sendQuotaPool.add(0) + // Wait until the transport has some quota to send the data. + tq, err := wait(s.ctx, t.shutdownChan, t.sendQuotaPool.acquire()) + if err != nil { + if _, ok := err.(StreamError); ok { + t.sendQuotaPool.cancel() + } + return err + } + if sq < size { + size = sq + } + if tq < size { + size = tq + } + p = r.Next(size) + ps := len(p) + if ps < sq { + // Overbooked stream quota. Return it back. + s.sendQuotaPool.add(sq - ps) + } + if ps < tq { + // Overbooked transport quota. Return it back. + t.sendQuotaPool.add(tq - ps) + } + } + var ( + endStream bool + forceFlush bool + ) + if opts.Last && r.Len() == 0 { + endStream = true + } + // Indicate there is a writer who is about to write a data frame. + t.framer.adjustNumWriters(1) + // Got some quota. Try to acquire writing privilege on the transport. + if _, err := wait(s.ctx, t.shutdownChan, t.writableChan); err != nil { + if t.framer.adjustNumWriters(-1) == 0 { + // This writer is the last one in this batch and has the + // responsibility to flush the buffered frames. It queues + // a flush request to controlBuf instead of flushing directly + // in order to avoid the race with other writing or flushing. + t.controlBuf.put(&flushIO{}) + } + return err + } + if r.Len() == 0 && t.framer.adjustNumWriters(0) == 1 { + // Do a force flush iff this is last frame for the entire gRPC message + // and the caller is the only writer at this moment. + forceFlush = true + } + // If WriteData fails, all the pending streams will be handled + // by http2Client.Close(). No explicit CloseStream() needs to be + // invoked. + if err := t.framer.writeData(forceFlush, s.id, endStream, p); err != nil { + t.notifyError(err) + return ConnectionErrorf("transport: %v", err) + } + if t.framer.adjustNumWriters(-1) == 0 { + t.framer.flushWrite() + } + t.writableChan <- 0 + if r.Len() == 0 { + break + } + } + if !opts.Last { + return nil + } + s.mu.Lock() + if s.state != streamDone { + if s.state == streamReadDone { + s.state = streamDone + } else { + s.state = streamWriteDone + } + } + s.mu.Unlock() + return nil +} + +func (t *http2Client) getStream(f http2.Frame) (*Stream, bool) { + t.mu.Lock() + defer t.mu.Unlock() + if t.activeStreams == nil { + // The transport is closing. + return nil, false + } + if s, ok := t.activeStreams[f.Header().StreamID]; ok { + return s, true + } + return nil, false +} + +// updateWindow adjusts the inbound quota for the stream and the transport. +// Window updates will deliver to the controller for sending when +// the cumulative quota exceeds the corresponding threshold. +func (t *http2Client) updateWindow(s *Stream, n uint32) { + swu, cwu := s.fc.onRead(n) + if swu > 0 { + t.controlBuf.put(&windowUpdate{s.id, swu}) + } + if cwu > 0 { + t.controlBuf.put(&windowUpdate{0, cwu}) + } +} + +func (t *http2Client) handleData(f *http2.DataFrame) { + // Select the right stream to dispatch. + s, ok := t.getStream(f) + if !ok { + return + } + size := len(f.Data()) + if size > 0 { + if err := s.fc.onData(uint32(size)); err != nil { + if _, ok := err.(ConnectionError); ok { + t.notifyError(err) + return + } + s.mu.Lock() + if s.state == streamDone { + s.mu.Unlock() + return + } + s.state = streamDone + s.statusCode = codes.Internal + s.statusDesc = err.Error() + s.mu.Unlock() + s.write(recvMsg{err: io.EOF}) + t.controlBuf.put(&resetStream{s.id, http2.ErrCodeFlowControl}) + return + } + // TODO(bradfitz, zhaoq): A copy is required here because there is no + // guarantee f.Data() is consumed before the arrival of next frame. + // Can this copy be eliminated? + data := make([]byte, size) + copy(data, f.Data()) + s.write(recvMsg{data: data}) + } + // The server has closed the stream without sending trailers. Record that + // the read direction is closed, and set the status appropriately. + if f.FrameHeader.Flags.Has(http2.FlagDataEndStream) { + s.mu.Lock() + if s.state == streamWriteDone { + s.state = streamDone + } else { + s.state = streamReadDone + } + s.statusCode = codes.Internal + s.statusDesc = "server closed the stream without sending trailers" + s.mu.Unlock() + s.write(recvMsg{err: io.EOF}) + } +} + +func (t *http2Client) handleRSTStream(f *http2.RSTStreamFrame) { + s, ok := t.getStream(f) + if !ok { + return + } + s.mu.Lock() + if s.state == streamDone { + s.mu.Unlock() + return + } + s.state = streamDone + if !s.headerDone { + close(s.headerChan) + s.headerDone = true + } + s.statusCode, ok = http2RSTErrConvTab[http2.ErrCode(f.ErrCode)] + if !ok { + grpclog.Println("transport: http2Client.handleRSTStream found no mapped gRPC status for the received http2 error ", f.ErrCode) + } + s.mu.Unlock() + s.write(recvMsg{err: io.EOF}) +} + +func (t *http2Client) handleSettings(f *http2.SettingsFrame) { + if f.IsAck() { + return + } + var ss []http2.Setting + f.ForeachSetting(func(s http2.Setting) error { + ss = append(ss, s) + return nil + }) + // The settings will be applied once the ack is sent. + t.controlBuf.put(&settings{ack: true, ss: ss}) +} + +func (t *http2Client) handlePing(f *http2.PingFrame) { + t.controlBuf.put(&ping{true}) +} + +func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { + // TODO(zhaoq): GoAwayFrame handler to be implemented +} + +func (t *http2Client) handleWindowUpdate(f *http2.WindowUpdateFrame) { + id := f.Header().StreamID + incr := f.Increment + if id == 0 { + t.sendQuotaPool.add(int(incr)) + return + } + if s, ok := t.getStream(f); ok { + s.sendQuotaPool.add(int(incr)) + } +} + +// operateHeader takes action on the decoded headers. It returns the current +// stream if there are remaining headers on the wire (in the following +// Continuation frame). +func (t *http2Client) operateHeaders(hDec *hpackDecoder, s *Stream, frame headerFrame, endStream bool) (pendingStream *Stream) { + defer func() { + if pendingStream == nil { + hDec.state = decodeState{} + } + }() + endHeaders, err := hDec.decodeClientHTTP2Headers(frame) + if s == nil { + // s has been closed. + return nil + } + if err != nil { + s.write(recvMsg{err: err}) + // Something wrong. Stops reading even when there is remaining. + return nil + } + if !endHeaders { + return s + } + + s.mu.Lock() + if !s.headerDone { + if !endStream && len(hDec.state.mdata) > 0 { + s.header = hDec.state.mdata + } + close(s.headerChan) + s.headerDone = true + } + if !endStream || s.state == streamDone { + s.mu.Unlock() + return nil + } + + if len(hDec.state.mdata) > 0 { + s.trailer = hDec.state.mdata + } + s.state = streamDone + s.statusCode = hDec.state.statusCode + s.statusDesc = hDec.state.statusDesc + s.mu.Unlock() + + s.write(recvMsg{err: io.EOF}) + return nil +} + +// reader runs as a separate goroutine in charge of reading data from network +// connection. +// +// TODO(zhaoq): currently one reader per transport. Investigate whether this is +// optimal. +// TODO(zhaoq): Check the validity of the incoming frame sequence. +func (t *http2Client) reader() { + // Check the validity of server preface. + frame, err := t.framer.readFrame() + if err != nil { + t.notifyError(err) + return + } + sf, ok := frame.(*http2.SettingsFrame) + if !ok { + t.notifyError(err) + return + } + t.handleSettings(sf) + + hDec := newHPACKDecoder() + var curStream *Stream + // loop to keep reading incoming messages on this transport. + for { + frame, err := t.framer.readFrame() + if err != nil { + t.notifyError(err) + return + } + switch frame := frame.(type) { + case *http2.HeadersFrame: + // operateHeaders has to be invoked regardless the value of curStream + // because the HPACK decoder needs to be updated using the received + // headers. + curStream, _ = t.getStream(frame) + endStream := frame.Header().Flags.Has(http2.FlagHeadersEndStream) + curStream = t.operateHeaders(hDec, curStream, frame, endStream) + case *http2.ContinuationFrame: + curStream = t.operateHeaders(hDec, curStream, frame, false) + case *http2.DataFrame: + t.handleData(frame) + case *http2.RSTStreamFrame: + t.handleRSTStream(frame) + case *http2.SettingsFrame: + t.handleSettings(frame) + case *http2.PingFrame: + t.handlePing(frame) + case *http2.GoAwayFrame: + t.handleGoAway(frame) + case *http2.WindowUpdateFrame: + t.handleWindowUpdate(frame) + default: + grpclog.Printf("transport: http2Client.reader got unhandled frame type %v.", frame) + } + } +} + +func (t *http2Client) applySettings(ss []http2.Setting) { + for _, s := range ss { + switch s.ID { + case http2.SettingMaxConcurrentStreams: + // TODO(zhaoq): This is a hack to avoid significant refactoring of the + // code to deal with the unrealistic int32 overflow. Probably will try + // to find a better way to handle this later. + if s.Val > math.MaxInt32 { + s.Val = math.MaxInt32 + } + t.mu.Lock() + reset := t.streamsQuota != nil + if !reset { + t.streamsQuota = newQuotaPool(int(s.Val) - len(t.activeStreams)) + } + ms := t.maxStreams + t.maxStreams = int(s.Val) + t.mu.Unlock() + if reset { + t.streamsQuota.reset(int(s.Val) - ms) + } + case http2.SettingInitialWindowSize: + t.mu.Lock() + for _, stream := range t.activeStreams { + // Adjust the sending quota for each stream. + stream.sendQuotaPool.reset(int(s.Val - t.streamSendQuota)) + } + t.streamSendQuota = s.Val + t.mu.Unlock() + } + } +} + +// controller running in a separate goroutine takes charge of sending control +// frames (e.g., window update, reset stream, setting, etc.) to the server. +func (t *http2Client) controller() { + for { + select { + case i := <-t.controlBuf.get(): + t.controlBuf.load() + select { + case <-t.writableChan: + switch i := i.(type) { + case *windowUpdate: + t.framer.writeWindowUpdate(true, i.streamID, i.increment) + case *settings: + if i.ack { + t.framer.writeSettingsAck(true) + t.applySettings(i.ss) + } else { + t.framer.writeSettings(true, i.ss...) + } + case *resetStream: + t.framer.writeRSTStream(true, i.streamID, i.code) + case *flushIO: + t.framer.flushWrite() + case *ping: + // TODO(zhaoq): Ack with all-0 data now. will change to some + // meaningful content when this is actually in use. + t.framer.writePing(true, i.ack, [8]byte{}) + default: + grpclog.Printf("transport: http2Client.controller got unexpected item type %v\n", i) + } + t.writableChan <- 0 + continue + case <-t.shutdownChan: + return + } + case <-t.shutdownChan: + return + } + } +} + +func (t *http2Client) Error() <-chan struct{} { + return t.errorChan +} + +func (t *http2Client) notifyError(err error) { + t.mu.Lock() + defer t.mu.Unlock() + // make sure t.errorChan is closed only once. + if t.state == reachable { + t.state = unreachable + close(t.errorChan) + grpclog.Printf("transport: http2Client.notifyError got notified that the client transport was broken %v.", err) + } +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/transport/http2_server.go b/components/engine/vendor/src/google.golang.org/grpc/transport/http2_server.go new file mode 100644 index 0000000000..f3488f83dc --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/transport/http2_server.go @@ -0,0 +1,695 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +package transport + +import ( + "bytes" + "errors" + "io" + "math" + "net" + "strconv" + "sync" + + "golang.org/x/net/context" + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" +) + +// ErrIllegalHeaderWrite indicates that setting header is illegal because of +// the stream's state. +var ErrIllegalHeaderWrite = errors.New("transport: the stream is done or WriteHeader was already called") + +// http2Server implements the ServerTransport interface with HTTP2. +type http2Server struct { + conn net.Conn + maxStreamID uint32 // max stream ID ever seen + authInfo credentials.AuthInfo // auth info about the connection + // writableChan synchronizes write access to the transport. + // A writer acquires the write lock by sending a value on writableChan + // and releases it by receiving from writableChan. + writableChan chan int + // shutdownChan is closed when Close is called. + // Blocking operations should select on shutdownChan to avoid + // blocking forever after Close. + shutdownChan chan struct{} + framer *framer + hBuf *bytes.Buffer // the buffer for HPACK encoding + hEnc *hpack.Encoder // HPACK encoder + + // The max number of concurrent streams. + maxStreams uint32 + // controlBuf delivers all the control related tasks (e.g., window + // updates, reset streams, and various settings) to the controller. + controlBuf *recvBuffer + fc *inFlow + // sendQuotaPool provides flow control to outbound message. + sendQuotaPool *quotaPool + + mu sync.Mutex // guard the following + state transportState + activeStreams map[uint32]*Stream + // the per-stream outbound flow control window size set by the peer. + streamSendQuota uint32 +} + +// newHTTP2Server constructs a ServerTransport based on HTTP2. ConnectionError is +// returned if something goes wrong. +func newHTTP2Server(conn net.Conn, maxStreams uint32, authInfo credentials.AuthInfo) (_ ServerTransport, err error) { + framer := newFramer(conn) + // Send initial settings as connection preface to client. + var settings []http2.Setting + // TODO(zhaoq): Have a better way to signal "no limit" because 0 is + // permitted in the HTTP2 spec. + if maxStreams == 0 { + maxStreams = math.MaxUint32 + } else { + settings = append(settings, http2.Setting{http2.SettingMaxConcurrentStreams, maxStreams}) + } + if initialWindowSize != defaultWindowSize { + settings = append(settings, http2.Setting{http2.SettingInitialWindowSize, uint32(initialWindowSize)}) + } + if err := framer.writeSettings(true, settings...); err != nil { + return nil, ConnectionErrorf("transport: %v", err) + } + // Adjust the connection flow control window if needed. + if delta := uint32(initialConnWindowSize - defaultWindowSize); delta > 0 { + if err := framer.writeWindowUpdate(true, 0, delta); err != nil { + return nil, ConnectionErrorf("transport: %v", err) + } + } + var buf bytes.Buffer + t := &http2Server{ + conn: conn, + authInfo: authInfo, + framer: framer, + hBuf: &buf, + hEnc: hpack.NewEncoder(&buf), + maxStreams: maxStreams, + controlBuf: newRecvBuffer(), + fc: &inFlow{limit: initialConnWindowSize}, + sendQuotaPool: newQuotaPool(defaultWindowSize), + state: reachable, + writableChan: make(chan int, 1), + shutdownChan: make(chan struct{}), + activeStreams: make(map[uint32]*Stream), + streamSendQuota: defaultWindowSize, + } + go t.controller() + t.writableChan <- 0 + return t, nil +} + +// operateHeader takes action on the decoded headers. It returns the current +// stream if there are remaining headers on the wire (in the following +// Continuation frame). +func (t *http2Server) operateHeaders(hDec *hpackDecoder, s *Stream, frame headerFrame, endStream bool, handle func(*Stream)) (pendingStream *Stream) { + defer func() { + if pendingStream == nil { + hDec.state = decodeState{} + } + }() + endHeaders, err := hDec.decodeServerHTTP2Headers(frame) + if s == nil { + // s has been closed. + return nil + } + if err != nil { + grpclog.Printf("transport: http2Server.operateHeader found %v", err) + if se, ok := err.(StreamError); ok { + t.controlBuf.put(&resetStream{s.id, statusCodeConvTab[se.Code]}) + } + return nil + } + if endStream { + // s is just created by the caller. No lock needed. + s.state = streamReadDone + } + if !endHeaders { + return s + } + t.mu.Lock() + if t.state != reachable { + t.mu.Unlock() + return nil + } + if uint32(len(t.activeStreams)) >= t.maxStreams { + t.mu.Unlock() + t.controlBuf.put(&resetStream{s.id, http2.ErrCodeRefusedStream}) + return nil + } + s.sendQuotaPool = newQuotaPool(int(t.streamSendQuota)) + t.activeStreams[s.id] = s + t.mu.Unlock() + s.windowHandler = func(n int) { + t.updateWindow(s, uint32(n)) + } + if hDec.state.timeoutSet { + s.ctx, s.cancel = context.WithTimeout(context.TODO(), hDec.state.timeout) + } else { + s.ctx, s.cancel = context.WithCancel(context.TODO()) + } + // Attach Auth info if there is any. + if t.authInfo != nil { + s.ctx = credentials.NewContext(s.ctx, t.authInfo) + } + // Cache the current stream to the context so that the server application + // can find out. Required when the server wants to send some metadata + // back to the client (unary call only). + s.ctx = newContextWithStream(s.ctx, s) + // Attach the received metadata to the context. + if len(hDec.state.mdata) > 0 { + s.ctx = metadata.NewContext(s.ctx, hDec.state.mdata) + } + + s.dec = &recvBufferReader{ + ctx: s.ctx, + recv: s.buf, + } + s.method = hDec.state.method + handle(s) + return nil +} + +// HandleStreams receives incoming streams using the given handler. This is +// typically run in a separate goroutine. +func (t *http2Server) HandleStreams(handle func(*Stream)) { + // Check the validity of client preface. + preface := make([]byte, len(clientPreface)) + if _, err := io.ReadFull(t.conn, preface); err != nil { + grpclog.Printf("transport: http2Server.HandleStreams failed to receive the preface from client: %v", err) + t.Close() + return + } + if !bytes.Equal(preface, clientPreface) { + grpclog.Printf("transport: http2Server.HandleStreams received bogus greeting from client: %q", preface) + t.Close() + return + } + + frame, err := t.framer.readFrame() + if err != nil { + grpclog.Printf("transport: http2Server.HandleStreams failed to read frame: %v", err) + t.Close() + return + } + sf, ok := frame.(*http2.SettingsFrame) + if !ok { + grpclog.Printf("transport: http2Server.HandleStreams saw invalid preface type %T from client", frame) + t.Close() + return + } + t.handleSettings(sf) + + hDec := newHPACKDecoder() + var curStream *Stream + for { + frame, err := t.framer.readFrame() + if err != nil { + t.Close() + return + } + switch frame := frame.(type) { + case *http2.HeadersFrame: + id := frame.Header().StreamID + if id%2 != 1 || id <= t.maxStreamID { + // illegal gRPC stream id. + grpclog.Println("transport: http2Server.HandleStreams received an illegal stream id: ", id) + t.Close() + break + } + t.maxStreamID = id + buf := newRecvBuffer() + fc := &inFlow{ + limit: initialWindowSize, + conn: t.fc, + } + curStream = &Stream{ + id: frame.Header().StreamID, + st: t, + buf: buf, + fc: fc, + } + endStream := frame.Header().Flags.Has(http2.FlagHeadersEndStream) + curStream = t.operateHeaders(hDec, curStream, frame, endStream, handle) + case *http2.ContinuationFrame: + curStream = t.operateHeaders(hDec, curStream, frame, false, handle) + case *http2.DataFrame: + t.handleData(frame) + case *http2.RSTStreamFrame: + t.handleRSTStream(frame) + case *http2.SettingsFrame: + t.handleSettings(frame) + case *http2.PingFrame: + t.handlePing(frame) + case *http2.WindowUpdateFrame: + t.handleWindowUpdate(frame) + case *http2.GoAwayFrame: + break + default: + grpclog.Printf("transport: http2Server.HandleStreams found unhandled frame type %v.", frame) + } + } +} + +func (t *http2Server) getStream(f http2.Frame) (*Stream, bool) { + t.mu.Lock() + defer t.mu.Unlock() + if t.activeStreams == nil { + // The transport is closing. + return nil, false + } + s, ok := t.activeStreams[f.Header().StreamID] + if !ok { + // The stream is already done. + return nil, false + } + return s, true +} + +// updateWindow adjusts the inbound quota for the stream and the transport. +// Window updates will deliver to the controller for sending when +// the cumulative quota exceeds the corresponding threshold. +func (t *http2Server) updateWindow(s *Stream, n uint32) { + swu, cwu := s.fc.onRead(n) + if swu > 0 { + t.controlBuf.put(&windowUpdate{s.id, swu}) + } + if cwu > 0 { + t.controlBuf.put(&windowUpdate{0, cwu}) + } +} + +func (t *http2Server) handleData(f *http2.DataFrame) { + // Select the right stream to dispatch. + s, ok := t.getStream(f) + if !ok { + return + } + size := len(f.Data()) + if size > 0 { + if err := s.fc.onData(uint32(size)); err != nil { + if _, ok := err.(ConnectionError); ok { + grpclog.Printf("transport: http2Server %v", err) + t.Close() + return + } + t.closeStream(s) + t.controlBuf.put(&resetStream{s.id, http2.ErrCodeFlowControl}) + return + } + // TODO(bradfitz, zhaoq): A copy is required here because there is no + // guarantee f.Data() is consumed before the arrival of next frame. + // Can this copy be eliminated? + data := make([]byte, size) + copy(data, f.Data()) + s.write(recvMsg{data: data}) + } + if f.Header().Flags.Has(http2.FlagDataEndStream) { + // Received the end of stream from the client. + s.mu.Lock() + if s.state != streamDone { + if s.state == streamWriteDone { + s.state = streamDone + } else { + s.state = streamReadDone + } + } + s.mu.Unlock() + s.write(recvMsg{err: io.EOF}) + } +} + +func (t *http2Server) handleRSTStream(f *http2.RSTStreamFrame) { + s, ok := t.getStream(f) + if !ok { + return + } + t.closeStream(s) +} + +func (t *http2Server) handleSettings(f *http2.SettingsFrame) { + if f.IsAck() { + return + } + var ss []http2.Setting + f.ForeachSetting(func(s http2.Setting) error { + ss = append(ss, s) + return nil + }) + // The settings will be applied once the ack is sent. + t.controlBuf.put(&settings{ack: true, ss: ss}) +} + +func (t *http2Server) handlePing(f *http2.PingFrame) { + t.controlBuf.put(&ping{true}) +} + +func (t *http2Server) handleWindowUpdate(f *http2.WindowUpdateFrame) { + id := f.Header().StreamID + incr := f.Increment + if id == 0 { + t.sendQuotaPool.add(int(incr)) + return + } + if s, ok := t.getStream(f); ok { + s.sendQuotaPool.add(int(incr)) + } +} + +func (t *http2Server) writeHeaders(s *Stream, b *bytes.Buffer, endStream bool) error { + first := true + endHeaders := false + var err error + // Sends the headers in a single batch. + for !endHeaders { + size := t.hBuf.Len() + if size > http2MaxFrameLen { + size = http2MaxFrameLen + } else { + endHeaders = true + } + if first { + p := http2.HeadersFrameParam{ + StreamID: s.id, + BlockFragment: b.Next(size), + EndStream: endStream, + EndHeaders: endHeaders, + } + err = t.framer.writeHeaders(endHeaders, p) + first = false + } else { + err = t.framer.writeContinuation(endHeaders, s.id, endHeaders, b.Next(size)) + } + if err != nil { + t.Close() + return ConnectionErrorf("transport: %v", err) + } + } + return nil +} + +// WriteHeader sends the header metedata md back to the client. +func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { + s.mu.Lock() + if s.headerOk || s.state == streamDone { + s.mu.Unlock() + return ErrIllegalHeaderWrite + } + s.headerOk = true + s.mu.Unlock() + if _, err := wait(s.ctx, t.shutdownChan, t.writableChan); err != nil { + return err + } + t.hBuf.Reset() + t.hEnc.WriteField(hpack.HeaderField{Name: ":status", Value: "200"}) + t.hEnc.WriteField(hpack.HeaderField{Name: "content-type", Value: "application/grpc"}) + for k, v := range md { + for _, entry := range v { + t.hEnc.WriteField(hpack.HeaderField{Name: k, Value: entry}) + } + } + if err := t.writeHeaders(s, t.hBuf, false); err != nil { + return err + } + t.writableChan <- 0 + return nil +} + +// WriteStatus sends stream status to the client and terminates the stream. +// There is no further I/O operations being able to perform on this stream. +// TODO(zhaoq): Now it indicates the end of entire stream. Revisit if early +// OK is adopted. +func (t *http2Server) WriteStatus(s *Stream, statusCode codes.Code, statusDesc string) error { + var headersSent bool + s.mu.Lock() + if s.state == streamDone { + s.mu.Unlock() + return nil + } + if s.headerOk { + headersSent = true + } + s.mu.Unlock() + if _, err := wait(s.ctx, t.shutdownChan, t.writableChan); err != nil { + return err + } + t.hBuf.Reset() + if !headersSent { + t.hEnc.WriteField(hpack.HeaderField{Name: ":status", Value: "200"}) + t.hEnc.WriteField(hpack.HeaderField{Name: "content-type", Value: "application/grpc"}) + } + t.hEnc.WriteField( + hpack.HeaderField{ + Name: "grpc-status", + Value: strconv.Itoa(int(statusCode)), + }) + t.hEnc.WriteField(hpack.HeaderField{Name: "grpc-message", Value: statusDesc}) + // Attach the trailer metadata. + for k, v := range s.trailer { + for _, entry := range v { + t.hEnc.WriteField(hpack.HeaderField{Name: k, Value: entry}) + } + } + if err := t.writeHeaders(s, t.hBuf, true); err != nil { + t.Close() + return err + } + t.closeStream(s) + t.writableChan <- 0 + return nil +} + +// Write converts the data into HTTP2 data frame and sends it out. Non-nil error +// is returns if it fails (e.g., framing error, transport error). +func (t *http2Server) Write(s *Stream, data []byte, opts *Options) error { + // TODO(zhaoq): Support multi-writers for a single stream. + var writeHeaderFrame bool + s.mu.Lock() + if !s.headerOk { + writeHeaderFrame = true + s.headerOk = true + } + s.mu.Unlock() + if writeHeaderFrame { + if _, err := wait(s.ctx, t.shutdownChan, t.writableChan); err != nil { + return err + } + t.hBuf.Reset() + t.hEnc.WriteField(hpack.HeaderField{Name: ":status", Value: "200"}) + t.hEnc.WriteField(hpack.HeaderField{Name: "content-type", Value: "application/grpc"}) + p := http2.HeadersFrameParam{ + StreamID: s.id, + BlockFragment: t.hBuf.Bytes(), + EndHeaders: true, + } + if err := t.framer.writeHeaders(false, p); err != nil { + t.Close() + return ConnectionErrorf("transport: %v", err) + } + t.writableChan <- 0 + } + r := bytes.NewBuffer(data) + for { + if r.Len() == 0 { + return nil + } + size := http2MaxFrameLen + s.sendQuotaPool.add(0) + // Wait until the stream has some quota to send the data. + sq, err := wait(s.ctx, t.shutdownChan, s.sendQuotaPool.acquire()) + if err != nil { + return err + } + t.sendQuotaPool.add(0) + // Wait until the transport has some quota to send the data. + tq, err := wait(s.ctx, t.shutdownChan, t.sendQuotaPool.acquire()) + if err != nil { + if _, ok := err.(StreamError); ok { + t.sendQuotaPool.cancel() + } + return err + } + if sq < size { + size = sq + } + if tq < size { + size = tq + } + p := r.Next(size) + ps := len(p) + if ps < sq { + // Overbooked stream quota. Return it back. + s.sendQuotaPool.add(sq - ps) + } + if ps < tq { + // Overbooked transport quota. Return it back. + t.sendQuotaPool.add(tq - ps) + } + t.framer.adjustNumWriters(1) + // Got some quota. Try to acquire writing privilege on the + // transport. + if _, err := wait(s.ctx, t.shutdownChan, t.writableChan); err != nil { + if t.framer.adjustNumWriters(-1) == 0 { + // This writer is the last one in this batch and has the + // responsibility to flush the buffered frames. It queues + // a flush request to controlBuf instead of flushing directly + // in order to avoid the race with other writing or flushing. + t.controlBuf.put(&flushIO{}) + } + return err + } + var forceFlush bool + if r.Len() == 0 && t.framer.adjustNumWriters(0) == 1 && !opts.Last { + forceFlush = true + } + if err := t.framer.writeData(forceFlush, s.id, false, p); err != nil { + t.Close() + return ConnectionErrorf("transport: %v", err) + } + if t.framer.adjustNumWriters(-1) == 0 { + t.framer.flushWrite() + } + t.writableChan <- 0 + } + +} + +func (t *http2Server) applySettings(ss []http2.Setting) { + for _, s := range ss { + if s.ID == http2.SettingInitialWindowSize { + t.mu.Lock() + defer t.mu.Unlock() + for _, stream := range t.activeStreams { + stream.sendQuotaPool.reset(int(s.Val - t.streamSendQuota)) + } + t.streamSendQuota = s.Val + } + + } +} + +// controller running in a separate goroutine takes charge of sending control +// frames (e.g., window update, reset stream, setting, etc.) to the server. +func (t *http2Server) controller() { + for { + select { + case i := <-t.controlBuf.get(): + t.controlBuf.load() + select { + case <-t.writableChan: + switch i := i.(type) { + case *windowUpdate: + t.framer.writeWindowUpdate(true, i.streamID, i.increment) + case *settings: + if i.ack { + t.framer.writeSettingsAck(true) + t.applySettings(i.ss) + } else { + t.framer.writeSettings(true, i.ss...) + } + case *resetStream: + t.framer.writeRSTStream(true, i.streamID, i.code) + case *flushIO: + t.framer.flushWrite() + case *ping: + // TODO(zhaoq): Ack with all-0 data now. will change to some + // meaningful content when this is actually in use. + t.framer.writePing(true, i.ack, [8]byte{}) + default: + grpclog.Printf("transport: http2Server.controller got unexpected item type %v\n", i) + } + t.writableChan <- 0 + continue + case <-t.shutdownChan: + return + } + case <-t.shutdownChan: + return + } + } +} + +// Close starts shutting down the http2Server transport. +// TODO(zhaoq): Now the destruction is not blocked on any pending streams. This +// could cause some resource issue. Revisit this later. +func (t *http2Server) Close() (err error) { + t.mu.Lock() + if t.state == closing { + t.mu.Unlock() + return errors.New("transport: Close() was already called") + } + t.state = closing + streams := t.activeStreams + t.activeStreams = nil + t.mu.Unlock() + close(t.shutdownChan) + err = t.conn.Close() + // Notify all active streams. + for _, s := range streams { + s.write(recvMsg{err: ErrConnClosing}) + } + return +} + +// closeStream clears the footprint of a stream when the stream is not needed +// any more. +func (t *http2Server) closeStream(s *Stream) { + t.mu.Lock() + delete(t.activeStreams, s.id) + t.mu.Unlock() + if q := s.fc.restoreConn(); q > 0 { + t.controlBuf.put(&windowUpdate{0, q}) + } + s.mu.Lock() + if s.state == streamDone { + s.mu.Unlock() + return + } + s.state = streamDone + s.mu.Unlock() + // In case stream sending and receiving are invoked in separate + // goroutines (e.g., bi-directional streaming), the caller needs + // to call cancel on the stream to interrupt the blocking on + // other goroutines. + s.cancel() +} + +func (t *http2Server) RemoteAddr() net.Addr { + return t.conn.RemoteAddr() +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/transport/http_util.go b/components/engine/vendor/src/google.golang.org/grpc/transport/http_util.go new file mode 100644 index 0000000000..fec4e4755d --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/transport/http_util.go @@ -0,0 +1,451 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +package transport + +import ( + "bufio" + "fmt" + "io" + "net" + "strconv" + "strings" + "sync/atomic" + "time" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" +) + +const ( + // The primary user agent + primaryUA = "grpc-go/0.11" + // http2MaxFrameLen specifies the max length of a HTTP2 frame. + http2MaxFrameLen = 16384 // 16KB frame + // http://http2.github.io/http2-spec/#SettingValues + http2InitHeaderTableSize = 4096 + // http2IOBufSize specifies the buffer size for sending frames. + http2IOBufSize = 32 * 1024 +) + +var ( + clientPreface = []byte(http2.ClientPreface) + http2RSTErrConvTab = map[http2.ErrCode]codes.Code{ + http2.ErrCodeNo: codes.Internal, + http2.ErrCodeProtocol: codes.Internal, + http2.ErrCodeInternal: codes.Internal, + http2.ErrCodeFlowControl: codes.ResourceExhausted, + http2.ErrCodeSettingsTimeout: codes.Internal, + http2.ErrCodeFrameSize: codes.Internal, + http2.ErrCodeRefusedStream: codes.Unavailable, + http2.ErrCodeCancel: codes.Canceled, + http2.ErrCodeCompression: codes.Internal, + http2.ErrCodeConnect: codes.Internal, + http2.ErrCodeEnhanceYourCalm: codes.ResourceExhausted, + http2.ErrCodeInadequateSecurity: codes.PermissionDenied, + } + statusCodeConvTab = map[codes.Code]http2.ErrCode{ + codes.Internal: http2.ErrCodeInternal, + codes.Canceled: http2.ErrCodeCancel, + codes.Unavailable: http2.ErrCodeRefusedStream, + codes.ResourceExhausted: http2.ErrCodeEnhanceYourCalm, + codes.PermissionDenied: http2.ErrCodeInadequateSecurity, + } +) + +// Records the states during HPACK decoding. Must be reset once the +// decoding of the entire headers are finished. +type decodeState struct { + // statusCode caches the stream status received from the trailer + // the server sent. Client side only. + statusCode codes.Code + statusDesc string + // Server side only fields. + timeoutSet bool + timeout time.Duration + method string + // key-value metadata map from the peer. + mdata map[string][]string +} + +// An hpackDecoder decodes HTTP2 headers which may span multiple frames. +type hpackDecoder struct { + h *hpack.Decoder + state decodeState + err error // The err when decoding +} + +// A headerFrame is either a http2.HeaderFrame or http2.ContinuationFrame. +type headerFrame interface { + Header() http2.FrameHeader + HeaderBlockFragment() []byte + HeadersEnded() bool +} + +// isReservedHeader checks whether hdr belongs to HTTP2 headers +// reserved by gRPC protocol. Any other headers are classified as the +// user-specified metadata. +func isReservedHeader(hdr string) bool { + if hdr[0] == ':' { + return true + } + switch hdr { + case "content-type", + "grpc-message-type", + "grpc-encoding", + "grpc-message", + "grpc-status", + "grpc-timeout", + "te": + return true + default: + return false + } +} + +func newHPACKDecoder() *hpackDecoder { + d := &hpackDecoder{} + d.h = hpack.NewDecoder(http2InitHeaderTableSize, func(f hpack.HeaderField) { + switch f.Name { + case "content-type": + if !strings.Contains(f.Value, "application/grpc") { + d.err = StreamErrorf(codes.FailedPrecondition, "transport: received the unexpected header") + return + } + case "grpc-status": + code, err := strconv.Atoi(f.Value) + if err != nil { + d.err = StreamErrorf(codes.Internal, "transport: malformed grpc-status: %v", err) + return + } + d.state.statusCode = codes.Code(code) + case "grpc-message": + d.state.statusDesc = f.Value + case "grpc-timeout": + d.state.timeoutSet = true + var err error + d.state.timeout, err = timeoutDecode(f.Value) + if err != nil { + d.err = StreamErrorf(codes.Internal, "transport: malformed time-out: %v", err) + return + } + case ":path": + d.state.method = f.Value + default: + if !isReservedHeader(f.Name) { + if f.Name == "user-agent" { + i := strings.LastIndex(f.Value, " ") + if i == -1 { + // There is no application user agent string being set. + return + } + // Extract the application user agent string. + f.Value = f.Value[:i] + } + if d.state.mdata == nil { + d.state.mdata = make(map[string][]string) + } + k, v, err := metadata.DecodeKeyValue(f.Name, f.Value) + if err != nil { + grpclog.Printf("Failed to decode (%q, %q): %v", f.Name, f.Value, err) + return + } + d.state.mdata[k] = append(d.state.mdata[k], v) + } + } + }) + return d +} + +func (d *hpackDecoder) decodeClientHTTP2Headers(frame headerFrame) (endHeaders bool, err error) { + d.err = nil + _, err = d.h.Write(frame.HeaderBlockFragment()) + if err != nil { + err = StreamErrorf(codes.Internal, "transport: HPACK header decode error: %v", err) + } + + if frame.HeadersEnded() { + if closeErr := d.h.Close(); closeErr != nil && err == nil { + err = StreamErrorf(codes.Internal, "transport: HPACK decoder close error: %v", closeErr) + } + endHeaders = true + } + + if err == nil && d.err != nil { + err = d.err + } + return +} + +func (d *hpackDecoder) decodeServerHTTP2Headers(frame headerFrame) (endHeaders bool, err error) { + d.err = nil + _, err = d.h.Write(frame.HeaderBlockFragment()) + if err != nil { + err = StreamErrorf(codes.Internal, "transport: HPACK header decode error: %v", err) + } + + if frame.HeadersEnded() { + if closeErr := d.h.Close(); closeErr != nil && err == nil { + err = StreamErrorf(codes.Internal, "transport: HPACK decoder close error: %v", closeErr) + } + endHeaders = true + } + + if err == nil && d.err != nil { + err = d.err + } + return +} + +type timeoutUnit uint8 + +const ( + hour timeoutUnit = 'H' + minute timeoutUnit = 'M' + second timeoutUnit = 'S' + millisecond timeoutUnit = 'm' + microsecond timeoutUnit = 'u' + nanosecond timeoutUnit = 'n' +) + +func timeoutUnitToDuration(u timeoutUnit) (d time.Duration, ok bool) { + switch u { + case hour: + return time.Hour, true + case minute: + return time.Minute, true + case second: + return time.Second, true + case millisecond: + return time.Millisecond, true + case microsecond: + return time.Microsecond, true + case nanosecond: + return time.Nanosecond, true + default: + } + return +} + +const maxTimeoutValue int64 = 100000000 - 1 + +// div does integer division and round-up the result. Note that this is +// equivalent to (d+r-1)/r but has less chance to overflow. +func div(d, r time.Duration) int64 { + if m := d % r; m > 0 { + return int64(d/r + 1) + } + return int64(d / r) +} + +// TODO(zhaoq): It is the simplistic and not bandwidth efficient. Improve it. +func timeoutEncode(t time.Duration) string { + if d := div(t, time.Nanosecond); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "n" + } + if d := div(t, time.Microsecond); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "u" + } + if d := div(t, time.Millisecond); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "m" + } + if d := div(t, time.Second); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "S" + } + if d := div(t, time.Minute); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "M" + } + // Note that maxTimeoutValue * time.Hour > MaxInt64. + return strconv.FormatInt(div(t, time.Hour), 10) + "H" +} + +func timeoutDecode(s string) (time.Duration, error) { + size := len(s) + if size < 2 { + return 0, fmt.Errorf("transport: timeout string is too short: %q", s) + } + unit := timeoutUnit(s[size-1]) + d, ok := timeoutUnitToDuration(unit) + if !ok { + return 0, fmt.Errorf("transport: timeout unit is not recognized: %q", s) + } + t, err := strconv.ParseInt(s[:size-1], 10, 64) + if err != nil { + return 0, err + } + return d * time.Duration(t), nil +} + +type framer struct { + numWriters int32 + reader io.Reader + writer *bufio.Writer + fr *http2.Framer +} + +func newFramer(conn net.Conn) *framer { + f := &framer{ + reader: conn, + writer: bufio.NewWriterSize(conn, http2IOBufSize), + } + f.fr = http2.NewFramer(f.writer, f.reader) + return f +} + +func (f *framer) adjustNumWriters(i int32) int32 { + return atomic.AddInt32(&f.numWriters, i) +} + +// The following writeXXX functions can only be called when the caller gets +// unblocked from writableChan channel (i.e., owns the privilege to write). + +func (f *framer) writeContinuation(forceFlush bool, streamID uint32, endHeaders bool, headerBlockFragment []byte) error { + if err := f.fr.WriteContinuation(streamID, endHeaders, headerBlockFragment); err != nil { + return err + } + if forceFlush { + return f.writer.Flush() + } + return nil +} + +func (f *framer) writeData(forceFlush bool, streamID uint32, endStream bool, data []byte) error { + if err := f.fr.WriteData(streamID, endStream, data); err != nil { + return err + } + if forceFlush { + return f.writer.Flush() + } + return nil +} + +func (f *framer) writeGoAway(forceFlush bool, maxStreamID uint32, code http2.ErrCode, debugData []byte) error { + if err := f.fr.WriteGoAway(maxStreamID, code, debugData); err != nil { + return err + } + if forceFlush { + return f.writer.Flush() + } + return nil +} + +func (f *framer) writeHeaders(forceFlush bool, p http2.HeadersFrameParam) error { + if err := f.fr.WriteHeaders(p); err != nil { + return err + } + if forceFlush { + return f.writer.Flush() + } + return nil +} + +func (f *framer) writePing(forceFlush, ack bool, data [8]byte) error { + if err := f.fr.WritePing(ack, data); err != nil { + return err + } + if forceFlush { + return f.writer.Flush() + } + return nil +} + +func (f *framer) writePriority(forceFlush bool, streamID uint32, p http2.PriorityParam) error { + if err := f.fr.WritePriority(streamID, p); err != nil { + return err + } + if forceFlush { + return f.writer.Flush() + } + return nil +} + +func (f *framer) writePushPromise(forceFlush bool, p http2.PushPromiseParam) error { + if err := f.fr.WritePushPromise(p); err != nil { + return err + } + if forceFlush { + return f.writer.Flush() + } + return nil +} + +func (f *framer) writeRSTStream(forceFlush bool, streamID uint32, code http2.ErrCode) error { + if err := f.fr.WriteRSTStream(streamID, code); err != nil { + return err + } + if forceFlush { + return f.writer.Flush() + } + return nil +} + +func (f *framer) writeSettings(forceFlush bool, settings ...http2.Setting) error { + if err := f.fr.WriteSettings(settings...); err != nil { + return err + } + if forceFlush { + return f.writer.Flush() + } + return nil +} + +func (f *framer) writeSettingsAck(forceFlush bool) error { + if err := f.fr.WriteSettingsAck(); err != nil { + return err + } + if forceFlush { + return f.writer.Flush() + } + return nil +} + +func (f *framer) writeWindowUpdate(forceFlush bool, streamID, incr uint32) error { + if err := f.fr.WriteWindowUpdate(streamID, incr); err != nil { + return err + } + if forceFlush { + return f.writer.Flush() + } + return nil +} + +func (f *framer) flushWrite() error { + return f.writer.Flush() +} + +func (f *framer) readFrame() (http2.Frame, error) { + return f.fr.ReadFrame() +} diff --git a/components/engine/vendor/src/google.golang.org/grpc/transport/transport.go b/components/engine/vendor/src/google.golang.org/grpc/transport/transport.go new file mode 100644 index 0000000000..e1e7f5761a --- /dev/null +++ b/components/engine/vendor/src/google.golang.org/grpc/transport/transport.go @@ -0,0 +1,465 @@ +/* + * + * Copyright 2014, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* +Package transport defines and implements message oriented communication channel +to complete various transactions (e.g., an RPC). +*/ +package transport // import "google.golang.org/grpc/transport" + +import ( + "bytes" + "errors" + "fmt" + "io" + "net" + "sync" + "time" + + "golang.org/x/net/context" + "golang.org/x/net/trace" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" +) + +// recvMsg represents the received msg from the transport. All transport +// protocol specific info has been removed. +type recvMsg struct { + data []byte + // nil: received some data + // io.EOF: stream is completed. data is nil. + // other non-nil error: transport failure. data is nil. + err error +} + +func (recvMsg) isItem() bool { + return true +} + +// All items in an out of a recvBuffer should be the same type. +type item interface { + isItem() bool +} + +// recvBuffer is an unbounded channel of item. +type recvBuffer struct { + c chan item + mu sync.Mutex + backlog []item +} + +func newRecvBuffer() *recvBuffer { + b := &recvBuffer{ + c: make(chan item, 1), + } + return b +} + +func (b *recvBuffer) put(r item) { + b.mu.Lock() + defer b.mu.Unlock() + b.backlog = append(b.backlog, r) + select { + case b.c <- b.backlog[0]: + b.backlog = b.backlog[1:] + default: + } +} + +func (b *recvBuffer) load() { + b.mu.Lock() + defer b.mu.Unlock() + if len(b.backlog) > 0 { + select { + case b.c <- b.backlog[0]: + b.backlog = b.backlog[1:] + default: + } + } +} + +// get returns the channel that receives an item in the buffer. +// +// Upon receipt of an item, the caller should call load to send another +// item onto the channel if there is any. +func (b *recvBuffer) get() <-chan item { + return b.c +} + +// recvBufferReader implements io.Reader interface to read the data from +// recvBuffer. +type recvBufferReader struct { + ctx context.Context + recv *recvBuffer + last *bytes.Reader // Stores the remaining data in the previous calls. + err error +} + +// Read reads the next len(p) bytes from last. If last is drained, it tries to +// read additional data from recv. It blocks if there no additional data available +// in recv. If Read returns any non-nil error, it will continue to return that error. +func (r *recvBufferReader) Read(p []byte) (n int, err error) { + if r.err != nil { + return 0, r.err + } + defer func() { r.err = err }() + if r.last != nil && r.last.Len() > 0 { + // Read remaining data left in last call. + return r.last.Read(p) + } + select { + case <-r.ctx.Done(): + return 0, ContextErr(r.ctx.Err()) + case i := <-r.recv.get(): + r.recv.load() + m := i.(*recvMsg) + if m.err != nil { + return 0, m.err + } + r.last = bytes.NewReader(m.data) + return r.last.Read(p) + } +} + +type streamState uint8 + +const ( + streamActive streamState = iota + streamWriteDone // EndStream sent + streamReadDone // EndStream received + streamDone // sendDone and recvDone or RSTStreamFrame is sent or received. +) + +// Stream represents an RPC in the transport layer. +type Stream struct { + id uint32 + // nil for client side Stream. + st ServerTransport + // ctx is the associated context of the stream. + ctx context.Context + cancel context.CancelFunc + // method records the associated RPC method of the stream. + method string + buf *recvBuffer + dec io.Reader + fc *inFlow + recvQuota uint32 + // The accumulated inbound quota pending for window update. + updateQuota uint32 + // The handler to control the window update procedure for both this + // particular stream and the associated transport. + windowHandler func(int) + + sendQuotaPool *quotaPool + // Close headerChan to indicate the end of reception of header metadata. + headerChan chan struct{} + // header caches the received header metadata. + header metadata.MD + // The key-value map of trailer metadata. + trailer metadata.MD + + mu sync.RWMutex // guard the following + // headerOK becomes true from the first header is about to send. + headerOk bool + state streamState + // true iff headerChan is closed. Used to avoid closing headerChan + // multiple times. + headerDone bool + // the status received from the server. + statusCode codes.Code + statusDesc string +} + +// Header acquires the key-value pairs of header metadata once it +// is available. It blocks until i) the metadata is ready or ii) there is no +// header metadata or iii) the stream is cancelled/expired. +func (s *Stream) Header() (metadata.MD, error) { + select { + case <-s.ctx.Done(): + return nil, ContextErr(s.ctx.Err()) + case <-s.headerChan: + return s.header.Copy(), nil + } +} + +// Trailer returns the cached trailer metedata. Note that if it is not called +// after the entire stream is done, it could return an empty MD. Client +// side only. +func (s *Stream) Trailer() metadata.MD { + s.mu.RLock() + defer s.mu.RUnlock() + return s.trailer.Copy() +} + +// ServerTransport returns the underlying ServerTransport for the stream. +// The client side stream always returns nil. +func (s *Stream) ServerTransport() ServerTransport { + return s.st +} + +// Context returns the context of the stream. +func (s *Stream) Context() context.Context { + return s.ctx +} + +// TraceContext recreates the context of s with a trace.Trace. +func (s *Stream) TraceContext(tr trace.Trace) { + s.ctx = trace.NewContext(s.ctx, tr) +} + +// Method returns the method for the stream. +func (s *Stream) Method() string { + return s.method +} + +// StatusCode returns statusCode received from the server. +func (s *Stream) StatusCode() codes.Code { + return s.statusCode +} + +// StatusDesc returns statusDesc received from the server. +func (s *Stream) StatusDesc() string { + return s.statusDesc +} + +// ErrIllegalTrailerSet indicates that the trailer has already been set or it +// is too late to do so. +var ErrIllegalTrailerSet = errors.New("transport: trailer has been set") + +// SetTrailer sets the trailer metadata which will be sent with the RPC status +// by the server. This can only be called at most once. Server side only. +func (s *Stream) SetTrailer(md metadata.MD) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.trailer != nil { + return ErrIllegalTrailerSet + } + s.trailer = md.Copy() + return nil +} + +func (s *Stream) write(m recvMsg) { + s.buf.put(&m) +} + +// Read reads all the data available for this Stream from the transport and +// passes them into the decoder, which converts them into a gRPC message stream. +// The error is io.EOF when the stream is done or another non-nil error if +// the stream broke. +func (s *Stream) Read(p []byte) (n int, err error) { + n, err = s.dec.Read(p) + if err != nil { + return + } + s.windowHandler(n) + return +} + +type key int + +// The key to save transport.Stream in the context. +const streamKey = key(0) + +// newContextWithStream creates a new context from ctx and attaches stream +// to it. +func newContextWithStream(ctx context.Context, stream *Stream) context.Context { + return context.WithValue(ctx, streamKey, stream) +} + +// StreamFromContext returns the stream saved in ctx. +func StreamFromContext(ctx context.Context) (s *Stream, ok bool) { + s, ok = ctx.Value(streamKey).(*Stream) + return +} + +// state of transport +type transportState int + +const ( + reachable transportState = iota + unreachable + closing +) + +// NewServerTransport creates a ServerTransport with conn or non-nil error +// if it fails. +func NewServerTransport(protocol string, conn net.Conn, maxStreams uint32, authInfo credentials.AuthInfo) (ServerTransport, error) { + return newHTTP2Server(conn, maxStreams, authInfo) +} + +// ConnectOptions covers all relevant options for dialing a server. +type ConnectOptions struct { + // UserAgent is the application user agent. + UserAgent string + // Dialer specifies how to dial a network address. + Dialer func(string, time.Duration) (net.Conn, error) + // AuthOptions stores the credentials required to setup a client connection and/or issue RPCs. + AuthOptions []credentials.Credentials + // Timeout specifies the timeout for dialing a client connection. + Timeout time.Duration +} + +// NewClientTransport establishes the transport with the required ConnectOptions +// and returns it to the caller. +func NewClientTransport(target string, opts *ConnectOptions) (ClientTransport, error) { + return newHTTP2Client(target, opts) +} + +// Options provides additional hints and information for message +// transmission. +type Options struct { + // Indicate whether it is the last piece for this stream. + Last bool + // The hint to transport impl whether the data could be buffered for + // batching write. Transport impl can feel free to ignore it. + Delay bool +} + +// CallHdr carries the information of a particular RPC. +type CallHdr struct { + Host string // peer host + Method string // the operation to perform on the specified host +} + +// ClientTransport is the common interface for all gRPC client side transport +// implementations. +type ClientTransport interface { + // Close tears down this transport. Once it returns, the transport + // should not be accessed any more. The caller must make sure this + // is called only once. + Close() error + + // Write sends the data for the given stream. A nil stream indicates + // the write is to be performed on the transport as a whole. + Write(s *Stream, data []byte, opts *Options) error + + // NewStream creates a Stream for an RPC. + NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, error) + + // CloseStream clears the footprint of a stream when the stream is + // not needed any more. The err indicates the error incurred when + // CloseStream is called. Must be called when a stream is finished + // unless the associated transport is closing. + CloseStream(stream *Stream, err error) + + // Error returns a channel that is closed when some I/O error + // happens. Typically the caller should have a goroutine to monitor + // this in order to take action (e.g., close the current transport + // and create a new one) in error case. It should not return nil + // once the transport is initiated. + Error() <-chan struct{} +} + +// ServerTransport is the common interface for all gRPC server side transport +// implementations. +type ServerTransport interface { + // WriteStatus sends the status of a stream to the client. + WriteStatus(s *Stream, statusCode codes.Code, statusDesc string) error + // Write sends the data for the given stream. + Write(s *Stream, data []byte, opts *Options) error + // WriteHeader sends the header metedata for the given stream. + WriteHeader(s *Stream, md metadata.MD) error + // HandleStreams receives incoming streams using the given handler. + HandleStreams(func(*Stream)) + // Close tears down the transport. Once it is called, the transport + // should not be accessed any more. All the pending streams and their + // handlers will be terminated asynchronously. + Close() error + // RemoteAddr returns the remote network address. + RemoteAddr() net.Addr +} + +// StreamErrorf creates an StreamError with the specified error code and description. +func StreamErrorf(c codes.Code, format string, a ...interface{}) StreamError { + return StreamError{ + Code: c, + Desc: fmt.Sprintf(format, a...), + } +} + +// ConnectionErrorf creates an ConnectionError with the specified error description. +func ConnectionErrorf(format string, a ...interface{}) ConnectionError { + return ConnectionError{ + Desc: fmt.Sprintf(format, a...), + } +} + +// ConnectionError is an error that results in the termination of the +// entire connection and the retry of all the active streams. +type ConnectionError struct { + Desc string +} + +func (e ConnectionError) Error() string { + return fmt.Sprintf("connection error: desc = %q", e.Desc) +} + +// Define some common ConnectionErrors. +var ErrConnClosing = ConnectionError{Desc: "transport is closing"} + +// StreamError is an error that only affects one stream within a connection. +type StreamError struct { + Code codes.Code + Desc string +} + +func (e StreamError) Error() string { + return fmt.Sprintf("stream error: code = %d desc = %q", e.Code, e.Desc) +} + +// ContextErr converts the error from context package into a StreamError. +func ContextErr(err error) StreamError { + switch err { + case context.DeadlineExceeded: + return StreamErrorf(codes.DeadlineExceeded, "%v", err) + case context.Canceled: + return StreamErrorf(codes.Canceled, "%v", err) + } + panic(fmt.Sprintf("Unexpected error from context packet: %v", err)) +} + +// wait blocks until it can receive from ctx.Done, closing, or proceed. +// If it receives from ctx.Done, it returns 0, the StreamError for ctx.Err. +// If it receives from closing, it returns 0, ErrConnClosing. +// If it receives from proceed, it returns the received integer, nil. +func wait(ctx context.Context, closing <-chan struct{}, proceed <-chan int) (int, error) { + select { + case <-ctx.Done(): + return 0, ContextErr(ctx.Err()) + case <-closing: + return 0, ErrConnClosing + case i := <-proceed: + return i, nil + } +} diff --git a/components/engine/volume/drivers/adapter.go b/components/engine/volume/drivers/adapter.go index 064dbffe36..1468b34344 100644 --- a/components/engine/volume/drivers/adapter.go +++ b/components/engine/volume/drivers/adapter.go @@ -1,7 +1,8 @@ package volumedrivers import ( - "github.com/docker/docker/pkg/plugins" + "fmt" + "github.com/docker/docker/volume" ) @@ -15,21 +16,7 @@ func (a *volumeDriverAdapter) Name() string { } func (a *volumeDriverAdapter) Create(name string, opts map[string]string) (volume.Volume, error) { - // First try a Get. For drivers that support Get this will return any - // existing volume. - v, err := a.proxy.Get(name) - if v != nil { - return &volumeAdapter{ - proxy: a.proxy, - name: v.Name, - driverName: a.Name(), - eMount: v.Mountpoint, - }, nil - } - - // Driver didn't support Get or volume didn't exist. Perform Create. - err = a.proxy.Create(name, opts) - if err != nil { + if err := a.proxy.Create(name, opts); err != nil { return nil, err } return &volumeAdapter{ @@ -63,11 +50,12 @@ func (a *volumeDriverAdapter) List() ([]volume.Volume, error) { func (a *volumeDriverAdapter) Get(name string) (volume.Volume, error) { v, err := a.proxy.Get(name) if err != nil { - // TODO: remove this hack. Allows back compat with volume drivers that don't support this call - if !plugins.IsNotFound(err) { - return nil, err - } - return a.Create(name, nil) + return nil, err + } + + // plugin may have returned no volume and no error + if v == nil { + return nil, fmt.Errorf("no such volume") } return &volumeAdapter{ diff --git a/components/engine/volume/drivers/extpoint.go b/components/engine/volume/drivers/extpoint.go index dd45d1365a..d705b8a597 100644 --- a/components/engine/volume/drivers/extpoint.go +++ b/components/engine/volume/drivers/extpoint.go @@ -43,7 +43,7 @@ type volumeDriver interface { Unmount(name string) (err error) // List lists all the volumes known to the driver List() (volumes list, err error) - // Get retreives the volume with the requested name + // Get retrieves the volume with the requested name Get(name string) (volume *proxyVolume, err error) } @@ -71,7 +71,7 @@ func Register(extension volume.Driver, name string) bool { return true } -// Unregister dissociates the name from it's driver, if the association exists. +// Unregister dissociates the name from its driver, if the association exists. func Unregister(name string) bool { drivers.Lock() defer drivers.Unlock() @@ -114,7 +114,7 @@ func Lookup(name string) (volume.Driver, error) { return d, nil } -// GetDriver returns a volume driver by it's name. +// GetDriver returns a volume driver by its name. // If the driver is empty, it looks for the local driver. func GetDriver(name string) (volume.Driver, error) { if name == "" { diff --git a/components/engine/volume/local/local.go b/components/engine/volume/local/local.go index 1a44c05858..b154a36e7b 100644 --- a/components/engine/volume/local/local.go +++ b/components/engine/volume/local/local.go @@ -4,15 +4,16 @@ package local import ( - "errors" + "encoding/json" "fmt" "io/ioutil" "os" "path/filepath" "sync" - derr "github.com/docker/docker/errors" + "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/idtools" + "github.com/docker/docker/pkg/mount" "github.com/docker/docker/utils" "github.com/docker/docker/volume" ) @@ -27,13 +28,26 @@ const ( var ( // ErrNotFound is the typed error returned when the requested volume name can't be found - ErrNotFound = errors.New("volume not found") + ErrNotFound = fmt.Errorf("volume not found") // volumeNameRegex ensures the name assigned for the volume is valid. // This name is used to create the bind directory, so we need to avoid characters that // would make the path to escape the root directory. volumeNameRegex = utils.RestrictedVolumeNamePattern ) +type validationError struct { + error +} + +func (validationError) IsValidationError() bool { + return true +} + +type activeMount struct { + count uint64 + mounted bool +} + // New instantiates a new Root instance with the provided scope. Scope // is the base path that the Root instance uses to store its // volumes. The base path is created here if it does not exist. @@ -57,13 +71,32 @@ func New(scope string, rootUID, rootGID int) (*Root, error) { return nil, err } + mountInfos, err := mount.GetMounts() + if err != nil { + logrus.Debugf("error looking up mounts for local volume cleanup: %v", err) + } + for _, d := range dirs { name := filepath.Base(d.Name()) - r.volumes[name] = &localVolume{ + v := &localVolume{ driverName: r.Name(), name: name, path: r.DataPath(name), } + r.volumes[name] = v + if b, err := ioutil.ReadFile(filepath.Join(name, "opts.json")); err == nil { + if err := json.Unmarshal(b, v.opts); err != nil { + return nil, err + } + + // unmount anything that may still be mounted (for example, from an unclean shutdown) + for _, info := range mountInfos { + if info.Mountpoint == v.path { + mount.Unmount(v.path) + break + } + } + } } return r, nil @@ -103,7 +136,7 @@ func (r *Root) Name() string { // Create creates a new volume.Volume with the provided name, creating // the underlying directory tree required for this volume in the // process. -func (r *Root) Create(name string, _ map[string]string) (volume.Volume, error) { +func (r *Root) Create(name string, opts map[string]string) (volume.Volume, error) { if err := r.validateName(name); err != nil { return nil, err } @@ -123,11 +156,34 @@ func (r *Root) Create(name string, _ map[string]string) (volume.Volume, error) { } return nil, err } + + var err error + defer func() { + if err != nil { + os.RemoveAll(filepath.Dir(path)) + } + }() + v = &localVolume{ driverName: r.Name(), name: name, path: path, } + + if opts != nil { + if err = setOpts(v, opts); err != nil { + return nil, err + } + var b []byte + b, err = json.Marshal(v.opts) + if err != nil { + return nil, err + } + if err = ioutil.WriteFile(filepath.Join(filepath.Dir(path), "opts.json"), b, 600); err != nil { + return nil, err + } + } + r.volumes[name] = v return v, nil } @@ -142,7 +198,7 @@ func (r *Root) Remove(v volume.Volume) error { lv, ok := v.(*localVolume) if !ok { - return errors.New("unknown volume type") + return fmt.Errorf("unknown volume type") } realPath, err := filepath.EvalSymlinks(lv.path) @@ -188,7 +244,7 @@ func (r *Root) Get(name string) (volume.Volume, error) { func (r *Root) validateName(name string) error { if !volumeNameRegex.MatchString(name) { - return derr.ErrorCodeVolumeName.WithArgs(name, utils.RestrictedNameChars) + return validationError{fmt.Errorf("%q includes invalid characters for a local volume name, only %q are allowed", name, utils.RestrictedNameChars)} } return nil } @@ -204,6 +260,10 @@ type localVolume struct { path string // driverName is the name of the driver that created the volume. driverName string + // opts is the parsed list of options used to create the volume + opts *optsConfig + // active refcounts the active mounts + active activeMount } // Name returns the name of the given Volume. @@ -223,10 +283,42 @@ func (v *localVolume) Path() string { // Mount implements the localVolume interface, returning the data location. func (v *localVolume) Mount() (string, error) { + v.m.Lock() + defer v.m.Unlock() + if v.opts != nil { + if !v.active.mounted { + if err := v.mount(); err != nil { + return "", err + } + v.active.mounted = true + } + v.active.count++ + } return v.path, nil } // Umount is for satisfying the localVolume interface and does not do anything in this driver. func (v *localVolume) Unmount() error { + v.m.Lock() + defer v.m.Unlock() + if v.opts != nil { + v.active.count-- + if v.active.count == 0 { + if err := mount.Unmount(v.path); err != nil { + v.active.count++ + return err + } + v.active.mounted = false + } + } + return nil +} + +func validateOpts(opts map[string]string) error { + for opt := range opts { + if !validOpts[opt] { + return validationError{fmt.Errorf("invalid option key: %q", opt)} + } + } return nil } diff --git a/components/engine/volume/local/local_test.go b/components/engine/volume/local/local_test.go index decf07cba1..1baa085457 100644 --- a/components/engine/volume/local/local_test.go +++ b/components/engine/volume/local/local_test.go @@ -3,10 +3,19 @@ package local import ( "io/ioutil" "os" + "runtime" + "strings" "testing" + + "github.com/docker/docker/pkg/mount" ) func TestRemove(t *testing.T) { + // TODO Windows: Investigate why this test fails on Windows under CI + // but passes locally. + if runtime.GOOS == "windows" { + t.Skip("Test failing on Windows CI") + } rootDir, err := ioutil.TempDir("", "local-volume-test") if err != nil { t.Fatal(err) @@ -145,3 +154,96 @@ func TestValidateName(t *testing.T) { } } } + +func TestCreateWithOpts(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + rootDir, err := ioutil.TempDir("", "local-volume-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(rootDir) + + r, err := New(rootDir, 0, 0) + if err != nil { + t.Fatal(err) + } + + if _, err := r.Create("test", map[string]string{"invalidopt": "notsupported"}); err == nil { + t.Fatal("expected invalid opt to cause error") + } + + vol, err := r.Create("test", map[string]string{"device": "tmpfs", "type": "tmpfs", "o": "size=1m,uid=1000"}) + if err != nil { + t.Fatal(err) + } + v := vol.(*localVolume) + + dir, err := v.Mount() + if err != nil { + t.Fatal(err) + } + defer func() { + if err := v.Unmount(); err != nil { + t.Fatal(err) + } + }() + + mountInfos, err := mount.GetMounts() + if err != nil { + t.Fatal(err) + } + + var found bool + for _, info := range mountInfos { + if info.Mountpoint == dir { + found = true + if info.Fstype != "tmpfs" { + t.Fatalf("expected tmpfs mount, got %q", info.Fstype) + } + if info.Source != "tmpfs" { + t.Fatalf("expected tmpfs mount, got %q", info.Source) + } + if !strings.Contains(info.VfsOpts, "uid=1000") { + t.Fatalf("expected mount info to have uid=1000: %q", info.VfsOpts) + } + if !strings.Contains(info.VfsOpts, "size=1024k") { + t.Fatalf("expected mount info to have size=1024k: %q", info.VfsOpts) + } + break + } + } + + if !found { + t.Fatal("mount not found") + } + + if v.active.count != 1 { + t.Fatalf("Expected active mount count to be 1, got %d", v.active.count) + } + + // test double mount + if _, err := v.Mount(); err != nil { + t.Fatal(err) + } + if v.active.count != 2 { + t.Fatalf("Expected active mount count to be 2, got %d", v.active.count) + } + + if err := v.Unmount(); err != nil { + t.Fatal(err) + } + if v.active.count != 1 { + t.Fatalf("Expected active mount count to be 1, got %d", v.active.count) + } + + mounted, err := mount.Mounted(v.path) + if err != nil { + t.Fatal(err) + } + if !mounted { + t.Fatal("expected mount to still be active") + } +} diff --git a/components/engine/volume/local/local_unix.go b/components/engine/volume/local/local_unix.go index 60f0e765d8..2e63777a19 100644 --- a/components/engine/volume/local/local_unix.go +++ b/components/engine/volume/local/local_unix.go @@ -6,11 +6,28 @@ package local import ( + "fmt" "path/filepath" "strings" + + "github.com/docker/docker/pkg/mount" ) -var oldVfsDir = filepath.Join("vfs", "dir") +var ( + oldVfsDir = filepath.Join("vfs", "dir") + + validOpts = map[string]bool{ + "type": true, // specify the filesystem type for mount, e.g. nfs + "o": true, // generic mount options + "device": true, // device to mount from + } +) + +type optsConfig struct { + MountType string + MountOpts string + MountDevice string +} // scopedPath verifies that the path where the volume is located // is under Docker's root and the valid local paths. @@ -27,3 +44,26 @@ func (r *Root) scopedPath(realPath string) bool { return false } + +func setOpts(v *localVolume, opts map[string]string) error { + if len(opts) == 0 { + return nil + } + if err := validateOpts(opts); err != nil { + return err + } + + v.opts = &optsConfig{ + MountType: opts["type"], + MountOpts: opts["o"], + MountDevice: opts["device"], + } + return nil +} + +func (v *localVolume) mount() error { + if v.opts.MountDevice == "" { + return fmt.Errorf("missing device in volume options") + } + return mount.Mount(v.opts.MountDevice, v.path, v.opts.MountType, v.opts.MountOpts) +} diff --git a/components/engine/volume/local/local_windows.go b/components/engine/volume/local/local_windows.go index 38812aa2f5..1bdb368a0f 100644 --- a/components/engine/volume/local/local_windows.go +++ b/components/engine/volume/local/local_windows.go @@ -4,10 +4,15 @@ package local import ( + "fmt" "path/filepath" "strings" ) +type optsConfig struct{} + +var validOpts map[string]bool + // scopedPath verifies that the path where the volume is located // is under Docker's root and the valid local paths. func (r *Root) scopedPath(realPath string) bool { @@ -16,3 +21,14 @@ func (r *Root) scopedPath(realPath string) bool { } return false } + +func setOpts(v *localVolume, opts map[string]string) error { + if len(opts) > 0 { + return fmt.Errorf("options are not supported on this platform") + } + return nil +} + +func (v *localVolume) mount() error { + return nil +} diff --git a/components/engine/volume/store/store.go b/components/engine/volume/store/store.go index 0d227ae01a..6fe0717b26 100644 --- a/components/engine/volume/store/store.go +++ b/components/engine/volume/store/store.go @@ -54,7 +54,7 @@ type VolumeStore struct { } // List proxies to all registered volume drivers to get the full list of volumes -// If a driver returns a volume that has name which conflicts with a another volume from a different driver, +// If a driver returns a volume that has name which conflicts with another volume from a different driver, // the first volume is chosen and the conflicting volume is dropped. func (s *VolumeStore) List() ([]volume.Volume, []string, error) { vols, warnings, err := s.list() @@ -168,7 +168,7 @@ func (s *VolumeStore) Create(name, driverName string, opts map[string]string) (v // create asks the given driver to create a volume with the name/opts. // If a volume with the name is already known, it will ask the stored driver for the volume. // If the passed in driver name does not match the driver name which is stored for the given volume name, an error is returned. -// It is expected that callers of this function hold any neccessary locks. +// It is expected that callers of this function hold any necessary locks. func (s *VolumeStore) create(name, driverName string, opts map[string]string) (volume.Volume, error) { // Validate the name in a platform-specific manner valid, err := volume.IsVolumeNameValid(name) @@ -186,18 +186,31 @@ func (s *VolumeStore) create(name, driverName string, opts map[string]string) (v return v, nil } - logrus.Debugf("Registering new volume reference: driver %s, name %s", driverName, name) + // Since there isn't a specified driver name, let's see if any of the existing drivers have this volume name + if driverName == "" { + v, _ := s.getVolume(name) + if v != nil { + return v, nil + } + } + vd, err := volumedrivers.GetDriver(driverName) + if err != nil { return nil, &OpErr{Op: "create", Name: name, Err: err} } + logrus.Debugf("Registering new volume reference: driver %q, name %q", vd.Name(), name) + + if v, _ := vd.Get(name); v != nil { + return v, nil + } return vd.Create(name, opts) } // GetWithRef gets a volume with the given name from the passed in driver and stores the ref // This is just like Get(), but we store the reference while holding the lock. -// This makes sure there are no races between checking for the existance of a volume and adding a reference for it +// This makes sure there are no races between checking for the existence of a volume and adding a reference for it func (s *VolumeStore) GetWithRef(name, driverName, ref string) (volume.Volume, error) { name = normaliseVolumeName(name) s.locks.Lock(name) @@ -231,9 +244,9 @@ func (s *VolumeStore) Get(name string) (volume.Volume, error) { return v, nil } -// get requests the volume, if the driver info is stored it just access that driver, +// getVolume requests the volume, if the driver info is stored it just accesses that driver, // if the driver is unknown it probes all drivers until it finds the first volume with that name. -// it is expected that callers of this function hold any neccessary locks +// it is expected that callers of this function hold any necessary locks func (s *VolumeStore) getVolume(name string) (volume.Volume, error) { logrus.Debugf("Getting volume reference for name: %s", name) if v, exists := s.names[name]; exists { @@ -291,16 +304,14 @@ func (s *VolumeStore) Dereference(v volume.Volume, ref string) { s.globalLock.Lock() defer s.globalLock.Unlock() - refs, exists := s.refs[v.Name()] - if !exists { - return - } + var refs []string - for i, r := range refs { - if r == ref { - s.refs[v.Name()] = append(s.refs[v.Name()][:i], s.refs[v.Name()][i+1:]...) + for _, r := range s.refs[v.Name()] { + if r != ref { + refs = append(refs, r) } } + s.refs[v.Name()] = refs } // Refs gets the current list of refs for the given volume diff --git a/components/engine/volume/store/store_test.go b/components/engine/volume/store/store_test.go index 83d49821be..7c3f730ef0 100644 --- a/components/engine/volume/store/store_test.go +++ b/components/engine/volume/store/store_test.go @@ -157,3 +157,22 @@ func TestFilterByUsed(t *testing.T) { t.Fatalf("expected used volume fake1, got %s", used[0].Name()) } } + +func TestDerefMultipleOfSameRef(t *testing.T) { + volumedrivers.Register(vt.NewFakeDriver("fake"), "fake") + + s := New() + v, err := s.CreateWithRef("fake1", "fake", "volReference", nil) + if err != nil { + t.Fatal(err) + } + + if _, err := s.GetWithRef("fake1", "fake", "volReference"); err != nil { + t.Fatal(err) + } + + s.Dereference(v, "volReference") + if err := s.Remove(v); err != nil { + t.Fatal(err) + } +} diff --git a/components/engine/volume/volume.go b/components/engine/volume/volume.go index d270b156f8..c3f23bbedd 100644 --- a/components/engine/volume/volume.go +++ b/components/engine/volume/volume.go @@ -1,12 +1,12 @@ package volume import ( + "fmt" "os" "runtime" "strings" "github.com/Sirupsen/logrus" - derr "github.com/docker/docker/errors" "github.com/docker/docker/pkg/system" ) @@ -24,7 +24,7 @@ type Driver interface { Remove(vol Volume) (err error) // List lists all the volumes the driver has List() ([]Volume, error) - // Get retreives the volume with the requested name + // Get retrieves the volume with the requested name Get(name string) (Volume, error) } @@ -82,7 +82,7 @@ func (m *MountPoint) Setup() (string, error) { } return m.Source, nil } - return "", derr.ErrorCodeMountSetup + return "", fmt.Errorf("Unable to setup mount point, neither source nor volume defined") } // Path returns the path of a volume in a mount point. @@ -93,10 +93,10 @@ func (m *MountPoint) Path() string { return m.Source } -// ParseVolumesFrom ensure that the supplied volumes-from is valid. +// ParseVolumesFrom ensures that the supplied volumes-from is valid. func ParseVolumesFrom(spec string) (string, string, error) { if len(spec) == 0 { - return "", "", derr.ErrorCodeVolumeFromBlank.WithArgs(spec) + return "", "", fmt.Errorf("malformed volumes-from specification: %s", spec) } specParts := strings.SplitN(spec, ":", 2) @@ -106,15 +106,23 @@ func ParseVolumesFrom(spec string) (string, string, error) { if len(specParts) == 2 { mode = specParts[1] if !ValidMountMode(mode) { - return "", "", derr.ErrorCodeVolumeInvalidMode.WithArgs(mode) + return "", "", errInvalidMode(mode) } // For now don't allow propagation properties while importing // volumes from data container. These volumes will inherit // the same propagation property as of the original volume // in data container. This probably can be relaxed in future. if HasPropagation(mode) { - return "", "", derr.ErrorCodeVolumeInvalidMode.WithArgs(mode) + return "", "", errInvalidMode(mode) } } return id, mode, nil } + +func errInvalidMode(mode string) error { + return fmt.Errorf("invalid mode: %v", mode) +} + +func errInvalidSpec(spec string) error { + return fmt.Errorf("Invalid volume specification: '%s'", spec) +} diff --git a/components/engine/volume/volume_propagation_linux.go b/components/engine/volume/volume_propagation_linux.go index 83f2df25be..f5f28205a0 100644 --- a/components/engine/volume/volume_propagation_linux.go +++ b/components/engine/volume/volume_propagation_linux.go @@ -32,7 +32,7 @@ func GetPropagation(mode string) string { } // HasPropagation checks if there is a valid propagation mode present in -// passed string. Returns true if a valid propagatio mode specifier is +// passed string. Returns true if a valid propagation mode specifier is // present, false otherwise. func HasPropagation(mode string) bool { for _, o := range strings.Split(mode, ",") { diff --git a/components/engine/volume/volume_propagation_linux_test.go b/components/engine/volume/volume_propagation_linux_test.go index 9cf82528bd..e579fa05ff 100644 --- a/components/engine/volume/volume_propagation_linux_test.go +++ b/components/engine/volume/volume_propagation_linux_test.go @@ -34,8 +34,8 @@ func TestParseMountSpecPropagation(t *testing.T) { "/hostPath:/containerPath:ro,Z,rprivate", } invalid = map[string]string{ - "/path:/path:ro,rshared,rslave": `invalid mode: "ro,rshared,rslave"`, - "/path:/path:ro,z,rshared,rslave": `invalid mode: "ro,z,rshared,rslave"`, + "/path:/path:ro,rshared,rslave": `invalid mode: ro,rshared,rslave`, + "/path:/path:ro,z,rshared,rslave": `invalid mode: ro,z,rshared,rslave`, "/path:shared": "Invalid volume specification", "/path:slave": "Invalid volume specification", "/path:private": "Invalid volume specification", diff --git a/components/engine/volume/volume_propagation_unsupported.go b/components/engine/volume/volume_propagation_unsupported.go index 85e4d46b33..0edc89abe3 100644 --- a/components/engine/volume/volume_propagation_unsupported.go +++ b/components/engine/volume/volume_propagation_unsupported.go @@ -15,7 +15,7 @@ func GetPropagation(mode string) string { } // HasPropagation checks if there is a valid propagation mode present in -// passed string. Returns true if a valid propagatio mode specifier is +// passed string. Returns true if a valid propagation mode specifier is // present, false otherwise. func HasPropagation(mode string) bool { return false diff --git a/components/engine/volume/volume_test.go b/components/engine/volume/volume_test.go index d3e7acc779..0077e82b99 100644 --- a/components/engine/volume/volume_test.go +++ b/components/engine/volume/volume_test.go @@ -111,8 +111,8 @@ func TestParseMountSpec(t *testing.T) { "/path:ro": "Invalid volume specification", "/rw:rw": "Invalid volume specification", "path:ro": "Invalid volume specification", - "/path:/path:sw": `invalid mode: "sw"`, - "/path:/path:rwz": `invalid mode: "rwz"`, + "/path:/path:sw": `invalid mode: sw`, + "/path:/path:rwz": `invalid mode: rwz`, } } @@ -167,10 +167,10 @@ func TestParseMountSpecSplit(t *testing.T) { {"/tmp:/tmp2:ro", "", "/tmp2", "/tmp", "", "", false, false}, {"/tmp:/tmp3:rw", "", "/tmp3", "/tmp", "", "", true, false}, {"/tmp:/tmp4:foo", "", "", "", "", "", false, true}, - {"name:/named1", "", "/named1", "", "name", "local", true, false}, + {"name:/named1", "", "/named1", "", "name", "", true, false}, {"name:/named2", "external", "/named2", "", "name", "external", true, false}, {"name:/named3:ro", "local", "/named3", "", "name", "local", false, false}, - {"local/name:/tmp:rw", "", "/tmp", "", "local/name", "local", true, false}, + {"local/name:/tmp:rw", "", "/tmp", "", "local/name", "", true, false}, {"/tmp:tmp", "", "", "", "", "", true, true}, } } diff --git a/components/engine/volume/volume_unix.go b/components/engine/volume/volume_unix.go index ddf278f07f..acd0b1bcd0 100644 --- a/components/engine/volume/volume_unix.go +++ b/components/engine/volume/volume_unix.go @@ -6,8 +6,6 @@ import ( "fmt" "path/filepath" "strings" - - derr "github.com/docker/docker/errors" ) // read-write modes @@ -47,12 +45,12 @@ func ParseMountSpec(spec, volumeDriver string) (*MountPoint, error) { Propagation: DefaultPropagationMode, } if strings.Count(spec, ":") > 2 { - return nil, derr.ErrorCodeVolumeInvalid.WithArgs(spec) + return nil, errInvalidSpec(spec) } arr := strings.SplitN(spec, ":", 3) if arr[0] == "" { - return nil, derr.ErrorCodeVolumeInvalid.WithArgs(spec) + return nil, errInvalidSpec(spec) } switch len(arr) { @@ -63,7 +61,7 @@ func ParseMountSpec(spec, volumeDriver string) (*MountPoint, error) { if isValid := ValidMountMode(arr[1]); isValid { // Destination + Mode is not a valid volume - volumes // cannot include a mode. eg /foo:rw - return nil, derr.ErrorCodeVolumeInvalid.WithArgs(spec) + return nil, errInvalidSpec(spec) } // Host Source Path or Name + Destination mp.Source = arr[0] @@ -74,32 +72,29 @@ func ParseMountSpec(spec, volumeDriver string) (*MountPoint, error) { mp.Destination = arr[1] mp.Mode = arr[2] // Mode field is used by SELinux to decide whether to apply label if !ValidMountMode(mp.Mode) { - return nil, derr.ErrorCodeVolumeInvalidMode.WithArgs(mp.Mode) + return nil, errInvalidMode(mp.Mode) } mp.RW = ReadWrite(mp.Mode) mp.Propagation = GetPropagation(mp.Mode) default: - return nil, derr.ErrorCodeVolumeInvalid.WithArgs(spec) + return nil, errInvalidSpec(spec) } //validate the volumes destination path mp.Destination = filepath.Clean(mp.Destination) if !filepath.IsAbs(mp.Destination) { - return nil, derr.ErrorCodeVolumeAbs.WithArgs(mp.Destination) + return nil, fmt.Errorf("Invalid volume destination path: '%s' mount path must be absolute.", mp.Destination) } // Destination cannot be "/" if mp.Destination == "/" { - return nil, derr.ErrorCodeVolumeSlash.WithArgs(spec) + return nil, fmt.Errorf("Invalid specification: destination can't be '/' in '%s'", spec) } name, source := ParseVolumeSource(mp.Source) if len(source) == 0 { mp.Source = "" // Clear it out as we previously assumed it was not a name mp.Driver = volumeDriver - if len(mp.Driver) == 0 { - mp.Driver = DefaultDriverName - } // Named volumes can't have propagation properties specified. // Their defaults will be decided by docker. This is just a // safeguard. Don't want to get into situations where named @@ -109,7 +104,7 @@ func ParseMountSpec(spec, volumeDriver string) (*MountPoint, error) { // cleanup becomes an issue if container does not unmount // submounts explicitly. if HasPropagation(mp.Mode) { - return nil, derr.ErrorCodeVolumeInvalid.WithArgs(spec) + return nil, errInvalidSpec(spec) } } else { mp.Source = filepath.Clean(source) @@ -166,7 +161,7 @@ func ValidMountMode(mode string) bool { // ReadWrite tells you if a mode string is a valid read-write mode or not. // If there are no specifications w.r.t read write mode, then by default -// it returs true. +// it returns true. func ReadWrite(mode string) bool { if !ValidMountMode(mode) { return false diff --git a/components/engine/volume/volume_windows.go b/components/engine/volume/volume_windows.go index ef6e6a1ffe..d01d10d671 100644 --- a/components/engine/volume/volume_windows.go +++ b/components/engine/volume/volume_windows.go @@ -1,13 +1,13 @@ package volume import ( + "fmt" "os" "path/filepath" "regexp" "strings" "github.com/Sirupsen/logrus" - derr "github.com/docker/docker/errors" ) // read-write modes @@ -96,7 +96,7 @@ func ParseMountSpec(spec string, volumeDriver string) (*MountPoint, error) { // Must have something back if len(match) == 0 { - return nil, derr.ErrorCodeVolumeInvalid.WithArgs(spec) + return nil, errInvalidSpec(spec) } // Pull out the sub expressions from the named capture groups @@ -116,7 +116,7 @@ func ParseMountSpec(spec string, volumeDriver string) (*MountPoint, error) { // Volumes cannot include an explicitly supplied mode eg c:\path:rw if mp.Source == "" && mp.Destination != "" && matchgroups["mode"] != "" { - return nil, derr.ErrorCodeVolumeInvalid.WithArgs(spec) + return nil, errInvalidSpec(spec) } // Note: No need to check if destination is absolute as it must be by @@ -125,14 +125,14 @@ func ParseMountSpec(spec string, volumeDriver string) (*MountPoint, error) { if filepath.VolumeName(mp.Destination) == mp.Destination { // Ensure the destination path, if a drive letter, is not the c drive if strings.ToLower(mp.Destination) == "c:" { - return nil, derr.ErrorCodeVolumeDestIsC.WithArgs(spec) + return nil, fmt.Errorf("Destination drive letter in '%s' cannot be c:", spec) } } else { // So we know the destination is a path, not drive letter. Clean it up. mp.Destination = filepath.Clean(mp.Destination) // Ensure the destination path, if a path, is not the c root directory if strings.ToLower(mp.Destination) == `c:\` { - return nil, derr.ErrorCodeVolumeDestIsCRoot.WithArgs(spec) + return nil, fmt.Errorf(`Destination path in '%s' cannot be c:\`, spec) } } @@ -163,10 +163,10 @@ func ParseMountSpec(spec string, volumeDriver string) (*MountPoint, error) { var fi os.FileInfo var err error if fi, err = os.Stat(mp.Source); err != nil { - return nil, derr.ErrorCodeVolumeSourceNotFound.WithArgs(mp.Source, err) + return nil, fmt.Errorf("Source directory '%s' could not be found: %s", mp.Source, err) } if !fi.IsDir() { - return nil, derr.ErrorCodeVolumeSourceNotDirectory.WithArgs(mp.Source) + return nil, fmt.Errorf("Source '%s' is not a directory", mp.Source) } } @@ -182,7 +182,7 @@ func IsVolumeNameValid(name string) (bool, error) { } nameExp = regexp.MustCompile(`^` + RXReservedNames + `$`) if nameExp.MatchString(name) { - return false, derr.ErrorCodeVolumeNameReservedWord.WithArgs(name) + return false, fmt.Errorf("Volume name %q cannot be a reserved word for Windows filenames", name) } return true, nil }