diff --git a/components/engine/.github/CODEOWNERS b/components/engine/.github/CODEOWNERS new file mode 100644 index 0000000000..79ff08905f --- /dev/null +++ b/components/engine/.github/CODEOWNERS @@ -0,0 +1,19 @@ +# Github code owners +# See https://help.github.com/articles/about-codeowners/ +# +# KEEP THIS FILE SORTED. Order is important. Last match takes precedence. + +builder/** @dnephin @tonistiigi +client/** @dnephin +contrib/mkimage/** @tianon +daemon/graphdriver/devmapper/** @rhvgoyal +daemon/graphdriver/lcow/** @johnstep @jhowardmsft +daemon/graphdriver/overlay/** @dmcgowan +daemon/graphdriver/overlay2/** @dmcgowan +daemon/graphdriver/windows/** @johnstep @jhowardmsft +daemon/logger/awslogs/** @samuelkarp +hack/** @dnephin @tianon +hack/integration-cli-on-swarm/** @AkihiroSuda +pkg/testutil/** @dnephin +plugin/** @cpuguy83 +project/** @thaJeztah diff --git a/components/engine/.gitignore b/components/engine/.gitignore index 943e7f3f68..93a6df48e7 100644 --- a/components/engine/.gitignore +++ b/components/engine/.gitignore @@ -4,7 +4,7 @@ *.exe *.exe~ *.orig -*.test +test.main .*.swp .DS_Store # a .bashrc may be added to customize the build environment diff --git a/components/engine/CONTRIBUTING.md b/components/engine/CONTRIBUTING.md index 917214cd1b..75d14cda41 100644 --- a/components/engine/CONTRIBUTING.md +++ b/components/engine/CONTRIBUTING.md @@ -160,6 +160,10 @@ it! Take a look at existing tests for inspiration. [Run the full test suite](https://docs.docker.com/opensource/project/test-and-docs/) on your branch before submitting a pull request. +If your changes need integration tests, write them against the API. The `cli` +integration tests are slowly either migrated to API tests or moved away as unit +tests in `docker/cli` and end-to-end tests for docker. + Update the documentation when creating or modifying features. Test your documentation changes for clarity, concision, and correctness, as well as a clean documentation build. See our contributors guide for [our style diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 33e88dce26..a26468201b 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -9,7 +9,7 @@ # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash # # # Run the test suite: -# docker run -e DOCKER_GITCOMMIT=foo --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py +# docker run -e DOCKER_GITCOMMIT=foo --privileged docker hack/make.sh test-unit test-integration test-docker-py # # # Publish a release: # docker run --privileged \ @@ -72,21 +72,21 @@ RUN apt-get update && apt-get install -y \ zip \ --no-install-recommends \ && pip install awscli==1.10.15 -# Get lvm2 source for compiling statically -ENV LVM2_VERSION 2.02.103 + +# Get lvm2 sources to build statically linked devmapper library +ENV LVM2_VERSION 2.02.173 RUN mkdir -p /usr/local/lvm2 \ && curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \ | tar -xzC /usr/local/lvm2 --strip-components=1 -# See https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags -# Compile and install lvm2 +# Compile and install (only the needed library) RUN cd /usr/local/lvm2 \ && ./configure \ --build="$(gcc -print-multiarch)" \ --enable-static_link \ - && make device-mapper \ - && make install_device-mapper -# See https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL + --enable-pkgconfig \ + && make -C include \ + && make -C libdm install_device-mapper # Install seccomp: the version shipped upstream is too old ENV SECCOMP_VERSION 2.3.2 diff --git a/components/engine/Dockerfile.aarch64 b/components/engine/Dockerfile.aarch64 index cabcda28bd..fd06e52265 100644 --- a/components/engine/Dockerfile.aarch64 +++ b/components/engine/Dockerfile.aarch64 @@ -9,7 +9,7 @@ # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash # # # Run the test suite: -# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py +# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py # # Note: AppArmor used to mess with privileged mode, but this is no longer # the case. Therefore, you don't have to disable it anymore. @@ -54,28 +54,20 @@ RUN apt-get update && apt-get install -y \ vim-common \ --no-install-recommends -# Get lvm2 source for compiling statically -ENV LVM2_VERSION 2.02.103 +# Get lvm2 sources to build statically linked devmapper library +ENV LVM2_VERSION 2.02.173 RUN mkdir -p /usr/local/lvm2 \ && curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \ | tar -xzC /usr/local/lvm2 --strip-components=1 -# See https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags -# Fix platform enablement in lvm2 to support aarch64 properly -RUN set -e \ - && for f in config.guess config.sub; do \ - curl -fsSL -o "/usr/local/lvm2/autoconf/$f" "http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=$f;hb=HEAD"; \ - done -# "arch.c:78:2: error: #error the arch code needs to know about your machine type" - -# Compile and install lvm2 +# Compile and install (only the needed library) RUN cd /usr/local/lvm2 \ && ./configure \ --build="$(gcc -print-multiarch)" \ --enable-static_link \ - && make device-mapper \ - && make install_device-mapper -# See https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL + --enable-pkgconfig \ + && make -C include \ + && make -C libdm install_device-mapper # Install seccomp: the version shipped upstream is too old ENV SECCOMP_VERSION 2.3.2 diff --git a/components/engine/Dockerfile.armhf b/components/engine/Dockerfile.armhf index dd1f536191..29503afdce 100644 --- a/components/engine/Dockerfile.armhf +++ b/components/engine/Dockerfile.armhf @@ -9,7 +9,7 @@ # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash # # # Run the test suite: -# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py +# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py # # Note: AppArmor used to mess with privileged mode, but this is no longer # the case. Therefore, you don't have to disable it anymore. @@ -53,21 +53,21 @@ RUN apt-get update && apt-get install -y \ --no-install-recommends \ && pip install awscli==1.10.15 -# Get lvm2 source for compiling statically -ENV LVM2_VERSION 2.02.103 +# Get lvm2 sources to build statically linked devmapper library +ENV LVM2_VERSION 2.02.173 RUN mkdir -p /usr/local/lvm2 \ && curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \ | tar -xzC /usr/local/lvm2 --strip-components=1 -# See https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags -# Compile and install lvm2 +# Compile and install (only the needed library) RUN cd /usr/local/lvm2 \ && ./configure \ --build="$(gcc -print-multiarch)" \ --enable-static_link \ - && make device-mapper \ - && make install_device-mapper -# See https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL + --enable-pkgconfig \ + && make -C include \ + && make -C libdm install_device-mapper + # Install Go # IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored diff --git a/components/engine/Dockerfile.ppc64le b/components/engine/Dockerfile.ppc64le index 43b84e4501..55c1aa13c8 100644 --- a/components/engine/Dockerfile.ppc64le +++ b/components/engine/Dockerfile.ppc64le @@ -9,7 +9,7 @@ # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash # # # Run the test suite: -# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py +# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py # # Note: AppArmor used to mess with privileged mode, but this is no longer # the case. Therefore, you don't have to disable it anymore. @@ -53,28 +53,20 @@ RUN apt-get update && apt-get install -y \ vim-common \ --no-install-recommends -# Get lvm2 source for compiling statically -ENV LVM2_VERSION 2.02.103 +# Get lvm2 sources to build statically linked devmapper library +ENV LVM2_VERSION 2.02.173 RUN mkdir -p /usr/local/lvm2 \ && curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \ | tar -xzC /usr/local/lvm2 --strip-components=1 -# See https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags -# Fix platform enablement in lvm2 to support ppc64le properly -RUN set -e \ - && for f in config.guess config.sub; do \ - curl -fsSL -o "/usr/local/lvm2/autoconf/$f" "http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=$f;hb=HEAD"; \ - done -# "arch.c:78:2: error: #error the arch code needs to know about your machine type" - -# Compile and install lvm2 +# Compile and install (only the needed library) RUN cd /usr/local/lvm2 \ && ./configure \ --build="$(gcc -print-multiarch)" \ --enable-static_link \ - && make device-mapper \ - && make install_device-mapper -# See https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL + --enable-pkgconfig \ + && make -C include \ + && make -C libdm install_device-mapper # Install seccomp: the version shipped upstream is too old ENV SECCOMP_VERSION 2.3.2 diff --git a/components/engine/Dockerfile.s390x b/components/engine/Dockerfile.s390x index 35ec683739..5a90f6c7c5 100644 --- a/components/engine/Dockerfile.s390x +++ b/components/engine/Dockerfile.s390x @@ -9,7 +9,7 @@ # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash # # # Run the test suite: -# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py +# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py # # Note: AppArmor used to mess with privileged mode, but this is no longer # the case. Therefore, you don't have to disable it anymore. @@ -64,28 +64,20 @@ RUN set -x \ ) \ && rm -rf "$SECCOMP_PATH" -# Get lvm2 source for compiling statically -ENV LVM2_VERSION 2.02.103 +# Get lvm2 sources to build statically linked devmapper library +ENV LVM2_VERSION 2.02.173 RUN mkdir -p /usr/local/lvm2 \ && curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \ | tar -xzC /usr/local/lvm2 --strip-components=1 -# See https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags -# Fix platform enablement in lvm2 to support s390x properly -RUN set -e \ - && for f in config.guess config.sub; do \ - curl -fsSL -o "/usr/local/lvm2/autoconf/$f" "http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=$f;hb=HEAD"; \ - done -# "arch.c:78:2: error: #error the arch code needs to know about your machine type" - -# Compile and install lvm2 +# Compile and install (only the needed library) RUN cd /usr/local/lvm2 \ && ./configure \ --build="$(gcc -print-multiarch)" \ --enable-static_link \ - && make device-mapper \ - && make install_device-mapper -# See https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL + --enable-pkgconfig \ + && make -C include \ + && make -C libdm install_device-mapper # IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored ENV GO_VERSION 1.8.3 diff --git a/components/engine/Dockerfile.simple b/components/engine/Dockerfile.simple index b4682d4cbc..f84f3c565c 100644 --- a/components/engine/Dockerfile.simple +++ b/components/engine/Dockerfile.simple @@ -1,7 +1,7 @@ # docker build -t docker:simple -f Dockerfile.simple . # docker run --rm docker:simple hack/make.sh dynbinary # docker run --rm --privileged docker:simple hack/dind hack/make.sh test-unit -# docker run --rm --privileged -v /var/lib/docker docker:simple hack/dind hack/make.sh dynbinary test-integration-cli +# docker run --rm --privileged -v /var/lib/docker docker:simple hack/dind hack/make.sh dynbinary test-integration # This represents the bare minimum required to build and test Docker. diff --git a/components/engine/MAINTAINERS b/components/engine/MAINTAINERS index dc4485da10..9b415b2a0a 100644 --- a/components/engine/MAINTAINERS +++ b/components/engine/MAINTAINERS @@ -90,6 +90,7 @@ "mgoelzer", "programmerq", "rheinwein", + "ripcurld0", "thajeztah" ] @@ -390,6 +391,11 @@ Email = "laura@codeship.com" GitHub = "rheinwein" + [people.ripcurld0] + Name = "Boaz Shuster" + Email = "ripcurld.github@gmail.com" + GitHub = "ripcurld0" + [people.runcom] Name = "Antonio Murdaca" Email = "runcom@redhat.com" diff --git a/components/engine/Makefile b/components/engine/Makefile index 0d99606cc2..85877bdd2e 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -1,4 +1,4 @@ -.PHONY: all binary dynbinary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration-cli test-unit tgz validate win +.PHONY: all binary dynbinary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration test-unit tgz validate win # set the graph driver as the current graphdriver if not set DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //')) @@ -149,13 +149,15 @@ shell: build ## start a shell inside the build env $(DOCKER_RUN_DOCKER) bash test: build ## run the unit, integration and docker-py tests - $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-unit test-integration-cli test-docker-py + $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-unit test-integration test-docker-py test-docker-py: build ## run the docker-py tests $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-docker-py -test-integration-cli: build ## run the integration tests - $(DOCKER_RUN_DOCKER) hack/make.sh build-integration-test-binary dynbinary test-integration-cli +test-integration-cli: test-integration ## (DEPRECATED) use test-integration + +test-integration: build ## run the integration tests + $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration test-unit: build ## run the unit tests $(DOCKER_RUN_DOCKER) hack/make.sh test-unit diff --git a/components/engine/api/common.go b/components/engine/api/common.go index 859daf602c..6e462aeda7 100644 --- a/components/engine/api/common.go +++ b/components/engine/api/common.go @@ -15,7 +15,7 @@ import ( // Common constants for daemon and client. const ( // DefaultVersion of Current REST API - DefaultVersion string = "1.31" + DefaultVersion string = "1.32" // NoBaseImageSpecifier is the symbol used by the FROM // command to specify that no base image is to be used. diff --git a/components/engine/api/server/httputils/errors.go b/components/engine/api/server/httputils/errors.go index 82da21c2a0..b677f95d6a 100644 --- a/components/engine/api/server/httputils/errors.go +++ b/components/engine/api/server/httputils/errors.go @@ -4,10 +4,10 @@ import ( "net/http" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/versions" "github.com/gorilla/mux" + "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/codes" ) diff --git a/components/engine/api/server/httputils/httputils.go b/components/engine/api/server/httputils/httputils.go index 92cb67c56c..4708d7d16e 100644 --- a/components/engine/api/server/httputils/httputils.go +++ b/components/engine/api/server/httputils/httputils.go @@ -7,7 +7,7 @@ import ( "net/http" "strings" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/api/server/httputils/write_log_stream.go b/components/engine/api/server/httputils/write_log_stream.go index 891e5f02b3..fd024e196e 100644 --- a/components/engine/api/server/httputils/write_log_stream.go +++ b/components/engine/api/server/httputils/write_log_stream.go @@ -5,7 +5,6 @@ import ( "io" "net/url" "sort" - "strings" "golang.org/x/net/context" @@ -53,7 +52,8 @@ func WriteLogStream(ctx context.Context, w io.Writer, msgs <-chan *backend.LogMe } logLine := msg.Line if config.Details { - logLine = append([]byte(stringAttrs(msg.Attrs)+" "), logLine...) + logLine = append(attrsByteSlice(msg.Attrs), ' ') + logLine = append(logLine, msg.Line...) } if config.Timestamps { // TODO(dperny) the format is defined in @@ -71,24 +71,26 @@ func WriteLogStream(ctx context.Context, w io.Writer, msgs <-chan *backend.LogMe } } -type byKey []string +type byKey []backend.LogAttr -func (s byKey) Len() int { return len(s) } -func (s byKey) Less(i, j int) bool { - keyI := strings.Split(s[i], "=") - keyJ := strings.Split(s[j], "=") - return keyI[0] < keyJ[0] -} -func (s byKey) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} +func (b byKey) Len() int { return len(b) } +func (b byKey) Less(i, j int) bool { return b[i].Key < b[j].Key } +func (b byKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] } -func stringAttrs(a backend.LogAttributes) string { - var ss byKey - for k, v := range a { - k, v := url.QueryEscape(k), url.QueryEscape(v) - ss = append(ss, k+"="+v) +func attrsByteSlice(a []backend.LogAttr) []byte { + // Note this sorts "a" in-place. That is fine here - nothing else is + // going to use Attrs or care about the order. + sort.Sort(byKey(a)) + + var ret []byte + for i, pair := range a { + k, v := url.QueryEscape(pair.Key), url.QueryEscape(pair.Value) + ret = append(ret, []byte(k)...) + ret = append(ret, '=') + ret = append(ret, []byte(v)...) + if i != len(a)-1 { + ret = append(ret, ',') + } } - sort.Sort(ss) - return strings.Join(ss, ",") + return ret } diff --git a/components/engine/api/server/middleware.go b/components/engine/api/server/middleware.go index 537ce8028f..653d554288 100644 --- a/components/engine/api/server/middleware.go +++ b/components/engine/api/server/middleware.go @@ -1,9 +1,9 @@ package server import ( - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/server/middleware" + "github.com/sirupsen/logrus" ) // handlerWithGlobalMiddlewares wraps the handler function for a request with diff --git a/components/engine/api/server/middleware/cors.go b/components/engine/api/server/middleware/cors.go index ea725dbc72..8ee19a343d 100644 --- a/components/engine/api/server/middleware/cors.go +++ b/components/engine/api/server/middleware/cors.go @@ -3,7 +3,7 @@ package middleware import ( "net/http" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/api/server/middleware/debug.go b/components/engine/api/server/middleware/debug.go index a9a94e7f33..7b867f1ebf 100644 --- a/components/engine/api/server/middleware/debug.go +++ b/components/engine/api/server/middleware/debug.go @@ -7,9 +7,9 @@ import ( "net/http" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/pkg/ioutils" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/api/server/router/build/build_routes.go b/components/engine/api/server/router/build/build_routes.go index baa1da303f..d9a123915d 100644 --- a/components/engine/api/server/router/build/build_routes.go +++ b/components/engine/api/server/router/build/build_routes.go @@ -12,7 +12,6 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" apierrors "github.com/docker/docker/api/errors" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types" @@ -24,6 +23,7 @@ import ( "github.com/docker/docker/pkg/streamformatter" units "github.com/docker/go-units" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/api/server/router/container/container_routes.go b/components/engine/api/server/router/container/container_routes.go index 96b1010e19..57204b2e88 100644 --- a/components/engine/api/server/router/container/container_routes.go +++ b/components/engine/api/server/router/container/container_routes.go @@ -8,7 +8,6 @@ import ( "strconv" "syscall" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types" @@ -19,6 +18,7 @@ import ( containerpkg "github.com/docker/docker/container" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/signal" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "golang.org/x/net/websocket" ) diff --git a/components/engine/api/server/router/container/exec.go b/components/engine/api/server/router/container/exec.go index 1134a0e797..64629b69ae 100644 --- a/components/engine/api/server/router/container/exec.go +++ b/components/engine/api/server/router/container/exec.go @@ -7,11 +7,11 @@ import ( "net/http" "strconv" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/versions" "github.com/docker/docker/pkg/stdcopy" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/api/server/router/network/network_routes.go b/components/engine/api/server/router/network/network_routes.go index 6f2041e35e..f439d65ad9 100644 --- a/components/engine/api/server/router/network/network_routes.go +++ b/components/engine/api/server/router/network/network_routes.go @@ -16,6 +16,7 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/versions" "github.com/docker/libnetwork" + netconst "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/networkdb" ) @@ -135,6 +136,17 @@ func (n *networkRouter) getNetwork(ctx context.Context, w http.ResponseWriter, r } } + nwk, err := n.cluster.GetNetwork(term) + if err == nil { + // If the get network is passed with a specific network ID / partial network ID + // or if the get network was passed with a network name and scope as swarm + // return the network. Skipped using isMatchingScope because it is true if the scope + // is not set which would be case if the client API v1.30 + if strings.HasPrefix(nwk.ID, term) || (netconst.SwarmScope == scope) { + return httputils.WriteJSON(w, http.StatusOK, nwk) + } + } + nr, _ := n.cluster.GetNetworks() for _, network := range nr { if network.ID == term && isMatchingScope(network.Scope, scope) { @@ -397,7 +409,9 @@ func buildIpamResources(r *types.NetworkResource, nwInfo libnetwork.NetworkInfo) for _, ip4Info := range ipv4Info { iData := network.IPAMConfig{} iData.Subnet = ip4Info.IPAMData.Pool.String() - iData.Gateway = ip4Info.IPAMData.Gateway.IP.String() + if ip4Info.IPAMData.Gateway != nil { + iData.Gateway = ip4Info.IPAMData.Gateway.IP.String() + } r.IPAM.Config = append(r.IPAM.Config, iData) } } diff --git a/components/engine/api/server/router/swarm/cluster_routes.go b/components/engine/api/server/router/swarm/cluster_routes.go index 91461da764..a1e18f5fad 100644 --- a/components/engine/api/server/router/swarm/cluster_routes.go +++ b/components/engine/api/server/router/swarm/cluster_routes.go @@ -6,7 +6,6 @@ import ( "net/http" "strconv" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/errors" "github.com/docker/docker/api/server/httputils" basictypes "github.com/docker/docker/api/types" @@ -14,6 +13,7 @@ import ( "github.com/docker/docker/api/types/filters" types "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/api/types/versions" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/api/server/router/system/system_routes.go b/components/engine/api/server/router/system/system_routes.go index 30fb000e1d..cd8c3a9fb8 100644 --- a/components/engine/api/server/router/system/system_routes.go +++ b/components/engine/api/server/router/system/system_routes.go @@ -6,7 +6,6 @@ import ( "net/http" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api" "github.com/docker/docker/api/errors" "github.com/docker/docker/api/server/httputils" @@ -18,6 +17,7 @@ import ( "github.com/docker/docker/api/types/versions" "github.com/docker/docker/pkg/ioutils" pkgerrors "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/api/server/server.go b/components/engine/api/server/server.go index e0f2d89d9a..55edcec335 100644 --- a/components/engine/api/server/server.go +++ b/components/engine/api/server/server.go @@ -7,7 +7,6 @@ import ( "net/http" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/errors" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/server/middleware" @@ -15,6 +14,7 @@ import ( "github.com/docker/docker/api/server/router/debug" "github.com/docker/docker/dockerversion" "github.com/gorilla/mux" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index 630278ecde..2ce4240956 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -19,10 +19,10 @@ produces: consumes: - "application/json" - "text/plain" -basePath: "/v1.31" +basePath: "/v1.32" info: title: "Docker Engine API" - version: "1.31" + version: "1.32" x-logo: url: "https://docs.docker.com/images/logo-docker-main.png" description: | @@ -44,7 +44,7 @@ info: The API is usually changed in each release of Docker, so API calls are versioned to ensure that clients don't break. - For Docker Engine 17.06, the API version is 1.30. To lock to this version, you prefix the URL with `/v1.30`. For example, calling `/info` is the same as calling `/v1.30/info`. + For Docker Engine 17.07, the API version is 1.31. To lock to this version, you prefix the URL with `/v1.31`. For example, calling `/info` is the same as calling `/v1.31/info`. Engine releases in the near future should support this version of the API, so your client will continue to work even if it is talking to a newer Engine. @@ -52,10 +52,11 @@ info: The API uses an open schema model, which means server may add extra properties to responses. Likewise, the server will ignore any extra query parameters and request body properties. When you write clients, you need to ignore additional properties in responses to ensure they do not break when talking to newer Docker daemons. - This documentation is for version 1.31 of the API. Use this table to find documentation for previous versions of the API: + This documentation is for version 1.32 of the API. Use this table to find documentation for previous versions of the API: Docker version | API version | Changes ----------------|-------------|--------- + 17.07.x | [1.31](https://docs.docker.com/engine/api/v1.31/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-31-api-changes) 17.06.x | [1.30](https://docs.docker.com/engine/api/v1.30/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-30-api-changes) 17.05.x | [1.29](https://docs.docker.com/engine/api/v1.29/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-29-api-changes) 17.04.x | [1.28](https://docs.docker.com/engine/api/v1.28/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-28-api-changes) @@ -322,7 +323,6 @@ definitions: MaximumRetryCount: type: "integer" description: "If `on-failure` is used, the number of times to retry before giving up" - default: {} Resources: description: "A container's resources (cgroups config, ulimits, etc)" @@ -487,6 +487,41 @@ definitions: type: "integer" format: "int64" + ResourceObject: + description: "An object describing the resources which can be advertised by a node and requested by a task" + type: "object" + properties: + NanoCPUs: + type: "integer" + format: "int64" + MemoryBytes: + type: "integer" + format: "int64" + GenericResources: + $ref: "#/definitions/GenericResources" + + GenericResources: + description: "User defined Resources, can be either Integer resources (e.g: SSD=3) or String resources (e.g: GPU={UUID1, UUID2})" + type: "array" + items: + type: "object" + properties: + NamedResourceSpec: + type: "object" + properties: + Kind: + type: "string" + Value: + type: "string" + DiscreteResourceSpec: + type: "object" + properties: + Kind: + type: "string" + Value: + type: "integer" + format: "int64" + HealthConfig: description: "A test to perform to check that the container is healthy." type: "object" @@ -630,7 +665,17 @@ definitions: type: "string" IpcMode: type: "string" - description: "IPC namespace to use for the container." + description: | + IPC sharing mode for the container. Possible values are: + + - `"none"`: own private IPC namespace, with /dev/shm not mounted + - `"private"`: own private IPC namespace + - `"shareable"`: own private IPC namespace, with a possibility to share it with other containers + - `"container:"`: join another (shareable) container's IPC namespace + - `"host"`: use the host system's IPC namespace + + If not specified, daemon default is used, which can either be `"private"` + or `"shareable"`, depending on daemon version and configuration. Cgroup: type: "string" description: "Cgroup to use for the container." @@ -1089,17 +1134,27 @@ definitions: type: "string" UsageData: type: "object" + x-nullable: true required: [Size, RefCount] + description: | + Usage details about the volume. This information is used by the + `GET /system/df` endpoint, and omitted in other endpoints. properties: Size: type: "integer" - description: "The disk space used by the volume (local driver only)" default: -1 + description: | + Amount of disk space used by the volume (in bytes). This information + is only available for volumes created with the `"local"` volume + driver. For volumes created with other volume drivers, this field + is set to `-1` ("not available") x-nullable: false RefCount: type: "integer" default: -1 - description: "The number of containers referencing this volume." + description: | + The number of containers referencing this volume. This field + is set to `-1` if the reference-count is not available. x-nullable: false example: @@ -1349,26 +1404,33 @@ definitions: Name: type: "string" x-nullable: false + example: "some-mount" Description: type: "string" x-nullable: false + example: "This is a mount that's used by the plugin." Settable: type: "array" items: type: "string" Source: type: "string" + example: "/var/lib/docker/plugins/" Destination: type: "string" x-nullable: false + example: "/mnt/state" Type: type: "string" x-nullable: false + example: "bind" Options: type: "array" items: type: "string" - + example: + - "rbind" + - "rw" PluginDevice: type: "object" required: [Name, Description, Settable, Path] @@ -1386,6 +1448,7 @@ definitions: type: "string" Path: type: "string" + example: "/dev/fuse" PluginEnv: type: "object" @@ -1427,13 +1490,16 @@ definitions: properties: Id: type: "string" + example: "5724e2c8652da337ab2eedd19fc6fc0ec908e4bd907c7421bf6a8dfc70c4c078" Name: type: "string" x-nullable: false + example: "tiborvass/sample-volume-plugin" Enabled: - description: "True when the plugin is running. False when the plugin is not running, only installed." + description: "True if the plugin is running. False if the plugin is not running, only installed." type: "boolean" x-nullable: false + example: true Settings: description: "Settings that can be modified by users." type: "object" @@ -1448,6 +1514,8 @@ definitions: type: "array" items: type: "string" + example: + - "DEBUG=0" Args: type: "array" items: @@ -1460,6 +1528,7 @@ definitions: description: "plugin remote reference used to push/pull the plugin" type: "string" x-nullable: false + example: "localhost:5000/tiborvass/sample-volume-plugin:latest" Config: description: "The config of a plugin." type: "object" @@ -1483,12 +1552,15 @@ definitions: description: "Docker Version used to create the plugin" type: "string" x-nullable: false + example: "17.06.0-ce" Description: type: "string" x-nullable: false + example: "A sample volume plugin for Docker" Documentation: type: "string" x-nullable: false + example: "https://docs.docker.com/engine/extend/plugins/" Interface: description: "The interface between Docker and the plugin" x-nullable: false @@ -1499,16 +1571,23 @@ definitions: type: "array" items: $ref: "#/definitions/PluginInterfaceType" + example: + - "docker.volumedriver/1.0" Socket: type: "string" x-nullable: false + example: "plugins.sock" Entrypoint: type: "array" items: type: "string" + example: + - "/usr/bin/sample-volume-plugin" + - "/data" WorkDir: type: "string" x-nullable: false + example: "/bin/" User: type: "object" x-nullable: false @@ -1516,9 +1595,11 @@ definitions: UID: type: "integer" format: "uint32" + example: 1000 GID: type: "integer" format: "uint32" + example: 1000 Network: type: "object" x-nullable: false @@ -1527,6 +1608,7 @@ definitions: Type: x-nullable: false type: "string" + example: "host" Linux: type: "object" x-nullable: false @@ -1536,9 +1618,13 @@ definitions: type: "array" items: type: "string" + example: + - "CAP_SYS_ADMIN" + - "CAP_SYSLOG" AllowAllDevices: type: "boolean" x-nullable: false + example: false Devices: type: "array" items: @@ -1546,12 +1632,15 @@ definitions: PropagatedMount: type: "string" x-nullable: false + example: "/mnt/volumes" IpcHost: type: "boolean" x-nullable: false + example: false PidHost: type: "boolean" x-nullable: false + example: false Mounts: type: "array" items: @@ -1560,6 +1649,11 @@ definitions: type: "array" items: $ref: "#/definitions/PluginEnv" + example: + - Name: "DEBUG" + Description: "If set, prints debug messages" + Settable: null + Value: "0" Args: type: "object" x-nullable: false @@ -1568,9 +1662,11 @@ definitions: Name: x-nullable: false type: "string" + example: "args" Description: x-nullable: false type: "string" + example: "command line arguments" Settable: type: "array" items: @@ -1584,50 +1680,14 @@ definitions: properties: type: type: "string" + example: "layers" diff_ids: type: "array" items: type: "string" - example: - Id: "5724e2c8652da337ab2eedd19fc6fc0ec908e4bd907c7421bf6a8dfc70c4c078" - Name: "tiborvass/sample-volume-plugin" - Tag: "latest" - Active: true - Settings: - Env: - - "DEBUG=0" - Args: null - Devices: null - Config: - Description: "A sample volume plugin for Docker" - Documentation: "https://docs.docker.com/engine/extend/plugins/" - Interface: - Types: - - "docker.volumedriver/1.0" - Socket: "plugins.sock" - Entrypoint: - - "/usr/bin/sample-volume-plugin" - - "/data" - WorkDir: "" - User: {} - Network: - Type: "" - Linux: - Capabilities: null - AllowAllDevices: false - Devices: null - Mounts: null - PropagatedMount: "/data" - Env: - - Name: "DEBUG" - Description: "If set, prints debug messages" - Settable: null - Value: "0" - Args: - Name: "args" - Description: "command line arguments" - Settable: null - Value: [] + example: + - "sha256:675532206fbf3030b8458f88d6e26d4eb1577688a25efec97154c94e8b6b4887" + - "sha256:e216a057b1cb1efc11f8a268f37ef62083e70b1b38323ba252e25ac88904a7e8" ObjectVersion: description: | @@ -1702,14 +1762,7 @@ definitions: OS: type: "string" Resources: - type: "object" - properties: - NanoCPUs: - type: "integer" - format: "int64" - MemoryBytes: - type: "integer" - format: "int64" + $ref: "#/definitions/ResourceObject" Engine: type: "object" properties: @@ -1750,6 +1803,16 @@ definitions: Resources: NanoCPUs: 4000000000 MemoryBytes: 8272408576 + GenericResources: + - DiscreteResourceSpec: + Kind: "SSD" + Value: 3 + - NamedResourceSpec: + Kind: "GPU" + Value: "UUID1" + - NamedResourceSpec: + Kind: "GPU" + Value: "UUID2" Engine: EngineVersion: "17.04.0" Labels: @@ -1977,7 +2040,7 @@ definitions: properties: PluginSpec: type: "object" - description: "Invalid when specified with `ContainerSpec`." + description: "Invalid when specified with `ContainerSpec`. *(Experimental release only.)*" properties: Name: description: "The name or 'alias' to use for the plugin." @@ -2214,27 +2277,10 @@ definitions: properties: Limits: description: "Define resources limits." - type: "object" - properties: - NanoCPUs: - description: "CPU limit in units of 10-9 CPU shares." - type: "integer" - format: "int64" - MemoryBytes: - description: "Memory limit in Bytes." - type: "integer" - format: "int64" + $ref: "#/definitions/ResourceObject" Reservation: description: "Define resources reservation." - properties: - NanoCPUs: - description: "CPU reservation in units of 10-9 CPU shares." - type: "integer" - format: "int64" - MemoryBytes: - description: "Memory reservation in Bytes." - type: "integer" - format: "int64" + $ref: "#/definitions/ResourceObject" RestartPolicy: description: "Specification for the restart policy which applies to containers created as part of this service." type: "object" @@ -2365,6 +2411,8 @@ definitions: NodeID: description: "The ID of the node that this task is on." type: "string" + AssignedGenericResources: + $ref: "#/definitions/GenericResources" Status: type: "object" properties: @@ -2444,6 +2492,16 @@ definitions: Gateway: "10.255.0.1" Addresses: - "10.255.0.10/16" + AssignedGenericResources: + - DiscreteResourceSpec: + Kind: "SSD" + Value: 3 + - NamedResourceSpec: + Kind: "GPU" + Value: "UUID1" + - NamedResourceSpec: + Kind: "GPU" + Value: "UUID2" ServiceSpec: description: "User modifiable configuration for a service." properties: @@ -2780,6 +2838,27 @@ definitions: type: "array" items: $ref: "#/definitions/Mount" + + Driver: + description: "Driver represents a driver (network, logging, secrets)." + type: "object" + required: [Name] + properties: + Name: + description: "Name of the driver." + type: "string" + x-nullable: false + example: "some-driver" + Options: + description: "Key/value map of driver-specific options." + type: "object" + x-nullable: false + additionalProperties: + type: "string" + example: + OptionA: "value for driver-specific option A" + OptionB: "value for driver-specific option B" + SecretSpec: type: "object" properties: @@ -2791,24 +2870,38 @@ definitions: type: "object" additionalProperties: type: "string" + example: + com.example.some-label: "some-value" + com.example.some-other-label: "some-other-value" Data: - description: "Base64-url-safe-encoded secret data" - type: "array" - items: - type: "string" + description: | + Base64-url-safe-encoded ([RFC 4648](https://tools.ietf.org/html/rfc4648#section-3.2)) + data to store as secret. + + This field is only used to _create_ a secret, and is not returned by + other endpoints. + type: "string" + example: "" + Driver: + description: "Name of the secrets driver used to fetch the secret's value from an external secret store" + $ref: "#/definitions/Driver" + Secret: type: "object" properties: ID: type: "string" + example: "blt1owaxmitz71s9v5zh81zun" Version: $ref: "#/definitions/ObjectVersion" CreatedAt: type: "string" format: "dateTime" + example: "2017-07-20T13:55:28.678958722Z" UpdatedAt: type: "string" format: "dateTime" + example: "2017-07-20T13:55:28.678958722Z" Spec: $ref: "#/definitions/SecretSpec" ConfigSpec: @@ -2823,10 +2916,10 @@ definitions: additionalProperties: type: "string" Data: - description: "Base64-url-safe-encoded config data" - type: "array" - items: - type: "string" + description: | + Base64-url-safe-encoded ([RFC 4648](https://tools.ietf.org/html/rfc4648#section-3.2)) + config data. + type: "string" Config: type: "object" properties: @@ -5506,6 +5599,8 @@ paths: type: "string" MemTotal: type: "integer" + GenericResources: + $ref: "#/definitions/GenericResources" MemoryLimit: type: "boolean" NCPU: @@ -5949,13 +6044,13 @@ paths: - Name: "my-volume" Driver: "local" - Mountpoint: "" + Mountpoint: "/var/lib/docker/volumes/my-volume/_data" Labels: null - Scope: "" + Scope: "local" Options: null UsageData: - Size: 0 - RefCount: 0 + Size: 10920104 + RefCount: 2 500: description: "server error" schema: @@ -6887,46 +6982,6 @@ paths: type: "array" items: $ref: "#/definitions/Plugin" - example: - - Id: "5724e2c8652da337ab2eedd19fc6fc0ec908e4bd907c7421bf6a8dfc70c4c078" - Name: "tiborvass/sample-volume-plugin" - Tag: "latest" - Active: true - Settings: - Env: - - "DEBUG=0" - Args: null - Devices: null - Config: - Description: "A sample volume plugin for Docker" - Documentation: "https://docs.docker.com/engine/extend/plugins/" - Interface: - Types: - - "docker.volumedriver/1.0" - Socket: "plugins.sock" - Entrypoint: - - "/usr/bin/sample-volume-plugin" - - "/data" - WorkDir: "" - User: {} - Network: - Type: "" - Linux: - Capabilities: null - AllowAllDevices: false - Devices: null - Mounts: null - PropagatedMount: "/data" - Env: - - Name: "DEBUG" - Description: "If set, prints debug messages" - Settable: null - Value: "0" - Args: - Name: "args" - Description: "command line arguments" - Settable: null - Value: [] 500: description: "Server error" schema: @@ -8416,6 +8471,20 @@ paths: items: $ref: "#/definitions/Secret" example: + - ID: "blt1owaxmitz71s9v5zh81zun" + Version: + Index: 85 + CreatedAt: "2017-07-20T13:55:28.678958722Z" + UpdatedAt: "2017-07-20T13:55:28.678958722Z" + Spec: + Name: "mysql-passwd" + Labels: + some.label: "some.value" + Driver: + Name: "secret-bucket" + Options: + OptionA: "value for driver option A" + OptionB: "value for driver option B" - ID: "ktnbjxoalbkvbvedmg1urrz8h" Version: Index: 11 @@ -8423,6 +8492,8 @@ paths: UpdatedAt: "2016-11-05T01:20:17.327670065Z" Spec: Name: "app-dev.crt" + Labels: + foo: "bar" 500: description: "server error" schema: @@ -8486,6 +8557,11 @@ paths: Labels: foo: "bar" Data: "VEhJUyBJUyBOT1QgQSBSRUFMIENFUlRJRklDQVRFCg==" + Driver: + Name: "secret-bucket" + Options: + OptionA: "value for driver option A" + OptionB: "value for driver option B" tags: ["Secret"] /secrets/{id}: get: @@ -8507,6 +8583,14 @@ paths: UpdatedAt: "2016-11-05T01:20:17.327670065Z" Spec: Name: "app-dev.crt" + Labels: + foo: "bar" + Driver: + Name: "secret-bucket" + Options: + OptionA: "value for driver option A" + OptionB: "value for driver option B" + 404: description: "secret not found" schema: diff --git a/components/engine/api/types/backend/backend.go b/components/engine/api/types/backend/backend.go index 368ad7b5ac..74cea50035 100644 --- a/components/engine/api/types/backend/backend.go +++ b/components/engine/api/types/backend/backend.go @@ -35,7 +35,7 @@ type LogMessage struct { Line []byte Source string Timestamp time.Time - Attrs LogAttributes + Attrs []LogAttr Partial bool // Err is an error associated with a message. Completeness of a message @@ -44,9 +44,11 @@ type LogMessage struct { Err error } -// LogAttributes is used to hold the extra attributes available in the log message -// Primarily used for converting the map type to string and sorting. -type LogAttributes map[string]string +// LogAttr is used to hold the extra attributes available in the log message. +type LogAttr struct { + Key string + Value string +} // LogSelector is a list of services and tasks that should be returned as part // of a log stream. It is similar to swarmapi.LogSelector, with the difference diff --git a/components/engine/api/types/container/host_config.go b/components/engine/api/types/container/host_config.go index 9fea9eb04b..bb421b3889 100644 --- a/components/engine/api/types/container/host_config.go +++ b/components/engine/api/types/container/host_config.go @@ -23,41 +23,46 @@ func (i Isolation) IsDefault() bool { // IpcMode represents the container ipc stack. type IpcMode string -// IsPrivate indicates whether the container uses its private ipc stack. +// IsPrivate indicates whether the container uses its own private ipc namespace which can not be shared. func (n IpcMode) IsPrivate() bool { - return !(n.IsHost() || n.IsContainer()) + return n == "private" } -// IsHost indicates whether the container uses the host's ipc stack. +// IsHost indicates whether the container shares the host's ipc namespace. func (n IpcMode) IsHost() bool { return n == "host" } -// IsContainer indicates whether the container uses a container's ipc stack. +// IsShareable indicates whether the container's ipc namespace can be shared with another container. +func (n IpcMode) IsShareable() bool { + return n == "shareable" +} + +// IsContainer indicates whether the container uses another container's ipc namespace. func (n IpcMode) IsContainer() bool { parts := strings.SplitN(string(n), ":", 2) return len(parts) > 1 && parts[0] == "container" } -// Valid indicates whether the ipc stack is valid. +// IsNone indicates whether container IpcMode is set to "none". +func (n IpcMode) IsNone() bool { + return n == "none" +} + +// IsEmpty indicates whether container IpcMode is empty +func (n IpcMode) IsEmpty() bool { + return n == "" +} + +// Valid indicates whether the ipc mode is valid. func (n IpcMode) Valid() bool { - parts := strings.Split(string(n), ":") - switch mode := parts[0]; mode { - case "", "host": - case "container": - if len(parts) != 2 || parts[1] == "" { - return false - } - default: - return false - } - return true + return n.IsEmpty() || n.IsNone() || n.IsPrivate() || n.IsHost() || n.IsShareable() || n.IsContainer() } // Container returns the name of the container ipc stack is going to be used. func (n IpcMode) Container() string { parts := strings.SplitN(string(n), ":", 2) - if len(parts) > 1 { + if len(parts) > 1 && parts[0] == "container" { return parts[1] } return "" diff --git a/components/engine/api/types/mount/mount.go b/components/engine/api/types/mount/mount.go index 2744f85d6d..71368643dd 100644 --- a/components/engine/api/types/mount/mount.go +++ b/components/engine/api/types/mount/mount.go @@ -15,6 +15,8 @@ const ( TypeVolume Type = "volume" // TypeTmpfs is the type for mounting tmpfs TypeTmpfs Type = "tmpfs" + // TypeNamedPipe is the type for mounting Windows named pipes + TypeNamedPipe Type = "npipe" ) // Mount represents a mount (volume). diff --git a/components/engine/api/types/plugin.go b/components/engine/api/types/plugin.go index ed3c2c26e4..cab333e01a 100644 --- a/components/engine/api/types/plugin.go +++ b/components/engine/api/types/plugin.go @@ -11,7 +11,7 @@ type Plugin struct { // Required: true Config PluginConfig `json:"Config"` - // True when the plugin is running. False when the plugin is not running, only installed. + // True if the plugin is running. False if the plugin is not running, only installed. // Required: true Enabled bool `json:"Enabled"` diff --git a/components/engine/api/types/plugin_responses.go b/components/engine/api/types/plugin_responses.go index 1c6461f2d9..18f743fcde 100644 --- a/components/engine/api/types/plugin_responses.go +++ b/components/engine/api/types/plugin_responses.go @@ -9,14 +9,6 @@ import ( // PluginsListResponse contains the response for the Engine API type PluginsListResponse []*Plugin -const ( - authzDriver = "AuthzDriver" - graphDriver = "GraphDriver" - ipamDriver = "IpamDriver" - networkDriver = "NetworkDriver" - volumeDriver = "VolumeDriver" -) - // UnmarshalJSON implements json.Unmarshaler for PluginInterfaceType func (t *PluginInterfaceType) UnmarshalJSON(p []byte) error { versionIndex := len(p) diff --git a/components/engine/api/types/swarm/common.go b/components/engine/api/types/swarm/common.go index 54af82b31b..2834cf2022 100644 --- a/components/engine/api/types/swarm/common.go +++ b/components/engine/api/types/swarm/common.go @@ -20,7 +20,7 @@ type Annotations struct { Labels map[string]string `json:"Labels"` } -// Driver represents a driver (network, logging). +// Driver represents a driver (network, logging, secrets backend). type Driver struct { Name string `json:",omitempty"` Options map[string]string `json:",omitempty"` diff --git a/components/engine/api/types/swarm/task.go b/components/engine/api/types/swarm/task.go index 1712c06cf6..ff11b07e74 100644 --- a/components/engine/api/types/swarm/task.go +++ b/components/engine/api/types/swarm/task.go @@ -51,6 +51,7 @@ type Task struct { Status TaskStatus `json:",omitempty"` DesiredState TaskState `json:",omitempty"` NetworksAttachments []NetworkAttachment `json:",omitempty"` + GenericResources []GenericResource `json:",omitempty"` } // TaskSpec represents the spec of a task. @@ -79,8 +80,34 @@ type TaskSpec struct { // Resources represents resources (CPU/Memory). type Resources struct { - NanoCPUs int64 `json:",omitempty"` - MemoryBytes int64 `json:",omitempty"` + NanoCPUs int64 `json:",omitempty"` + MemoryBytes int64 `json:",omitempty"` + GenericResources []GenericResource `json:",omitempty"` +} + +// GenericResource represents a "user defined" resource which can +// be either an integer (e.g: SSD=3) or a string (e.g: SSD=sda1) +type GenericResource struct { + NamedResourceSpec *NamedGenericResource `json:",omitempty"` + DiscreteResourceSpec *DiscreteGenericResource `json:",omitempty"` +} + +// NamedGenericResource represents a "user defined" resource which is defined +// as a string. +// "Kind" is used to describe the Kind of a resource (e.g: "GPU", "FPGA", "SSD", ...) +// Value is used to identify the resource (GPU="UUID-1", FPGA="/dev/sdb5", ...) +type NamedGenericResource struct { + Kind string `json:",omitempty"` + Value string `json:",omitempty"` +} + +// DiscreteGenericResource represents a "user defined" resource which is defined +// as an integer +// "Kind" is used to describe the Kind of a resource (e.g: "GPU", "FPGA", "SSD", ...) +// Value is used to count the resource (SSD=5, HDD=3, ...) +type DiscreteGenericResource struct { + Kind string `json:",omitempty"` + Value int64 `json:",omitempty"` } // ResourceRequirements represents resources requirements. diff --git a/components/engine/api/types/types.go b/components/engine/api/types/types.go index c96df27332..f7ac772971 100644 --- a/components/engine/api/types/types.go +++ b/components/engine/api/types/types.go @@ -168,6 +168,7 @@ type Info struct { RegistryConfig *registry.ServiceConfig NCPU int MemTotal int64 + GenericResources []swarm.GenericResource DockerRootDir string HTTPProxy string `json:"HttpProxy"` HTTPSProxy string `json:"HttpsProxy"` diff --git a/components/engine/api/types/volume.go b/components/engine/api/types/volume.go index 8fab786759..b5ee96a500 100644 --- a/components/engine/api/types/volume.go +++ b/components/engine/api/types/volume.go @@ -47,15 +47,23 @@ type Volume struct { UsageData *VolumeUsageData `json:"UsageData,omitempty"` } -// VolumeUsageData volume usage data +// VolumeUsageData Usage details about the volume. This information is used by the +// `GET /system/df` endpoint, and omitted in other endpoints. +// // swagger:model VolumeUsageData type VolumeUsageData struct { - // The number of containers referencing this volume. + // The number of containers referencing this volume. This field + // is set to `-1` if the reference-count is not available. + // // Required: true RefCount int64 `json:"RefCount"` - // The disk space used by the volume (local driver only) + // Amount of disk space used by the volume (in bytes). This information + // is only available for volumes created with the `"local"` volume + // driver. For volumes created with other volume drivers, this field + // is set to `-1` ("not available") + // // Required: true Size int64 `json:"Size"` } diff --git a/components/engine/builder/dockerfile/builder.go b/components/engine/builder/dockerfile/builder.go index fb1786225a..1d7fa413f8 100644 --- a/components/engine/builder/dockerfile/builder.go +++ b/components/engine/builder/dockerfile/builder.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/container" @@ -18,14 +17,15 @@ import ( "github.com/docker/docker/builder/dockerfile/parser" "github.com/docker/docker/builder/fscache" "github.com/docker/docker/builder/remotecontext" - "github.com/docker/docker/client/session" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/system" + "github.com/moby/buildkit/session" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "golang.org/x/sync/syncmap" ) diff --git a/components/engine/builder/dockerfile/clientsession.go b/components/engine/builder/dockerfile/clientsession.go index a7709ce517..9a5411685a 100644 --- a/components/engine/builder/dockerfile/clientsession.go +++ b/components/engine/builder/dockerfile/clientsession.go @@ -5,8 +5,8 @@ import ( "github.com/docker/docker/builder/fscache" "github.com/docker/docker/builder/remotecontext" - "github.com/docker/docker/client/session" - "github.com/docker/docker/client/session/filesync" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/filesync" "github.com/pkg/errors" "golang.org/x/net/context" ) diff --git a/components/engine/builder/dockerfile/containerbackend.go b/components/engine/builder/dockerfile/containerbackend.go index 7b241f3d3b..d11c95270b 100644 --- a/components/engine/builder/dockerfile/containerbackend.go +++ b/components/engine/builder/dockerfile/containerbackend.go @@ -4,13 +4,13 @@ import ( "fmt" "io" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/builder" containerpkg "github.com/docker/docker/container" "github.com/docker/docker/pkg/stringid" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/builder/dockerfile/dispatchers.go b/components/engine/builder/dockerfile/dispatchers.go index 1f74241248..24a0a1c4df 100644 --- a/components/engine/builder/dockerfile/dispatchers.go +++ b/components/engine/builder/dockerfile/dispatchers.go @@ -17,7 +17,6 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/strslice" @@ -29,6 +28,7 @@ import ( "github.com/docker/docker/pkg/system" "github.com/docker/go-connections/nat" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // ENV foo bar diff --git a/components/engine/builder/dockerfile/imagecontext.go b/components/engine/builder/dockerfile/imagecontext.go index 64b2572b85..fedad6fdf3 100644 --- a/components/engine/builder/dockerfile/imagecontext.go +++ b/components/engine/builder/dockerfile/imagecontext.go @@ -4,12 +4,12 @@ import ( "strconv" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" "github.com/docker/docker/builder/remotecontext" dockerimage "github.com/docker/docker/image" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/builder/dockerfile/imageprobe.go b/components/engine/builder/dockerfile/imageprobe.go index 3433612de5..239eb9e38f 100644 --- a/components/engine/builder/dockerfile/imageprobe.go +++ b/components/engine/builder/dockerfile/imageprobe.go @@ -1,9 +1,9 @@ package dockerfile import ( - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types/container" "github.com/docker/docker/builder" + "github.com/sirupsen/logrus" ) // ImageProber exposes an Image cache to the Builder. It supports resetting a diff --git a/components/engine/builder/fscache/fscache.go b/components/engine/builder/fscache/fscache.go index 63331091a6..faef7d1ac1 100644 --- a/components/engine/builder/fscache/fscache.go +++ b/components/engine/builder/fscache/fscache.go @@ -8,14 +8,14 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/boltdb/bolt" "github.com/docker/docker/builder" "github.com/docker/docker/builder/remotecontext" - "github.com/docker/docker/client/session/filesync" "github.com/docker/docker/pkg/directory" "github.com/docker/docker/pkg/stringid" + "github.com/moby/buildkit/session/filesync" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/tonistiigi/fsutil" "golang.org/x/net/context" "golang.org/x/sync/singleflight" diff --git a/components/engine/builder/fscache/fscache_test.go b/components/engine/builder/fscache/fscache_test.go index 2532a218c8..3f6a1b02af 100644 --- a/components/engine/builder/fscache/fscache_test.go +++ b/components/engine/builder/fscache/fscache_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/docker/docker/client/session/filesync" + "github.com/moby/buildkit/session/filesync" "github.com/stretchr/testify/assert" "golang.org/x/net/context" ) diff --git a/components/engine/builder/remotecontext/detect.go b/components/engine/builder/remotecontext/detect.go index 4345736805..ec32dbed7a 100644 --- a/components/engine/builder/remotecontext/detect.go +++ b/components/engine/builder/remotecontext/detect.go @@ -8,7 +8,6 @@ import ( "path/filepath" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" "github.com/docker/docker/builder/dockerfile/parser" @@ -17,6 +16,7 @@ import ( "github.com/docker/docker/pkg/symlink" "github.com/docker/docker/pkg/urlutil" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // ClientSessionRemote is identifier for client-session context transport diff --git a/components/engine/builder/remotecontext/git/gitutils.go b/components/engine/builder/remotecontext/git/gitutils.go index b94d462cd5..7bc2268155 100644 --- a/components/engine/builder/remotecontext/git/gitutils.go +++ b/components/engine/builder/remotecontext/git/gitutils.go @@ -1,7 +1,6 @@ package git import ( - "fmt" "io/ioutil" "net/http" "net/url" @@ -98,22 +97,46 @@ func getRefAndSubdir(fragment string) (ref string, subdir string) { func fetchArgs(remoteURL string, ref string) []string { args := []string{"fetch", "--recurse-submodules=yes"} - shallow := true - if urlutil.IsURL(remoteURL) { - res, err := http.Head(fmt.Sprintf("%s/info/refs?service=git-upload-pack", remoteURL)) - if err != nil || res.Header.Get("Content-Type") != "application/x-git-upload-pack-advertisement" { - shallow = false - } - } - - if shallow { + if supportsShallowClone(remoteURL) { args = append(args, "--depth", "1") } return append(args, "origin", ref) } +// Check if a given git URL supports a shallow git clone, +// i.e. it is a non-HTTP server or a smart HTTP server. +func supportsShallowClone(remoteURL string) bool { + if urlutil.IsURL(remoteURL) { + // Check if the HTTP server is smart + + // Smart servers must correctly respond to a query for the git-upload-pack service + serviceURL := remoteURL + "/info/refs?service=git-upload-pack" + + // Try a HEAD request and fallback to a Get request on error + res, err := http.Head(serviceURL) + if err != nil || res.StatusCode != http.StatusOK { + res, err = http.Get(serviceURL) + if err == nil { + res.Body.Close() + } + if err != nil || res.StatusCode != http.StatusOK { + // request failed + return false + } + } + + if res.Header.Get("Content-Type") != "application/x-git-upload-pack-advertisement" { + // Fallback, not a smart server + return false + } + return true + } + // Non-HTTP protocols always support shallow clones + return true +} + func checkoutGit(root, ref, subdir string) (string, error) { // Try checking out by ref name first. This will work on branches and sets // .git/HEAD to the current branch name diff --git a/components/engine/cli/debug/debug.go b/components/engine/cli/debug/debug.go index 51dfab2a97..b00ea63ad8 100644 --- a/components/engine/cli/debug/debug.go +++ b/components/engine/cli/debug/debug.go @@ -3,7 +3,7 @@ package debug import ( "os" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // Enable sets the DEBUG env var to true diff --git a/components/engine/cli/debug/debug_test.go b/components/engine/cli/debug/debug_test.go index ad8412a944..903115283c 100644 --- a/components/engine/cli/debug/debug_test.go +++ b/components/engine/cli/debug/debug_test.go @@ -4,7 +4,7 @@ import ( "os" "testing" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) func TestEnable(t *testing.T) { diff --git a/components/engine/client/client.go b/components/engine/client/client.go index 7e14531723..c4e3914b1e 100644 --- a/components/engine/client/client.go +++ b/components/engine/client/client.go @@ -165,7 +165,7 @@ func NewClient(host string, version string, client *http.Client, httpHeaders map } if client != nil { - if _, ok := client.Transport.(*http.Transport); !ok { + if _, ok := client.Transport.(http.RoundTripper); !ok { return nil, fmt.Errorf("unable to verify TLS configuration, invalid transport %v", client.Transport) } } else { diff --git a/components/engine/client/hijack.go b/components/engine/client/hijack.go index 346c74ae82..8cf0119f3d 100644 --- a/components/engine/client/hijack.go +++ b/components/engine/client/hijack.go @@ -177,12 +177,14 @@ func (cli *Client) setupHijackConn(req *http.Request, proto string) (net.Conn, e // Server hijacks the connection, error 'connection closed' expected resp, err := clientconn.Do(req) - if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusSwitchingProtocols { - resp.Body.Close() - return nil, fmt.Errorf("unable to upgrade to %s, received %d", proto, resp.StatusCode) + if err != httputil.ErrPersistEOF { + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusSwitchingProtocols { + resp.Body.Close() + return nil, fmt.Errorf("unable to upgrade to %s, received %d", proto, resp.StatusCode) + } } c, br := clientconn.Hijack() diff --git a/components/engine/client/session/filesync/filesync_test.go b/components/engine/client/session/filesync/filesync_test.go deleted file mode 100644 index b48c08b826..0000000000 --- a/components/engine/client/session/filesync/filesync_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package filesync - -import ( - "context" - "io/ioutil" - "path/filepath" - "testing" - - "github.com/docker/docker/client/session" - "github.com/docker/docker/client/session/testutil" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/sync/errgroup" -) - -func TestFileSyncIncludePatterns(t *testing.T) { - tmpDir, err := ioutil.TempDir("", "fsynctest") - require.NoError(t, err) - - destDir, err := ioutil.TempDir("", "fsynctest") - require.NoError(t, err) - - err = ioutil.WriteFile(filepath.Join(tmpDir, "foo"), []byte("content1"), 0600) - require.NoError(t, err) - - err = ioutil.WriteFile(filepath.Join(tmpDir, "bar"), []byte("content2"), 0600) - require.NoError(t, err) - - s, err := session.NewSession("foo", "bar") - require.NoError(t, err) - - m, err := session.NewManager() - require.NoError(t, err) - - fs := NewFSSyncProvider(tmpDir, nil) - s.Allow(fs) - - dialer := session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn))) - - g, ctx := errgroup.WithContext(context.Background()) - - g.Go(func() error { - return s.Run(ctx, dialer) - }) - - g.Go(func() (reterr error) { - c, err := m.Get(ctx, s.UUID()) - if err != nil { - return err - } - if err := FSSync(ctx, c, FSSendRequestOpt{ - DestDir: destDir, - IncludePatterns: []string{"ba*"}, - }); err != nil { - return err - } - - _, err = ioutil.ReadFile(filepath.Join(destDir, "foo")) - assert.Error(t, err) - - dt, err := ioutil.ReadFile(filepath.Join(destDir, "bar")) - if err != nil { - return err - } - assert.Equal(t, "content2", string(dt)) - return s.Close() - }) - - err = g.Wait() - require.NoError(t, err) -} diff --git a/components/engine/client/session/testutil/testutil.go b/components/engine/client/session/testutil/testutil.go deleted file mode 100644 index 2e145d9006..0000000000 --- a/components/engine/client/session/testutil/testutil.go +++ /dev/null @@ -1,70 +0,0 @@ -package testutil - -import ( - "io" - "net" - "time" - - "github.com/Sirupsen/logrus" - "golang.org/x/net/context" -) - -// Handler is function called to handle incoming connection -type Handler func(ctx context.Context, conn net.Conn, meta map[string][]string) error - -// Dialer is a function for dialing an outgoing connection -type Dialer func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) - -// TestStream creates an in memory session dialer for a handler function -func TestStream(handler Handler) Dialer { - s1, s2 := sockPair() - return func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) { - go func() { - err := handler(context.TODO(), s1, meta) - if err != nil { - logrus.Error(err) - } - s1.Close() - }() - return s2, nil - } -} - -func sockPair() (*sock, *sock) { - pr1, pw1 := io.Pipe() - pr2, pw2 := io.Pipe() - return &sock{pw1, pr2, pw1}, &sock{pw2, pr1, pw2} -} - -type sock struct { - io.Writer - io.Reader - io.Closer -} - -func (s *sock) LocalAddr() net.Addr { - return dummyAddr{} -} -func (s *sock) RemoteAddr() net.Addr { - return dummyAddr{} -} -func (s *sock) SetDeadline(t time.Time) error { - return nil -} -func (s *sock) SetReadDeadline(t time.Time) error { - return nil -} -func (s *sock) SetWriteDeadline(t time.Time) error { - return nil -} - -type dummyAddr struct { -} - -func (d dummyAddr) Network() string { - return "tcp" -} - -func (d dummyAddr) String() string { - return "localhost" -} diff --git a/components/engine/cmd/dockerd/config.go b/components/engine/cmd/dockerd/config.go index 440497bfbc..fa5565ee46 100644 --- a/components/engine/cmd/dockerd/config.go +++ b/components/engine/cmd/dockerd/config.go @@ -62,6 +62,9 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) { flags.StringVar(&conf.MetricsAddress, "metrics-addr", "", "Set default address and port to serve the metrics api on") + flags.StringVar(&conf.NodeGenericResources, "node-generic-resources", "", "user defined resources (e.g. fpga=2;gpu={UUID1,UUID2,UUID3})") + flags.IntVar(&conf.NetworkControlPlaneMTU, "network-control-plane-mtu", config.DefaultNetworkMtu, "Network Control plane MTU") + // "--deprecated-key-path" is to allow configuration of the key used // for the daemon ID and the deprecated image signing. It was never // exposed as a command line option but is added here to allow diff --git a/components/engine/cmd/dockerd/config_unix.go b/components/engine/cmd/dockerd/config_unix.go index ba37121e96..148fa87459 100644 --- a/components/engine/cmd/dockerd/config_unix.go +++ b/components/engine/cmd/dockerd/config_unix.go @@ -47,6 +47,7 @@ func installConfigFlags(conf *config.Config, flags *pflag.FlagSet) { flags.StringVar(&conf.SeccompProfile, "seccomp-profile", "", "Path to seccomp profile") flags.Var(&conf.ShmSize, "default-shm-size", "Default shm size for containers") flags.BoolVar(&conf.NoNewPrivileges, "no-new-privileges", false, "Set no-new-privileges by default for new containers") + flags.StringVar(&conf.IpcMode, "default-ipc-mode", config.DefaultIpcMode, `Default mode for containers ipc ("shareable" | "private")`) attachExperimentalFlags(conf, flags) } diff --git a/components/engine/cmd/dockerd/daemon.go b/components/engine/cmd/dockerd/daemon.go index 215f9c2d59..d36c294ec3 100644 --- a/components/engine/cmd/dockerd/daemon.go +++ b/components/engine/cmd/dockerd/daemon.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/uuid" "github.com/docker/docker/api" apiserver "github.com/docker/docker/api/server" @@ -30,17 +29,16 @@ import ( "github.com/docker/docker/builder/dockerfile" "github.com/docker/docker/builder/fscache" "github.com/docker/docker/cli/debug" - "github.com/docker/docker/client/session" "github.com/docker/docker/daemon" "github.com/docker/docker/daemon/cluster" "github.com/docker/docker/daemon/config" + "github.com/docker/docker/daemon/listeners" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/dockerversion" "github.com/docker/docker/libcontainerd" dopts "github.com/docker/docker/opts" "github.com/docker/docker/pkg/authorization" "github.com/docker/docker/pkg/jsonlog" - "github.com/docker/docker/pkg/listeners" "github.com/docker/docker/pkg/pidfile" "github.com/docker/docker/pkg/plugingetter" "github.com/docker/docker/pkg/signal" @@ -50,7 +48,9 @@ import ( "github.com/docker/docker/runconfig" "github.com/docker/go-connections/tlsconfig" swarmapi "github.com/docker/swarmkit/api" + "github.com/moby/buildkit/session" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/spf13/pflag" ) @@ -96,6 +96,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { logrus.SetFormatter(&logrus.TextFormatter{ TimestampFormat: jsonlog.RFC3339NanoFixed, DisableColors: cli.Config.RawLogs, + FullTimestamp: true, }) if err := setDefaultUmask(); err != nil { @@ -205,7 +206,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { signal.Trap(func() { cli.stop() <-stopc // wait for daemonCli.start() to return - }) + }, logrus.StandardLogger()) // Notify that the API is active, but before daemon is set up. preNotifySystem() diff --git a/components/engine/cmd/dockerd/daemon_test.go b/components/engine/cmd/dockerd/daemon_test.go index 7ae91fe1a7..1b81aa5734 100644 --- a/components/engine/cmd/dockerd/daemon_test.go +++ b/components/engine/cmd/dockerd/daemon_test.go @@ -3,10 +3,10 @@ package main import ( "testing" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/config" "github.com/docker/docker/pkg/testutil" "github.com/docker/docker/pkg/testutil/tempfile" + "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/components/engine/cmd/dockerd/daemon_windows.go b/components/engine/cmd/dockerd/daemon_windows.go index 2e9598d4cf..7dd0920ce7 100644 --- a/components/engine/cmd/dockerd/daemon_windows.go +++ b/components/engine/cmd/dockerd/daemon_windows.go @@ -6,9 +6,9 @@ import ( "os" "path/filepath" - "github.com/Sirupsen/logrus" "github.com/docker/docker/libcontainerd" "github.com/docker/docker/pkg/system" + "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) diff --git a/components/engine/cmd/dockerd/docker.go b/components/engine/cmd/dockerd/docker.go index 8a5c8f5433..2ccca46c18 100644 --- a/components/engine/cmd/dockerd/docker.go +++ b/components/engine/cmd/dockerd/docker.go @@ -6,12 +6,12 @@ import ( "path/filepath" "runtime" - "github.com/Sirupsen/logrus" "github.com/docker/docker/cli" "github.com/docker/docker/daemon/config" "github.com/docker/docker/dockerversion" "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/pkg/term" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) diff --git a/components/engine/cmd/dockerd/metrics.go b/components/engine/cmd/dockerd/metrics.go index 0c8860408b..99d41cb3aa 100644 --- a/components/engine/cmd/dockerd/metrics.go +++ b/components/engine/cmd/dockerd/metrics.go @@ -4,8 +4,8 @@ import ( "net" "net/http" - "github.com/Sirupsen/logrus" metrics "github.com/docker/go-metrics" + "github.com/sirupsen/logrus" ) func startMetricsServer(addr string) error { diff --git a/components/engine/cmd/dockerd/options.go b/components/engine/cmd/dockerd/options.go index 629b0223e9..4fd597d610 100644 --- a/components/engine/cmd/dockerd/options.go +++ b/components/engine/cmd/dockerd/options.go @@ -5,11 +5,11 @@ import ( "os" "path/filepath" - "github.com/Sirupsen/logrus" cliconfig "github.com/docker/docker/cli/config" "github.com/docker/docker/daemon/config" "github.com/docker/docker/opts" "github.com/docker/go-connections/tlsconfig" + "github.com/sirupsen/logrus" "github.com/spf13/pflag" ) diff --git a/components/engine/cmd/dockerd/service_windows.go b/components/engine/cmd/dockerd/service_windows.go index 0172169517..00432af643 100644 --- a/components/engine/cmd/dockerd/service_windows.go +++ b/components/engine/cmd/dockerd/service_windows.go @@ -12,8 +12,8 @@ import ( "time" "unsafe" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/system" + "github.com/sirupsen/logrus" "github.com/spf13/pflag" "golang.org/x/sys/windows" "golang.org/x/sys/windows/svc" diff --git a/components/engine/container/container.go b/components/engine/container/container.go index 7d339c688f..19079adbc7 100644 --- a/components/engine/container/container.go +++ b/components/engine/container/container.go @@ -15,7 +15,6 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" containertypes "github.com/docker/docker/api/types/container" mounttypes "github.com/docker/docker/api/types/mount" networktypes "github.com/docker/docker/api/types/network" @@ -44,6 +43,7 @@ import ( "github.com/docker/libnetwork/options" "github.com/docker/libnetwork/types" agentexec "github.com/docker/swarmkit/agent/exec" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) @@ -167,7 +167,7 @@ func (container *Container) toDisk() (*Container, error) { } // Save container settings - f, err := ioutils.NewAtomicFileWriter(pth, 0644) + f, err := ioutils.NewAtomicFileWriter(pth, 0600) if err != nil { return nil, err } @@ -745,6 +745,9 @@ func (container *Container) BuildCreateEndpointOptions(n libnetwork.Network, epC for _, alias := range epConfig.Aliases { createOptions = append(createOptions, libnetwork.CreateOptionMyAlias(alias)) } + for k, v := range epConfig.DriverOpts { + createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(options.Generic{k: v})) + } } if container.NetworkSettings.Service != nil { @@ -790,9 +793,6 @@ func (container *Container) BuildCreateEndpointOptions(n libnetwork.Network, epC createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(genericOption)) } - for k, v := range epConfig.DriverOpts { - createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(options.Generic{k: v})) - } } diff --git a/components/engine/container/container_unix.go b/components/engine/container/container_unix.go index 327f950ff8..e7a27d212c 100644 --- a/components/engine/container/container_unix.go +++ b/components/engine/container/container_unix.go @@ -7,9 +7,7 @@ import ( "io/ioutil" "os" "path/filepath" - "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" mounttypes "github.com/docker/docker/api/types/mount" @@ -20,6 +18,8 @@ import ( "github.com/docker/docker/pkg/system" "github.com/docker/docker/volume" "github.com/opencontainers/selinux/go-selinux/label" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -171,47 +171,47 @@ func (container *Container) HasMountFor(path string) bool { return exists } -// UnmountIpcMounts uses the provided unmount function to unmount shm and mqueue if they were mounted -func (container *Container) UnmountIpcMounts(unmount func(pth string) error) { - if container.HostConfig.IpcMode.IsContainer() || container.HostConfig.IpcMode.IsHost() { - return +// UnmountIpcMount uses the provided unmount function to unmount shm if it was mounted +func (container *Container) UnmountIpcMount(unmount func(pth string) error) error { + if container.HasMountFor("/dev/shm") { + return nil } - var warnings []string - - if !container.HasMountFor("/dev/shm") { - shmPath, err := container.ShmResourcePath() - if err != nil { - logrus.Error(err) - warnings = append(warnings, err.Error()) - } else if shmPath != "" { - if err := unmount(shmPath); err != nil && !os.IsNotExist(err) { - if mounted, mErr := mount.Mounted(shmPath); mounted || mErr != nil { - warnings = append(warnings, fmt.Sprintf("failed to umount %s: %v", shmPath, err)) - } - } - + // container.ShmPath should not be used here as it may point + // to the host's or other container's /dev/shm + shmPath, err := container.ShmResourcePath() + if err != nil { + return err + } + if shmPath == "" { + return nil + } + if err = unmount(shmPath); err != nil && !os.IsNotExist(err) { + if mounted, mErr := mount.Mounted(shmPath); mounted || mErr != nil { + return errors.Wrapf(err, "umount %s", shmPath) } } - - if len(warnings) > 0 { - logrus.Warnf("failed to cleanup ipc mounts:\n%v", strings.Join(warnings, "\n")) - } + return nil } // IpcMounts returns the list of IPC mounts func (container *Container) IpcMounts() []Mount { var mounts []Mount - if !container.HasMountFor("/dev/shm") { - label.SetFileLabel(container.ShmPath, container.MountLabel) - mounts = append(mounts, Mount{ - Source: container.ShmPath, - Destination: "/dev/shm", - Writable: true, - Propagation: string(volume.DefaultPropagationMode), - }) + if container.HasMountFor("/dev/shm") { + return mounts } + if container.ShmPath == "" { + return mounts + } + + label.SetFileLabel(container.ShmPath, container.MountLabel) + mounts = append(mounts, Mount{ + Source: container.ShmPath, + Destination: "/dev/shm", + Writable: true, + Propagation: string(volume.DefaultPropagationMode), + }) return mounts } diff --git a/components/engine/container/container_windows.go b/components/engine/container/container_windows.go index 0f2a45df99..9c52d00a46 100644 --- a/components/engine/container/container_windows.go +++ b/components/engine/container/container_windows.go @@ -24,9 +24,10 @@ type ExitStatus struct { ExitCode int } -// UnmountIpcMounts unmounts Ipc related mounts. +// UnmountIpcMount unmounts Ipc related mounts. // This is a NOOP on windows. -func (container *Container) UnmountIpcMounts(unmount func(pth string) error) { +func (container *Container) UnmountIpcMount(unmount func(pth string) error) error { + return nil } // IpcMounts returns the list of Ipc related mounts. diff --git a/components/engine/container/health.go b/components/engine/container/health.go index 31c5600d25..5919008d27 100644 --- a/components/engine/container/health.go +++ b/components/engine/container/health.go @@ -1,8 +1,8 @@ package container import ( - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" + "github.com/sirupsen/logrus" ) // Health holds the current container health-check state diff --git a/components/engine/container/monitor.go b/components/engine/container/monitor.go index f05e72b25f..f35097b2cd 100644 --- a/components/engine/container/monitor.go +++ b/components/engine/container/monitor.go @@ -3,7 +3,7 @@ package container import ( "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/container/stream/attach.go b/components/engine/container/stream/attach.go index 3dd53d3354..24b68863d7 100644 --- a/components/engine/container/stream/attach.go +++ b/components/engine/container/stream/attach.go @@ -6,10 +6,10 @@ import ( "golang.org/x/net/context" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/promise" "github.com/docker/docker/pkg/term" + "github.com/sirupsen/logrus" ) var defaultEscapeSequence = []byte{16, 17} // ctrl-p, ctrl-q diff --git a/components/engine/container/stream/streams.go b/components/engine/container/stream/streams.go index 735bab5107..7e734d81c4 100644 --- a/components/engine/container/stream/streams.go +++ b/components/engine/container/stream/streams.go @@ -7,11 +7,11 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/docker/libcontainerd" "github.com/docker/docker/pkg/broadcaster" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/pools" + "github.com/sirupsen/logrus" ) // Config holds information about I/O streams managed together. diff --git a/components/engine/container/view.go b/components/engine/container/view.go index e865e4d5df..8d885268c2 100644 --- a/components/engine/container/view.go +++ b/components/engine/container/view.go @@ -6,11 +6,11 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/network" "github.com/docker/go-connections/nat" "github.com/hashicorp/go-memdb" + "github.com/sirupsen/logrus" ) const ( @@ -168,9 +168,9 @@ func (db *memDB) Delete(c *Container) error { txn.Delete(memdbNamesTable, nameAssociation{name: name}) } - if err := txn.Delete(memdbContainersTable, NewBaseContainer(c.ID, c.Root)); err != nil { - return err - } + // Ignore error - the container may not actually exist in the + // db, but we still need to clean up associated names. + txn.Delete(memdbContainersTable, NewBaseContainer(c.ID, c.Root)) return nil }) } diff --git a/components/engine/container/view_test.go b/components/engine/container/view_test.go index 09ba343830..4535a8fade 100644 --- a/components/engine/container/view_test.go +++ b/components/engine/container/view_test.go @@ -150,4 +150,12 @@ func TestNames(t *testing.T) { view = db.Snapshot() assert.Equal(t, map[string][]string{"containerid1": {"name1", "name3", "name4"}, "containerid4": {"name2"}}, view.GetAllNames()) + + // Release containerid1's names with Delete even though no container exists + assert.NoError(t, db.Delete(&Container{ID: "containerid1"})) + + // Reusing one of those names should work + assert.NoError(t, db.ReserveName("name1", "containerid4")) + view = db.Snapshot() + assert.Equal(t, map[string][]string{"containerid4": {"name1", "name2"}}, view.GetAllNames()) } diff --git a/components/engine/contrib/docker-device-tool/device_tool.go b/components/engine/contrib/docker-device-tool/device_tool.go index fc171666fa..8f6251582a 100644 --- a/components/engine/contrib/docker-device-tool/device_tool.go +++ b/components/engine/contrib/docker-device-tool/device_tool.go @@ -11,9 +11,9 @@ import ( "strconv" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver/devmapper" "github.com/docker/docker/pkg/devicemapper" + "github.com/sirupsen/logrus" ) func usage() { diff --git a/components/engine/contrib/download-frozen-image-v2.sh b/components/engine/contrib/download-frozen-image-v2.sh index 232679019d..2dfe247b5a 100755 --- a/components/engine/contrib/download-frozen-image-v2.sh +++ b/components/engine/contrib/download-frozen-image-v2.sh @@ -64,7 +64,7 @@ fetch_blob() { -D- )" curlHeaders="$(echo "$curlHeaders" | tr -d '\r')" - if [ "$(echo "$curlHeaders" | awk 'NR == 1 { print $2; exit }')" != '200' ]; then + if echo "$curlHeaders" | grep -qE "^HTTP/[0-9].[0-9] 3"; then rm -f "$targetFile" local blobRedirect="$(echo "$curlHeaders" | awk -F ': ' 'tolower($1) == "location" { print $2; exit }')" diff --git a/components/engine/contrib/mkimage-yum.sh b/components/engine/contrib/mkimage-yum.sh index 29da170480..901280451b 100755 --- a/components/engine/contrib/mkimage-yum.sh +++ b/components/engine/contrib/mkimage-yum.sh @@ -81,13 +81,13 @@ fi if [[ -n "$install_groups" ]]; then yum -c "$yum_config" --installroot="$target" --releasever=/ --setopt=tsflags=nodocs \ - --setopt=group_package_types=mandatory -y groupinstall $install_groups + --setopt=group_package_types=mandatory -y groupinstall "$install_groups" fi if [[ -n "$install_packages" ]]; then yum -c "$yum_config" --installroot="$target" --releasever=/ --setopt=tsflags=nodocs \ - --setopt=group_package_types=mandatory -y install $install_packages + --setopt=group_package_types=mandatory -y install "$install_packages" fi yum -c "$yum_config" --installroot="$target" -y clean all diff --git a/components/engine/contrib/syntax/textmate/Docker.tmbundle/Syntaxes/Dockerfile.tmLanguage b/components/engine/contrib/syntax/textmate/Docker.tmbundle/Syntaxes/Dockerfile.tmLanguage index 5a273337f0..a4a7b7ae8d 100644 --- a/components/engine/contrib/syntax/textmate/Docker.tmbundle/Syntaxes/Dockerfile.tmLanguage +++ b/components/engine/contrib/syntax/textmate/Docker.tmbundle/Syntaxes/Dockerfile.tmLanguage @@ -25,7 +25,7 @@ match - ^\s*\b(FROM)\b.*?\b(AS)\b + ^\s*\b(?i:(FROM))\b.*?\b(?i:(AS))\b captures @@ -42,7 +42,7 @@ match - ^\s*(?:(ONBUILD)\s+)?(ADD|ARG|CMD|COPY|ENTRYPOINT|ENV|EXPOSE|FROM|HEALTHCHECK|LABEL|MAINTAINER|RUN|SHELL|STOPSIGNAL|USER|VOLUME|WORKDIR)\s + ^\s*(?i:(ONBUILD)\s+)?(?i:(ADD|ARG|CMD|COPY|ENTRYPOINT|ENV|EXPOSE|FROM|HEALTHCHECK|LABEL|MAINTAINER|RUN|SHELL|STOPSIGNAL|USER|VOLUME|WORKDIR))\s captures @@ -59,7 +59,7 @@ match - ^\s*(?:(ONBUILD)\s+)?(CMD|ENTRYPOINT)\s + ^\s*(?i:(ONBUILD)\s+)?(?i:(CMD|ENTRYPOINT))\s begin diff --git a/components/engine/daemon/attach.go b/components/engine/daemon/attach.go index 32410393a3..115c315764 100644 --- a/components/engine/daemon/attach.go +++ b/components/engine/daemon/attach.go @@ -5,7 +5,6 @@ import ( "fmt" "io" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/errors" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/container" @@ -13,6 +12,7 @@ import ( "github.com/docker/docker/daemon/logger" "github.com/docker/docker/pkg/stdcopy" "github.com/docker/docker/pkg/term" + "github.com/sirupsen/logrus" ) // ContainerAttach attaches to logs according to the config passed in. See ContainerAttachConfig. diff --git a/components/engine/daemon/build.go b/components/engine/daemon/build.go index 9b518d64f3..39269ab5a2 100644 --- a/components/engine/daemon/build.go +++ b/components/engine/daemon/build.go @@ -4,7 +4,6 @@ import ( "io" "runtime" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" @@ -15,6 +14,7 @@ import ( "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/registry" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) @@ -27,6 +27,7 @@ type releaseableLayer struct { func (rl *releaseableLayer) Mount() (string, error) { var err error + var mountPath string var chainID layer.ChainID if rl.roLayer != nil { chainID = rl.roLayer.ChainID() @@ -38,7 +39,19 @@ func (rl *releaseableLayer) Mount() (string, error) { return "", errors.Wrap(err, "failed to create rwlayer") } - return rl.rwLayer.Mount("") + mountPath, err = rl.rwLayer.Mount("") + if err != nil { + // Clean up the layer if we fail to mount it here. + metadata, err := rl.layerStore.ReleaseRWLayer(rl.rwLayer) + layer.LogReleaseMetadata(metadata) + if err != nil { + logrus.Errorf("Failed to release RWLayer: %s", err) + } + rl.rwLayer = nil + return "", err + } + + return mountPath, nil } func (rl *releaseableLayer) Commit(platform string) (builder.ReleaseableLayer, error) { @@ -51,6 +64,7 @@ func (rl *releaseableLayer) Commit(platform string) (builder.ReleaseableLayer, e if err != nil { return nil, err } + defer stream.Close() newLayer, err := rl.layerStore.Register(stream, chainID, layer.Platform(platform)) if err != nil { @@ -75,20 +89,32 @@ func (rl *releaseableLayer) Release() error { if rl.released { return nil } + if err := rl.releaseRWLayer(); err != nil { + // Best effort attempt at releasing read-only layer before returning original error. + rl.releaseROLayer() + return err + } + if err := rl.releaseROLayer(); err != nil { + return err + } rl.released = true - rl.releaseRWLayer() - return rl.releaseROLayer() + return nil } func (rl *releaseableLayer) releaseRWLayer() error { if rl.rwLayer == nil { return nil } + if err := rl.rwLayer.Unmount(); err != nil { + logrus.Errorf("Failed to unmount RWLayer: %s", err) + return err + } metadata, err := rl.layerStore.ReleaseRWLayer(rl.rwLayer) layer.LogReleaseMetadata(metadata) if err != nil { logrus.Errorf("Failed to release RWLayer: %s", err) } + rl.rwLayer = nil return err } @@ -98,6 +124,10 @@ func (rl *releaseableLayer) releaseROLayer() error { } metadata, err := rl.layerStore.Release(rl.roLayer) layer.LogReleaseMetadata(metadata) + if err != nil { + logrus.Errorf("Failed to release ROLayer: %s", err) + } + rl.roLayer = nil return err } diff --git a/components/engine/daemon/cache.go b/components/engine/daemon/cache.go index 219b0b38da..5ad68ec0af 100644 --- a/components/engine/daemon/cache.go +++ b/components/engine/daemon/cache.go @@ -1,9 +1,9 @@ package daemon import ( - "github.com/Sirupsen/logrus" "github.com/docker/docker/builder" "github.com/docker/docker/image/cache" + "github.com/sirupsen/logrus" ) // MakeImageCache creates a stateful image cache. diff --git a/components/engine/daemon/cluster/cluster.go b/components/engine/daemon/cluster/cluster.go index 57fc4d2d6f..55540fc06c 100644 --- a/components/engine/daemon/cluster/cluster.go +++ b/components/engine/daemon/cluster/cluster.go @@ -46,7 +46,6 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types/network" types "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/daemon/cluster/controllers/plugin" @@ -56,6 +55,7 @@ import ( swarmapi "github.com/docker/swarmkit/api" swarmnode "github.com/docker/swarmkit/node" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/daemon/cluster/controllers/plugin/controller.go b/components/engine/daemon/cluster/controllers/plugin/controller.go index e72edcdd75..8cc9b04e36 100644 --- a/components/engine/daemon/cluster/controllers/plugin/controller.go +++ b/components/engine/daemon/cluster/controllers/plugin/controller.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "net/http" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" enginetypes "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/swarm/runtime" @@ -14,6 +13,7 @@ import ( "github.com/docker/swarmkit/api" "github.com/gogo/protobuf/proto" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/daemon/cluster/controllers/plugin/controller_test.go b/components/engine/daemon/cluster/controllers/plugin/controller_test.go index 17b77cc89f..70bb91b12e 100644 --- a/components/engine/daemon/cluster/controllers/plugin/controller_test.go +++ b/components/engine/daemon/cluster/controllers/plugin/controller_test.go @@ -9,13 +9,13 @@ import ( "testing" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" enginetypes "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/swarm/runtime" "github.com/docker/docker/pkg/pubsub" "github.com/docker/docker/plugin" "github.com/docker/docker/plugin/v2" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/daemon/cluster/convert/container.go b/components/engine/daemon/cluster/convert/container.go index 6ac6f331f2..795e944ae1 100644 --- a/components/engine/daemon/cluster/convert/container.go +++ b/components/engine/daemon/cluster/convert/container.go @@ -5,12 +5,12 @@ import ( "fmt" "strings" - "github.com/Sirupsen/logrus" container "github.com/docker/docker/api/types/container" mounttypes "github.com/docker/docker/api/types/mount" types "github.com/docker/docker/api/types/swarm" swarmapi "github.com/docker/swarmkit/api" gogotypes "github.com/gogo/protobuf/types" + "github.com/sirupsen/logrus" ) func containerSpecFromGRPC(c *swarmapi.ContainerSpec) *types.ContainerSpec { diff --git a/components/engine/daemon/cluster/convert/node.go b/components/engine/daemon/cluster/convert/node.go index f075783e88..528ec54a60 100644 --- a/components/engine/daemon/cluster/convert/node.go +++ b/components/engine/daemon/cluster/convert/node.go @@ -42,6 +42,7 @@ func NodeFromGRPC(n swarmapi.Node) types.Node { if n.Description.Resources != nil { node.Description.Resources.NanoCPUs = n.Description.Resources.NanoCPUs node.Description.Resources.MemoryBytes = n.Description.Resources.MemoryBytes + node.Description.Resources.GenericResources = GenericResourcesFromGRPC(n.Description.Resources.Generic) } if n.Description.Engine != nil { node.Description.Engine.EngineVersion = n.Description.Engine.EngineVersion diff --git a/components/engine/daemon/cluster/convert/service.go b/components/engine/daemon/cluster/convert/service.go index 947debdf52..f0dee57bd4 100644 --- a/components/engine/daemon/cluster/convert/service.go +++ b/components/engine/daemon/cluster/convert/service.go @@ -8,6 +8,7 @@ import ( "github.com/docker/docker/api/types/swarm/runtime" "github.com/docker/docker/pkg/namesgenerator" swarmapi "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/api/genericresource" "github.com/gogo/protobuf/proto" gogotypes "github.com/gogo/protobuf/types" "github.com/pkg/errors" @@ -301,6 +302,31 @@ func annotationsFromGRPC(ann swarmapi.Annotations) types.Annotations { return a } +// GenericResourcesFromGRPC converts a GRPC GenericResource to a GenericResource +func GenericResourcesFromGRPC(genericRes []*swarmapi.GenericResource) []types.GenericResource { + var generic []types.GenericResource + for _, res := range genericRes { + var current types.GenericResource + + switch r := res.Resource.(type) { + case *swarmapi.GenericResource_DiscreteResourceSpec: + current.DiscreteResourceSpec = &types.DiscreteGenericResource{ + Kind: r.DiscreteResourceSpec.Kind, + Value: r.DiscreteResourceSpec.Value, + } + case *swarmapi.GenericResource_NamedResourceSpec: + current.NamedResourceSpec = &types.NamedGenericResource{ + Kind: r.NamedResourceSpec.Kind, + Value: r.NamedResourceSpec.Value, + } + } + + generic = append(generic, current) + } + + return generic +} + func resourcesFromGRPC(res *swarmapi.ResourceRequirements) *types.ResourceRequirements { var resources *types.ResourceRequirements if res != nil { @@ -313,8 +339,9 @@ func resourcesFromGRPC(res *swarmapi.ResourceRequirements) *types.ResourceRequir } if res.Reservations != nil { resources.Reservations = &types.Resources{ - NanoCPUs: res.Reservations.NanoCPUs, - MemoryBytes: res.Reservations.MemoryBytes, + NanoCPUs: res.Reservations.NanoCPUs, + MemoryBytes: res.Reservations.MemoryBytes, + GenericResources: GenericResourcesFromGRPC(res.Reservations.Generic), } } } @@ -322,6 +349,24 @@ func resourcesFromGRPC(res *swarmapi.ResourceRequirements) *types.ResourceRequir return resources } +// GenericResourcesToGRPC converts a GenericResource to a GRPC GenericResource +func GenericResourcesToGRPC(genericRes []types.GenericResource) []*swarmapi.GenericResource { + var generic []*swarmapi.GenericResource + for _, res := range genericRes { + var r *swarmapi.GenericResource + + if res.DiscreteResourceSpec != nil { + r = genericresource.NewDiscrete(res.DiscreteResourceSpec.Kind, res.DiscreteResourceSpec.Value) + } else if res.NamedResourceSpec != nil { + r = genericresource.NewString(res.NamedResourceSpec.Kind, res.NamedResourceSpec.Value) + } + + generic = append(generic, r) + } + + return generic +} + func resourcesToGRPC(res *types.ResourceRequirements) *swarmapi.ResourceRequirements { var reqs *swarmapi.ResourceRequirements if res != nil { @@ -336,6 +381,7 @@ func resourcesToGRPC(res *types.ResourceRequirements) *swarmapi.ResourceRequirem reqs.Reservations = &swarmapi.Resources{ NanoCPUs: res.Reservations.NanoCPUs, MemoryBytes: res.Reservations.MemoryBytes, + Generic: GenericResourcesToGRPC(res.Reservations.GenericResources), } } diff --git a/components/engine/daemon/cluster/convert/task.go b/components/engine/daemon/cluster/convert/task.go index e301415c6c..bedf2dba9e 100644 --- a/components/engine/daemon/cluster/convert/task.go +++ b/components/engine/daemon/cluster/convert/task.go @@ -30,7 +30,8 @@ func TaskFromGRPC(t swarmapi.Task) (types.Task, error) { Message: t.Status.Message, Err: t.Status.Err, }, - DesiredState: types.TaskState(strings.ToLower(t.DesiredState.String())), + DesiredState: types.TaskState(strings.ToLower(t.DesiredState.String())), + GenericResources: GenericResourcesFromGRPC(t.AssignedGenericResources), } // Meta diff --git a/components/engine/daemon/cluster/executor/container/adapter.go b/components/engine/daemon/cluster/executor/container/adapter.go index 7444057c35..0a33739669 100644 --- a/components/engine/daemon/cluster/executor/container/adapter.go +++ b/components/engine/daemon/cluster/executor/container/adapter.go @@ -12,7 +12,6 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" @@ -28,6 +27,7 @@ import ( "github.com/docker/swarmkit/log" gogotypes "github.com/gogo/protobuf/types" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "golang.org/x/time/rate" ) diff --git a/components/engine/daemon/cluster/executor/container/container.go b/components/engine/daemon/cluster/executor/container/container.go index d25e59acf6..8ea4126d73 100644 --- a/components/engine/daemon/cluster/executor/container/container.go +++ b/components/engine/daemon/cluster/executor/container/container.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" @@ -25,6 +25,7 @@ import ( netconst "github.com/docker/libnetwork/datastore" "github.com/docker/swarmkit/agent/exec" "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/api/genericresource" "github.com/docker/swarmkit/template" gogotypes "github.com/gogo/protobuf/types" ) @@ -186,13 +187,16 @@ func (c *containerConfig) exposedPorts() map[nat.Port]struct{} { } func (c *containerConfig) config() *enginecontainer.Config { + genericEnvs := genericresource.EnvFormat(c.task.AssignedGenericResources, "DOCKER_RESOURCE") + env := append(c.spec().Env, genericEnvs...) + config := &enginecontainer.Config{ Labels: c.labels(), StopSignal: c.spec().StopSignal, Tty: c.spec().TTY, OpenStdin: c.spec().OpenStdin, User: c.spec().User, - Env: c.spec().Env, + Env: env, Hostname: c.spec().Hostname, WorkingDir: c.spec().Dir, Image: c.image(), diff --git a/components/engine/daemon/cluster/executor/container/controller.go b/components/engine/daemon/cluster/executor/container/controller.go index 4d6134da37..7fa4a864d7 100644 --- a/components/engine/daemon/cluster/executor/container/controller.go +++ b/components/engine/daemon/cluster/executor/container/controller.go @@ -524,10 +524,12 @@ func (r *controller) Logs(ctx context.Context, publisher exec.LogPublisher, opti } // parse the details out of the Attrs map - attrs := []api.LogAttr{} - for k, v := range msg.Attrs { - attr := api.LogAttr{Key: k, Value: v} - attrs = append(attrs, attr) + var attrs []api.LogAttr + if len(msg.Attrs) != 0 { + attrs = make([]api.LogAttr, 0, len(msg.Attrs)) + for _, attr := range msg.Attrs { + attrs = append(attrs, api.LogAttr{Key: attr.Key, Value: attr.Value}) + } } if err := publisher.Publish(ctx, api.LogMessage{ diff --git a/components/engine/daemon/cluster/executor/container/executor.go b/components/engine/daemon/cluster/executor/container/executor.go index a71a9412e3..fb339f14fc 100644 --- a/components/engine/daemon/cluster/executor/container/executor.go +++ b/components/engine/daemon/cluster/executor/container/executor.go @@ -5,12 +5,12 @@ import ( "sort" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/network" swarmtypes "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/daemon/cluster/controllers/plugin" + "github.com/docker/docker/daemon/cluster/convert" executorpkg "github.com/docker/docker/daemon/cluster/executor" clustertypes "github.com/docker/docker/daemon/cluster/provider" networktypes "github.com/docker/libnetwork/types" @@ -18,6 +18,7 @@ import ( "github.com/docker/swarmkit/agent/exec" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/api/naming" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) @@ -119,6 +120,7 @@ func (e *executor) Describe(ctx context.Context) (*api.NodeDescription, error) { Resources: &api.Resources{ NanoCPUs: int64(info.NCPU) * 1e9, MemoryBytes: info.MemTotal, + Generic: convert.GenericResourcesToGRPC(info.GenericResources), }, } @@ -183,13 +185,17 @@ func (e *executor) Controller(t *api.Task) (exec.Controller, error) { } switch runtimeKind { case string(swarmtypes.RuntimePlugin): + info, _ := e.backend.SystemInfo() + if !info.ExperimentalBuild { + return ctlr, fmt.Errorf("runtime type %q only supported in experimental", swarmtypes.RuntimePlugin) + } c, err := plugin.NewController(e.pluginBackend, t) if err != nil { return ctlr, err } ctlr = c default: - return ctlr, fmt.Errorf("unsupported runtime type: %q", r.Generic.Kind) + return ctlr, fmt.Errorf("unsupported runtime type: %q", runtimeKind) } case *api.TaskSpec_Container: c, err := newController(e.backend, t, dependencyGetter) diff --git a/components/engine/daemon/cluster/networks.go b/components/engine/daemon/cluster/networks.go index 1906c37bd2..3b271dbc8d 100644 --- a/components/engine/daemon/cluster/networks.go +++ b/components/engine/daemon/cluster/networks.go @@ -3,7 +3,6 @@ package cluster import ( "fmt" - "github.com/Sirupsen/logrus" apierrors "github.com/docker/docker/api/errors" apitypes "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/network" @@ -12,6 +11,7 @@ import ( "github.com/docker/docker/runconfig" swarmapi "github.com/docker/swarmkit/api" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/daemon/cluster/noderunner.go b/components/engine/daemon/cluster/noderunner.go index b970e7b217..aa12737cdd 100644 --- a/components/engine/daemon/cluster/noderunner.go +++ b/components/engine/daemon/cluster/noderunner.go @@ -8,13 +8,13 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" types "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/daemon/cluster/executor/container" lncluster "github.com/docker/libnetwork/cluster" swarmapi "github.com/docker/swarmkit/api" swarmnode "github.com/docker/swarmkit/node" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "google.golang.org/grpc" ) diff --git a/components/engine/daemon/cluster/services.go b/components/engine/daemon/cluster/services.go index 42397fa00b..ba5ef040ac 100644 --- a/components/engine/daemon/cluster/services.go +++ b/components/engine/daemon/cluster/services.go @@ -10,7 +10,6 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" apierrors "github.com/docker/docker/api/errors" apitypes "github.com/docker/docker/api/types" @@ -22,6 +21,7 @@ import ( swarmapi "github.com/docker/swarmkit/api" gogotypes "github.com/gogo/protobuf/types" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) @@ -139,9 +139,16 @@ func (c *Cluster) CreateService(s types.ServiceSpec, encodedAuth string, queryRe case *swarmapi.TaskSpec_Generic: switch serviceSpec.Task.GetGeneric().Kind { case string(types.RuntimePlugin): + info, _ := c.config.Backend.SystemInfo() + if !info.ExperimentalBuild { + return fmt.Errorf("runtime type %q only supported in experimental", types.RuntimePlugin) + } if s.TaskTemplate.PluginSpec == nil { return errors.New("plugin spec must be set") } + + default: + return fmt.Errorf("unsupported runtime type: %q", serviceSpec.Task.GetGeneric().Kind) } r, err := state.controlClient.CreateService(ctx, &swarmapi.CreateServiceRequest{Spec: &serviceSpec}) @@ -458,22 +465,33 @@ func (c *Cluster) ServiceLogs(ctx context.Context, selector *backend.LogSelector for _, msg := range subscribeMsg.Messages { // make a new message m := new(backend.LogMessage) - m.Attrs = make(backend.LogAttributes) + m.Attrs = make([]backend.LogAttr, 0, len(msg.Attrs)+3) // add the timestamp, adding the error if it fails m.Timestamp, err = gogotypes.TimestampFromProto(msg.Timestamp) if err != nil { m.Err = err } + + nodeKey := contextPrefix + ".node.id" + serviceKey := contextPrefix + ".service.id" + taskKey := contextPrefix + ".task.id" + // copy over all of the details for _, d := range msg.Attrs { - m.Attrs[d.Key] = d.Value + switch d.Key { + case nodeKey, serviceKey, taskKey: + // we have the final say over context details (in case there + // is a conflict (if the user added a detail with a context's + // key for some reason)) + default: + m.Attrs = append(m.Attrs, backend.LogAttr{Key: d.Key, Value: d.Value}) + } } - // we have the final say over context details (in case there - // is a conflict (if the user added a detail with a context's - // key for some reason)) - m.Attrs[contextPrefix+".node.id"] = msg.Context.NodeID - m.Attrs[contextPrefix+".service.id"] = msg.Context.ServiceID - m.Attrs[contextPrefix+".task.id"] = msg.Context.TaskID + m.Attrs = append(m.Attrs, + backend.LogAttr{Key: nodeKey, Value: msg.Context.NodeID}, + backend.LogAttr{Key: serviceKey, Value: msg.Context.ServiceID}, + backend.LogAttr{Key: taskKey, Value: msg.Context.TaskID}, + ) switch msg.Stream { case swarmapi.LogStreamStdout: diff --git a/components/engine/daemon/cluster/swarm.go b/components/engine/daemon/cluster/swarm.go index ef0596b6cc..2b34d89a6a 100644 --- a/components/engine/daemon/cluster/swarm.go +++ b/components/engine/daemon/cluster/swarm.go @@ -6,7 +6,6 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" apierrors "github.com/docker/docker/api/errors" apitypes "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" @@ -18,6 +17,7 @@ import ( "github.com/docker/swarmkit/manager/encryption" swarmnode "github.com/docker/swarmkit/node" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/daemon/cluster/tasks.go b/components/engine/daemon/cluster/tasks.go index f0d6621dc5..26706a2fa5 100644 --- a/components/engine/daemon/cluster/tasks.go +++ b/components/engine/daemon/cluster/tasks.go @@ -11,57 +11,50 @@ import ( // GetTasks returns a list of tasks matching the filter options. func (c *Cluster) GetTasks(options apitypes.TaskListOptions) ([]types.Task, error) { - c.mu.RLock() - defer c.mu.RUnlock() + var r *swarmapi.ListTasksResponse - state := c.currentNodeState() - if !state.IsActiveManager() { - return nil, c.errNoManager(state) - } - - filterTransform := func(filter filters.Args) error { - if filter.Include("service") { - serviceFilters := filter.Get("service") - for _, serviceFilter := range serviceFilters { - service, err := c.GetService(serviceFilter, false) - if err != nil { - return err + if err := c.lockedManagerAction(func(ctx context.Context, state nodeState) error { + filterTransform := func(filter filters.Args) error { + if filter.Include("service") { + serviceFilters := filter.Get("service") + for _, serviceFilter := range serviceFilters { + service, err := getService(ctx, state.controlClient, serviceFilter, false) + if err != nil { + return err + } + filter.Del("service", serviceFilter) + filter.Add("service", service.ID) } - filter.Del("service", serviceFilter) - filter.Add("service", service.ID) } - } - if filter.Include("node") { - nodeFilters := filter.Get("node") - for _, nodeFilter := range nodeFilters { - node, err := c.GetNode(nodeFilter) - if err != nil { - return err + if filter.Include("node") { + nodeFilters := filter.Get("node") + for _, nodeFilter := range nodeFilters { + node, err := getNode(ctx, state.controlClient, nodeFilter) + if err != nil { + return err + } + filter.Del("node", nodeFilter) + filter.Add("node", node.ID) } - filter.Del("node", nodeFilter) - filter.Add("node", node.ID) } + if !filter.Include("runtime") { + // default to only showing container tasks + filter.Add("runtime", "container") + filter.Add("runtime", "") + } + return nil } - if !filter.Include("runtime") { - // default to only showing container tasks - filter.Add("runtime", "container") - filter.Add("runtime", "") + + filters, err := newListTasksFilters(options.Filters, filterTransform) + if err != nil { + return err } - return nil - } - filters, err := newListTasksFilters(options.Filters, filterTransform) - if err != nil { - return nil, err - } - - ctx, cancel := c.getRequestContext() - defer cancel() - - r, err := state.controlClient.ListTasks( - ctx, - &swarmapi.ListTasksRequest{Filters: filters}) - if err != nil { + r, err = state.controlClient.ListTasks( + ctx, + &swarmapi.ListTasksRequest{Filters: filters}) + return err + }); err != nil { return nil, err } diff --git a/components/engine/daemon/config/config.go b/components/engine/daemon/config/config.go index beb147224f..74441d1a5c 100644 --- a/components/engine/daemon/config/config.go +++ b/components/engine/daemon/config/config.go @@ -12,13 +12,13 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" daemondiscovery "github.com/docker/docker/daemon/discovery" "github.com/docker/docker/opts" "github.com/docker/docker/pkg/authorization" "github.com/docker/docker/pkg/discovery" "github.com/docker/docker/registry" "github.com/imdario/mergo" + "github.com/sirupsen/logrus" "github.com/spf13/pflag" ) @@ -168,6 +168,11 @@ type CommonConfig struct { ValuesSet map[string]interface{} Experimental bool `json:"experimental"` // Experimental indicates whether experimental features should be exposed or not + + // Exposed node Generic Resources + NodeGenericResources string `json:"node-generic-resources,omitempty"` + // NetworkControlPlaneMTU allows to specify the control plane MTU, this will allow to optimize the network use in some components + NetworkControlPlaneMTU int `json:"network-control-plane-mtu,omitempty"` } // IsValueSet returns true if a configuration value @@ -497,6 +502,10 @@ func Validate(config *Config) error { } } + if _, err := opts.ParseGenericResources(config.NodeGenericResources); err != nil { + return err + } + if defaultRuntime := config.GetDefaultRuntimeName(); defaultRuntime != "" && defaultRuntime != StockRuntimeName { runtimes := config.GetAllRuntimes() if _, ok := runtimes[defaultRuntime]; !ok { @@ -504,6 +513,11 @@ func Validate(config *Config) error { } } + // validate platform-specific settings + if err := config.ValidatePlatformConfig(); err != nil { + return err + } + return nil } diff --git a/components/engine/daemon/config/config_solaris.go b/components/engine/daemon/config/config_solaris.go index f4f0802701..4741befac9 100644 --- a/components/engine/daemon/config/config_solaris.go +++ b/components/engine/daemon/config/config_solaris.go @@ -27,3 +27,8 @@ type BridgeConfig struct { func (conf *Config) IsSwarmCompatible() error { return nil } + +// ValidatePlatformConfig checks if any platform-specific configuration settings are invalid. +func (conf *Config) ValidatePlatformConfig() error { + return nil +} diff --git a/components/engine/daemon/config/config_unix.go b/components/engine/daemon/config/config_unix.go index 8f1da59198..3a9e1cf53c 100644 --- a/components/engine/daemon/config/config_unix.go +++ b/components/engine/daemon/config/config_unix.go @@ -5,10 +5,16 @@ package config import ( "fmt" + containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/opts" units "github.com/docker/go-units" ) +const ( + // DefaultIpcMode is default for container's IpcMode, if not set otherwise + DefaultIpcMode = "shareable" // TODO: change to private +) + // Config defines the configuration of a docker daemon. // It includes json tags to deserialize configuration from a file // using the same names that the flags in the command line uses. @@ -31,6 +37,7 @@ type Config struct { SeccompProfile string `json:"seccomp-profile,omitempty"` ShmSize opts.MemBytes `json:"default-shm-size,omitempty"` NoNewPrivileges bool `json:"no-new-privileges,omitempty"` + IpcMode string `json:"default-ipc-mode,omitempty"` } // BridgeConfig stores all the bridge driver specific @@ -61,3 +68,21 @@ func (conf *Config) IsSwarmCompatible() error { } return nil } + +func verifyDefaultIpcMode(mode string) error { + const hint = "Use \"shareable\" or \"private\"." + + dm := containertypes.IpcMode(mode) + if !dm.Valid() { + return fmt.Errorf("Default IPC mode setting (%v) is invalid. "+hint, dm) + } + if dm != "" && !dm.IsPrivate() && !dm.IsShareable() { + return fmt.Errorf("IPC mode \"%v\" is not supported as default value. "+hint, dm) + } + return nil +} + +// ValidatePlatformConfig checks if any platform-specific configuration settings are invalid. +func (conf *Config) ValidatePlatformConfig() error { + return verifyDefaultIpcMode(conf.IpcMode) +} diff --git a/components/engine/daemon/config/config_windows.go b/components/engine/daemon/config/config_windows.go index 849acc1ac6..6f994e391d 100644 --- a/components/engine/daemon/config/config_windows.go +++ b/components/engine/daemon/config/config_windows.go @@ -50,3 +50,8 @@ func (conf *Config) GetExecRoot() string { func (conf *Config) IsSwarmCompatible() error { return nil } + +// ValidatePlatformConfig checks if any platform-specific configuration settings are invalid. +func (conf *Config) ValidatePlatformConfig() error { + return nil +} diff --git a/components/engine/daemon/configs.go b/components/engine/daemon/configs.go index 31da56b2d3..9d9137c42f 100644 --- a/components/engine/daemon/configs.go +++ b/components/engine/daemon/configs.go @@ -1,8 +1,8 @@ package daemon import ( - "github.com/Sirupsen/logrus" swarmtypes "github.com/docker/docker/api/types/swarm" + "github.com/sirupsen/logrus" ) // SetContainerConfigReferences sets the container config references needed diff --git a/components/engine/daemon/container_operations.go b/components/engine/daemon/container_operations.go index 7c7dcc7ced..70eee6e003 100644 --- a/components/engine/daemon/container_operations.go +++ b/components/engine/daemon/container_operations.go @@ -10,7 +10,6 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" derr "github.com/docker/docker/api/errors" containertypes "github.com/docker/docker/api/types/container" networktypes "github.com/docker/docker/api/types/network" @@ -24,6 +23,7 @@ import ( "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/options" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/daemon/container_operations_unix.go b/components/engine/daemon/container_operations_unix.go index 8c1b44b601..f47e9e2835 100644 --- a/components/engine/daemon/container_operations_unix.go +++ b/components/engine/daemon/container_operations_unix.go @@ -11,7 +11,6 @@ import ( "strconv" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/container" "github.com/docker/docker/daemon/links" "github.com/docker/docker/pkg/idtools" @@ -21,6 +20,7 @@ import ( "github.com/docker/libnetwork" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -57,13 +57,27 @@ func (daemon *Daemon) setupLinkedContainers(container *container.Container) ([]s return env, nil } -func (daemon *Daemon) getIpcContainer(container *container.Container) (*container.Container, error) { - containerID := container.HostConfig.IpcMode.Container() - container, err := daemon.GetContainer(containerID) +func (daemon *Daemon) getIpcContainer(id string) (*container.Container, error) { + errMsg := "can't join IPC of container " + id + // Check the container exists + container, err := daemon.GetContainer(id) if err != nil { - return nil, errors.Wrapf(err, "cannot join IPC of a non running container: %s", container.ID) + return nil, errors.Wrap(err, errMsg) } - return container, daemon.checkContainer(container, containerIsRunning, containerIsNotRestarting) + // Check the container is running and not restarting + if err := daemon.checkContainer(container, containerIsRunning, containerIsNotRestarting); err != nil { + return nil, errors.Wrap(err, errMsg) + } + // Check the container ipc is shareable + if st, err := os.Stat(container.ShmPath); err != nil || !st.IsDir() { + if err == nil || os.IsNotExist(err) { + return nil, errors.New(errMsg + ": non-shareable IPC") + } + // stat() failed? + return nil, errors.Wrap(err, errMsg+": unexpected error from stat "+container.ShmPath) + } + + return container, nil } func (daemon *Daemon) getPidContainer(container *container.Container) (*container.Container, error) { @@ -90,25 +104,33 @@ func containerIsNotRestarting(c *container.Container) error { } func (daemon *Daemon) setupIpcDirs(c *container.Container) error { - var err error + ipcMode := c.HostConfig.IpcMode - c.ShmPath, err = c.ShmResourcePath() - if err != nil { - return err - } - - if c.HostConfig.IpcMode.IsContainer() { - ic, err := daemon.getIpcContainer(c) + switch { + case ipcMode.IsContainer(): + ic, err := daemon.getIpcContainer(ipcMode.Container()) if err != nil { return err } c.ShmPath = ic.ShmPath - } else if c.HostConfig.IpcMode.IsHost() { + + case ipcMode.IsHost(): if _, err := os.Stat("/dev/shm"); err != nil { return fmt.Errorf("/dev/shm is not mounted, but must be for --ipc=host") } c.ShmPath = "/dev/shm" - } else { + + case ipcMode.IsPrivate(), ipcMode.IsNone(): + // c.ShmPath will/should not be used, so make it empty. + // Container's /dev/shm mount comes from OCI spec. + c.ShmPath = "" + + case ipcMode.IsEmpty(): + // A container was created by an older version of the daemon. + // The default behavior used to be what is now called "shareable". + fallthrough + + case ipcMode.IsShareable(): rootIDs := daemon.idMappings.RootPair() if !c.HasMountFor("/dev/shm") { shmPath, err := c.ShmResourcePath() @@ -120,19 +142,18 @@ func (daemon *Daemon) setupIpcDirs(c *container.Container) error { return err } - shmSize := int64(daemon.configStore.ShmSize) - if c.HostConfig.ShmSize != 0 { - shmSize = c.HostConfig.ShmSize - } - shmproperty := "mode=1777,size=" + strconv.FormatInt(shmSize, 10) + shmproperty := "mode=1777,size=" + strconv.FormatInt(c.HostConfig.ShmSize, 10) if err := unix.Mount("shm", shmPath, "tmpfs", uintptr(unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV), label.FormatMountLabel(shmproperty, c.GetMountLabel())); err != nil { return fmt.Errorf("mounting shm tmpfs: %s", err) } if err := os.Chown(shmPath, rootIDs.UID, rootIDs.GID); err != nil { return err } + c.ShmPath = shmPath } + default: + return fmt.Errorf("invalid IPC mode: %v", ipcMode) } return nil diff --git a/components/engine/daemon/container_operations_windows.go b/components/engine/daemon/container_operations_windows.go index 2788f1a7cd..51762a2441 100644 --- a/components/engine/daemon/container_operations_windows.go +++ b/components/engine/daemon/container_operations_windows.go @@ -5,11 +5,11 @@ import ( "io/ioutil" "os" - "github.com/Sirupsen/logrus" "github.com/docker/docker/container" "github.com/docker/docker/pkg/system" "github.com/docker/libnetwork" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) func (daemon *Daemon) setupLinkedContainers(container *container.Container) ([]string, error) { diff --git a/components/engine/daemon/create.go b/components/engine/daemon/create.go index 78070fd29d..cf61336597 100644 --- a/components/engine/daemon/create.go +++ b/components/engine/daemon/create.go @@ -9,7 +9,6 @@ import ( "github.com/pkg/errors" - "github.com/Sirupsen/logrus" apierrors "github.com/docker/docker/api/errors" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" @@ -22,6 +21,7 @@ import ( "github.com/docker/docker/pkg/system" "github.com/docker/docker/runconfig" "github.com/opencontainers/selinux/go-selinux/label" + "github.com/sirupsen/logrus" ) // CreateManagedContainer creates a container that is managed by a Service diff --git a/components/engine/daemon/create_unix.go b/components/engine/daemon/create_unix.go index 2501a3374a..1940e9c64e 100644 --- a/components/engine/daemon/create_unix.go +++ b/components/engine/daemon/create_unix.go @@ -7,12 +7,12 @@ import ( "os" "path/filepath" - "github.com/Sirupsen/logrus" containertypes "github.com/docker/docker/api/types/container" mounttypes "github.com/docker/docker/api/types/mount" "github.com/docker/docker/container" "github.com/docker/docker/pkg/stringid" "github.com/opencontainers/selinux/go-selinux/label" + "github.com/sirupsen/logrus" ) // createContainerPlatformSpecificSettings performs platform specific container create functionality diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go index 93d871d6df..ebfdb68d1d 100644 --- a/components/engine/daemon/daemon.go +++ b/components/engine/daemon/daemon.go @@ -18,17 +18,19 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" containerd "github.com/containerd/containerd/api/grpc/types" "github.com/docker/docker/api" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/container" "github.com/docker/docker/daemon/config" "github.com/docker/docker/daemon/discovery" "github.com/docker/docker/daemon/events" "github.com/docker/docker/daemon/exec" "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/opts" + "github.com/sirupsen/logrus" // register graph drivers _ "github.com/docker/docker/daemon/graphdriver/register" "github.com/docker/docker/daemon/initlayer" @@ -109,6 +111,7 @@ type Daemon struct { defaultIsolation containertypes.Isolation // Default isolation mode on Windows clusterProvider cluster.Provider cluster Cluster + genericResources []swarm.GenericResource metricsPluginListener net.Listener machineMemory uint64 @@ -566,6 +569,9 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe } }() + if err := d.setGenericResources(config); err != nil { + return nil, err + } // set up SIGUSR1 handler on Unix-like systems, or a Win32 global event // on Windows to dump Go routine stacks stackDumpDir := config.Root @@ -620,6 +626,8 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe driverName := os.Getenv("DOCKER_DRIVER") if driverName == "" { driverName = config.GraphDriver + } else { + logrus.Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", driverName) } d.stores[runtime.GOOS] = daemonStore{graphDriver: driverName} // May still be empty. Layerstore init determines instead. } @@ -1033,6 +1041,17 @@ func (daemon *Daemon) setupInitLayer(initPath string) error { return initlayer.Setup(initPath, rootIDs) } +func (daemon *Daemon) setGenericResources(conf *config.Config) error { + genericResources, err := opts.ParseGenericResources(conf.NodeGenericResources) + if err != nil { + return err + } + + daemon.genericResources = genericResources + + return nil +} + func setDefaultMtu(conf *config.Config) { // do nothing if the config does not have the default 0 value. if conf.Mtu != 0 { @@ -1130,6 +1149,8 @@ func (daemon *Daemon) networkOptions(dconfig *config.Config, pg plugingetter.Plu options = append(options, nwconfig.OptionPluginGetter(pg)) } + options = append(options, nwconfig.OptionNetworkControlPlaneMTU(dconfig.NetworkControlPlaneMTU)) + return options, nil } diff --git a/components/engine/daemon/daemon_linux.go b/components/engine/daemon/daemon_linux.go index 000a048699..56ed4ce95a 100644 --- a/components/engine/daemon/daemon_linux.go +++ b/components/engine/daemon/daemon_linux.go @@ -8,9 +8,9 @@ import ( "regexp" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/mount" + "github.com/sirupsen/logrus" ) // On Linux, plugins use a static path for storing execution state, diff --git a/components/engine/daemon/daemon_solaris.go b/components/engine/daemon/daemon_solaris.go index f464ee34b8..905ed5843a 100644 --- a/components/engine/daemon/daemon_solaris.go +++ b/components/engine/daemon/daemon_solaris.go @@ -7,17 +7,15 @@ import ( "net" "strconv" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" + "github.com/docker/docker/daemon/config" "github.com/docker/docker/image" - "github.com/docker/docker/layer" "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/sysinfo" - refstore "github.com/docker/docker/reference" "github.com/docker/libnetwork" nwconfig "github.com/docker/libnetwork/config" "github.com/docker/libnetwork/drivers/solaris/bridge" @@ -27,6 +25,7 @@ import ( "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) //#include @@ -72,7 +71,7 @@ func (daemon *Daemon) parseSecurityOpt(container *container.Container, hostConfi } func parseSecurityOpt(container *container.Container, config *containertypes.HostConfig) error { - //Since config.SecurityOpt is specifically defined as a "List of string values to + //Since hostConfig.SecurityOpt is specifically defined as a "List of string values to //customize labels for MLs systems, such as SELinux" //until we figure out how to map to Trusted Extensions //this is being disabled for now on Solaris @@ -89,11 +88,11 @@ func parseSecurityOpt(container *container.Container, config *containertypes.Hos return err } -func setupRemappedRoot(config *Config) ([]idtools.IDMap, []idtools.IDMap, error) { - return nil, nil, nil +func setupRemappedRoot(config *config.Config) (*idtools.IDMappings, error) { + return nil, nil } -func setupDaemonRoot(config *Config, rootDir string, rootUID, rootGID int) error { +func setupDaemonRoot(config *config.Config, rootDir string, rootIDs idtools.IDPair) error { return nil } @@ -136,7 +135,7 @@ func (daemon *Daemon) adaptContainerSettings(hostConfig *containertypes.HostConf } // UsingSystemd returns true if cli option includes native.cgroupdriver=systemd -func UsingSystemd(config *Config) bool { +func UsingSystemd(config *config.Config) bool { return false } @@ -309,25 +308,26 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes. // reloadPlatform updates configuration with platform specific options // and updates the passed attributes -func (daemon *Daemon) reloadPlatform(config *Config, attributes map[string]string) { -} - -// verifyDaemonSettings performs validation of daemon config struct -func verifyDaemonSettings(config *Config) error { - - if config.DefaultRuntime == "" { - config.DefaultRuntime = stockRuntimeName - } - if config.Runtimes == nil { - config.Runtimes = make(map[string]types.Runtime) - } - stockRuntimeOpts := []string{} - config.Runtimes[stockRuntimeName] = types.Runtime{Path: DefaultRuntimeBinary, Args: stockRuntimeOpts} - - // checkSystem validates platform-specific requirements +func (daemon *Daemon) reloadPlatform(conf *config.Config, attributes map[string]string) error { return nil } +// verifyDaemonSettings performs validation of daemon config struct +func verifyDaemonSettings(conf *config.Config) error { + + if conf.DefaultRuntime == "" { + conf.DefaultRuntime = stockRuntimeName + } + if conf.Runtimes == nil { + conf.Runtimes = make(map[string]types.Runtime) + } + stockRuntimeOpts := []string{} + conf.Runtimes[stockRuntimeName] = types.Runtime{Path: DefaultRuntimeBinary, Args: stockRuntimeOpts} + + return nil +} + +// checkSystem validates platform-specific requirements func checkSystem() error { // check OS version for compatibility, ensure running in global zone var err error @@ -349,16 +349,16 @@ func checkSystem() error { // configureMaxThreads sets the Go runtime max threads threshold // which is 90% of the kernel setting from /proc/sys/kernel/threads-max -func configureMaxThreads(config *Config) error { +func configureMaxThreads(config *config.Config) error { return nil } -// configureKernelSecuritySupport configures and validate security support for the kernel +// configureKernelSecuritySupport configures and validates security support for the kernel func configureKernelSecuritySupport(config *config.Config, driverNames []string) error { return nil } -func (daemon *Daemon) initNetworkController(config *Config, activeSandboxes map[string]interface{}) (libnetwork.NetworkController, error) { +func (daemon *Daemon) initNetworkController(config *config.Config, activeSandboxes map[string]interface{}) (libnetwork.NetworkController, error) { netOptions, err := daemon.networkOptions(config, daemon.PluginStore, activeSandboxes) if err != nil { return nil, err @@ -384,7 +384,7 @@ func (daemon *Daemon) initNetworkController(config *Config, activeSandboxes map[ return controller, nil } -func initBridgeDriver(controller libnetwork.NetworkController, config *Config) error { +func initBridgeDriver(controller libnetwork.NetworkController, config *config.Config) error { if n, err := controller.NetworkByName("bridge"); err == nil { if err = n.Delete(); err != nil { return fmt.Errorf("could not delete the default bridge network: %v", err) @@ -497,12 +497,7 @@ func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container return daemon.Unmount(container) } -func restoreCustomImage(is image.Store, ls layer.Store, rs refstore.Store) error { - // Solaris has no custom images to register - return nil -} - -func driverOptions(config *Config) []nwconfig.Option { +func driverOptions(config *config.Config) []nwconfig.Option { return []nwconfig.Option{} } @@ -520,7 +515,7 @@ func rootFSToAPIType(rootfs *image.RootFS) types.RootFS { return types.RootFS{} } -func setupDaemonProcess(config *Config) error { +func setupDaemonProcess(config *config.Config) error { return nil } diff --git a/components/engine/daemon/daemon_unix.go b/components/engine/daemon/daemon_unix.go index ff9eadf75b..5b317293c2 100644 --- a/components/engine/daemon/daemon_unix.go +++ b/components/engine/daemon/daemon_unix.go @@ -14,10 +14,8 @@ import ( "runtime/debug" "strconv" "strings" - "syscall" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/blkiodev" pblkiodev "github.com/docker/docker/api/types/blkiodev" @@ -45,7 +43,9 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" ) const ( @@ -146,11 +146,11 @@ func getCPUResources(config containertypes.Resources) (*specs.LinuxCPU, error) { } func getBlkioWeightDevices(config containertypes.Resources) ([]specs.LinuxWeightDevice, error) { - var stat syscall.Stat_t + var stat unix.Stat_t var blkioWeightDevices []specs.LinuxWeightDevice for _, weightDevice := range config.BlkioWeightDevice { - if err := syscall.Stat(weightDevice.Path, &stat); err != nil { + if err := unix.Stat(weightDevice.Path, &stat); err != nil { return nil, err } weight := weightDevice.Weight @@ -219,10 +219,10 @@ func parseSecurityOpt(container *container.Container, config *containertypes.Hos func getBlkioThrottleDevices(devs []*blkiodev.ThrottleDevice) ([]specs.LinuxThrottleDevice, error) { var throttleDevices []specs.LinuxThrottleDevice - var stat syscall.Stat_t + var stat unix.Stat_t for _, d := range devs { - if err := syscall.Stat(d.Path, &stat); err != nil { + if err := unix.Stat(d.Path, &stat); err != nil { return nil, err } d := specs.LinuxThrottleDevice{Rate: d.Rate} @@ -276,6 +276,15 @@ func (daemon *Daemon) adaptContainerSettings(hostConfig *containertypes.HostConf hostConfig.ShmSize = int64(daemon.configStore.ShmSize) } } + // Set default IPC mode, if unset for container + if hostConfig.IpcMode.IsEmpty() { + m := config.DefaultIpcMode + if daemon.configStore != nil { + m = daemon.configStore.IpcMode + } + hostConfig.IpcMode = containertypes.IpcMode(m) + } + var err error opts, err := daemon.generateSecurityOpt(hostConfig) if err != nil { @@ -547,7 +556,7 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes. // check for various conflicting options with user namespaces if daemon.configStore.RemappedRoot != "" && hostConfig.UsernsMode.IsPrivate() { if hostConfig.Privileged { - return warnings, fmt.Errorf("Privileged mode is incompatible with user namespaces") + return warnings, fmt.Errorf("Privileged mode is incompatible with user namespaces. You must run the container in the host namespace when running privileged mode.") } if hostConfig.NetworkMode.IsHost() && !hostConfig.UsernsMode.IsHost() { return warnings, fmt.Errorf("Cannot share the host's network namespace when user namespaces are enabled") @@ -581,7 +590,11 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes. // reloadPlatform updates configuration with platform specific options // and updates the passed attributes -func (daemon *Daemon) reloadPlatform(conf *config.Config, attributes map[string]string) { +func (daemon *Daemon) reloadPlatform(conf *config.Config, attributes map[string]string) error { + if err := conf.ValidatePlatformConfig(); err != nil { + return err + } + if conf.IsValueSet("runtimes") { daemon.configStore.Runtimes = conf.Runtimes // Always set the default one @@ -596,6 +609,10 @@ func (daemon *Daemon) reloadPlatform(conf *config.Config, attributes map[string] daemon.configStore.ShmSize = conf.ShmSize } + if conf.IpcMode != "" { + daemon.configStore.IpcMode = conf.IpcMode + } + // Update attributes var runtimeList bytes.Buffer for name, rt := range daemon.configStore.Runtimes { @@ -608,6 +625,9 @@ func (daemon *Daemon) reloadPlatform(conf *config.Config, attributes map[string] attributes["runtimes"] = runtimeList.String() attributes["default-runtime"] = daemon.configStore.DefaultRuntime attributes["default-shm-size"] = fmt.Sprintf("%d", daemon.configStore.ShmSize) + attributes["default-ipc-mode"] = daemon.configStore.IpcMode + + return nil } // verifyDaemonSettings performs validation of daemon config struct diff --git a/components/engine/daemon/daemon_windows.go b/components/engine/daemon/daemon_windows.go index 798ba39971..4cb7560fc2 100644 --- a/components/engine/daemon/daemon_windows.go +++ b/components/engine/daemon/daemon_windows.go @@ -8,7 +8,6 @@ import ( "syscall" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" @@ -28,6 +27,7 @@ import ( "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/options" blkiodev "github.com/opencontainers/runc/libcontainer/configs" + "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) @@ -231,7 +231,8 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes. // reloadPlatform updates configuration with platform specific options // and updates the passed attributes -func (daemon *Daemon) reloadPlatform(config *config.Config, attributes map[string]string) { +func (daemon *Daemon) reloadPlatform(config *config.Config, attributes map[string]string) error { + return nil } // verifyDaemonSettings performs validation of daemon config struct diff --git a/components/engine/daemon/debugtrap_unix.go b/components/engine/daemon/debugtrap_unix.go index 39298dfd62..8ea696d961 100644 --- a/components/engine/daemon/debugtrap_unix.go +++ b/components/engine/daemon/debugtrap_unix.go @@ -6,8 +6,8 @@ import ( "os" "os/signal" - "github.com/Sirupsen/logrus" stackdump "github.com/docker/docker/pkg/signal" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/daemon/debugtrap_windows.go b/components/engine/daemon/debugtrap_windows.go index dfaf953244..6bd83e5446 100644 --- a/components/engine/daemon/debugtrap_windows.go +++ b/components/engine/daemon/debugtrap_windows.go @@ -6,9 +6,9 @@ import ( "unsafe" winio "github.com/Microsoft/go-winio" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/signal" "github.com/docker/docker/pkg/system" + "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) diff --git a/components/engine/daemon/delete.go b/components/engine/daemon/delete.go index c57a89654b..aea33c84c8 100644 --- a/components/engine/daemon/delete.go +++ b/components/engine/daemon/delete.go @@ -7,7 +7,6 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" apierrors "github.com/docker/docker/api/errors" "github.com/docker/docker/api/types" "github.com/docker/docker/container" @@ -15,6 +14,7 @@ import ( "github.com/docker/docker/pkg/system" volumestore "github.com/docker/docker/volume/store" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // ContainerRm removes the container id from the filesystem. An error @@ -119,7 +119,7 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, forceRemo if container.RWLayer != nil { metadata, err := daemon.stores[container.Platform].layerStore.ReleaseRWLayer(container.RWLayer) layer.LogReleaseMetadata(metadata) - if err != nil && err != layer.ErrMountDoesNotExist { + if err != nil && err != layer.ErrMountDoesNotExist && !os.IsNotExist(errors.Cause(err)) { return errors.Wrapf(err, "driver %q failed to remove root filesystem for %s", daemon.GraphDriverName(container.Platform), container.ID) } } diff --git a/components/engine/daemon/discovery/discovery.go b/components/engine/daemon/discovery/discovery.go index 509155cbbd..4b1b15faa0 100644 --- a/components/engine/daemon/discovery/discovery.go +++ b/components/engine/daemon/discovery/discovery.go @@ -6,8 +6,8 @@ import ( "strconv" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/discovery" + "github.com/sirupsen/logrus" // Register the libkv backends for discovery. _ "github.com/docker/docker/pkg/discovery/kv" diff --git a/components/engine/daemon/disk_usage.go b/components/engine/daemon/disk_usage.go index c64a243304..23bce4f70b 100644 --- a/components/engine/daemon/disk_usage.go +++ b/components/engine/daemon/disk_usage.go @@ -6,13 +6,13 @@ import ( "golang.org/x/net/context" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/directory" "github.com/docker/docker/volume" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" ) func (daemon *Daemon) getLayerRefs(platform string) map[layer.ChainID]int { diff --git a/components/engine/daemon/events.go b/components/engine/daemon/events.go index 7ae851802e..e302a4f43d 100644 --- a/components/engine/daemon/events.go +++ b/components/engine/daemon/events.go @@ -6,7 +6,6 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types/events" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/container" @@ -14,6 +13,7 @@ import ( "github.com/docker/libnetwork" swarmapi "github.com/docker/swarmkit/api" gogotypes "github.com/gogo/protobuf/types" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/daemon/exec.go b/components/engine/daemon/exec.go index 72d01c8c23..e77970968c 100644 --- a/components/engine/daemon/exec.go +++ b/components/engine/daemon/exec.go @@ -8,7 +8,6 @@ import ( "golang.org/x/net/context" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/errors" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/strslice" @@ -19,6 +18,7 @@ import ( "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/signal" "github.com/docker/docker/pkg/term" + "github.com/sirupsen/logrus" ) // Seconds to wait after sending TERM before trying KILL diff --git a/components/engine/daemon/exec/exec.go b/components/engine/daemon/exec/exec.go index 933136f965..471113cc4a 100644 --- a/components/engine/daemon/exec/exec.go +++ b/components/engine/daemon/exec/exec.go @@ -4,10 +4,10 @@ import ( "runtime" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/docker/container/stream" "github.com/docker/docker/libcontainerd" "github.com/docker/docker/pkg/stringid" + "github.com/sirupsen/logrus" ) // Config holds the configurations for execs. The Daemon keeps diff --git a/components/engine/daemon/export.go b/components/engine/daemon/export.go index 402e67583d..081e1639b7 100644 --- a/components/engine/daemon/export.go +++ b/components/engine/daemon/export.go @@ -13,15 +13,15 @@ import ( // ContainerExport writes the contents of the container to the given // writer. An error is returned if the container cannot be found. func (daemon *Daemon) ContainerExport(name string, out io.Writer) error { - if runtime.GOOS == "windows" { - return fmt.Errorf("the daemon on this platform does not support export of a container") - } - container, err := daemon.GetContainer(name) if err != nil { return err } + if runtime.GOOS == "windows" && container.Platform == "windows" { + return fmt.Errorf("the daemon on this platform does not support exporting Windows containers") + } + data, err := daemon.containerExport(container) if err != nil { return fmt.Errorf("Error exporting container %s: %v", name, err) diff --git a/components/engine/daemon/getsize_unix.go b/components/engine/daemon/getsize_unix.go index 434fa4388a..e47e646df3 100644 --- a/components/engine/daemon/getsize_unix.go +++ b/components/engine/daemon/getsize_unix.go @@ -5,7 +5,7 @@ package daemon import ( "runtime" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // getSize returns the real size & virtual size of the container. diff --git a/components/engine/daemon/graphdriver/aufs/aufs.go b/components/engine/daemon/graphdriver/aufs/aufs.go index c68c98cf80..81edb8ea94 100644 --- a/components/engine/daemon/graphdriver/aufs/aufs.go +++ b/components/engine/daemon/graphdriver/aufs/aufs.go @@ -35,7 +35,6 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" @@ -46,6 +45,8 @@ import ( "github.com/docker/docker/pkg/system" rsystem "github.com/opencontainers/runc/libcontainer/system" "github.com/opencontainers/selinux/go-selinux/label" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/vbatts/tar-split/tar/storage" "golang.org/x/sys/unix" ) @@ -282,30 +283,41 @@ func (a *Driver) Remove(id string) error { mountpoint = a.getMountpoint(id) } + logger := logrus.WithFields(logrus.Fields{ + "module": "graphdriver", + "driver": "aufs", + "layer": id, + }) + var retries int for { mounted, err := a.mounted(mountpoint) if err != nil { + if os.IsNotExist(err) { + break + } return err } if !mounted { break } - if err := a.unmount(mountpoint); err != nil { - if err != unix.EBUSY { - return fmt.Errorf("aufs: unmount error: %s: %v", mountpoint, err) - } - if retries >= 5 { - return fmt.Errorf("aufs: unmount error after retries: %s: %v", mountpoint, err) - } - // If unmount returns EBUSY, it could be a transient error. Sleep and retry. - retries++ - logrus.Warnf("unmount failed due to EBUSY: retry count: %d", retries) - time.Sleep(100 * time.Millisecond) - continue + err = a.unmount(mountpoint) + if err == nil { + break } - break + + if err != unix.EBUSY { + return errors.Wrapf(err, "aufs: unmount error: %s", mountpoint) + } + if retries >= 5 { + return errors.Wrapf(err, "aufs: unmount error after retries: %s", mountpoint) + } + // If unmount returns EBUSY, it could be a transient error. Sleep and retry. + retries++ + logger.Warnf("unmount failed due to EBUSY: retry count: %d", retries) + time.Sleep(100 * time.Millisecond) + continue } // Atomically remove each directory in turn by first moving it out of the @@ -314,21 +326,22 @@ func (a *Driver) Remove(id string) error { tmpMntPath := path.Join(a.mntPath(), fmt.Sprintf("%s-removing", id)) if err := os.Rename(mountpoint, tmpMntPath); err != nil && !os.IsNotExist(err) { if err == unix.EBUSY { - logrus.Warn("os.Rename err due to EBUSY") + logger.WithField("dir", mountpoint).WithError(err).Warn("os.Rename err due to EBUSY") } - return err + return errors.Wrapf(err, "error preparing atomic delete of aufs mountpoint for id: %s", id) + } + if err := system.EnsureRemoveAll(tmpMntPath); err != nil { + return errors.Wrapf(err, "error removing aufs layer %s", id) } - defer system.EnsureRemoveAll(tmpMntPath) tmpDiffpath := path.Join(a.diffPath(), fmt.Sprintf("%s-removing", id)) if err := os.Rename(a.getDiffPath(id), tmpDiffpath); err != nil && !os.IsNotExist(err) { - return err + return errors.Wrapf(err, "error preparing atomic delete of aufs diff dir for id: %s", id) } - defer system.EnsureRemoveAll(tmpDiffpath) // Remove the layers file for the id if err := os.Remove(path.Join(a.rootPath(), "layers", id)); err != nil && !os.IsNotExist(err) { - return err + return errors.Wrapf(err, "error removing layers dir for %s", id) } a.pathCacheLock.Lock() diff --git a/components/engine/daemon/graphdriver/aufs/mount.go b/components/engine/daemon/graphdriver/aufs/mount.go index 890213b80e..100e7537a9 100644 --- a/components/engine/daemon/graphdriver/aufs/mount.go +++ b/components/engine/daemon/graphdriver/aufs/mount.go @@ -5,7 +5,7 @@ package aufs import ( "os/exec" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/daemon/graphdriver/btrfs/btrfs.go b/components/engine/daemon/graphdriver/btrfs/btrfs.go index 25998422c5..dcdfc9a867 100644 --- a/components/engine/daemon/graphdriver/btrfs/btrfs.go +++ b/components/engine/daemon/graphdriver/btrfs/btrfs.go @@ -26,7 +26,6 @@ import ( "sync" "unsafe" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/mount" @@ -34,6 +33,7 @@ import ( "github.com/docker/docker/pkg/system" "github.com/docker/go-units" "github.com/opencontainers/selinux/go-selinux/label" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/daemon/graphdriver/devmapper/device_setup.go b/components/engine/daemon/graphdriver/devmapper/device_setup.go index ef6cffbf20..30463f23a2 100644 --- a/components/engine/daemon/graphdriver/devmapper/device_setup.go +++ b/components/engine/daemon/graphdriver/devmapper/device_setup.go @@ -12,8 +12,8 @@ import ( "reflect" "strings" - "github.com/Sirupsen/logrus" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) type directLVMConfig struct { @@ -172,29 +172,18 @@ func writeLVMConfig(root string, cfg directLVMConfig) error { } func setupDirectLVM(cfg directLVMConfig) error { - pvCreate, err := exec.LookPath("pvcreate") - if err != nil { - return errors.Wrap(err, "error looking up command `pvcreate` while setting up direct lvm") + lvmProfileDir := "/etc/lvm/profile" + binaries := []string{"pvcreate", "vgcreate", "lvcreate", "lvconvert", "lvchange", "thin_check"} + + for _, bin := range binaries { + if _, err := exec.LookPath(bin); err != nil { + return errors.Wrap(err, "error looking up command `"+bin+"` while setting up direct lvm") + } } - vgCreate, err := exec.LookPath("vgcreate") + err := os.MkdirAll(lvmProfileDir, 0755) if err != nil { - return errors.Wrap(err, "error looking up command `vgcreate` while setting up direct lvm") - } - - lvCreate, err := exec.LookPath("lvcreate") - if err != nil { - return errors.Wrap(err, "error looking up command `lvcreate` while setting up direct lvm") - } - - lvConvert, err := exec.LookPath("lvconvert") - if err != nil { - return errors.Wrap(err, "error looking up command `lvconvert` while setting up direct lvm") - } - - lvChange, err := exec.LookPath("lvchange") - if err != nil { - return errors.Wrap(err, "error looking up command `lvchange` while setting up direct lvm") + return errors.Wrap(err, "error creating lvm profile directory") } if cfg.AutoExtendPercent == 0 { @@ -212,36 +201,36 @@ func setupDirectLVM(cfg directLVMConfig) error { cfg.ThinpMetaPercent = 1 } - out, err := exec.Command(pvCreate, "-f", cfg.Device).CombinedOutput() + out, err := exec.Command("pvcreate", "-f", cfg.Device).CombinedOutput() if err != nil { return errors.Wrap(err, string(out)) } - out, err = exec.Command(vgCreate, "docker", cfg.Device).CombinedOutput() + out, err = exec.Command("vgcreate", "docker", cfg.Device).CombinedOutput() if err != nil { return errors.Wrap(err, string(out)) } - out, err = exec.Command(lvCreate, "--wipesignatures", "y", "-n", "thinpool", "docker", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpPercent)).CombinedOutput() + out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpool", "docker", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpPercent)).CombinedOutput() if err != nil { return errors.Wrap(err, string(out)) } - out, err = exec.Command(lvCreate, "--wipesignatures", "y", "-n", "thinpoolmeta", "docker", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpMetaPercent)).CombinedOutput() + out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpoolmeta", "docker", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpMetaPercent)).CombinedOutput() if err != nil { return errors.Wrap(err, string(out)) } - out, err = exec.Command(lvConvert, "-y", "--zero", "n", "-c", "512K", "--thinpool", "docker/thinpool", "--poolmetadata", "docker/thinpoolmeta").CombinedOutput() + out, err = exec.Command("lvconvert", "-y", "--zero", "n", "-c", "512K", "--thinpool", "docker/thinpool", "--poolmetadata", "docker/thinpoolmeta").CombinedOutput() if err != nil { return errors.Wrap(err, string(out)) } profile := fmt.Sprintf("activation{\nthin_pool_autoextend_threshold=%d\nthin_pool_autoextend_percent=%d\n}", cfg.AutoExtendThreshold, cfg.AutoExtendPercent) - err = ioutil.WriteFile("/etc/lvm/profile/docker-thinpool.profile", []byte(profile), 0600) + err = ioutil.WriteFile(lvmProfileDir+"/docker-thinpool.profile", []byte(profile), 0600) if err != nil { return errors.Wrap(err, "error writing docker thinp autoextend profile") } - out, err = exec.Command(lvChange, "--metadataprofile", "docker-thinpool", "docker/thinpool").CombinedOutput() + out, err = exec.Command("lvchange", "--metadataprofile", "docker-thinpool", "docker/thinpool").CombinedOutput() return errors.Wrap(err, string(out)) } diff --git a/components/engine/daemon/graphdriver/devmapper/deviceset.go b/components/engine/daemon/graphdriver/devmapper/deviceset.go index a7e06e1f95..95d9df7cb0 100644 --- a/components/engine/daemon/graphdriver/devmapper/deviceset.go +++ b/components/engine/daemon/graphdriver/devmapper/deviceset.go @@ -16,10 +16,8 @@ import ( "strconv" "strings" "sync" - "syscall" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/dockerversion" "github.com/docker/docker/pkg/devicemapper" @@ -30,6 +28,7 @@ import ( units "github.com/docker/go-units" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -1480,12 +1479,9 @@ func (devices *DeviceSet) closeTransaction() error { } func determineDriverCapabilities(version string) error { - /* - * Driver version 4.27.0 and greater support deferred activation - * feature. - */ + // Kernel driver version >= 4.27.0 support deferred removal - logrus.Debugf("devicemapper: driver version is %s", version) + logrus.Debugf("devicemapper: kernel dm driver version is %s", version) versionSplit := strings.Split(version, ".") major, err := strconv.Atoi(versionSplit[0]) @@ -1521,12 +1517,13 @@ func determineDriverCapabilities(version string) error { // Determine the major and minor number of loopback device func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) { - stat, err := file.Stat() + var stat unix.Stat_t + err := unix.Stat(file.Name(), &stat) if err != nil { return 0, 0, err } - dev := stat.Sys().(*syscall.Stat_t).Rdev + dev := stat.Rdev majorNum := major(dev) minorNum := minor(dev) @@ -1725,18 +1722,17 @@ func (devices *DeviceSet) initDevmapper(doInit bool) (retErr error) { } // Set the device prefix from the device id and inode of the docker root dir - st, err := os.Stat(devices.root) - if err != nil { + var st unix.Stat_t + if err := unix.Stat(devices.root, &st); err != nil { return fmt.Errorf("devmapper: Error looking up dir %s: %s", devices.root, err) } - sysSt := st.Sys().(*syscall.Stat_t) // "reg-" stands for "regular file". // In the future we might use "dev-" for "device file", etc. // docker-maj,min[-inode] stands for: // - Managed by docker // - The target of this device is at major and minor // - If is defined, use that file inside the device as a loopback image. Otherwise use the device itself. - devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino) + devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(st.Dev), minor(st.Dev), st.Ino) logrus.Debugf("devmapper: Generated prefix: %s", devices.devicePrefix) // Check for the existence of the thin-pool device @@ -1864,7 +1860,7 @@ func (devices *DeviceSet) initDevmapper(doInit bool) (retErr error) { if devices.thinPoolDevice == "" { if devices.metadataLoopFile != "" || devices.dataLoopFile != "" { - logrus.Warn("devmapper: Usage of loopback devices is strongly discouraged for production use. Please use `--storage-opt dm.thinpooldev` or use `man docker` to refer to dm.thinpooldev section.") + logrus.Warn("devmapper: Usage of loopback devices is strongly discouraged for production use. Please use `--storage-opt dm.thinpooldev` or use `man dockerd` to refer to dm.thinpooldev section.") } } diff --git a/components/engine/daemon/graphdriver/devmapper/driver.go b/components/engine/daemon/graphdriver/devmapper/driver.go index 243d88a8bb..02ee0124f1 100644 --- a/components/engine/daemon/graphdriver/devmapper/driver.go +++ b/components/engine/daemon/graphdriver/devmapper/driver.go @@ -9,7 +9,7 @@ import ( "path" "strconv" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/devicemapper" diff --git a/components/engine/daemon/graphdriver/devmapper/mount.go b/components/engine/daemon/graphdriver/devmapper/mount.go index cca1fe1b38..1dc3262d27 100644 --- a/components/engine/daemon/graphdriver/devmapper/mount.go +++ b/components/engine/daemon/graphdriver/devmapper/mount.go @@ -7,7 +7,8 @@ import ( "fmt" "os" "path/filepath" - "syscall" + + "golang.org/x/sys/unix" ) // FIXME: this is copy-pasted from the aufs driver. @@ -15,19 +16,17 @@ import ( // Mounted returns true if a mount point exists. func Mounted(mountpoint string) (bool, error) { - mntpoint, err := os.Stat(mountpoint) - if err != nil { + var mntpointSt unix.Stat_t + if err := unix.Stat(mountpoint, &mntpointSt); err != nil { if os.IsNotExist(err) { return false, nil } return false, err } - parent, err := os.Stat(filepath.Join(mountpoint, "..")) - if err != nil { + var parentSt unix.Stat_t + if err := unix.Stat(filepath.Join(mountpoint, ".."), &parentSt); err != nil { return false, err } - mntpointSt := mntpoint.Sys().(*syscall.Stat_t) - parentSt := parent.Sys().(*syscall.Stat_t) return mntpointSt.Dev != parentSt.Dev, nil } diff --git a/components/engine/daemon/graphdriver/driver.go b/components/engine/daemon/graphdriver/driver.go index 88f190d9e1..94c52094c3 100644 --- a/components/engine/daemon/graphdriver/driver.go +++ b/components/engine/daemon/graphdriver/driver.go @@ -8,7 +8,7 @@ import ( "path/filepath" "strings" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/vbatts/tar-split/tar/storage" "github.com/docker/docker/pkg/archive" diff --git a/components/engine/daemon/graphdriver/driver_linux.go b/components/engine/daemon/graphdriver/driver_linux.go index a92993d45a..50c8e6a6ef 100644 --- a/components/engine/daemon/graphdriver/driver_linux.go +++ b/components/engine/daemon/graphdriver/driver_linux.go @@ -53,10 +53,10 @@ const ( var ( // Slice of drivers that should be used in an order priority = []string{ - "aufs", "btrfs", "zfs", "overlay2", + "aufs", "overlay", "devicemapper", "vfs", diff --git a/components/engine/daemon/graphdriver/driver_solaris.go b/components/engine/daemon/graphdriver/driver_solaris.go index 06dc360cf5..121fd9230d 100644 --- a/components/engine/daemon/graphdriver/driver_solaris.go +++ b/components/engine/daemon/graphdriver/driver_solaris.go @@ -19,8 +19,8 @@ import ( "path/filepath" "unsafe" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/mount" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/daemon/graphdriver/fsdiff.go b/components/engine/daemon/graphdriver/fsdiff.go index 20826cd7d2..e8c7ff27a4 100644 --- a/components/engine/daemon/graphdriver/fsdiff.go +++ b/components/engine/daemon/graphdriver/fsdiff.go @@ -4,11 +4,11 @@ import ( "io" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/ioutils" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/daemon/graphdriver/lcow/lcow.go b/components/engine/daemon/graphdriver/lcow/lcow.go index 75c775bb59..86beb3d5f0 100644 --- a/components/engine/daemon/graphdriver/lcow/lcow.go +++ b/components/engine/daemon/graphdriver/lcow/lcow.go @@ -13,7 +13,43 @@ // operations. The downside of safe-mode is that operations are slower as // a new service utility VM has to be started and torn-down when needed. // -// To enable global mode, run with --storage-opt lcow.globalmode=true +// Options (needs official documentation, but lets get full functionality first...) @jhowardmsft +// +// The following options are read by the graphdriver itself: +// +// * lcow.globalmode - Enables global service VM Mode +// -- Possible values: true/false +// -- Default if omitted: false +// +// * lcow.sandboxsize - Specifies a custom sandbox size in GB for starting a container +// -- Possible values: >= default sandbox size (opengcs defined, currently 20) +// -- Default if ommitted: 20 +// +// The following options are read by opengcs: +// +// * lcow.kirdpath - Specifies a custom path to a kernel/initrd pair +// -- Possible values: Any local path that is not a mapped drive +// -- Default if ommitted: %ProgramFiles%\Linux Containers +// +// * lcow.kernel - Specifies a custom kernel file located in the `lcow.kirdpath` path +// -- Possible values: Any valid filename +// -- Default if ommitted: bootx64.efi +// +// * lcow.initrd - Specifies a custom initrd file located in the `lcow.kirdpath` path +// -- Possible values: Any valid filename +// -- Default if ommitted: initrd.img +// +// * lcow.bootparameters - Specifies additional boot parameters for booting in kernel+initrd mode +// -- Possible values: Any valid linux kernel boot options +// -- Default if ommitted: +// +// * lcow.vhdx - Specifies a custom vhdx file to boot (instead of a kernel+initrd) +// -- Possible values: Any valid filename +// -- Default if ommitted: uvm.vhdx under `lcow.kirdpath` +// +// * lcow.timeout - Specifies a timeout for utility VM operations in seconds +// -- Possible values: >=0 +// -- Default if ommitted: 300 // TODO: Grab logs from SVM at terminate or errors @@ -32,13 +68,13 @@ import ( "time" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" + "github.com/Microsoft/opengcs/client" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/system" - "github.com/jhowardmsft/opengcs/gogcs/client" + "github.com/sirupsen/logrus" ) // init registers this driver to the register. It gets initialised by the @@ -60,8 +96,7 @@ const ( // toolsScratchPath is a location in a service utility VM that the tools can use as a // scratch space to avoid running out of memory. - // TODO @jhowardmsft. I really dislike this path! But needs a platform change or passing parameters to the tools. - toolsScratchPath = "/mnt/gcs/LinuxServiceVM/scratch" + toolsScratchPath = "/tmp/scratch" // svmGlobalID is the ID used in the serviceVMs map for the global service VM when running in "global" mode. svmGlobalID = "_lcow_global_svm_" @@ -84,6 +119,34 @@ type cacheItem struct { isMounted bool // True when mounted in a service VM } +// setIsMounted is a helper function for a cacheItem which does exactly what it says +func (ci *cacheItem) setIsMounted() { + logrus.Debugf("locking cache item for set isMounted") + ci.Lock() + defer ci.Unlock() + ci.isMounted = true + logrus.Debugf("set isMounted on cache item") +} + +// incrementRefCount is a helper function for a cacheItem which does exactly what it says +func (ci *cacheItem) incrementRefCount() { + logrus.Debugf("locking cache item for increment") + ci.Lock() + defer ci.Unlock() + ci.refCount++ + logrus.Debugf("incremented refcount on cache item %+v", ci) +} + +// decrementRefCount is a helper function for a cacheItem which does exactly what it says +func (ci *cacheItem) decrementRefCount() int { + logrus.Debugf("locking cache item for decrement") + ci.Lock() + defer ci.Unlock() + ci.refCount-- + logrus.Debugf("decremented refcount on cache item %+v", ci) + return ci.refCount +} + // serviceVMItem is our internal structure representing an item in our // map of service VMs we are maintaining. type serviceVMItem struct { @@ -111,6 +174,14 @@ type Driver struct { cache map[string]*cacheItem // Map holding a cache of all the IDs we've mounted/unmounted. } +// layerDetails is the structure returned by a helper function `getLayerDetails` +// for getting information about a layer folder +type layerDetails struct { + filename string // \path\to\sandbox.vhdx or \path\to\layer.vhd + size int64 // size of the above file + isSandbox bool // true if sandbox.vhdx +} + // deletefiles is a helper function for initialisation where we delete any // left-over scratch files in case we were previously forcibly terminated. func deletefiles(path string, f os.FileInfo, err error) error { @@ -204,7 +275,7 @@ func (d *Driver) startServiceVMIfNotRunning(id string, mvdToAdd *hcsshim.MappedV logrus.Debugf("%s locking serviceVmItem %s", title, svm.config.Name) svm.Lock() - if err := svm.config.HotAddVhd(mvdToAdd.HostPath, mvdToAdd.ContainerPath); err != nil { + if err := svm.config.HotAddVhd(mvdToAdd.HostPath, mvdToAdd.ContainerPath, false, true); err != nil { logrus.Debugf("%s releasing serviceVmItem %s on hot-add failure %s", title, svm.config.Name, err) svm.Unlock() return nil, fmt.Errorf("%s hot add %s to %s failed: %s", title, mvdToAdd.HostPath, mvdToAdd.ContainerPath, err) @@ -274,7 +345,7 @@ func (d *Driver) startServiceVMIfNotRunning(id string, mvdToAdd *hcsshim.MappedV // Start it. logrus.Debugf("lcowdriver: startServiceVmIfNotRunning: (%s) starting %s", context, svm.config.Name) - if err := svm.config.Create(); err != nil { + if err := svm.config.StartUtilityVM(); err != nil { return nil, fmt.Errorf("failed to start service utility VM (%s): %s", context, err) } @@ -295,18 +366,22 @@ func (d *Driver) startServiceVMIfNotRunning(id string, mvdToAdd *hcsshim.MappedV logrus.Debugf("%s locking cachedScratchMutex", title) d.cachedScratchMutex.Lock() if _, err := os.Stat(d.cachedScratchFile); err != nil { - // TODO: Not a typo, but needs fixing when the platform sandbox stuff has been sorted out. logrus.Debugf("%s (%s): creating an SVM scratch - locking serviceVM", title, context) svm.Lock() - if err := svm.config.CreateSandbox(d.cachedScratchFile, client.DefaultSandboxSizeMB, d.cachedSandboxFile); err != nil { - logrus.Debugf("%s (%s): releasing serviceVM on error path", title, context) + if err := svm.config.CreateExt4Vhdx(scratchTargetFile, client.DefaultVhdxSizeGB, d.cachedScratchFile); err != nil { + logrus.Debugf("%s (%s): releasing serviceVM on error path from CreateExt4Vhdx: %s", title, context, err) svm.Unlock() logrus.Debugf("%s (%s): releasing cachedScratchMutex on error path", title, context) d.cachedScratchMutex.Unlock() - // TODO: NEED TO REMOVE FROM MAP HERE AND STOP IT + + // Do a force terminate and remove it from the map on failure, ignoring any errors + if err2 := d.terminateServiceVM(id, "error path from CreateExt4Vhdx", true); err2 != nil { + logrus.Warnf("failed to terminate service VM on error path from CreateExt4Vhdx: %s", err2) + } + return nil, fmt.Errorf("failed to create SVM scratch VHDX (%s): %s", context, err) } - logrus.Debugf("%s (%s): releasing serviceVM on error path", title, context) + logrus.Debugf("%s (%s): releasing serviceVM after %s created and cached to %s", title, context, scratchTargetFile, d.cachedScratchFile) svm.Unlock() } logrus.Debugf("%s (%s): releasing cachedScratchMutex", title, context) @@ -314,19 +389,17 @@ func (d *Driver) startServiceVMIfNotRunning(id string, mvdToAdd *hcsshim.MappedV // Hot-add the scratch-space if not already attached if !svm.scratchAttached { - // Make a copy of it to the layer directory - logrus.Debugf("lcowdriver: startServiceVmIfNotRunning: (%s) cloning cached scratch for hot-add", context) - if err := client.CopyFile(d.cachedScratchFile, scratchTargetFile, true); err != nil { - // TODO: NEED TO REMOVE FROM MAP HERE AND STOP IT - return nil, err - } - logrus.Debugf("lcowdriver: startServiceVmIfNotRunning: (%s) hot-adding scratch %s - locking serviceVM", context, scratchTargetFile) svm.Lock() - if err := svm.config.HotAddVhd(scratchTargetFile, toolsScratchPath); err != nil { - logrus.Debugf("%s (%s): releasing serviceVM on error path", title, context) + if err := svm.config.HotAddVhd(scratchTargetFile, toolsScratchPath, false, true); err != nil { + logrus.Debugf("%s (%s): releasing serviceVM on error path of HotAddVhd: %s", title, context, err) svm.Unlock() - // TODOL NEED TO REMOVE FROM MAP HERE AND STOP IT + + // Do a force terminate and remove it from the map on failure, ignoring any errors + if err2 := d.terminateServiceVM(id, "error path from HotAddVhd", true); err2 != nil { + logrus.Warnf("failed to terminate service VM on error path from HotAddVhd: %s", err2) + } + return nil, fmt.Errorf("failed to hot-add %s failed: %s", scratchTargetFile, err) } logrus.Debugf("%s (%s): releasing serviceVM", title, context) @@ -441,26 +514,43 @@ func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts return err } + // Look for an explicit sandbox size option. + sandboxSize := uint64(client.DefaultVhdxSizeGB) + for k, v := range opts.StorageOpt { + switch strings.ToLower(k) { + case "lcow.sandboxsize": + var err error + sandboxSize, err = strconv.ParseUint(v, 10, 32) + if err != nil { + return fmt.Errorf("%s failed to parse value '%s' for 'lcow.sandboxsize'", title, v) + } + if sandboxSize < client.DefaultVhdxSizeGB { + return fmt.Errorf("%s 'lcow.sandboxsize' option cannot be less than %d", title, client.DefaultVhdxSizeGB) + } + break + } + } + // Massive perf optimisation here. If we know that the RW layer is the default size, // and that the cached sandbox already exists, and we are running in safe mode, we // can just do a simple copy into the layers sandbox file without needing to start a - // unique service VM. For a global service VM, it doesn't really matter. + // unique service VM. For a global service VM, it doesn't really matter. Of course, + // this is only the case where the sandbox is the default size. // - // TODO: @jhowardmsft Where are we going to get the required size from? - // We need to look at the CreateOpts for that, I think.... - // Make sure we have the sandbox mutex taken while we are examining it. - logrus.Debugf("%s: locking cachedSandboxMutex", title) - d.cachedSandboxMutex.Lock() - _, err := os.Stat(d.cachedSandboxFile) - logrus.Debugf("%s: releasing cachedSandboxMutex", title) - d.cachedSandboxMutex.Unlock() - if err == nil { - logrus.Debugf("%s: using cached sandbox to populate", title) - if err := client.CopyFile(d.cachedSandboxFile, filepath.Join(d.dir(id), sandboxFilename), true); err != nil { - return err + if sandboxSize == client.DefaultVhdxSizeGB { + logrus.Debugf("%s: locking cachedSandboxMutex", title) + d.cachedSandboxMutex.Lock() + _, err := os.Stat(d.cachedSandboxFile) + logrus.Debugf("%s: releasing cachedSandboxMutex", title) + d.cachedSandboxMutex.Unlock() + if err == nil { + logrus.Debugf("%s: using cached sandbox to populate", title) + if err := client.CopyFile(d.cachedSandboxFile, filepath.Join(d.dir(id), sandboxFilename), true); err != nil { + return err + } + return nil } - return nil } logrus.Debugf("%s: creating SVM to create sandbox", title) @@ -470,13 +560,16 @@ func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts } defer d.terminateServiceVM(id, "createreadwrite", false) - // So the cached sandbox needs creating. Ensure we are the only thread creating it. - logrus.Debugf("%s: locking cachedSandboxMutex for creation", title) - d.cachedSandboxMutex.Lock() - defer func() { - logrus.Debugf("%s: releasing cachedSandboxMutex for creation", title) - d.cachedSandboxMutex.Unlock() - }() + // So the sandbox needs creating. If default size ensure we are the only thread populating the cache. + // Non-default size we don't store, just create them one-off so no need to lock the cachedSandboxMutex. + if sandboxSize == client.DefaultVhdxSizeGB { + logrus.Debugf("%s: locking cachedSandboxMutex for creation", title) + d.cachedSandboxMutex.Lock() + defer func() { + logrus.Debugf("%s: releasing cachedSandboxMutex for creation", title) + d.cachedSandboxMutex.Unlock() + }() + } // Synchronise the operation in the service VM. logrus.Debugf("%s: locking svm for sandbox creation", title) @@ -485,7 +578,15 @@ func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts logrus.Debugf("%s: releasing svm for sandbox creation", title) svm.Unlock() }() - if err := svm.config.CreateSandbox(filepath.Join(d.dir(id), sandboxFilename), client.DefaultSandboxSizeMB, d.cachedSandboxFile); err != nil { + + // Make sure we don't write to our local cached copy if this is for a non-default size request. + targetCacheFile := d.cachedSandboxFile + if sandboxSize != client.DefaultVhdxSizeGB { + targetCacheFile = "" + } + + // Actually do the creation. + if err := svm.config.CreateExt4Vhdx(filepath.Join(d.dir(id), sandboxFilename), uint32(sandboxSize), targetCacheFile); err != nil { return err } @@ -563,42 +664,37 @@ func (d *Driver) Get(id, mountLabel string) (string, error) { logrus.Debugf(title) // Work out what we are working on - vhdFilename, vhdSize, isSandbox, err := getLayerDetails(d.dir(id)) + ld, err := getLayerDetails(d.dir(id)) if err != nil { logrus.Debugf("%s failed to get layer details from %s: %s", title, d.dir(id), err) return "", fmt.Errorf("%s failed to open layer or sandbox VHD to open in %s: %s", title, d.dir(id), err) } - logrus.Debugf("%s %s, size %d, isSandbox %t", title, vhdFilename, vhdSize, isSandbox) + logrus.Debugf("%s %s, size %d, isSandbox %t", title, ld.filename, ld.size, ld.isSandbox) // Add item to cache, or update existing item, but ensure we have the // lock while updating items. logrus.Debugf("%s: locking cacheMutex", title) d.cacheMutex.Lock() - var cacheEntry *cacheItem - if entry, ok := d.cache[id]; !ok { + var ci *cacheItem + if item, ok := d.cache[id]; !ok { // The item is not currently in the cache. - cacheEntry = &cacheItem{ + ci = &cacheItem{ refCount: 1, - isSandbox: isSandbox, - hostPath: vhdFilename, + isSandbox: ld.isSandbox, + hostPath: ld.filename, uvmPath: fmt.Sprintf("/mnt/%s", id), isMounted: false, // we defer this as an optimisation } - d.cache[id] = cacheEntry - logrus.Debugf("%s: added cache entry %+v", title, cacheEntry) + d.cache[id] = ci + logrus.Debugf("%s: added cache item %+v", title, ci) } else { // Increment the reference counter in the cache. - logrus.Debugf("%s: locking cache item for increment", title) - entry.Lock() - entry.refCount++ - logrus.Debugf("%s: releasing cache item for increment", title) - entry.Unlock() - logrus.Debugf("%s: incremented refcount on cache entry %+v", title, cacheEntry) + item.incrementRefCount() } logrus.Debugf("%s: releasing cacheMutex", title) d.cacheMutex.Unlock() - logrus.Debugf("%s %s success. %s: %+v: size %d", title, id, d.dir(id), cacheEntry, vhdSize) + logrus.Debugf("%s %s success. %s: %+v: size %d", title, id, d.dir(id), ci, ld.size) return d.dir(id), nil } @@ -609,67 +705,53 @@ func (d *Driver) Put(id string) error { logrus.Debugf("%s: locking cacheMutex", title) d.cacheMutex.Lock() - entry, ok := d.cache[id] + item, ok := d.cache[id] if !ok { logrus.Debugf("%s: releasing cacheMutex on error path", title) d.cacheMutex.Unlock() return fmt.Errorf("%s possible ref-count error, or invalid id was passed to the graphdriver. Cannot handle id %s as it's not in the cache", title, id) } - // Are we just decrementing the reference count? - logrus.Debugf("%s: locking cache item for possible decrement", title) - entry.Lock() - if entry.refCount > 1 { - entry.refCount-- - logrus.Debugf("%s: releasing cache item for decrement and early get-out as refCount is now %d", title, entry.refCount) - entry.Unlock() - logrus.Debugf("%s: refCount decremented to %d. Releasing cacheMutex", title, entry.refCount) + // Decrement the ref-count, and nothing more to do if still in use. + if item.decrementRefCount() > 0 { + logrus.Debugf("%s: releasing cacheMutex. Cache item is still in use", title) d.cacheMutex.Unlock() return nil } - logrus.Debugf("%s: releasing cache item", title) - entry.Unlock() - logrus.Debugf("%s: releasing cacheMutex. Ref count has dropped to zero", title) + + // Remove from the cache map. + delete(d.cache, id) + logrus.Debugf("%s: releasing cacheMutex. Ref count on cache item has dropped to zero, removed from cache", title) d.cacheMutex.Unlock() - // To reach this point, the reference count has dropped to zero. If we have - // done a mount and we are in global mode, then remove it. We don't - // need to remove in safe mode as the service VM is going to be torn down - // anyway. - + // If we have done a mount and we are in global mode, then remove it. We don't + // need to remove in safe mode as the service VM is going to be torn down anyway. if d.globalMode { logrus.Debugf("%s: locking cache item at zero ref-count", title) - entry.Lock() + item.Lock() defer func() { logrus.Debugf("%s: releasing cache item at zero ref-count", title) - entry.Unlock() + item.Unlock() }() - if entry.isMounted { + if item.isMounted { svm, err := d.getServiceVM(id, false) if err != nil { return err } - logrus.Debugf("%s: Hot-Removing %s. Locking svm", title, entry.hostPath) + logrus.Debugf("%s: Hot-Removing %s. Locking svm", title, item.hostPath) svm.Lock() - if err := svm.config.HotRemoveVhd(entry.hostPath); err != nil { + if err := svm.config.HotRemoveVhd(item.hostPath); err != nil { logrus.Debugf("%s: releasing svm on error path", title) svm.Unlock() - return fmt.Errorf("%s failed to hot-remove %s from global service utility VM: %s", title, entry.hostPath, err) + return fmt.Errorf("%s failed to hot-remove %s from global service utility VM: %s", title, item.hostPath, err) } logrus.Debugf("%s: releasing svm", title) svm.Unlock() } } - // Remove from the cache map. - logrus.Debugf("%s: Locking cacheMutex to delete item from cache", title) - d.cacheMutex.Lock() - delete(d.cache, id) - logrus.Debugf("%s: releasing cacheMutex after item deleted from cache", title) - d.cacheMutex.Unlock() - - logrus.Debugf("%s %s: refCount 0. %s (%s) completed successfully", title, id, entry.hostPath, entry.uvmPath) + logrus.Debugf("%s %s: refCount 0. %s (%s) completed successfully", title, id, item.hostPath, item.uvmPath) return nil } @@ -681,7 +763,7 @@ func (d *Driver) Cleanup() error { d.cacheMutex.Lock() for k, v := range d.cache { - logrus.Debugf("%s cache entry: %s: %+v", title, k, v) + logrus.Debugf("%s cache item: %s: %+v", title, k, v) if v.refCount > 0 { logrus.Warnf("%s leaked %s: %+v", title, k, v) } @@ -713,7 +795,7 @@ func (d *Driver) Cleanup() error { // Cleanup any service VMs we have running, along with their scratch spaces. // We don't take the lock for this as it's taken in terminateServiceVm. for k, v := range d.serviceVms { - logrus.Debugf("%s svm entry: %s: %+v", title, k, v) + logrus.Debugf("%s svm: %s: %+v", title, k, v) d.terminateServiceVM(k, "cleanup", true) } @@ -737,28 +819,28 @@ func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) { d.cacheMutex.Unlock() return nil, fmt.Errorf("%s fail as %s is not in the cache", title, id) } - cacheEntry := d.cache[id] + ci := d.cache[id] logrus.Debugf("%s: releasing cacheMutex", title) d.cacheMutex.Unlock() // Stat to get size - logrus.Debugf("%s: locking cacheEntry", title) - cacheEntry.Lock() - fileInfo, err := os.Stat(cacheEntry.hostPath) + logrus.Debugf("%s: locking cacheItem", title) + ci.Lock() + fileInfo, err := os.Stat(ci.hostPath) if err != nil { - logrus.Debugf("%s: releasing cacheEntry on error path", title) - cacheEntry.Unlock() - return nil, fmt.Errorf("%s failed to stat %s: %s", title, cacheEntry.hostPath, err) + logrus.Debugf("%s: releasing cacheItem on error path", title) + ci.Unlock() + return nil, fmt.Errorf("%s failed to stat %s: %s", title, ci.hostPath, err) } - logrus.Debugf("%s: releasing cacheEntry", title) - cacheEntry.Unlock() + logrus.Debugf("%s: releasing cacheItem", title) + ci.Unlock() // Start the SVM with a mapped virtual disk. Note that if the SVM is // already runing and we are in global mode, this will be // hot-added. mvd := &hcsshim.MappedVirtualDisk{ - HostPath: cacheEntry.hostPath, - ContainerPath: cacheEntry.uvmPath, + HostPath: ci.hostPath, + ContainerPath: ci.uvmPath, CreateInUtilityVM: true, ReadOnly: true, } @@ -769,8 +851,8 @@ func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) { return nil, err } - // Set `isMounted` for the cache entry. Note that we re-scan the cache - // at this point as it's possible the cacheEntry changed during the long- + // Set `isMounted` for the cache item. Note that we re-scan the cache + // at this point as it's possible the cacheItem changed during the long- // running operation above when we weren't holding the cacheMutex lock. logrus.Debugf("%s: locking cacheMutex for updating isMounted", title) d.cacheMutex.Lock() @@ -780,18 +862,14 @@ func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) { d.terminateServiceVM(id, fmt.Sprintf("diff %s", id), false) return nil, fmt.Errorf("%s fail as %s is not in the cache when updating isMounted", title, id) } - cacheEntry = d.cache[id] - logrus.Debugf("%s: locking cacheEntry for updating isMounted", title) - cacheEntry.Lock() - cacheEntry.isMounted = true - logrus.Debugf("%s: releasing cacheEntry for updating isMounted", title) - cacheEntry.Unlock() + ci = d.cache[id] + ci.setIsMounted() logrus.Debugf("%s: releasing cacheMutex for updating isMounted", title) d.cacheMutex.Unlock() // Obtain the tar stream for it - logrus.Debugf("%s %s, size %d, isSandbox %t", title, cacheEntry.hostPath, fileInfo.Size(), cacheEntry.isSandbox) - tarReadCloser, err := svm.config.VhdToTar(cacheEntry.hostPath, cacheEntry.uvmPath, cacheEntry.isSandbox, fileInfo.Size()) + logrus.Debugf("%s %s, size %d, isSandbox %t", title, ci.hostPath, fileInfo.Size(), ci.isSandbox) + tarReadCloser, err := svm.config.VhdToTar(ci.hostPath, ci.uvmPath, ci.isSandbox, fileInfo.Size()) if err != nil { d.terminateServiceVM(id, fmt.Sprintf("diff %s", id), false) return nil, fmt.Errorf("%s failed to export layer to tar stream for id: %s, parent: %s : %s", title, id, parent, err) @@ -909,21 +987,22 @@ func (d *Driver) setLayerChain(id string, chain []string) error { // getLayerDetails is a utility for getting a file name, size and indication of // sandbox for a VHD(x) in a folder. A read-only layer will be layer.vhd. A // read-write layer will be sandbox.vhdx. -func getLayerDetails(folder string) (string, int64, bool, error) { +func getLayerDetails(folder string) (*layerDetails, error) { var fileInfo os.FileInfo - isSandbox := false - filename := filepath.Join(folder, layerFilename) - var err error - - if fileInfo, err = os.Stat(filename); err != nil { - filename = filepath.Join(folder, sandboxFilename) - if fileInfo, err = os.Stat(filename); err != nil { - if os.IsNotExist(err) { - return "", 0, isSandbox, fmt.Errorf("could not find layer or sandbox in %s", folder) - } - return "", 0, isSandbox, fmt.Errorf("error locating layer or sandbox in %s: %s", folder, err) - } - isSandbox = true + ld := &layerDetails{ + isSandbox: false, + filename: filepath.Join(folder, layerFilename), } - return filename, fileInfo.Size(), isSandbox, nil + + fileInfo, err := os.Stat(ld.filename) + if err != nil { + ld.filename = filepath.Join(folder, sandboxFilename) + if fileInfo, err = os.Stat(ld.filename); err != nil { + return nil, fmt.Errorf("failed to locate layer or sandbox in %s", folder) + } + ld.isSandbox = true + } + ld.size = fileInfo.Size() + + return ld, nil } diff --git a/components/engine/daemon/graphdriver/overlay/overlay.go b/components/engine/daemon/graphdriver/overlay/overlay.go index 9db2e9405f..0d14b045f9 100644 --- a/components/engine/daemon/graphdriver/overlay/overlay.go +++ b/components/engine/daemon/graphdriver/overlay/overlay.go @@ -12,7 +12,6 @@ import ( "path" "strconv" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/daemon/graphdriver/overlayutils" "github.com/docker/docker/pkg/archive" @@ -22,6 +21,7 @@ import ( "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/system" "github.com/opencontainers/selinux/go-selinux/label" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/daemon/graphdriver/overlay2/check.go b/components/engine/daemon/graphdriver/overlay2/check.go index 35e088aa32..851c4099c7 100644 --- a/components/engine/daemon/graphdriver/overlay2/check.go +++ b/components/engine/daemon/graphdriver/overlay2/check.go @@ -9,9 +9,9 @@ import ( "path" "path/filepath" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/system" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/daemon/graphdriver/overlay2/overlay.go b/components/engine/daemon/graphdriver/overlay2/overlay.go index 4ef2a8caaf..6b0d4be869 100644 --- a/components/engine/daemon/graphdriver/overlay2/overlay.go +++ b/components/engine/daemon/graphdriver/overlay2/overlay.go @@ -16,7 +16,7 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/daemon/graphdriver/overlayutils" diff --git a/components/engine/daemon/graphdriver/overlay2/randomid.go b/components/engine/daemon/graphdriver/overlay2/randomid.go index 04212c0694..30d6f3b1cf 100644 --- a/components/engine/daemon/graphdriver/overlay2/randomid.go +++ b/components/engine/daemon/graphdriver/overlay2/randomid.go @@ -11,7 +11,7 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/daemon/graphdriver/quota/projectquota.go b/components/engine/daemon/graphdriver/quota/projectquota.go index 194c30e9af..0e70515434 100644 --- a/components/engine/daemon/graphdriver/quota/projectquota.go +++ b/components/engine/daemon/graphdriver/quota/projectquota.go @@ -52,13 +52,11 @@ import "C" import ( "fmt" "io/ioutil" - "os" "path" "path/filepath" - "syscall" "unsafe" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -323,15 +321,14 @@ func getDirFd(dir *C.DIR) uintptr { // and create a block device node under the home directory // to be used by quotactl commands func makeBackingFsDev(home string) (string, error) { - fileinfo, err := os.Stat(home) - if err != nil { + var stat unix.Stat_t + if err := unix.Stat(home, &stat); err != nil { return "", err } backingFsBlockDev := path.Join(home, "backingFsBlockDev") // Re-create just in case someone copied the home directory over to a new device unix.Unlink(backingFsBlockDev) - stat := fileinfo.Sys().(*syscall.Stat_t) if err := unix.Mknod(backingFsBlockDev, unix.S_IFBLK|0600, int(stat.Dev)); err != nil { return "", fmt.Errorf("Failed to mknod %s: %v", backingFsBlockDev, err) } diff --git a/components/engine/daemon/graphdriver/windows/windows.go b/components/engine/daemon/graphdriver/windows/windows.go index 49c8d34a57..44114051bb 100644 --- a/components/engine/daemon/graphdriver/windows/windows.go +++ b/components/engine/daemon/graphdriver/windows/windows.go @@ -24,7 +24,6 @@ import ( "github.com/Microsoft/go-winio/archive/tar" "github.com/Microsoft/go-winio/backuptar" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/idtools" @@ -33,6 +32,7 @@ import ( "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/pkg/system" units "github.com/docker/go-units" + "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) diff --git a/components/engine/daemon/graphdriver/zfs/zfs.go b/components/engine/daemon/graphdriver/zfs/zfs.go index a772cc9ee6..729099a6cd 100644 --- a/components/engine/daemon/graphdriver/zfs/zfs.go +++ b/components/engine/daemon/graphdriver/zfs/zfs.go @@ -12,13 +12,13 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/parsers" zfs "github.com/mistifyio/go-zfs" "github.com/opencontainers/selinux/go-selinux/label" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/daemon/graphdriver/zfs/zfs_freebsd.go b/components/engine/daemon/graphdriver/zfs/zfs_freebsd.go index e02012afe2..bbc3216fc2 100644 --- a/components/engine/daemon/graphdriver/zfs/zfs_freebsd.go +++ b/components/engine/daemon/graphdriver/zfs/zfs_freebsd.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/daemon/graphdriver/zfs/zfs_linux.go b/components/engine/daemon/graphdriver/zfs/zfs_linux.go index 53aa4c8c6f..71d6d3cb4d 100644 --- a/components/engine/daemon/graphdriver/zfs/zfs_linux.go +++ b/components/engine/daemon/graphdriver/zfs/zfs_linux.go @@ -3,8 +3,8 @@ package zfs import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/daemon/graphdriver/zfs/zfs_solaris.go b/components/engine/daemon/graphdriver/zfs/zfs_solaris.go index bb4a85bd64..d63642252d 100644 --- a/components/engine/daemon/graphdriver/zfs/zfs_solaris.go +++ b/components/engine/daemon/graphdriver/zfs/zfs_solaris.go @@ -20,8 +20,8 @@ import ( "strings" "unsafe" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" + "github.com/sirupsen/logrus" ) func checkRootdirFs(rootdir string) error { diff --git a/components/engine/daemon/health.go b/components/engine/daemon/health.go index 61b531484e..c8e84a2d1f 100644 --- a/components/engine/daemon/health.go +++ b/components/engine/daemon/health.go @@ -10,12 +10,12 @@ import ( "golang.org/x/net/context" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/strslice" "github.com/docker/docker/container" "github.com/docker/docker/daemon/exec" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/daemon/info.go b/components/engine/daemon/info.go index 2cb3e84798..1c4aa561bd 100644 --- a/components/engine/daemon/info.go +++ b/components/engine/daemon/info.go @@ -7,7 +7,6 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api" "github.com/docker/docker/api/types" "github.com/docker/docker/cli/debug" @@ -22,6 +21,7 @@ import ( "github.com/docker/docker/registry" "github.com/docker/docker/volume/drivers" "github.com/docker/go-connections/sockets" + "github.com/sirupsen/logrus" ) // SystemInfo returns information about the host server the daemon is running on. @@ -123,6 +123,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { RegistryConfig: daemon.RegistryService.ServiceConfig(), NCPU: sysinfo.NumCPU(), MemTotal: meminfo.MemTotal, + GenericResources: daemon.genericResources, DockerRootDir: daemon.configStore.Root, Labels: daemon.configStore.Labels, ExperimentalBuild: daemon.configStore.Experimental, diff --git a/components/engine/daemon/info_unix.go b/components/engine/daemon/info_unix.go index e816f8dff1..f43af6274f 100644 --- a/components/engine/daemon/info_unix.go +++ b/components/engine/daemon/info_unix.go @@ -7,11 +7,11 @@ import ( "os/exec" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/dockerversion" "github.com/docker/docker/pkg/sysinfo" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // FillPlatformInfo fills the platform related info. diff --git a/components/engine/daemon/kill.go b/components/engine/daemon/kill.go index b118160f84..586c9661fb 100644 --- a/components/engine/daemon/kill.go +++ b/components/engine/daemon/kill.go @@ -8,9 +8,9 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" containerpkg "github.com/docker/docker/container" "github.com/docker/docker/pkg/signal" + "github.com/sirupsen/logrus" ) type errNoSuchProcess struct { diff --git a/components/engine/daemon/list.go b/components/engine/daemon/list.go index 6889c55889..5906ed3a69 100644 --- a/components/engine/daemon/list.go +++ b/components/engine/daemon/list.go @@ -7,13 +7,13 @@ import ( "strconv" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/container" "github.com/docker/docker/image" "github.com/docker/docker/volume" "github.com/docker/go-connections/nat" + "github.com/sirupsen/logrus" ) var acceptedVolumeFilterTags = map[string]bool{ diff --git a/components/engine/pkg/listeners/group_unix.go b/components/engine/daemon/listeners/group_unix.go similarity index 100% rename from components/engine/pkg/listeners/group_unix.go rename to components/engine/daemon/listeners/group_unix.go diff --git a/components/engine/pkg/listeners/listeners_solaris.go b/components/engine/daemon/listeners/listeners_solaris.go similarity index 96% rename from components/engine/pkg/listeners/listeners_solaris.go rename to components/engine/daemon/listeners/listeners_solaris.go index c9003bcf68..ee1bd0fccf 100644 --- a/components/engine/pkg/listeners/listeners_solaris.go +++ b/components/engine/daemon/listeners/listeners_solaris.go @@ -6,8 +6,8 @@ import ( "net" "os" - "github.com/Sirupsen/logrus" "github.com/docker/go-connections/sockets" + "github.com/sirupsen/logrus" ) // Init creates new listeners for the server. diff --git a/components/engine/pkg/listeners/listeners_unix.go b/components/engine/daemon/listeners/listeners_unix.go similarity index 98% rename from components/engine/pkg/listeners/listeners_unix.go rename to components/engine/daemon/listeners/listeners_unix.go index 25c98fba17..dc54606894 100644 --- a/components/engine/pkg/listeners/listeners_unix.go +++ b/components/engine/daemon/listeners/listeners_unix.go @@ -9,9 +9,9 @@ import ( "os" "strconv" - "github.com/Sirupsen/logrus" "github.com/coreos/go-systemd/activation" "github.com/docker/go-connections/sockets" + "github.com/sirupsen/logrus" ) // Init creates new listeners for the server. diff --git a/components/engine/pkg/listeners/listeners_windows.go b/components/engine/daemon/listeners/listeners_windows.go similarity index 100% rename from components/engine/pkg/listeners/listeners_windows.go rename to components/engine/daemon/listeners/listeners_windows.go diff --git a/components/engine/daemon/logger/adapter.go b/components/engine/daemon/logger/adapter.go index a187b30fdd..98852e89c1 100644 --- a/components/engine/daemon/logger/adapter.go +++ b/components/engine/daemon/logger/adapter.go @@ -7,10 +7,10 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types/plugins/logdriver" "github.com/docker/docker/pkg/plugingetter" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // pluginAdapter takes a plugin and implements the Logger interface for logger diff --git a/components/engine/daemon/logger/awslogs/cloudwatchlogs.go b/components/engine/daemon/logger/awslogs/cloudwatchlogs.go index 4d98468a79..3a7f2f631d 100644 --- a/components/engine/daemon/logger/awslogs/cloudwatchlogs.go +++ b/components/engine/daemon/logger/awslogs/cloudwatchlogs.go @@ -2,7 +2,6 @@ package awslogs import ( - "bytes" "fmt" "os" "regexp" @@ -13,7 +12,6 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/ec2metadata" @@ -23,8 +21,8 @@ import ( "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/loggerutils" "github.com/docker/docker/dockerversion" - "github.com/docker/docker/pkg/templates" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) const ( @@ -194,19 +192,6 @@ var strftimeToRegex = map[string]string{ /*milliseconds */ `%L`: `\.\d{3}`, } -func parseLogGroup(info logger.Info, groupTemplate string) (string, error) { - tmpl, err := templates.NewParse("log-group", groupTemplate) - if err != nil { - return "", err - } - buf := new(bytes.Buffer) - if err := tmpl.Execute(buf, &info); err != nil { - return "", err - } - - return buf.String(), nil -} - // newRegionFinder is a variable such that the implementation // can be swapped out for unit tests. var newRegionFinder = func() regionFinder { @@ -384,15 +369,18 @@ func (l *logStream) collectBatch() { eventBufferNegative := eventBufferAge < 0 if eventBufferExpired || eventBufferNegative { events = l.processEvent(events, eventBuffer, eventBufferTimestamp) + eventBuffer = eventBuffer[:0] } } l.publishBatch(events) events = events[:0] case msg, more := <-l.messages: if !more { - // Flush event buffer + // Flush event buffer and release resources events = l.processEvent(events, eventBuffer, eventBufferTimestamp) + eventBuffer = eventBuffer[:0] l.publishBatch(events) + events = events[:0] return } if eventBufferTimestamp == 0 { @@ -400,17 +388,13 @@ func (l *logStream) collectBatch() { } unprocessedLine := msg.Line if l.multilinePattern != nil { - if l.multilinePattern.Match(unprocessedLine) { - // This is a new log event so flush the current eventBuffer to events + if l.multilinePattern.Match(unprocessedLine) || len(eventBuffer)+len(unprocessedLine) > maximumBytesPerEvent { + // This is a new log event or we will exceed max bytes per event + // so flush the current eventBuffer to events and reset timestamp events = l.processEvent(events, eventBuffer, eventBufferTimestamp) eventBufferTimestamp = msg.Timestamp.UnixNano() / int64(time.Millisecond) eventBuffer = eventBuffer[:0] } - // If we will exceed max bytes per event flush the current event buffer before appending - if len(eventBuffer)+len(unprocessedLine) > maximumBytesPerEvent { - events = l.processEvent(events, eventBuffer, eventBufferTimestamp) - eventBuffer = eventBuffer[:0] - } // Append new line processedLine := append(unprocessedLine, "\n"...) eventBuffer = append(eventBuffer, processedLine...) diff --git a/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go b/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go index e3862ffebe..8688026a0d 100644 --- a/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go +++ b/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go @@ -641,7 +641,7 @@ func TestCollectBatchMultilinePatternMaxEventAge(t *testing.T) { }) // Fire ticker batchPublishFrequency seconds later - ticks <- time.Now().Add(batchPublishFrequency * time.Second) + ticks <- time.Now().Add(batchPublishFrequency + time.Second) // Verify single multiline event is flushed after maximum event buffer age (batchPublishFrequency) argument := <-mockClient.putLogEventsArgument @@ -649,6 +649,20 @@ func TestCollectBatchMultilinePatternMaxEventAge(t *testing.T) { assert.Equal(t, 1, len(argument.LogEvents), "Expected single multiline event") assert.Equal(t, logline+"\n"+logline+"\n", *argument.LogEvents[0].Message, "Received incorrect multiline message") + // Log an event 1 second later + stream.Log(&logger.Message{ + Line: []byte(logline), + Timestamp: time.Now().Add(time.Second), + }) + + // Fire ticker another batchPublishFrequency seconds later + ticks <- time.Now().Add(2*batchPublishFrequency + time.Second) + + // Verify the event buffer is truly flushed - we should only receive a single event + argument = <-mockClient.putLogEventsArgument + assert.NotNil(t, argument, "Expected non-nil PutLogEventsInput") + assert.Equal(t, 1, len(argument.LogEvents), "Expected single multiline event") + assert.Equal(t, logline+"\n", *argument.LogEvents[0].Message, "Received incorrect multiline message") stream.Close() } diff --git a/components/engine/daemon/logger/copier.go b/components/engine/daemon/logger/copier.go index 65d8fb148e..c773fc6a29 100644 --- a/components/engine/daemon/logger/copier.go +++ b/components/engine/daemon/logger/copier.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/daemon/logger/etwlogs/etwlogs_windows.go b/components/engine/daemon/logger/etwlogs/etwlogs_windows.go index 8608f15ca1..aa530e7d9a 100644 --- a/components/engine/daemon/logger/etwlogs/etwlogs_windows.go +++ b/components/engine/daemon/logger/etwlogs/etwlogs_windows.go @@ -18,8 +18,8 @@ import ( "sync" "unsafe" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/logger" + "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) diff --git a/components/engine/daemon/logger/fluentd/fluentd.go b/components/engine/daemon/logger/fluentd/fluentd.go index c8977ec0da..6a0653ef26 100644 --- a/components/engine/daemon/logger/fluentd/fluentd.go +++ b/components/engine/daemon/logger/fluentd/fluentd.go @@ -11,13 +11,13 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/loggerutils" "github.com/docker/docker/pkg/urlutil" "github.com/docker/go-units" "github.com/fluent/fluent-logger-golang/fluent" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) type fluentd struct { diff --git a/components/engine/daemon/logger/gcplogs/gcplogging.go b/components/engine/daemon/logger/gcplogs/gcplogging.go index a33566ae1f..5fdd137dd2 100644 --- a/components/engine/daemon/logger/gcplogs/gcplogging.go +++ b/components/engine/daemon/logger/gcplogs/gcplogging.go @@ -10,7 +10,7 @@ import ( "cloud.google.com/go/compute/metadata" "cloud.google.com/go/logging" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "golang.org/x/net/context" mrpb "google.golang.org/genproto/googleapis/api/monitoredres" ) diff --git a/components/engine/daemon/logger/gcplogs/gcplogging_linux.go b/components/engine/daemon/logger/gcplogs/gcplogging_linux.go index 9af8b3c17e..8917bdd4c7 100644 --- a/components/engine/daemon/logger/gcplogs/gcplogging_linux.go +++ b/components/engine/daemon/logger/gcplogs/gcplogging_linux.go @@ -5,9 +5,9 @@ package gcplogs import ( "os" - "github.com/Sirupsen/logrus" "github.com/docker/docker/dockerversion" "github.com/docker/docker/pkg/homedir" + "github.com/sirupsen/logrus" ) // ensureHomeIfIAmStatic ensure $HOME to be set if dockerversion.IAmStatic is "true". diff --git a/components/engine/daemon/logger/gelf/gelf.go b/components/engine/daemon/logger/gelf/gelf.go index 4b0130dfbe..de209ce9bf 100644 --- a/components/engine/daemon/logger/gelf/gelf.go +++ b/components/engine/daemon/logger/gelf/gelf.go @@ -14,10 +14,10 @@ import ( "time" "github.com/Graylog2/go-gelf/gelf" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/loggerutils" "github.com/docker/docker/pkg/urlutil" + "github.com/sirupsen/logrus" ) const name = "gelf" diff --git a/components/engine/daemon/logger/journald/journald.go b/components/engine/daemon/logger/journald/journald.go index 86d7378b5b..46aac8328b 100644 --- a/components/engine/daemon/logger/journald/journald.go +++ b/components/engine/daemon/logger/journald/journald.go @@ -9,10 +9,10 @@ import ( "sync" "unicode" - "github.com/Sirupsen/logrus" "github.com/coreos/go-systemd/journal" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/loggerutils" + "github.com/sirupsen/logrus" ) const name = "journald" diff --git a/components/engine/daemon/logger/journald/read.go b/components/engine/daemon/logger/journald/read.go index 9ecc3b521d..4d9b999a50 100644 --- a/components/engine/daemon/logger/journald/read.go +++ b/components/engine/daemon/logger/journald/read.go @@ -155,9 +155,10 @@ import ( "time" "unsafe" - "github.com/Sirupsen/logrus" "github.com/coreos/go-systemd/journal" + "github.com/docker/docker/api/types/backend" "github.com/docker/docker/daemon/logger" + "github.com/sirupsen/logrus" ) func (s *journald) Close() error { @@ -213,14 +214,11 @@ drain: source = "stdout" } // Retrieve the values of any variables we're adding to the journal. - attrs := make(map[string]string) + var attrs []backend.LogAttr C.sd_journal_restart_data(j) for C.get_attribute_field(j, &data, &length) > C.int(0) { kv := strings.SplitN(C.GoStringN(data, C.int(length)), "=", 2) - attrs[kv[0]] = kv[1] - } - if len(attrs) == 0 { - attrs = nil + attrs = append(attrs, backend.LogAttr{Key: kv[0], Value: kv[1]}) } // Send the log message. logWatcher.Msg <- &logger.Message{ diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go index e8df0ecbd0..c5f12d0021 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go @@ -11,12 +11,12 @@ import ( "strconv" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/loggerutils" "github.com/docker/docker/pkg/jsonlog" "github.com/docker/go-units" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // Name is the name of the file that the jsonlogger logs to. diff --git a/components/engine/daemon/logger/jsonfilelog/read.go b/components/engine/daemon/logger/jsonfilelog/read.go index 3fe5967241..32425b5128 100644 --- a/components/engine/daemon/logger/jsonfilelog/read.go +++ b/components/engine/daemon/logger/jsonfilelog/read.go @@ -11,13 +11,14 @@ import ( "github.com/fsnotify/fsnotify" "golang.org/x/net/context" - "github.com/Sirupsen/logrus" + "github.com/docker/docker/api/types/backend" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/jsonfilelog/multireader" "github.com/docker/docker/pkg/filenotify" "github.com/docker/docker/pkg/jsonlog" "github.com/docker/docker/pkg/tailfile" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) const maxJSONDecodeRetry = 20000 @@ -27,11 +28,18 @@ func decodeLogLine(dec *json.Decoder, l *jsonlog.JSONLog) (*logger.Message, erro if err := dec.Decode(l); err != nil { return nil, err } + var attrs []backend.LogAttr + if len(l.Attrs) != 0 { + attrs = make([]backend.LogAttr, 0, len(l.Attrs)) + for k, v := range l.Attrs { + attrs = append(attrs, backend.LogAttr{Key: k, Value: v}) + } + } msg := &logger.Message{ Source: l.Stream, Timestamp: l.Created, Line: []byte(l.Log), - Attrs: l.Attrs, + Attrs: attrs, } return msg, nil } diff --git a/components/engine/daemon/logger/logentries/logentries.go b/components/engine/daemon/logger/logentries/logentries.go index a353d9d49a..7f6ff471db 100644 --- a/components/engine/daemon/logger/logentries/logentries.go +++ b/components/engine/daemon/logger/logentries/logentries.go @@ -5,9 +5,9 @@ package logentries import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/bsphere/le_go" "github.com/docker/docker/daemon/logger" + "github.com/sirupsen/logrus" ) type logentries struct { diff --git a/components/engine/daemon/logger/logger.go b/components/engine/daemon/logger/logger.go index daa5403de2..258a5dc5f0 100644 --- a/components/engine/daemon/logger/logger.go +++ b/components/engine/daemon/logger/logger.go @@ -68,11 +68,6 @@ func (m *Message) AsLogMessage() *backend.LogMessage { return (*backend.LogMessage)(m) } -// LogAttributes is used to hold the extra attributes available in the log message -// Primarily used for converting the map type to string and sorting. -// Imported here so it can be used internally with less refactoring -type LogAttributes backend.LogAttributes - // Logger is the interface for docker logging drivers. type Logger interface { Log(*Message) error diff --git a/components/engine/daemon/logger/logger_test.go b/components/engine/daemon/logger/logger_test.go index 4d6e079308..15f9b8145b 100644 --- a/components/engine/daemon/logger/logger_test.go +++ b/components/engine/daemon/logger/logger_test.go @@ -1,5 +1,9 @@ package logger +import ( + "github.com/docker/docker/api/types/backend" +) + func (m *Message) copy() *Message { msg := &Message{ Source: m.Source, @@ -8,10 +12,8 @@ func (m *Message) copy() *Message { } if m.Attrs != nil { - msg.Attrs = make(map[string]string, len(m.Attrs)) - for k, v := range m.Attrs { - msg.Attrs[k] = v - } + msg.Attrs = make([]backend.LogAttr, len(m.Attrs)) + copy(msg.Attrs, m.Attrs) } msg.Line = append(make([]byte, 0, len(m.Line)), m.Line...) diff --git a/components/engine/daemon/logger/loggerutils/log_tag.go b/components/engine/daemon/logger/loggerutils/log_tag.go index f801047c41..7c170909a7 100644 --- a/components/engine/daemon/logger/loggerutils/log_tag.go +++ b/components/engine/daemon/logger/loggerutils/log_tag.go @@ -4,7 +4,7 @@ import ( "bytes" "github.com/docker/docker/daemon/logger" - "github.com/docker/docker/pkg/templates" + "github.com/docker/docker/daemon/logger/templates" ) // DefaultTemplate defines the defaults template logger should use. diff --git a/components/engine/daemon/logger/ring.go b/components/engine/daemon/logger/ring.go index 5c55955474..dea8f6b6a4 100644 --- a/components/engine/daemon/logger/ring.go +++ b/components/engine/daemon/logger/ring.go @@ -5,7 +5,7 @@ import ( "sync" "sync/atomic" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/daemon/logger/splunk/splunk.go b/components/engine/daemon/logger/splunk/splunk.go index 233a2db96f..7319a3e3d5 100644 --- a/components/engine/daemon/logger/splunk/splunk.go +++ b/components/engine/daemon/logger/splunk/splunk.go @@ -18,10 +18,10 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/loggerutils" "github.com/docker/docker/pkg/urlutil" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/daemon/logger/syslog/syslog.go b/components/engine/daemon/logger/syslog/syslog.go index 42855e1174..b4cacd7af8 100644 --- a/components/engine/daemon/logger/syslog/syslog.go +++ b/components/engine/daemon/logger/syslog/syslog.go @@ -14,11 +14,11 @@ import ( syslog "github.com/RackSec/srslog" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/loggerutils" "github.com/docker/docker/pkg/urlutil" "github.com/docker/go-connections/tlsconfig" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/pkg/templates/templates.go b/components/engine/daemon/logger/templates/templates.go similarity index 65% rename from components/engine/pkg/templates/templates.go rename to components/engine/daemon/logger/templates/templates.go index d2d7e0c3d7..f632e6e4b5 100644 --- a/components/engine/pkg/templates/templates.go +++ b/components/engine/daemon/logger/templates/templates.go @@ -27,34 +27,6 @@ var basicFunctions = template.FuncMap{ "truncate": truncateWithLength, } -// HeaderFunctions are used to created headers of a table. -// This is a replacement of basicFunctions for header generation -// because we want the header to remain intact. -// Some functions like `split` are irrelevant so not added. -var HeaderFunctions = template.FuncMap{ - "json": func(v string) string { - return v - }, - "title": func(v string) string { - return v - }, - "lower": func(v string) string { - return v - }, - "upper": func(v string) string { - return v - }, - "truncate": func(v string, l int) string { - return v - }, -} - -// Parse creates a new anonymous template with the basic functions -// and parses the given format. -func Parse(format string) (*template.Template, error) { - return NewParse("", format) -} - // NewParse creates a new tagged template with the basic functions // and parses the given format. func NewParse(tag, format string) (*template.Template, error) { diff --git a/components/engine/daemon/logger/templates/templates_test.go b/components/engine/daemon/logger/templates/templates_test.go new file mode 100644 index 0000000000..6205e1dc71 --- /dev/null +++ b/components/engine/daemon/logger/templates/templates_test.go @@ -0,0 +1,18 @@ +package templates + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewParse(t *testing.T) { + tm, err := NewParse("foo", "this is a {{ . }}") + assert.NoError(t, err) + + var b bytes.Buffer + assert.NoError(t, tm.Execute(&b, "string")) + want := "this is a string" + assert.Equal(t, want, b.String()) +} diff --git a/components/engine/daemon/logs.go b/components/engine/daemon/logs.go index 96e1b8a491..2bd57ee533 100644 --- a/components/engine/daemon/logs.go +++ b/components/engine/daemon/logs.go @@ -7,13 +7,13 @@ import ( "golang.org/x/net/context" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" containertypes "github.com/docker/docker/api/types/container" timetypes "github.com/docker/docker/api/types/time" "github.com/docker/docker/container" "github.com/docker/docker/daemon/logger" + "github.com/sirupsen/logrus" ) // ContainerLogs copies the container's log channel to the channel provided in diff --git a/components/engine/daemon/metrics.go b/components/engine/daemon/metrics.go index bf9e49d044..0d9f37e8db 100644 --- a/components/engine/daemon/metrics.go +++ b/components/engine/daemon/metrics.go @@ -4,12 +4,12 @@ import ( "path/filepath" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/plugingetter" "github.com/docker/go-metrics" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" ) const metricsPluginType = "MetricsCollector" diff --git a/components/engine/daemon/metrics_unix.go b/components/engine/daemon/metrics_unix.go index 84166d1a80..6045939a58 100644 --- a/components/engine/daemon/metrics_unix.go +++ b/components/engine/daemon/metrics_unix.go @@ -8,12 +8,12 @@ import ( "os" "path/filepath" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/plugingetter" "github.com/docker/docker/pkg/plugins" metrics "github.com/docker/go-metrics" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/daemon/monitor.go b/components/engine/daemon/monitor.go index 5156d9a8e1..3fd7bffb87 100644 --- a/components/engine/daemon/monitor.go +++ b/components/engine/daemon/monitor.go @@ -7,11 +7,11 @@ import ( "strconv" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/container" "github.com/docker/docker/libcontainerd" "github.com/docker/docker/restartmanager" + "github.com/sirupsen/logrus" ) func (daemon *Daemon) setStateCounter(c *container.Container) { diff --git a/components/engine/daemon/names.go b/components/engine/daemon/names.go index 7cdabeba9f..845c4e9953 100644 --- a/components/engine/daemon/names.go +++ b/components/engine/daemon/names.go @@ -4,11 +4,11 @@ import ( "fmt" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api" "github.com/docker/docker/container" "github.com/docker/docker/pkg/namesgenerator" "github.com/docker/docker/pkg/stringid" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/daemon/network.go b/components/engine/daemon/network.go index 366c2a59e4..22f94bcb32 100644 --- a/components/engine/daemon/network.go +++ b/components/engine/daemon/network.go @@ -8,7 +8,6 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" apierrors "github.com/docker/docker/api/errors" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/network" @@ -21,6 +20,7 @@ import ( "github.com/docker/libnetwork/ipamapi" networktypes "github.com/docker/libnetwork/types" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/daemon/oci_linux.go b/components/engine/daemon/oci_linux.go index 6d74301a05..a80de1be31 100644 --- a/components/engine/daemon/oci_linux.go +++ b/components/engine/daemon/oci_linux.go @@ -11,7 +11,6 @@ import ( "strconv" "strings" - "github.com/Sirupsen/logrus" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" "github.com/docker/docker/daemon/caps" @@ -27,6 +26,7 @@ import ( "github.com/opencontainers/runc/libcontainer/devices" "github.com/opencontainers/runc/libcontainer/user" specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" ) var ( @@ -301,10 +301,13 @@ func setNamespaces(daemon *Daemon, s *specs.Spec, c *container.Container) error } setNamespace(s, ns) } + // ipc - if c.HostConfig.IpcMode.IsContainer() { + ipcMode := c.HostConfig.IpcMode + switch { + case ipcMode.IsContainer(): ns := specs.LinuxNamespace{Type: "ipc"} - ic, err := daemon.getIpcContainer(c) + ic, err := daemon.getIpcContainer(ipcMode.Container()) if err != nil { return err } @@ -316,12 +319,19 @@ func setNamespaces(daemon *Daemon, s *specs.Spec, c *container.Container) error nsUser.Path = fmt.Sprintf("/proc/%d/ns/user", ic.State.GetPID()) setNamespace(s, nsUser) } - } else if c.HostConfig.IpcMode.IsHost() { + case ipcMode.IsHost(): oci.RemoveNamespace(s, specs.LinuxNamespaceType("ipc")) - } else { + case ipcMode.IsEmpty(): + // A container was created by an older version of the daemon. + // The default behavior used to be what is now called "shareable". + fallthrough + case ipcMode.IsPrivate(), ipcMode.IsShareable(), ipcMode.IsNone(): ns := specs.LinuxNamespace{Type: "ipc"} setNamespace(s, ns) + default: + return fmt.Errorf("Invalid IPC mode: %v", ipcMode) } + // pid if c.HostConfig.PidMode.IsContainer() { ns := specs.LinuxNamespace{Type: "pid"} @@ -486,10 +496,16 @@ func setMounts(daemon *Daemon, s *specs.Spec, c *container.Container, mounts []c userMounts[m.Destination] = struct{}{} } - // Filter out mounts that are overridden by user supplied mounts + // Filter out mounts from spec + noIpc := c.HostConfig.IpcMode.IsNone() var defaultMounts []specs.Mount _, mountDev := userMounts["/dev"] for _, m := range s.Mounts { + // filter out /dev/shm mount if case IpcMode is none + if noIpc && m.Destination == "/dev/shm" { + continue + } + // filter out mount overridden by a user supplied mount if _, ok := userMounts[m.Destination]; !ok { if mountDev && strings.HasPrefix(m.Destination, "/dev/") { continue @@ -589,6 +605,14 @@ func setMounts(daemon *Daemon, s *specs.Spec, c *container.Container, mounts []c s.Linux.MaskedPaths = nil } + // Set size for /dev/shm mount that comes from spec (IpcMode: private only) + for i, m := range s.Mounts { + if m.Destination == "/dev/shm" { + sizeOpt := "size=" + strconv.FormatInt(c.HostConfig.ShmSize, 10) + s.Mounts[i].Options = append(s.Mounts[i].Options, sizeOpt) + } + } + // TODO: until a kernel/mount solution exists for handling remount in a user namespace, // we must clear the readonly flag for the cgroups mount (@mrunalp concurs) if uidMap := daemon.idMappings.UIDs(); uidMap != nil || c.HostConfig.Privileged { @@ -745,7 +769,9 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { return nil, err } - ms = append(ms, c.IpcMounts()...) + if !c.HostConfig.IpcMode.IsPrivate() && !c.HostConfig.IpcMode.IsEmpty() { + ms = append(ms, c.IpcMounts()...) + } tmpfsMounts, err := c.TmpfsMounts() if err != nil { diff --git a/components/engine/daemon/prune.go b/components/engine/daemon/prune.go index 9f8a545e7c..824e0cd833 100644 --- a/components/engine/daemon/prune.go +++ b/components/engine/daemon/prune.go @@ -7,7 +7,6 @@ import ( "sync/atomic" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" @@ -20,6 +19,7 @@ import ( "github.com/docker/docker/volume" "github.com/docker/libnetwork" digest "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/daemon/reload.go b/components/engine/daemon/reload.go index 0200bcf06e..a6674ec951 100644 --- a/components/engine/daemon/reload.go +++ b/components/engine/daemon/reload.go @@ -4,10 +4,10 @@ import ( "encoding/json" "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/config" "github.com/docker/docker/daemon/discovery" "github.com/docker/docker/libcontainerd" + "github.com/sirupsen/logrus" ) // Reload reads configuration changes and modifies the @@ -37,7 +37,9 @@ func (daemon *Daemon) Reload(conf *config.Config) (err error) { } }() - daemon.reloadPlatform(conf, attributes) + if err := daemon.reloadPlatform(conf, attributes); err != nil { + return err + } daemon.reloadDebug(conf, attributes) daemon.reloadMaxConcurrentDownloadsAndUploads(conf, attributes) daemon.reloadShutdownTimeout(conf, attributes) diff --git a/components/engine/daemon/rename.go b/components/engine/daemon/rename.go index 686fbd3b99..b714ece7f9 100644 --- a/components/engine/daemon/rename.go +++ b/components/engine/daemon/rename.go @@ -5,9 +5,9 @@ import ( "fmt" "strings" - "github.com/Sirupsen/logrus" dockercontainer "github.com/docker/docker/container" "github.com/docker/libnetwork" + "github.com/sirupsen/logrus" ) // ContainerRename changes the name of a container, using the oldName diff --git a/components/engine/daemon/restart.go b/components/engine/daemon/restart.go index 9f2ef569af..e030cfaed5 100644 --- a/components/engine/daemon/restart.go +++ b/components/engine/daemon/restart.go @@ -3,8 +3,8 @@ package daemon import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/docker/container" + "github.com/sirupsen/logrus" ) // ContainerRestart stops and starts a container. It attempts to diff --git a/components/engine/daemon/seccomp_linux.go b/components/engine/daemon/seccomp_linux.go index 472e3133c0..3fce213490 100644 --- a/components/engine/daemon/seccomp_linux.go +++ b/components/engine/daemon/seccomp_linux.go @@ -5,10 +5,10 @@ package daemon import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/docker/container" "github.com/docker/docker/profiles/seccomp" "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" ) var supportsSeccomp = true diff --git a/components/engine/daemon/secrets.go b/components/engine/daemon/secrets.go index 90fa99e987..214391fe84 100644 --- a/components/engine/daemon/secrets.go +++ b/components/engine/daemon/secrets.go @@ -1,8 +1,8 @@ package daemon import ( - "github.com/Sirupsen/logrus" swarmtypes "github.com/docker/docker/api/types/swarm" + "github.com/sirupsen/logrus" ) // SetContainerSecretReferences sets the container secret references needed diff --git a/components/engine/daemon/start.go b/components/engine/daemon/start.go index 8d938519c4..61af1cf9ec 100644 --- a/components/engine/daemon/start.go +++ b/components/engine/daemon/start.go @@ -10,11 +10,11 @@ import ( "google.golang.org/grpc" - "github.com/Sirupsen/logrus" apierrors "github.com/docker/docker/api/errors" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" + "github.com/sirupsen/logrus" ) // ContainerStart starts a container. @@ -198,7 +198,9 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint func (daemon *Daemon) Cleanup(container *container.Container) { daemon.releaseNetwork(container) - container.UnmountIpcMounts(detachMounted) + if err := container.UnmountIpcMount(detachMounted); err != nil { + logrus.Warnf("%s cleanup: failed to unmount IPC: %s", container.ID, err) + } if err := daemon.conditionalUnmountOnCleanup(container); err != nil { // FIXME: remove once reference counting for graphdrivers has been refactored diff --git a/components/engine/daemon/start_windows.go b/components/engine/daemon/start_windows.go index 74129bd612..098380d00b 100644 --- a/components/engine/daemon/start_windows.go +++ b/components/engine/daemon/start_windows.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" + "github.com/Microsoft/opengcs/client" "github.com/docker/docker/container" "github.com/docker/docker/layer" "github.com/docker/docker/libcontainerd" @@ -142,6 +143,36 @@ func (daemon *Daemon) getLibcontainerdCreateOptions(container *container.Contain } } + // LCOW options. + if container.Platform == "linux" { + config := &client.Config{} + if err := config.GenerateDefault(daemon.configStore.GraphOptions); err != nil { + return nil, err + } + // Override from user-supplied options. + for k, v := range container.HostConfig.StorageOpt { + switch k { + case "lcow.kirdpath": + config.KirdPath = v + case "lcow.kernel": + config.KernelFile = v + case "lcow.initrd": + config.InitrdFile = v + case "lcow.vhdx": + config.Vhdx = v + case "lcow.bootparameters": + config.BootParameters = v + } + } + if err := config.Validate(); err != nil { + return nil, err + } + lcowOpts := &libcontainerd.LCOWOption{ + Config: config, + } + createOptions = append(createOptions, lcowOpts) + } + // Now add the remaining options. createOptions = append(createOptions, &libcontainerd.FlushOption{IgnoreFlushesDuringBoot: !container.HasBeenStartedBefore}) createOptions = append(createOptions, hvOpts) diff --git a/components/engine/daemon/stats/collector.go b/components/engine/daemon/stats/collector.go index 0520efa238..c06d80c172 100644 --- a/components/engine/daemon/stats/collector.go +++ b/components/engine/daemon/stats/collector.go @@ -5,10 +5,10 @@ package stats import ( "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/container" "github.com/docker/docker/pkg/pubsub" + "github.com/sirupsen/logrus" ) // Collect registers the container with the collector and adds it to diff --git a/components/engine/daemon/stop.go b/components/engine/daemon/stop.go index 6a4776d155..af3020eaba 100644 --- a/components/engine/daemon/stop.go +++ b/components/engine/daemon/stop.go @@ -6,9 +6,9 @@ import ( "net/http" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/errors" containerpkg "github.com/docker/docker/container" + "github.com/sirupsen/logrus" ) // ContainerStop looks for the given container and terminates it, diff --git a/components/engine/daemon/volumes.go b/components/engine/daemon/volumes.go index 6f24f05910..c2372ef572 100644 --- a/components/engine/daemon/volumes.go +++ b/components/engine/daemon/volumes.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" dockererrors "github.com/docker/docker/api/errors" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" @@ -17,6 +16,7 @@ import ( "github.com/docker/docker/container" "github.com/docker/docker/volume" "github.com/docker/docker/volume/drivers" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/distribution/errors.go b/components/engine/distribution/errors.go index f453c01cc0..0e4942d6a6 100644 --- a/components/engine/distribution/errors.go +++ b/components/engine/distribution/errors.go @@ -5,7 +5,6 @@ import ( "strings" "syscall" - "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/api/errcode" @@ -14,6 +13,7 @@ import ( "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/distribution/xfer" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // ErrNoSupport is an error type used for errors indicating that an operation diff --git a/components/engine/distribution/pull.go b/components/engine/distribution/pull.go index c5bdbd6d3b..690551c842 100644 --- a/components/engine/distribution/pull.go +++ b/components/engine/distribution/pull.go @@ -3,7 +3,6 @@ package distribution import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/docker/api" "github.com/docker/docker/distribution/metadata" @@ -11,6 +10,7 @@ import ( refstore "github.com/docker/docker/reference" "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/distribution/pull_v1.go b/components/engine/distribution/pull_v1.go index 7151a7584d..eb62146422 100644 --- a/components/engine/distribution/pull_v1.go +++ b/components/engine/distribution/pull_v1.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/distribution/metadata" @@ -24,6 +23,7 @@ import ( "github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/registry" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/distribution/pull_v2.go b/components/engine/distribution/pull_v2.go index 50257f5cb4..713e9b7da7 100644 --- a/components/engine/distribution/pull_v2.go +++ b/components/engine/distribution/pull_v2.go @@ -10,7 +10,6 @@ import ( "os" "runtime" - "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/distribution/manifest/manifestlist" "github.com/docker/distribution/manifest/schema1" @@ -31,6 +30,7 @@ import ( refstore "github.com/docker/docker/reference" "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/distribution/pull_v2_windows.go b/components/engine/distribution/pull_v2_windows.go index 543ecc10eb..e10070d53a 100644 --- a/components/engine/distribution/pull_v2_windows.go +++ b/components/engine/distribution/pull_v2_windows.go @@ -6,11 +6,11 @@ import ( "net/http" "os" - "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/registry/client/transport" + "github.com/sirupsen/logrus" ) var _ distribution.Describable = &v2LayerDescriptor{} diff --git a/components/engine/distribution/push.go b/components/engine/distribution/push.go index 395e4d1512..25b49faf5e 100644 --- a/components/engine/distribution/push.go +++ b/components/engine/distribution/push.go @@ -6,11 +6,11 @@ import ( "fmt" "io" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/docker/distribution/metadata" "github.com/docker/docker/pkg/progress" "github.com/docker/docker/registry" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/distribution/push_v1.go b/components/engine/distribution/push_v1.go index 431faaf28c..4da4d5eac4 100644 --- a/components/engine/distribution/push_v1.go +++ b/components/engine/distribution/push_v1.go @@ -4,7 +4,6 @@ import ( "fmt" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/distribution/metadata" @@ -17,6 +16,7 @@ import ( "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/distribution/push_v2.go b/components/engine/distribution/push_v2.go index ffc7d68107..3d32061e8a 100644 --- a/components/engine/distribution/push_v2.go +++ b/components/engine/distribution/push_v2.go @@ -11,7 +11,6 @@ import ( "golang.org/x/net/context" - "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" @@ -26,6 +25,7 @@ import ( "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/distribution/registry_unit_test.go b/components/engine/distribution/registry_unit_test.go index 910061f45a..d6b6ee8832 100644 --- a/components/engine/distribution/registry_unit_test.go +++ b/components/engine/distribution/registry_unit_test.go @@ -11,13 +11,13 @@ import ( "strings" "testing" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/registry" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/distribution/utils/progress.go b/components/engine/distribution/utils/progress.go index cc3632a534..96380fdb24 100644 --- a/components/engine/distribution/utils/progress.go +++ b/components/engine/distribution/utils/progress.go @@ -6,9 +6,9 @@ import ( "os" "syscall" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/streamformatter" + "github.com/sirupsen/logrus" ) // WriteDistributionProgress is a helper for writing progress from chan to JSON diff --git a/components/engine/distribution/xfer/download.go b/components/engine/distribution/xfer/download.go index 6769ee1cdf..6ee0aa57b8 100644 --- a/components/engine/distribution/xfer/download.go +++ b/components/engine/distribution/xfer/download.go @@ -7,13 +7,13 @@ import ( "runtime" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/progress" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/distribution/xfer/upload.go b/components/engine/distribution/xfer/upload.go index 58422e57a1..3310a3c684 100644 --- a/components/engine/distribution/xfer/upload.go +++ b/components/engine/distribution/xfer/upload.go @@ -4,10 +4,10 @@ import ( "errors" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/progress" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/docs/api/version-history.md b/components/engine/docs/api/version-history.md index e5d9d8d4ca..387ce2950c 100644 --- a/components/engine/docs/api/version-history.md +++ b/components/engine/docs/api/version-history.md @@ -13,12 +13,28 @@ keywords: "API, Docker, rcli, REST, documentation" will be rejected. --> +## v1.32 API changes + +[Docker Engine API v1.32](https://docs.docker.com/engine/api/v1.32/) documentation + +* `POST /containers/create` now accepts additional values for the + `HostConfig.IpcMode` property. New values are `private`, `shareable`, + and `none`. + ## v1.31 API changes [Docker Engine API v1.31](https://docs.docker.com/engine/api/v1.31/) documentation * `DELETE /secrets/(name)` now returns status code 404 instead of 500 when the secret does not exist. * `POST /secrets/create` now returns status code 409 instead of 500 when creating an already existing secret. +* `POST /secrets/create` now accepts a `Driver` struct, allowing the + `Name` and driver-specific `Options` to be passed to store a secrets + in an external secrets store. The `Driver` property can be omitted + if the default (internal) secrets store is used. +* `GET /secrets/(id)` and `GET /secrets` now return a `Driver` struct, + containing the `Name` and driver-specific `Options` of the external + secrets store used to store the secret. The `Driver` property is + omitted if no external store is used. * `POST /secrets/(name)/update` now returns status code 400 instead of 500 when updating a secret's content which is not the labels. * `POST /nodes/(name)/update` now returns status code 400 instead of 500 when demoting last node fails. * `GET /networks/(id or name)` now takes an optional query parameter `scope` that will filter the network based on the scope (`local`, `swarm`, or `global`). diff --git a/components/engine/hack/README.md b/components/engine/hack/README.md index 802395d533..9e588db256 100644 --- a/components/engine/hack/README.md +++ b/components/engine/hack/README.md @@ -37,14 +37,14 @@ More information is found within `make.ps1` by the author, @jhowardmsft - Referenced via `make test` when running tests on a local machine, or directly referenced when running tests inside a Docker development container. - When running on a local machine, `make test` to run all tests found in -`test`, `test-unit`, `test-integration-cli`, and `test-docker-py` on +`test`, `test-unit`, `test-integration`, and `test-docker-py` on your local machine. The default timeout is set in `make.sh` to 60 minutes (`${TIMEOUT:=60m}`), since it currently takes up to an hour to run all of the tests. - When running inside a Docker development container, `hack/make.sh` does not have a single target that runs all the tests. You need to provide a single command line with multiple targets that performs the same thing. -An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary cross test-unit test-integration-cli test-docker-py` +An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary cross test-unit test-integration test-docker-py` - For more information related to testing outside the scope of this README, refer to [Run tests and test documentation](https://docs.docker.com/opensource/project/test-and-docs/) diff --git a/components/engine/hack/integration-cli-on-swarm/host/dockercmd.go b/components/engine/hack/integration-cli-on-swarm/host/dockercmd.go index 10ea0ecc24..c08b763a2b 100644 --- a/components/engine/hack/integration-cli-on-swarm/host/dockercmd.go +++ b/components/engine/hack/integration-cli-on-swarm/host/dockercmd.go @@ -43,7 +43,7 @@ func deployStack(unusedCli *client.Client, stackName, composeFilePath string) er func hasStack(unusedCli *client.Client, stackName string) bool { // FIXME: eliminate os/exec (but stack is implemented in CLI ...) - out, err := exec.Command("docker", "stack", "ls").Output() + out, err := exec.Command("docker", "stack", "ls").CombinedOutput() if err != nil { panic(fmt.Errorf("`docker stack ls` failed with: %s", string(out))) } diff --git a/components/engine/hack/integration-cli-on-swarm/host/host.go b/components/engine/hack/integration-cli-on-swarm/host/host.go index 6823a76686..fdc2a83e7f 100644 --- a/components/engine/hack/integration-cli-on-swarm/host/host.go +++ b/components/engine/hack/integration-cli-on-swarm/host/host.go @@ -10,11 +10,11 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/hack/make.ps1 b/components/engine/hack/make.ps1 index ac3e369048..f3f45e3d00 100644 --- a/components/engine/hack/make.ps1 +++ b/components/engine/hack/make.ps1 @@ -318,7 +318,7 @@ Function Run-UnitTests() { $pkgList = $pkgList | Select-String -Pattern "github.com/docker/docker" $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/vendor" $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man" - $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration-cli" + $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration" $pkgList = $pkgList -replace "`r`n", " " $goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList" Invoke-Expression $goTestCommand diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index b7d59ba94a..c0a6467f75 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -60,7 +60,7 @@ DEFAULT_BUNDLES=( dynbinary test-unit - test-integration-cli + test-integration test-docker-py cross @@ -68,7 +68,7 @@ DEFAULT_BUNDLES=( ) VERSION=$(< ./VERSION) -! BUILDTIME=$(date --rfc-3339 ns 2> /dev/null | sed -e 's/ /T/') +! BUILDTIME=$(date -u -d "@${SOURCE_DATE_EPOCH:-$(date +%s)}" --rfc-3339 ns 2> /dev/null | sed -e 's/ /T/') if [ "$DOCKER_GITCOMMIT" ]; then GITCOMMIT="$DOCKER_GITCOMMIT" elif command -v git &> /dev/null && [ -d .git ] && git rev-parse &> /dev/null; then @@ -130,9 +130,9 @@ fi # functionality. if \ command -v gcc &> /dev/null \ - && ! ( echo -e '#include \nint main() { dm_task_deferred_remove(NULL); }'| gcc -xc - -o /dev/null -ldevmapper &> /dev/null ) \ + && ! ( echo -e '#include \nint main() { dm_task_deferred_remove(NULL); }'| gcc -xc - -o /dev/null $(pkg-config --libs devmapper) &> /dev/null ) \ ; then - DOCKER_BUILDTAGS+=' libdm_no_deferred_remove' + DOCKER_BUILDTAGS+=' libdm_no_deferred_remove' fi # Use these flags when compiling the tests and final binary @@ -158,8 +158,8 @@ fi ORIG_BUILDFLAGS+=( $REBUILD_FLAG ) BUILDFLAGS=( $BUILDFLAGS "${ORIG_BUILDFLAGS[@]}" ) -# Test timeout. +# Test timeout. if [ "${DOCKER_ENGINE_GOARCH}" == "arm" ]; then : ${TIMEOUT:=10m} elif [ "${DOCKER_ENGINE_GOARCH}" == "windows" ]; then @@ -183,79 +183,12 @@ if [ "$(uname -s)" = 'FreeBSD' ]; then LDFLAGS="$LDFLAGS -extld clang" fi -HAVE_GO_TEST_COVER= -if \ - go help testflag | grep -- -cover > /dev/null \ - && go tool -n cover > /dev/null 2>&1 \ -; then - HAVE_GO_TEST_COVER=1 -fi - -# a helper to provide ".exe" when it's appropriate -binary_extension() { - if [ "$(go env GOOS)" = 'windows' ]; then - echo -n '.exe' - fi -} - -hash_files() { - while [ $# -gt 0 ]; do - f="$1" - shift - dir="$(dirname "$f")" - base="$(basename "$f")" - for hashAlgo in md5 sha256; do - if command -v "${hashAlgo}sum" &> /dev/null; then - ( - # subshell and cd so that we get output files like: - # $HASH docker-$VERSION - # instead of: - # $HASH /go/src/github.com/.../$VERSION/binary/docker-$VERSION - cd "$dir" - "${hashAlgo}sum" "$base" > "$base.$hashAlgo" - ) - fi - done - done -} - bundle() { local bundle="$1"; shift echo "---> Making bundle: $(basename "$bundle") (in $DEST)" source "$SCRIPTDIR/make/$bundle" "$@" } -copy_binaries() { - dir="$1" - # Add nested executables to bundle dir so we have complete set of - # them available, but only if the native OS/ARCH is the same as the - # OS/ARCH of the build target - if [ "$(go env GOOS)/$(go env GOARCH)" == "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" ]; then - if [ -x /usr/local/bin/docker-runc ]; then - echo "Copying nested executables into $dir" - for file in containerd containerd-shim containerd-ctr runc init proxy; do - cp -f `which "docker-$file"` "$dir/" - if [ "$2" == "hash" ]; then - hash_files "$dir/docker-$file" - fi - done - fi - fi -} - -install_binary() { - file="$1" - target="${DOCKER_MAKE_INSTALL_PREFIX:=/usr/local}/bin/" - if [ "$(go env GOOS)" == "linux" ]; then - echo "Installing $(basename $file) to ${target}" - mkdir -p "$target" - cp -f -L "$file" "$target" - else - echo "Install is only supported on linux" - return 1 - fi -} - main() { # We want this to fail if the bundles already exist and cannot be removed. # This is to avoid mixing bundles from different versions of the code. diff --git a/components/engine/hack/make/.binary b/components/engine/hack/make/.binary index 8d4265cb65..ff5cbff722 100644 --- a/components/engine/hack/make/.binary +++ b/components/engine/hack/make/.binary @@ -1,6 +1,13 @@ #!/usr/bin/env bash set -e +# a helper to provide ".exe" when it's appropriate +binary_extension() { + if [ "$(go env GOOS)" = 'windows' ]; then + echo -n '.exe' + fi +} + GO_PACKAGE='github.com/docker/docker/cmd/dockerd' BINARY_SHORT_NAME='dockerd' BINARY_NAME="$BINARY_SHORT_NAME-$VERSION" @@ -9,6 +16,27 @@ BINARY_FULLNAME="$BINARY_NAME$BINARY_EXTENSION" source "${MAKEDIR}/.go-autogen" +hash_files() { + while [ $# -gt 0 ]; do + f="$1" + shift + dir="$(dirname "$f")" + base="$(basename "$f")" + for hashAlgo in md5 sha256; do + if command -v "${hashAlgo}sum" &> /dev/null; then + ( + # subshell and cd so that we get output files like: + # $HASH docker-$VERSION + # instead of: + # $HASH /go/src/github.com/.../$VERSION/binary/docker-$VERSION + cd "$dir" + "${hashAlgo}sum" "$base" > "$base.$hashAlgo" + ) + fi + done + done +} + ( export GOGC=${DOCKER_BUILD_GOGC:-1000} diff --git a/components/engine/hack/make/.ensure-emptyfs b/components/engine/hack/make/.ensure-emptyfs index 7b00b9d455..898cc22834 100644 --- a/components/engine/hack/make/.ensure-emptyfs +++ b/components/engine/hack/make/.ensure-emptyfs @@ -1,23 +1,23 @@ #!/usr/bin/env bash set -e -if ! docker inspect -t image emptyfs &> /dev/null; then - # let's build a "docker save" tarball for "emptyfs" +if ! docker image inspect emptyfs > /dev/null; then + # build a "docker save" tarball for "emptyfs" # see https://github.com/docker/docker/pull/5262 # and also https://github.com/docker/docker/issues/4242 dir="$DEST/emptyfs" - mkdir -p "$dir" + uuid=511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158 + mkdir -p "$dir/$uuid" ( - cd "$dir" - echo '{"emptyfs":{"latest":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158"}}' > repositories - mkdir -p 511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158 - ( - cd 511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158 - echo '{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}' > json - echo '1.0' > VERSION - tar -cf layer.tar --files-from /dev/null - ) + echo '{"emptyfs":{"latest":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158"}}' > "$dir/repositories" + cd "$dir/$uuid" + echo '{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}' > json + echo '1.0' > VERSION + tar -cf layer.tar --files-from /dev/null + ) + ( + [ -n "$TESTDEBUG" ] && set -x + tar -cC "$dir" . | docker load ) - ( set -x; tar -cC "$dir" . | docker load ) rm -rf "$dir" fi diff --git a/components/engine/hack/make/.integration-daemon-setup b/components/engine/hack/make/.integration-daemon-setup index 5134e4c2db..c130e23560 100644 --- a/components/engine/hack/make/.integration-daemon-setup +++ b/components/engine/hack/make/.integration-daemon-setup @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -bundle .detect-daemon-osarch +source "$MAKEDIR/.detect-daemon-osarch" if [ "$DOCKER_ENGINE_GOOS" != "windows" ]; then bundle .ensure-emptyfs fi diff --git a/components/engine/hack/make/.integration-daemon-start b/components/engine/hack/make/.integration-daemon-start index dafd0533d9..b4e6684854 100644 --- a/components/engine/hack/make/.integration-daemon-start +++ b/components/engine/hack/make/.integration-daemon-start @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# see test-integration-cli for example usage of this script +# see test-integration for example usage of this script base="$ABS_DEST/.." export PATH="$base/binary-daemon:$base/dynbinary-daemon:$PATH" @@ -76,24 +76,26 @@ if [ -z "$DOCKER_TEST_HOST" ]; then # see https://github.com/docker/libcontainer/blob/master/apparmor/apparmor.go#L16 export container="" ( - set -x + [ -n "$TESTDEBUG" ] && set -x /etc/init.d/apparmor start ) fi - export DOCKER_HOST="unix://$(cd "$DEST" && pwd)/docker.sock" # "pwd" tricks to make sure $DEST is an absolute path, not a relative one - ( set -x; exec \ - dockerd --debug \ - --host "$DOCKER_HOST" \ - --storage-driver "$DOCKER_GRAPHDRIVER" \ - --pidfile "$DEST/docker.pid" \ - --userland-proxy="$DOCKER_USERLANDPROXY" \ - $storage_params \ - $extra_params \ - &> "$DEST/docker.log" + # "pwd" tricks to make sure $DEST is an absolute path, not a relative one + export DOCKER_HOST="unix://$(cd "$DEST" && pwd)/docker.sock" + ( + echo "Starting dockerd" + [ -n "$TESTDEBUG" ] && set -x + exec \ + dockerd --debug \ + --host "$DOCKER_HOST" \ + --storage-driver "$DOCKER_GRAPHDRIVER" \ + --pidfile "$DEST/docker.pid" \ + --userland-proxy="$DOCKER_USERLANDPROXY" \ + $storage_params \ + $extra_params \ + &> "$DEST/docker.log" ) & - # make sure that if the script exits unexpectedly, we stop this daemon we just started - trap 'bundle .integration-daemon-stop' EXIT else export DOCKER_HOST="$DOCKER_TEST_HOST" fi diff --git a/components/engine/hack/make/.integration-daemon-stop b/components/engine/hack/make/.integration-daemon-stop index 591a8d6bec..c1d43e1a5e 100644 --- a/components/engine/hack/make/.integration-daemon-stop +++ b/components/engine/hack/make/.integration-daemon-stop @@ -1,11 +1,12 @@ #!/usr/bin/env bash if [ ! "$(go env GOOS)" = 'windows' ]; then - trap - EXIT # reset EXIT trap applied in .integration-daemon-start - for pidFile in $(find "$DEST" -name docker.pid); do - pid=$(set -x; cat "$pidFile") - ( set -x; kill "$pid" ) + pid=$([ -n "$TESTDEBUG" ] && set -x; cat "$pidFile") + ( + [ -n "$TESTDEBUG" ] && set -x + kill "$pid" + ) if ! wait "$pid"; then echo >&2 "warning: PID $pid from $pidFile had a nonzero exit code" fi @@ -15,7 +16,7 @@ if [ ! "$(go env GOOS)" = 'windows' ]; then # Stop apparmor if it is enabled if [ -e "/sys/module/apparmor/parameters/enabled" ] && [ "$(cat /sys/module/apparmor/parameters/enabled)" == "Y" ]; then ( - set -x + [ -n "$TESTDEBUG" ] && set -x /etc/init.d/apparmor stop ) fi diff --git a/components/engine/hack/make/.integration-test-helpers b/components/engine/hack/make/.integration-test-helpers index 4ff9677c79..288be863ce 100644 --- a/components/engine/hack/make/.integration-test-helpers +++ b/components/engine/hack/make/.integration-test-helpers @@ -1,65 +1,72 @@ #!/usr/bin/env bash - -: ${TEST_REPEAT:=0} - -bundle_test_integration_cli() { - TESTFLAGS="$TESTFLAGS -check.v -check.timeout=${TIMEOUT} -test.timeout=360m" - go_test_dir integration-cli $DOCKER_INTEGRATION_TESTS_VERIFIED -} - -# If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. -# You can use this to select certain tests to run, e.g. -# -# TESTFLAGS='-test.run ^TestBuild$' ./hack/make.sh test-unit # # For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want # to run certain tests on your local host, you should run with command: # -# TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration-cli +# TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration # -go_test_dir() { - dir=$1 - precompiled=$2 - testbinary="$ABS_DEST/test.main" - testcover=() - testcoverprofile=() + +source "$SCRIPTDIR/make/.go-autogen" + +: ${TEST_REPEAT:=1} + +integration_api_dirs=("$( + find ./integration -type d | + grep -vE '^(./integration$|./integration/util)')") + +run_test_integration() { + local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS" + for dir in $integration_api_dirs; do + ( + cd $dir + echo "Running $PWD" + test_env ./test.main $flags + ) + done + ( - set -e - mkdir -p "$DEST/coverprofiles" - export DEST="$ABS_DEST" # in a subshell this is safe -- our integration-cli tests need DEST, and "cd" screws it up - if [ -z $precompiled ]; then - ensure_test_dir $1 $testbinary - fi - cd "$dir" - i=0 - while ((++i)); do - test_env "$testbinary" $TESTFLAGS - if [ $i -gt "$TEST_REPEAT" ]; then - break - fi - echo "Repeating test ($i)" - done + flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS" + cd integration-cli + echo "Running $PWD" + test_env ./test.main $flags ) } -ensure_test_dir() { - ( - # make sure a test dir will compile - dir="$1" - out="$2" - echo Building test dir: "$dir" - set -xe - cd "$dir" - go test -c -o "$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" - ) +build_test_suite_binaries() { + build_test_suite_binary ./integration-cli "test.main" + for dir in $integration_api_dirs; do + build_test_suite_binary "$dir" "test.main" + done } +# Build a binary for a test suite package +build_test_suite_binary() { + local dir="$1" + local out="$2" + echo Building test suite binary "$dir/$out" + go test -c -o "$dir/$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" "$dir" +} + +cleanup_test_suite_binaries() { + [ -n "$TESTDEBUG" ] && return + echo "Removing test suite binaries" + find integration* -name test.main | xargs -r rm +} + +repeat() { + for i in $(seq 1 $TEST_REPEAT); do + echo "Running integration-test (iteration $i)" + $@ + done +} + +# use "env -i" to tightly control the environment variables that bleed into the tests test_env() { ( - set -xe - # use "env -i" to tightly control the environment variables that bleed into the tests + set -e + [ -n "$TESTDEBUG" ] && set -x env -i \ - DEST="$DEST" \ + DEST="$ABS_DEST" \ DOCKER_CLI_VERSION="$DOCKER_CLI_VERSION" \ DOCKER_API_VERSION="$DOCKER_API_VERSION" \ DOCKER_INTEGRATION_DAEMON_DEST="$DOCKER_INTEGRATION_DAEMON_DEST" \ @@ -82,3 +89,19 @@ test_env() { "$@" ) } + + +error_on_leaked_containerd_shims() { + if [ "$(go env GOOS)" == 'windows' ]; then + return + fi + + leftovers=$(ps -ax -o pid,cmd | + awk '$2 == "docker-containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration/ { print $1 }') + if [ -n "$leftovers" ]; then + ps aux + kill -9 $leftovers 2> /dev/null + echo "!!!! WARNING you have left over shim(s), Cleanup your test !!!!" + exit 1 + fi +} diff --git a/components/engine/hack/make/README.md b/components/engine/hack/make/README.md index 6574b0efe6..3d069fa165 100644 --- a/components/engine/hack/make/README.md +++ b/components/engine/hack/make/README.md @@ -4,10 +4,9 @@ Each script is named after the bundle it creates. They should not be called directly - instead, pass it as argument to make.sh, for example: ``` -./hack/make.sh test ./hack/make.sh binary ubuntu -# Or to run all bundles: +# Or to run all default bundles: ./hack/make.sh ``` diff --git a/components/engine/hack/make/binary-daemon b/components/engine/hack/make/binary-daemon index 736f308a83..f68163636b 100644 --- a/components/engine/hack/make/binary-daemon +++ b/components/engine/hack/make/binary-daemon @@ -1,9 +1,27 @@ #!/usr/bin/env bash set -e -[ -z "$KEEPDEST" ] && rm -rf "$DEST" +copy_binaries() { + local dir="$1" + local hash="$2" + # Add nested executables to bundle dir so we have complete set of + # them available, but only if the native OS/ARCH is the same as the + # OS/ARCH of the build target + if [ "$(go env GOOS)/$(go env GOARCH)" != "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" ]; then + return + fi + if [ ! -x /usr/local/bin/docker-runc ]; then + return + fi + echo "Copying nested executables into $dir" + for file in containerd containerd-shim containerd-ctr runc init proxy; do + cp -f `which "docker-$file"` "$dir/" + if [ "$hash" == "hash" ]; then + hash_files "$dir/docker-$file" + fi + done +} -( - source "${MAKEDIR}/.binary" - copy_binaries "$DEST" 'hash' -) +[ -z "$KEEPDEST" ] && rm -rf "$DEST" +source "${MAKEDIR}/.binary" +copy_binaries "$DEST" 'hash' diff --git a/components/engine/hack/make/build-integration-test-binary b/components/engine/hack/make/build-integration-test-binary deleted file mode 100644 index a842e8cce8..0000000000 --- a/components/engine/hack/make/build-integration-test-binary +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -e - -rm -rf "$DEST" -DEST="$ABS_DEST/../test-integration-cli" - -source "$SCRIPTDIR/make/.go-autogen" - -if [ -z $DOCKER_INTEGRATION_TESTS_VERIFIED ]; then - source ${MAKEDIR}/.integration-test-helpers - ensure_test_dir integration-cli "$DEST/test.main" - export DOCKER_INTEGRATION_TESTS_VERIFIED=1 -fi diff --git a/components/engine/hack/make/install-binary-daemon b/components/engine/hack/make/install-binary-daemon index 12126ffacc..f6a4361fdb 100644 --- a/components/engine/hack/make/install-binary-daemon +++ b/components/engine/hack/make/install-binary-daemon @@ -3,6 +3,19 @@ set -e rm -rf "$DEST" +install_binary() { + local file="$1" + local target="${DOCKER_MAKE_INSTALL_PREFIX:=/usr/local}/bin/" + if [ "$(go env GOOS)" == "linux" ]; then + echo "Installing $(basename $file) to ${target}" + mkdir -p "$target" + cp -f -L "$file" "$target" + else + echo "Install is only supported on linux" + return 1 + fi +} + ( DEST="$(dirname $DEST)/binary-daemon" source "${MAKEDIR}/.binary-setup" diff --git a/components/engine/hack/make/test-integration b/components/engine/hack/make/test-integration new file mode 100755 index 0000000000..e419d66c6e --- /dev/null +++ b/components/engine/hack/make/test-integration @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e + +source "${MAKEDIR}/.go-autogen" +source hack/make/.integration-test-helpers + +( + build_test_suite_binaries + bundle .integration-daemon-start + bundle .integration-daemon-setup + + local testexit=0 + ( repeat run_test_integration ) || testexit=$? + + # Always run cleanup, even if the subshell fails + bundle .integration-daemon-stop + cleanup_test_suite_binaries + error_on_leaked_containerd_shims + + exit $testexit + +) 2>&1 | tee -a "$DEST/test.log" diff --git a/components/engine/hack/make/test-integration-cli b/components/engine/hack/make/test-integration-cli index c8137f4313..4cc79d095d 100755 --- a/components/engine/hack/make/test-integration-cli +++ b/components/engine/hack/make/test-integration-cli @@ -1,28 +1,6 @@ #!/usr/bin/env bash set -e +echo "WARNING: test-integration-cli is DEREPCATED. Use test-integration." >&2 -source hack/make/.integration-test-helpers - -# subshell so that we can export PATH without breaking other things -( - bundle .integration-daemon-start - - bundle .integration-daemon-setup - - bundle_test_integration_cli - - bundle .integration-daemon-stop - - if [ "$(go env GOOS)" != 'windows' ] - then - leftovers=$(ps -ax -o pid,cmd | awk '$2 == "docker-containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration-cli/ { print $1 }') - if [ -n "$leftovers" ] - then - ps aux - kill -9 $leftovers 2> /dev/null - echo "!!!! WARNING you have left over shim(s), Cleanup your test !!!!" - exit 1 - fi - fi - -) 2>&1 | tee -a "$DEST/test.log" +# TODO: remove this and exit 1 once CI has changed to use test-integration +bundle test-integration diff --git a/components/engine/hack/make/test-integration-shell b/components/engine/hack/make/test-integration-shell index 2201f5eb37..bcfa4682eb 100644 --- a/components/engine/hack/make/test-integration-shell +++ b/components/engine/hack/make/test-integration-shell @@ -5,3 +5,5 @@ bundle .integration-daemon-setup export ABS_DEST bash +e + +bundle .integration-daemon-stop diff --git a/components/engine/hack/make/test-unit b/components/engine/hack/make/test-unit index 85eef5b5b6..d985a6ec2d 100644 --- a/components/engine/hack/make/test-unit +++ b/components/engine/hack/make/test-unit @@ -1,58 +1,6 @@ #!/usr/bin/env bash set -e -# Run Docker's test suite, including sub-packages, and store their output as a bundle -# If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. -# You can use this to select certain tests to run, e.g. -# -# TESTFLAGS='-test.run ^TestBuild$' ./hack/make.sh test-unit -# -bundle_test_unit() { - TESTFLAGS+=" -test.timeout=${TIMEOUT}" - INCBUILD="-i" - count=0 - for flag in "${BUILDFLAGS[@]}"; do - if [ "${flag}" == ${INCBUILD} ]; then - unset BUILDFLAGS[${count}] - break - fi - count=$[ ${count} + 1 ] - done +echo "DEPRECATED: use hack/test/unit instead of hack/make.sh test-unit" >&2 - date - if [ -z "$TESTDIRS" ]; then - TEST_PATH=./... - else - TEST_PATH=./${TESTDIRS} - fi - - source "${MAKEDIR}/.go-autogen" - - if [ "$(go env GOHOSTOS)" = 'solaris' ]; then - pkg_list=$(go list -e \ - -f '{{if ne .Name "github.com/docker/docker"}} - {{.ImportPath}} - {{end}}' \ - "${BUILDFLAGS[@]}" $TEST_PATH \ - | grep github.com/docker/docker \ - | grep -v github.com/docker/docker/vendor \ - | grep -v github.com/docker/docker/daemon/graphdriver \ - | grep -v github.com/docker/docker/man \ - | grep -v github.com/docker/docker/integration-cli) - else - pkg_list=$(go list -e \ - -f '{{if ne .Name "github.com/docker/docker"}} - {{.ImportPath}} - {{end}}' \ - "${BUILDFLAGS[@]}" $TEST_PATH \ - | grep github.com/docker/docker \ - | grep -v github.com/docker/docker/vendor \ - | grep -v github.com/docker/docker/man \ - | grep -v github.com/docker/docker/integration-cli) - fi - - go test -cover -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS $pkg_list - go test -cover -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS github.com/docker/docker/pkg/term -test.root -} - -bundle_test_unit 2>&1 | tee -a "$DEST/test.log" +$SCRIPTDIR/test/unit 2>&1 | tee -a "$DEST/test.log" diff --git a/components/engine/hack/release.sh b/components/engine/hack/release.sh index 5d7363044b..04f15be02b 100755 --- a/components/engine/hack/release.sh +++ b/components/engine/hack/release.sh @@ -57,7 +57,7 @@ if [ "$1" != '--release-regardless-of-test-failure' ]; then RELEASE_BUNDLES=( test-unit "${RELEASE_BUNDLES[@]}" - test-integration-cli + test-integration ) fi diff --git a/components/engine/hack/test/unit b/components/engine/hack/test/unit new file mode 100755 index 0000000000..2b07089bcf --- /dev/null +++ b/components/engine/hack/test/unit @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Run unit tests +# +# TESTFLAGS - add additional test flags. Ex: +# +# TESTFLAGS="-v -run TestBuild" hack/test/unit +# +# TESTDIRS - run tests for specified packages. Ex: +# +# TESTDIRS="./pkg/term" hack/test/unit +# +set -eu -o pipefail + +TESTFLAGS+=" -test.timeout=${TIMEOUT:-5m}" +BUILDFLAGS=( -tags "netgo seccomp libdm_no_deferred_remove" ) +TESTDIRS="${TESTDIRS:-"./..."}" + +exclude_paths="/vendor/|/integration" +if [ "$(go env GOHOSTOS)" = 'solaris' ]; then + exclude_paths="$exclude_paths|/daemon/graphdriver" +fi +pkg_list=$(go list $TESTDIRS | grep -vE "($exclude_paths)") + +go test -cover "${BUILDFLAGS[@]}" $TESTFLAGS $pkg_list diff --git a/components/engine/image/fs.go b/components/engine/image/fs.go index 10f6dab5fc..92cdb2662a 100644 --- a/components/engine/image/fs.go +++ b/components/engine/image/fs.go @@ -7,10 +7,10 @@ import ( "path/filepath" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/ioutils" "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // DigestWalkFunc is function called by StoreBackend.Walk diff --git a/components/engine/image/rootfs.go b/components/engine/image/rootfs.go index 5a9020f0ed..fb5f674c84 100644 --- a/components/engine/image/rootfs.go +++ b/components/engine/image/rootfs.go @@ -3,8 +3,8 @@ package image import ( "runtime" - "github.com/Sirupsen/logrus" "github.com/docker/docker/layer" + "github.com/sirupsen/logrus" ) // TypeLayers is used for RootFS.Type for filesystems organized into layers. diff --git a/components/engine/image/store.go b/components/engine/image/store.go index c85f8d6830..1a3045f58a 100644 --- a/components/engine/image/store.go +++ b/components/engine/image/store.go @@ -7,12 +7,12 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/digestset" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/system" "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // Store is an interface for creating and accessing images diff --git a/components/engine/image/tarexport/load.go b/components/engine/image/tarexport/load.go index af8cefc6ad..d69c9342ad 100644 --- a/components/engine/image/tarexport/load.go +++ b/components/engine/image/tarexport/load.go @@ -11,7 +11,6 @@ import ( "reflect" "runtime" - "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/distribution/reference" "github.com/docker/docker/image" @@ -25,6 +24,7 @@ import ( "github.com/docker/docker/pkg/symlink" "github.com/docker/docker/pkg/system" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" ) func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool) error { diff --git a/components/engine/image/v1/imagev1.go b/components/engine/image/v1/imagev1.go index 0e8a23cb5d..697d59e335 100644 --- a/components/engine/image/v1/imagev1.go +++ b/components/engine/image/v1/imagev1.go @@ -5,12 +5,12 @@ import ( "reflect" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types/versions" "github.com/docker/docker/image" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/stringid" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" ) // noFallbackMinVersion is the minimum version for which v1compatibility diff --git a/components/engine/integration-cli/check_test.go b/components/engine/integration-cli/check_test.go index f05b6504e9..6af5229f04 100644 --- a/components/engine/integration-cli/check_test.go +++ b/components/engine/integration-cli/check_test.go @@ -4,10 +4,8 @@ import ( "fmt" "net/http/httptest" "os" - "os/exec" "path" "path/filepath" - "strings" "sync" "syscall" "testing" @@ -72,17 +70,7 @@ func TestMain(m *testing.M) { func Test(t *testing.T) { cli.EnsureTestEnvIsLoaded(t) fakestorage.EnsureTestEnvIsLoaded(t) - cmd := exec.Command(dockerBinary, "images", "-f", "dangling=false", "--format", "{{.Repository}}:{{.Tag}}") - cmd.Env = appendBaseEnv(true) - out, err := cmd.CombinedOutput() - if err != nil { - panic(fmt.Errorf("err=%v\nout=%s\n", err, out)) - } - images := strings.Split(strings.TrimSpace(string(out)), "\n") - testEnv.ProtectImage(t, images...) - if testEnv.DaemonPlatform() == "linux" { - ensureFrozenImagesLinux(t) - } + environment.ProtectImages(t, testEnv) check.TestingT(t) } diff --git a/components/engine/integration-cli/docker_api_build_test.go b/components/engine/integration-cli/docker_api_build_test.go index c1ab7661e0..fb62917823 100644 --- a/components/engine/integration-cli/docker_api_build_test.go +++ b/components/engine/integration-cli/docker_api_build_test.go @@ -12,8 +12,6 @@ import ( "strings" "github.com/docker/docker/api/types" - "github.com/docker/docker/client/session" - "github.com/docker/docker/client/session/filesync" "github.com/docker/docker/integration-cli/checker" "github.com/docker/docker/integration-cli/cli/build/fakecontext" "github.com/docker/docker/integration-cli/cli/build/fakegit" @@ -21,6 +19,8 @@ import ( "github.com/docker/docker/integration-cli/request" "github.com/docker/docker/pkg/testutil" "github.com/go-check/check" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/filesync" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/context" diff --git a/components/engine/integration-cli/docker_api_containers_windows_test.go b/components/engine/integration-cli/docker_api_containers_windows_test.go new file mode 100644 index 0000000000..4cbe067cd5 --- /dev/null +++ b/components/engine/integration-cli/docker_api_containers_windows_test.go @@ -0,0 +1,71 @@ +// +build windows + +package main + +import ( + "fmt" + "io/ioutil" + "math/rand" + "net/http" + "strings" + + winio "github.com/Microsoft/go-winio" + "github.com/docker/docker/integration-cli/checker" + "github.com/docker/docker/integration-cli/request" + "github.com/go-check/check" +) + +func (s *DockerSuite) TestContainersAPICreateMountsBindNamedPipe(c *check.C) { + testRequires(c, SameHostDaemon, DaemonIsWindowsAtLeastBuild(16210)) // Named pipe support was added in RS3 + + // Create a host pipe to map into the container + hostPipeName := fmt.Sprintf(`\\.\pipe\docker-cli-test-pipe-%x`, rand.Uint64()) + pc := &winio.PipeConfig{ + SecurityDescriptor: "D:P(A;;GA;;;AU)", // Allow all users access to the pipe + } + l, err := winio.ListenPipe(hostPipeName, pc) + if err != nil { + c.Fatal(err) + } + defer l.Close() + + // Asynchronously read data that the container writes to the mapped pipe. + var b []byte + ch := make(chan error) + go func() { + conn, err := l.Accept() + if err == nil { + b, err = ioutil.ReadAll(conn) + conn.Close() + } + ch <- err + }() + + containerPipeName := `\\.\pipe\docker-cli-test-pipe` + text := "hello from a pipe" + cmd := fmt.Sprintf("echo %s > %s", text, containerPipeName) + + name := "test-bind-npipe" + data := map[string]interface{}{ + "Image": testEnv.MinimalBaseImage(), + "Cmd": []string{"cmd", "/c", cmd}, + "HostConfig": map[string]interface{}{"Mounts": []map[string]interface{}{{"Type": "npipe", "Source": hostPipeName, "Target": containerPipeName}}}, + } + + status, resp, err := request.SockRequest("POST", "/containers/create?name="+name, data, daemonHost()) + c.Assert(err, checker.IsNil, check.Commentf(string(resp))) + c.Assert(status, checker.Equals, http.StatusCreated, check.Commentf(string(resp))) + + status, _, err = request.SockRequest("POST", "/containers/"+name+"/start", nil, daemonHost()) + c.Assert(err, checker.IsNil) + c.Assert(status, checker.Equals, http.StatusNoContent) + + err = <-ch + if err != nil { + c.Fatal(err) + } + result := strings.TrimSpace(string(b)) + if result != text { + c.Errorf("expected pipe to contain %s, got %s", text, result) + } +} diff --git a/components/engine/integration-cli/docker_api_create_test.go b/components/engine/integration-cli/docker_api_create_test.go index e404b6cf58..c2152d32a4 100644 --- a/components/engine/integration-cli/docker_api_create_test.go +++ b/components/engine/integration-cli/docker_api_create_test.go @@ -11,82 +11,6 @@ import ( "github.com/go-check/check" ) -func (s *DockerSuite) TestAPICreateWithNotExistImage(c *check.C) { - name := "test" - config := map[string]interface{}{ - "Image": "test456:v1", - "Volumes": map[string]struct{}{"/tmp": {}}, - } - - status, body, err := request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusNotFound) - expected := "No such image: test456:v1" - c.Assert(getErrorMessage(c, body), checker.Contains, expected) - - config2 := map[string]interface{}{ - "Image": "test456", - "Volumes": map[string]struct{}{"/tmp": {}}, - } - - status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config2, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusNotFound) - expected = "No such image: test456:latest" - c.Assert(getErrorMessage(c, body), checker.Equals, expected) - - config3 := map[string]interface{}{ - "Image": "sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa", - } - - status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config3, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusNotFound) - expected = "No such image: sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa" - c.Assert(getErrorMessage(c, body), checker.Equals, expected) - -} - -// Test for #25099 -func (s *DockerSuite) TestAPICreateEmptyEnv(c *check.C) { - name := "test1" - config := map[string]interface{}{ - "Image": "busybox", - "Env": []string{"", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, - "Cmd": []string{"true"}, - } - - status, body, err := request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusInternalServerError) - expected := "invalid environment variable:" - c.Assert(getErrorMessage(c, body), checker.Contains, expected) - - name = "test2" - config = map[string]interface{}{ - "Image": "busybox", - "Env": []string{"=", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, - "Cmd": []string{"true"}, - } - status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusInternalServerError) - expected = "invalid environment variable: =" - c.Assert(getErrorMessage(c, body), checker.Contains, expected) - - name = "test3" - config = map[string]interface{}{ - "Image": "busybox", - "Env": []string{"=foo", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, - "Cmd": []string{"true"}, - } - status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusInternalServerError) - expected = "invalid environment variable: =foo" - c.Assert(getErrorMessage(c, body), checker.Contains, expected) -} - func (s *DockerSuite) TestAPICreateWithInvalidHealthcheckParams(c *check.C) { // test invalid Interval in Healthcheck: less than 0s name := "test1" diff --git a/components/engine/integration-cli/docker_api_images_test.go b/components/engine/integration-cli/docker_api_images_test.go index d44b307fa1..0cdd50685a 100644 --- a/components/engine/integration-cli/docker_api_images_test.go +++ b/components/engine/integration-cli/docker_api_images_test.go @@ -53,9 +53,7 @@ func (s *DockerSuite) TestAPIImagesFilter(c *check.C) { } func (s *DockerSuite) TestAPIImagesSaveAndLoad(c *check.C) { - // TODO Windows to Windows CI: Investigate further why this test fails. testRequires(c, Network) - testRequires(c, DaemonIsLinux) buildImageSuccessfully(c, "saveandload", build.WithDockerfile("FROM busybox\nENV FOO bar")) id := getIDByName(c, "saveandload") diff --git a/components/engine/integration-cli/docker_api_ipcmode_test.go b/components/engine/integration-cli/docker_api_ipcmode_test.go new file mode 100644 index 0000000000..52dfee85a7 --- /dev/null +++ b/components/engine/integration-cli/docker_api_ipcmode_test.go @@ -0,0 +1,222 @@ +// build +linux +package main + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/integration-cli/checker" + "github.com/docker/docker/integration-cli/cli" + "github.com/docker/docker/integration-cli/request" + "github.com/go-check/check" + "golang.org/x/net/context" +) + +/* testIpcCheckDevExists checks whether a given mount (identified by its + * major:minor pair from /proc/self/mountinfo) exists on the host system. + * + * The format of /proc/self/mountinfo is like: + * + * 29 23 0:24 / /dev/shm rw,nosuid,nodev shared:4 - tmpfs tmpfs rw + * ^^^^\ + * - this is the minor:major we look for + */ +func testIpcCheckDevExists(mm string) (bool, error) { + f, err := os.Open("/proc/self/mountinfo") + if err != nil { + return false, err + } + defer f.Close() + + s := bufio.NewScanner(f) + for s.Scan() { + fields := strings.Fields(s.Text()) + if len(fields) < 7 { + continue + } + if fields[2] == mm { + return true, nil + } + } + + if err := s.Err(); err != nil { + return false, err + } + + return false, nil +} + +// testIpcNonePrivateShareable is a helper function to test "none", +// "private" and "shareable" modes. +func testIpcNonePrivateShareable(c *check.C, mode string, mustBeMounted bool, mustBeShared bool) { + cfg := container.Config{ + Image: "busybox", + Cmd: []string{"top"}, + } + hostCfg := container.HostConfig{ + IpcMode: container.IpcMode(mode), + } + ctx := context.Background() + + client, err := request.NewClient() + c.Assert(err, checker.IsNil) + + resp, err := client.ContainerCreate(ctx, &cfg, &hostCfg, nil, "") + c.Assert(err, checker.IsNil) + c.Assert(len(resp.Warnings), checker.Equals, 0) + + err = client.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}) + c.Assert(err, checker.IsNil) + + // get major:minor pair for /dev/shm from container's /proc/self/mountinfo + cmd := "awk '($5 == \"/dev/shm\") {printf $3}' /proc/self/mountinfo" + mm := cli.DockerCmd(c, "exec", "-i", resp.ID, "sh", "-c", cmd).Combined() + if !mustBeMounted { + c.Assert(mm, checker.Equals, "") + // no more checks to perform + return + } + c.Assert(mm, checker.Matches, "^[0-9]+:[0-9]+$") + + shared, err := testIpcCheckDevExists(mm) + c.Assert(err, checker.IsNil) + c.Logf("[testIpcPrivateShareable] ipcmode: %v, ipcdev: %v, shared: %v, mustBeShared: %v\n", mode, mm, shared, mustBeShared) + c.Assert(shared, checker.Equals, mustBeShared) +} + +/* TestAPIIpcModeNone checks the container "none" IPC mode + * (--ipc none) works as expected. It makes sure there is no + * /dev/shm mount inside the container. + */ +func (s *DockerSuite) TestAPIIpcModeNone(c *check.C) { + testRequires(c, DaemonIsLinux) + testIpcNonePrivateShareable(c, "none", false, false) +} + +/* TestAPIIpcModePrivate checks the container private IPC mode + * (--ipc private) works as expected. It gets the minor:major pair + * of /dev/shm mount from the container, and makes sure there is no + * such pair on the host. + */ +func (s *DockerSuite) TestAPIIpcModePrivate(c *check.C) { + testRequires(c, DaemonIsLinux) + testIpcNonePrivateShareable(c, "private", true, false) +} + +/* TestAPIIpcModeShareable checks the container shareable IPC mode + * (--ipc shareable) works as expected. It gets the minor:major pair + * of /dev/shm mount from the container, and makes sure such pair + * also exists on the host. + */ +func (s *DockerSuite) TestAPIIpcModeShareable(c *check.C) { + testRequires(c, DaemonIsLinux) + testIpcNonePrivateShareable(c, "shareable", true, true) +} + +// testIpcContainer is a helper function to test --ipc container:NNN mode in various scenarios +func testIpcContainer(s *DockerSuite, c *check.C, donorMode string, mustWork bool) { + cfg := container.Config{ + Image: "busybox", + Cmd: []string{"top"}, + } + hostCfg := container.HostConfig{ + IpcMode: container.IpcMode(donorMode), + } + ctx := context.Background() + + client, err := request.NewClient() + c.Assert(err, checker.IsNil) + + // create and start the "donor" container + resp, err := client.ContainerCreate(ctx, &cfg, &hostCfg, nil, "") + c.Assert(err, checker.IsNil) + c.Assert(len(resp.Warnings), checker.Equals, 0) + name1 := resp.ID + + err = client.ContainerStart(ctx, name1, types.ContainerStartOptions{}) + c.Assert(err, checker.IsNil) + + // create and start the second container + hostCfg.IpcMode = container.IpcMode("container:" + name1) + resp, err = client.ContainerCreate(ctx, &cfg, &hostCfg, nil, "") + c.Assert(err, checker.IsNil) + c.Assert(len(resp.Warnings), checker.Equals, 0) + name2 := resp.ID + + err = client.ContainerStart(ctx, name2, types.ContainerStartOptions{}) + if !mustWork { + // start should fail with a specific error + c.Assert(err, checker.NotNil) + c.Assert(fmt.Sprintf("%v", err), checker.Contains, "non-shareable IPC") + // no more checks to perform here + return + } + + // start should succeed + c.Assert(err, checker.IsNil) + + // check that IPC is shared + // 1. create a file in the first container + cli.DockerCmd(c, "exec", name1, "sh", "-c", "printf covfefe > /dev/shm/bar") + // 2. check it's the same file in the second one + out := cli.DockerCmd(c, "exec", "-i", name2, "cat", "/dev/shm/bar").Combined() + c.Assert(out, checker.Matches, "^covfefe$") +} + +/* TestAPIIpcModeShareableAndContainer checks that a container created with + * --ipc container:ID can use IPC of another shareable container. + */ +func (s *DockerSuite) TestAPIIpcModeShareableAndContainer(c *check.C) { + testRequires(c, DaemonIsLinux) + testIpcContainer(s, c, "shareable", true) +} + +/* TestAPIIpcModePrivateAndContainer checks that a container created with + * --ipc container:ID can NOT use IPC of another private container. + */ +func (s *DockerSuite) TestAPIIpcModePrivateAndContainer(c *check.C) { + testRequires(c, DaemonIsLinux) + testIpcContainer(s, c, "private", false) +} + +/* TestAPIIpcModeHost checks that a container created with --ipc host + * can use IPC of the host system. + */ +func (s *DockerSuite) TestAPIIpcModeHost(c *check.C) { + testRequires(c, DaemonIsLinux) + + cfg := container.Config{ + Image: "busybox", + Cmd: []string{"top"}, + } + hostCfg := container.HostConfig{ + IpcMode: container.IpcMode("host"), + } + ctx := context.Background() + + client, err := request.NewClient() + c.Assert(err, checker.IsNil) + + resp, err := client.ContainerCreate(ctx, &cfg, &hostCfg, nil, "") + c.Assert(err, checker.IsNil) + c.Assert(len(resp.Warnings), checker.Equals, 0) + name := resp.ID + + err = client.ContainerStart(ctx, name, types.ContainerStartOptions{}) + c.Assert(err, checker.IsNil) + + // check that IPC is shared + // 1. create a file inside container + cli.DockerCmd(c, "exec", name, "sh", "-c", "printf covfefe > /dev/shm/."+name) + // 2. check it's the same on the host + bytes, err := ioutil.ReadFile("/dev/shm/." + name) + c.Assert(err, checker.IsNil) + c.Assert(string(bytes), checker.Matches, "^covfefe$") + // 3. clean up + cli.DockerCmd(c, "exec", name, "rm", "-f", "/dev/shm/."+name) +} diff --git a/components/engine/integration-cli/docker_api_swarm_service_test.go b/components/engine/integration-cli/docker_api_swarm_service_test.go index 2ec56ccd8d..1530ea1ab3 100644 --- a/components/engine/integration-cli/docker_api_swarm_service_test.go +++ b/components/engine/integration-cli/docker_api_swarm_service_test.go @@ -603,7 +603,8 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) { // Test plugins deployed via swarm services func (s *DockerSwarmSuite) TestAPISwarmServicesPlugin(c *check.C) { - testRequires(c, DaemonIsLinux, IsAmd64) + testRequires(c, ExperimentalDaemon, DaemonIsLinux, IsAmd64) + reg := setupRegistry(c, false, "", "") defer reg.Close() diff --git a/components/engine/integration-cli/docker_cli_daemon_test.go b/components/engine/integration-cli/docker_cli_daemon_test.go index 6a98cabdfa..55b45a9724 100644 --- a/components/engine/integration-cli/docker_cli_daemon_test.go +++ b/components/engine/integration-cli/docker_cli_daemon_test.go @@ -2985,6 +2985,165 @@ func (s *DockerDaemonSuite) TestShmSizeReload(c *check.C) { c.Assert(strings.TrimSpace(out), check.Equals, fmt.Sprintf("%v", size)) } +// this is used to test both "private" and "shareable" daemon default ipc modes +func testDaemonIpcPrivateShareable(d *daemon.Daemon, c *check.C, mustExist bool) { + name := "test-ipcmode" + _, err := d.Cmd("run", "-d", "--name", name, "busybox", "top") + c.Assert(err, checker.IsNil) + + // get major:minor pair for /dev/shm from container's /proc/self/mountinfo + cmd := "awk '($5 == \"/dev/shm\") {printf $3}' /proc/self/mountinfo" + mm, err := d.Cmd("exec", "-i", name, "sh", "-c", cmd) + c.Assert(err, checker.IsNil) + c.Assert(mm, checker.Matches, "^[0-9]+:[0-9]+$") + + exists, err := testIpcCheckDevExists(mm) + c.Assert(err, checker.IsNil) + c.Logf("[testDaemonIpcPrivateShareable] ipcdev: %v, exists: %v, mustExist: %v\n", mm, exists, mustExist) + c.Assert(exists, checker.Equals, mustExist) +} + +// TestDaemonIpcModeShareable checks that --default-ipc-mode shareable works as intended. +func (s *DockerDaemonSuite) TestDaemonIpcModeShareable(c *check.C) { + testRequires(c, DaemonIsLinux) + + s.d.StartWithBusybox(c, "--default-ipc-mode", "shareable") + testDaemonIpcPrivateShareable(s.d, c, true) +} + +// TestDaemonIpcModePrivate checks that --default-ipc-mode private works as intended. +func (s *DockerDaemonSuite) TestDaemonIpcModePrivate(c *check.C) { + testRequires(c, DaemonIsLinux) + + s.d.StartWithBusybox(c, "--default-ipc-mode", "private") + testDaemonIpcPrivateShareable(s.d, c, false) +} + +// used to check if an IpcMode given in config works as intended +func testDaemonIpcFromConfig(s *DockerDaemonSuite, c *check.C, mode string, mustExist bool) { + f, err := ioutil.TempFile("", "test-daemon-ipc-config") + c.Assert(err, checker.IsNil) + defer os.Remove(f.Name()) + + config := `{"default-ipc-mode": "` + mode + `"}` + _, err = f.WriteString(config) + c.Assert(f.Close(), checker.IsNil) + c.Assert(err, checker.IsNil) + + s.d.StartWithBusybox(c, "--config-file", f.Name()) + testDaemonIpcPrivateShareable(s.d, c, mustExist) +} + +// TestDaemonIpcModePrivateFromConfig checks that "default-ipc-mode: private" config works as intended. +func (s *DockerDaemonSuite) TestDaemonIpcModePrivateFromConfig(c *check.C) { + testDaemonIpcFromConfig(s, c, "private", false) +} + +// TestDaemonIpcModeShareableFromConfig checks that "default-ipc-mode: shareable" config works as intended. +func (s *DockerDaemonSuite) TestDaemonIpcModeShareableFromConfig(c *check.C) { + testDaemonIpcFromConfig(s, c, "shareable", true) +} + +func testDaemonStartIpcMode(c *check.C, from, mode string, valid bool) { + testRequires(c, DaemonIsLinux) + + d := daemon.New(c, dockerBinary, dockerdBinary, daemon.Config{ + Experimental: testEnv.ExperimentalDaemon(), + }) + c.Logf("Checking IpcMode %s set from %s\n", mode, from) + var serr error + switch from { + case "config": + f, err := ioutil.TempFile("", "test-daemon-ipc-config") + c.Assert(err, checker.IsNil) + defer os.Remove(f.Name()) + config := `{"default-ipc-mode": "` + mode + `"}` + _, err = f.WriteString(config) + c.Assert(f.Close(), checker.IsNil) + c.Assert(err, checker.IsNil) + + serr = d.StartWithError("--config-file", f.Name()) + case "cli": + serr = d.StartWithError("--default-ipc-mode", mode) + default: + c.Fatalf("testDaemonStartIpcMode: invalid 'from' argument") + } + if serr == nil { + d.Stop(c) + } + + if valid { + c.Assert(serr, check.IsNil) + } else { + c.Assert(serr, check.NotNil) + icmd.RunCommand("grep", "-E", "IPC .* is (invalid|not supported)", d.LogFileName()).Assert(c, icmd.Success) + } +} + +// TestDaemonStartWithIpcModes checks that daemon starts fine given correct +// arguments for default IPC mode, and bails out with incorrect ones. +// Both CLI option (--default-ipc-mode) and config parameter are tested. +func (s *DockerDaemonSuite) TestDaemonStartWithIpcModes(c *check.C) { + ipcModes := []struct { + mode string + valid bool + }{ + {"private", true}, + {"shareable", true}, + + {"host", false}, + {"container:123", false}, + {"nosuchmode", false}, + } + + for _, from := range []string{"config", "cli"} { + for _, m := range ipcModes { + testDaemonStartIpcMode(c, from, m.mode, m.valid) + } + } +} + +// TestDaemonRestartIpcMode makes sure a container keeps its ipc mode +// (derived from daemon default) even after the daemon is restarted +// with a different default ipc mode. +func (s *DockerDaemonSuite) TestDaemonRestartIpcMode(c *check.C) { + f, err := ioutil.TempFile("", "test-daemon-ipc-config-restart") + c.Assert(err, checker.IsNil) + file := f.Name() + defer os.Remove(file) + c.Assert(f.Close(), checker.IsNil) + + config := []byte(`{"default-ipc-mode": "private"}`) + c.Assert(ioutil.WriteFile(file, config, 0644), checker.IsNil) + s.d.StartWithBusybox(c, "--config-file", file) + + // check the container is created with private ipc mode as per daemon default + name := "ipc1" + _, err = s.d.Cmd("run", "-d", "--name", name, "--restart=always", "busybox", "top") + c.Assert(err, checker.IsNil) + m, err := s.d.InspectField(name, ".HostConfig.IpcMode") + c.Assert(err, check.IsNil) + c.Assert(m, checker.Equals, "private") + + // restart the daemon with shareable default ipc mode + config = []byte(`{"default-ipc-mode": "shareable"}`) + c.Assert(ioutil.WriteFile(file, config, 0644), checker.IsNil) + s.d.Restart(c, "--config-file", file) + + // check the container is still having private ipc mode + m, err = s.d.InspectField(name, ".HostConfig.IpcMode") + c.Assert(err, check.IsNil) + c.Assert(m, checker.Equals, "private") + + // check a new container is created with shareable ipc mode as per new daemon default + name = "ipc2" + _, err = s.d.Cmd("run", "-d", "--name", name, "busybox", "top") + c.Assert(err, checker.IsNil) + m, err = s.d.InspectField(name, ".HostConfig.IpcMode") + c.Assert(err, check.IsNil) + c.Assert(m, checker.Equals, "shareable") +} + // TestFailedPluginRemove makes sure that a failed plugin remove does not block // the daemon from starting func (s *DockerDaemonSuite) TestFailedPluginRemove(c *check.C) { diff --git a/components/engine/integration-cli/docker_cli_events_unix_test.go b/components/engine/integration-cli/docker_cli_events_unix_test.go index 1f87d5fe8c..a2d67069de 100644 --- a/components/engine/integration-cli/docker_cli_events_unix_test.go +++ b/components/engine/integration-cli/docker_cli_events_unix_test.go @@ -428,7 +428,32 @@ func (s *DockerDaemonSuite) TestDaemonEvents(c *check.C) { out, err = s.d.Cmd("events", "--since=0", "--until", daemonUnixTime(c)) c.Assert(err, checker.IsNil) - c.Assert(out, checker.Contains, fmt.Sprintf("daemon reload %s (allow-nondistributable-artifacts=[], cluster-advertise=, cluster-store=, cluster-store-opts={}, debug=true, default-runtime=runc, default-shm-size=67108864, insecure-registries=[], labels=[\"bar=foo\"], live-restore=false, max-concurrent-downloads=1, max-concurrent-uploads=5, name=%s, registry-mirrors=[], runtimes=runc:{docker-runc []}, shutdown-timeout=10)", daemonID, daemonName)) + // only check for values known (daemon ID/name) or explicitly set above, + // otherwise just check for names being present. + expectedSubstrings := []string{ + " daemon reload " + daemonID + " ", + "(allow-nondistributable-artifacts=[", + " cluster-advertise=, ", + " cluster-store=, ", + " cluster-store-opts={", + " debug=true, ", + " default-ipc-mode=", + " default-runtime=", + " default-shm-size=", + " insecure-registries=[", + " labels=[\"bar=foo\"], ", + " live-restore=", + " max-concurrent-downloads=1, ", + " max-concurrent-uploads=5, ", + " name=" + daemonName, + " registry-mirrors=[", + " runtimes=", + " shutdown-timeout=10)", + } + + for _, s := range expectedSubstrings { + c.Assert(out, checker.Contains, s) + } } func (s *DockerDaemonSuite) TestDaemonEventsWithFilters(c *check.C) { diff --git a/components/engine/integration-cli/docker_cli_network_unix_test.go b/components/engine/integration-cli/docker_cli_network_unix_test.go index 05cc078bcb..5685c7bd0f 100644 --- a/components/engine/integration-cli/docker_cli_network_unix_test.go +++ b/components/engine/integration-cli/docker_cli_network_unix_test.go @@ -690,6 +690,21 @@ func (s *DockerNetworkSuite) TestDockerNetworkIPAMOptions(c *check.C) { c.Assert(opts["opt2"], checker.Equals, "drv2") } +func (s *DockerNetworkSuite) TestDockerNetworkNullIPAMDriver(c *check.C) { + // Create a network with null ipam driver + _, _, err := dockerCmdWithError("network", "create", "-d", dummyNetworkDriver, "--ipam-driver", "null", "test000") + c.Assert(err, check.IsNil) + assertNwIsAvailable(c, "test000") + + // Verify the inspect data contains the default subnet provided by the null + // ipam driver and no gateway, as the null ipam driver does not provide one + nr := getNetworkResource(c, "test000") + c.Assert(nr.IPAM.Driver, checker.Equals, "null") + c.Assert(len(nr.IPAM.Config), checker.Equals, 1) + c.Assert(nr.IPAM.Config[0].Subnet, checker.Equals, "0.0.0.0/0") + c.Assert(nr.IPAM.Config[0].Gateway, checker.Equals, "") +} + func (s *DockerNetworkSuite) TestDockerNetworkInspectDefault(c *check.C) { nr := getNetworkResource(c, "none") c.Assert(nr.Driver, checker.Equals, "null") diff --git a/components/engine/integration-cli/docker_cli_run_test.go b/components/engine/integration-cli/docker_cli_run_test.go index 544cfdf9a8..6032de4d93 100644 --- a/components/engine/integration-cli/docker_cli_run_test.go +++ b/components/engine/integration-cli/docker_cli_run_test.go @@ -4610,10 +4610,7 @@ func (s *DockerSuite) TestRunAddDeviceCgroupRule(c *check.C) { // Verifies that running as local system is operating correctly on Windows func (s *DockerSuite) TestWindowsRunAsSystem(c *check.C) { - testRequires(c, DaemonIsWindows) - if testEnv.DaemonKernelVersionNumeric() < 15000 { - c.Skip("Requires build 15000 or later") - } + testRequires(c, DaemonIsWindowsAtLeastBuild(15000)) out, _ := dockerCmd(c, "run", "--net=none", `--user=nt authority\system`, "--hostname=XYZZY", minimalBaseImage(), "cmd", "/c", `@echo %USERNAME%`) c.Assert(strings.TrimSpace(out), checker.Equals, "XYZZY$") } diff --git a/components/engine/integration-cli/environment/protect.go b/components/engine/integration-cli/environment/protect.go index 2b0dd6df2f..05c86b7e4a 100644 --- a/components/engine/integration-cli/environment/protect.go +++ b/components/engine/integration-cli/environment/protect.go @@ -1,5 +1,16 @@ package environment +import ( + "strings" + + "github.com/docker/docker/integration-cli/fixtures/load" + icmd "github.com/docker/docker/pkg/testutil/cmd" +) + +type protectedElements struct { + images map[string]struct{} +} + // ProtectImage adds the specified image(s) to be protected in case of clean func (e *Execution) ProtectImage(t testingT, images ...string) { for _, image := range images { @@ -7,6 +18,31 @@ func (e *Execution) ProtectImage(t testingT, images ...string) { } } -type protectedElements struct { - images map[string]struct{} +// ProtectImages protects existing images and on linux frozen images from being +// cleaned up at the end of test runs +func ProtectImages(t testingT, testEnv *Execution) { + images := getExistingImages(t, testEnv) + + if testEnv.DaemonPlatform() == "linux" { + images = append(images, ensureFrozenImagesLinux(t, testEnv)...) + } + testEnv.ProtectImage(t, images...) +} + +func getExistingImages(t testingT, testEnv *Execution) []string { + // TODO: use API instead of cli + result := icmd.RunCommand(testEnv.dockerBinary, "images", "-f", "dangling=false", "--format", "{{.Repository}}:{{.Tag}}") + result.Assert(t, icmd.Success) + return strings.Split(strings.TrimSpace(result.Stdout()), "\n") +} + +func ensureFrozenImagesLinux(t testingT, testEnv *Execution) []string { + images := []string{"busybox:latest", "hello-world:frozen", "debian:jessie"} + err := load.FrozenImagesLinux(testEnv.DockerBinary(), images...) + if err != nil { + result := icmd.RunCommand(testEnv.DockerBinary(), "image", "ls") + t.Logf(result.String()) + t.Fatalf("%+v", err) + } + return images } diff --git a/components/engine/integration-cli/events_utils_test.go b/components/engine/integration-cli/events_utils_test.go index 9350edcb96..580188950a 100644 --- a/components/engine/integration-cli/events_utils_test.go +++ b/components/engine/integration-cli/events_utils_test.go @@ -9,10 +9,10 @@ import ( "strconv" "strings" - "github.com/Sirupsen/logrus" eventstestutils "github.com/docker/docker/daemon/events/testutils" "github.com/docker/docker/integration-cli/checker" "github.com/go-check/check" + "github.com/sirupsen/logrus" ) // eventMatcher is a function that tries to match an event input. diff --git a/components/engine/integration-cli/fixtures_linux_daemon_test.go b/components/engine/integration-cli/fixtures_linux_daemon_test.go index 895f976a18..0011797c0e 100644 --- a/components/engine/integration-cli/fixtures_linux_daemon_test.go +++ b/components/engine/integration-cli/fixtures_linux_daemon_test.go @@ -24,16 +24,6 @@ type logT interface { Logf(string, ...interface{}) } -func ensureFrozenImagesLinux(t testingT) { - images := []string{"busybox:latest", "hello-world:frozen", "debian:jessie"} - err := load.FrozenImagesLinux(dockerBinary, images...) - if err != nil { - t.Logf(dockerCmdWithError("images")) - t.Fatalf("%+v", err) - } - defer testEnv.ProtectImage(t, images...) -} - var ensureSyscallTestOnce sync.Once func ensureSyscallTest(c *check.C) { diff --git a/components/engine/integration-cli/requirements_test.go b/components/engine/integration-cli/requirements_test.go index 15b3df2265..d6cc27b1d0 100644 --- a/components/engine/integration-cli/requirements_test.go +++ b/components/engine/integration-cli/requirements_test.go @@ -37,6 +37,12 @@ func DaemonIsWindows() bool { return PlatformIs("windows") } +func DaemonIsWindowsAtLeastBuild(buildNumber int) func() bool { + return func() bool { + return DaemonIsWindows() && testEnv.DaemonKernelVersionNumeric() >= buildNumber + } +} + func DaemonIsLinux() bool { return PlatformIs("linux") } diff --git a/components/engine/integration/container/create_test.go b/components/engine/integration/container/create_test.go new file mode 100644 index 0000000000..4b83af41d4 --- /dev/null +++ b/components/engine/integration/container/create_test.go @@ -0,0 +1,93 @@ +package container + +import ( + "context" + "strconv" + "testing" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/integration/util/request" + "github.com/docker/docker/pkg/testutil" +) + +func TestCreateFailsWhenIdentifierDoesNotExist(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + + testCases := []struct { + doc string + image string + expectedError string + }{ + { + doc: "image and tag", + image: "test456:v1", + expectedError: "No such image: test456:v1", + }, + { + doc: "image no tag", + image: "test456", + expectedError: "No such image: test456", + }, + { + doc: "digest", + image: "sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa", + expectedError: "No such image: sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa", + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.doc, func(t *testing.T) { + t.Parallel() + _, err := client.ContainerCreate(context.Background(), + &container.Config{Image: tc.image}, + &container.HostConfig{}, + &network.NetworkingConfig{}, + "foo", + ) + testutil.ErrorContains(t, err, tc.expectedError) + }) + } +} + +func TestCreateWithInvalidEnv(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + + testCases := []struct { + env string + expectedError string + }{ + { + env: "", + expectedError: "invalid environment variable:", + }, + { + env: "=", + expectedError: "invalid environment variable: =", + }, + { + env: "=foo", + expectedError: "invalid environment variable: =foo", + }, + } + + for index, tc := range testCases { + tc := tc + t.Run(strconv.Itoa(index), func(t *testing.T) { + t.Parallel() + _, err := client.ContainerCreate(context.Background(), + &container.Config{ + Image: "busybox", + Env: []string{tc.env}, + }, + &container.HostConfig{}, + &network.NetworkingConfig{}, + "foo", + ) + testutil.ErrorContains(t, err, tc.expectedError) + }) + } +} diff --git a/components/engine/integration/container/main_test.go b/components/engine/integration/container/main_test.go new file mode 100644 index 0000000000..a54b8d9094 --- /dev/null +++ b/components/engine/integration/container/main_test.go @@ -0,0 +1,37 @@ +package container + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/integration-cli/environment" +) + +var ( + testEnv *environment.Execution +) + +func TestMain(m *testing.M) { + var err error + testEnv, err = environment.New() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + // TODO: replace this with `testEnv.Print()` to print the full env + if testEnv.LocalDaemon() { + fmt.Println("INFO: Testing against a local daemon") + } else { + fmt.Println("INFO: Testing against a remote daemon") + } + + res := m.Run() + os.Exit(res) +} + +func setupTest(t *testing.T) func() { + environment.ProtectImages(t, testEnv) + return func() { testEnv.Clean(t, testEnv.DockerBinary()) } +} diff --git a/components/engine/integration/doc.go b/components/engine/integration/doc.go new file mode 100644 index 0000000000..2fdf62eef7 --- /dev/null +++ b/components/engine/integration/doc.go @@ -0,0 +1,3 @@ +// Package integration provides integrations tests for Moby (API). +// These tests require a daemon (dockerd for now) to run. +package integration diff --git a/components/engine/integration/util/request/client.go b/components/engine/integration/util/request/client.go new file mode 100644 index 0000000000..771563673e --- /dev/null +++ b/components/engine/integration/util/request/client.go @@ -0,0 +1,15 @@ +package request + +import ( + "testing" + + "github.com/docker/docker/client" + "github.com/stretchr/testify/require" +) + +// NewAPIClient returns a docker API client configured from environment variables +func NewAPIClient(t *testing.T) client.APIClient { + clt, err := client.NewEnvClient() + require.NoError(t, err) + return clt +} diff --git a/components/engine/layer/filestore.go b/components/engine/layer/filestore.go index 533f45481a..d813555525 100644 --- a/components/engine/layer/filestore.go +++ b/components/engine/layer/filestore.go @@ -13,10 +13,10 @@ import ( "strconv" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/docker/pkg/ioutils" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/layer/layer.go b/components/engine/layer/layer.go index b3480a0cc1..e269ef8a4a 100644 --- a/components/engine/layer/layer.go +++ b/components/engine/layer/layer.go @@ -13,10 +13,10 @@ import ( "errors" "io" - "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/docker/pkg/archive" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/layer/layer_store.go b/components/engine/layer/layer_store.go index 75ac1e4f48..7283014459 100644 --- a/components/engine/layer/layer_store.go +++ b/components/engine/layer/layer_store.go @@ -8,7 +8,6 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/idtools" @@ -16,6 +15,7 @@ import ( "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/system" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" ) diff --git a/components/engine/layer/migration.go b/components/engine/layer/migration.go index 4803a1ae56..f4c4adec4a 100644 --- a/components/engine/layer/migration.go +++ b/components/engine/layer/migration.go @@ -7,8 +7,8 @@ import ( "io" "os" - "github.com/Sirupsen/logrus" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" ) diff --git a/components/engine/libcontainerd/client_linux.go b/components/engine/libcontainerd/client_linux.go index 54eaf35530..6c3460a8c9 100644 --- a/components/engine/libcontainerd/client_linux.go +++ b/components/engine/libcontainerd/client_linux.go @@ -7,7 +7,6 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" containerd "github.com/containerd/containerd/api/grpc/types" containerd_runtime_types "github.com/containerd/containerd/runtime" "github.com/docker/docker/pkg/ioutils" @@ -15,6 +14,7 @@ import ( "github.com/golang/protobuf/ptypes" "github.com/golang/protobuf/ptypes/timestamp" specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "golang.org/x/sys/unix" ) diff --git a/components/engine/libcontainerd/client_unix.go b/components/engine/libcontainerd/client_unix.go index 6dbf3af06e..129e87bd72 100644 --- a/components/engine/libcontainerd/client_unix.go +++ b/components/engine/libcontainerd/client_unix.go @@ -10,10 +10,10 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" containerd "github.com/containerd/containerd/api/grpc/types" "github.com/docker/docker/pkg/idtools" specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/libcontainerd/client_windows.go b/components/engine/libcontainerd/client_windows.go index 455e8e5e6c..db84061254 100644 --- a/components/engine/libcontainerd/client_windows.go +++ b/components/engine/libcontainerd/client_windows.go @@ -15,9 +15,11 @@ import ( "golang.org/x/net/context" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" + opengcs "github.com/Microsoft/opengcs/client" "github.com/docker/docker/pkg/sysinfo" + "github.com/docker/docker/pkg/system" specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" ) type client struct { @@ -229,20 +231,35 @@ func (clnt *client) createWindows(containerID string, checkpoint string, checkpo } // Add the mounts (volumes, bind mounts etc) to the structure - mds := make([]hcsshim.MappedDir, len(spec.Mounts)) - for i, mount := range spec.Mounts { - mds[i] = hcsshim.MappedDir{ - HostPath: mount.Source, - ContainerPath: mount.Destination, - ReadOnly: false, - } - for _, o := range mount.Options { - if strings.ToLower(o) == "ro" { - mds[i].ReadOnly = true + var mds []hcsshim.MappedDir + var mps []hcsshim.MappedPipe + for _, mount := range spec.Mounts { + const pipePrefix = `\\.\pipe\` + if strings.HasPrefix(mount.Destination, pipePrefix) { + mp := hcsshim.MappedPipe{ + HostPath: mount.Source, + ContainerPipeName: mount.Destination[len(pipePrefix):], } + mps = append(mps, mp) + } else { + md := hcsshim.MappedDir{ + HostPath: mount.Source, + ContainerPath: mount.Destination, + ReadOnly: false, + } + for _, o := range mount.Options { + if strings.ToLower(o) == "ro" { + md.ReadOnly = true + } + } + mds = append(mds, md) } } configuration.MappedDirectories = mds + if len(mps) > 0 && system.GetOSVersion().Build < 16210 { // replace with Win10 RS3 build number at RTM + return errors.New("named pipe mounts are not supported on this version of Windows") + } + configuration.MappedPipes = mps hcsContainer, err := hcsshim.CreateContainer(containerID, configuration) if err != nil { @@ -289,8 +306,20 @@ func (clnt *client) createWindows(containerID string, checkpoint string, checkpo func (clnt *client) createLinux(containerID string, checkpoint string, checkpointDir string, spec specs.Spec, attachStdio StdioCallback, options ...CreateOption) error { logrus.Debugf("libcontainerd: createLinux(): containerId %s ", containerID) - // TODO @jhowardmsft LCOW Support: This needs to be configurable, not hard-coded. - // However, good-enough for the LCOW bring-up. + var layerOpt *LayerOption + var lcowOpt *LCOWOption + for _, option := range options { + if layer, ok := option.(*LayerOption); ok { + layerOpt = layer + } + if lcow, ok := option.(*LCOWOption); ok { + lcowOpt = lcow + } + } + if lcowOpt == nil || lcowOpt.Config == nil { + return fmt.Errorf("lcow option must be supplied to the runtime") + } + configuration := &hcsshim.ContainerConfig{ HvPartition: true, Name: containerID, @@ -298,17 +327,18 @@ func (clnt *client) createLinux(containerID string, checkpoint string, checkpoin ContainerType: "linux", Owner: defaultOwner, TerminateOnLastHandleClosed: true, - HvRuntime: &hcsshim.HvRuntime{ - ImagePath: `c:\Program Files\Linux Containers`, - LinuxKernelFile: `bootx64.efi`, - LinuxInitrdFile: `initrd.img`, - }, } - var layerOpt *LayerOption - for _, option := range options { - if l, ok := option.(*LayerOption); ok { - layerOpt = l + if lcowOpt.Config.ActualMode == opengcs.ModeActualVhdx { + configuration.HvRuntime = &hcsshim.HvRuntime{ + ImagePath: lcowOpt.Config.Vhdx, + } + } else { + configuration.HvRuntime = &hcsshim.HvRuntime{ + ImagePath: lcowOpt.Config.KirdPath, + LinuxKernelFile: lcowOpt.Config.KernelFile, + LinuxInitrdFile: lcowOpt.Config.InitrdFile, + LinuxBootParameters: lcowOpt.Config.BootParameters, } } diff --git a/components/engine/libcontainerd/container_unix.go b/components/engine/libcontainerd/container_unix.go index 869f88523f..9a7dbf01cd 100644 --- a/components/engine/libcontainerd/container_unix.go +++ b/components/engine/libcontainerd/container_unix.go @@ -11,10 +11,10 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" containerd "github.com/containerd/containerd/api/grpc/types" "github.com/docker/docker/pkg/ioutils" specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" "github.com/tonistiigi/fifo" "golang.org/x/net/context" "golang.org/x/sys/unix" diff --git a/components/engine/libcontainerd/container_windows.go b/components/engine/libcontainerd/container_windows.go index e895fa0302..33480514e5 100644 --- a/components/engine/libcontainerd/container_windows.go +++ b/components/engine/libcontainerd/container_windows.go @@ -9,9 +9,8 @@ import ( "time" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" - "github.com/docker/docker/pkg/system" "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) @@ -89,8 +88,8 @@ func (ctr *container) start(attachStdio StdioCallback) error { } createProcessParms.User = ctr.ociSpec.Process.User.Username - // LCOW requires the raw OCI spec passed through HCS and onwards to GCS for the utility VM. - if system.LCOWSupported() && ctr.ociSpec.Platform.OS == "linux" { + // Linux containers requires the raw OCI spec passed through HCS and onwards to GCS for the utility VM. + if ctr.ociSpec.Platform.OS == "linux" { ociBuf, err := json.Marshal(ctr.ociSpec) if err != nil { return err diff --git a/components/engine/libcontainerd/oom_linux.go b/components/engine/libcontainerd/oom_linux.go index e126b7a550..70f0daca5f 100644 --- a/components/engine/libcontainerd/oom_linux.go +++ b/components/engine/libcontainerd/oom_linux.go @@ -5,8 +5,8 @@ import ( "os" "strconv" - "github.com/Sirupsen/logrus" "github.com/opencontainers/runc/libcontainer/system" + "github.com/sirupsen/logrus" ) func setOOMScore(pid, score int) error { diff --git a/components/engine/libcontainerd/remote_unix.go b/components/engine/libcontainerd/remote_unix.go index 24fbc5ad60..7bab53e796 100644 --- a/components/engine/libcontainerd/remote_unix.go +++ b/components/engine/libcontainerd/remote_unix.go @@ -17,12 +17,12 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" containerd "github.com/containerd/containerd/api/grpc/types" "github.com/docker/docker/pkg/locker" "github.com/docker/docker/pkg/system" "github.com/golang/protobuf/ptypes" "github.com/golang/protobuf/ptypes/timestamp" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "golang.org/x/sys/unix" "google.golang.org/grpc" diff --git a/components/engine/libcontainerd/types_windows.go b/components/engine/libcontainerd/types_windows.go index 317bfb0209..1ffe0f40aa 100644 --- a/components/engine/libcontainerd/types_windows.go +++ b/components/engine/libcontainerd/types_windows.go @@ -2,6 +2,7 @@ package libcontainerd import ( "github.com/Microsoft/hcsshim" + opengcs "github.com/Microsoft/opengcs/client" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -25,6 +26,11 @@ type Stats hcsshim.Statistics // Resources defines updatable container resource values. type Resources struct{} +// LCOWOption is a CreateOption required for LCOW configuration +type LCOWOption struct { + Config *opengcs.Config +} + // ServicingOption is a CreateOption with a no-op application that signifies // the container needs to be used for a Windows servicing operation. type ServicingOption struct { diff --git a/components/engine/libcontainerd/utils_windows.go b/components/engine/libcontainerd/utils_windows.go index 41ac40d2c2..e741a296f0 100644 --- a/components/engine/libcontainerd/utils_windows.go +++ b/components/engine/libcontainerd/utils_windows.go @@ -44,3 +44,8 @@ func (s *NetworkEndpointsOption) Apply(interface{}) error { func (s *CredentialsOption) Apply(interface{}) error { return nil } + +// Apply for the LCOW option is a no-op. +func (s *LCOWOption) Apply(interface{}) error { + return nil +} diff --git a/components/engine/migrate/v1/migratev1.go b/components/engine/migrate/v1/migratev1.go index 3cb8828915..60dbfe0787 100644 --- a/components/engine/migrate/v1/migratev1.go +++ b/components/engine/migrate/v1/migratev1.go @@ -13,7 +13,6 @@ import ( "encoding/json" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/docker/distribution/metadata" "github.com/docker/docker/image" @@ -22,6 +21,7 @@ import ( "github.com/docker/docker/pkg/ioutils" refstore "github.com/docker/docker/reference" "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" ) type graphIDRegistrar interface { diff --git a/components/engine/oci/defaults.go b/components/engine/oci/defaults.go index 4376faf5a7..c1d5909713 100644 --- a/components/engine/oci/defaults.go +++ b/components/engine/oci/defaults.go @@ -91,7 +91,7 @@ func DefaultLinuxSpec() specs.Spec { Destination: "/dev", Type: "tmpfs", Source: "tmpfs", - Options: []string{"nosuid", "strictatime", "mode=755"}, + Options: []string{"nosuid", "strictatime", "mode=755", "size=65536k"}, }, { Destination: "/dev/pts", @@ -117,6 +117,12 @@ func DefaultLinuxSpec() specs.Spec { Source: "mqueue", Options: []string{"nosuid", "noexec", "nodev"}, }, + { + Destination: "/dev/shm", + Type: "tmpfs", + Source: "shm", + Options: []string{"nosuid", "noexec", "nodev", "mode=1777"}, + }, } s.Process.Capabilities = &specs.LinuxCapabilities{ Bounding: defaultCapabilities(), diff --git a/components/engine/opts/opts.go b/components/engine/opts/opts.go index 8d82f76792..300fb426ad 100644 --- a/components/engine/opts/opts.go +++ b/components/engine/opts/opts.go @@ -7,7 +7,10 @@ import ( "regexp" "strings" + "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/daemon/cluster/convert" units "github.com/docker/go-units" + "github.com/docker/swarmkit/api/genericresource" ) var ( @@ -325,3 +328,19 @@ func (m *MemBytes) UnmarshalJSON(s []byte) error { *m = MemBytes(val) return err } + +// ParseGenericResources parses and validates the specified string as a list of GenericResource +func ParseGenericResources(value string) ([]swarm.GenericResource, error) { + if value == "" { + return nil, nil + } + + resources, err := genericresource.Parse(value) + if err != nil { + return nil, err + } + + obj := convert.GenericResourcesFromGRPC(resources) + + return obj, nil +} diff --git a/components/engine/pkg/archive/archive.go b/components/engine/pkg/archive/archive.go index 6cbc2e2bd2..b3c84ee156 100644 --- a/components/engine/pkg/archive/archive.go +++ b/components/engine/pkg/archive/archive.go @@ -16,13 +16,13 @@ import ( "strings" "syscall" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/promise" "github.com/docker/docker/pkg/system" + "github.com/sirupsen/logrus" ) type ( diff --git a/components/engine/pkg/archive/archive_linux_test.go b/components/engine/pkg/archive/archive_linux_test.go index f219b3e67e..da683c8c96 100644 --- a/components/engine/pkg/archive/archive_linux_test.go +++ b/components/engine/pkg/archive/archive_linux_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/docker/docker/pkg/system" + "github.com/stretchr/testify/require" "golang.org/x/sys/unix" ) @@ -22,46 +23,37 @@ import ( // └── f1 # whiteout, 0644 func setupOverlayTestDir(t *testing.T, src string) { // Create opaque directory containing single file and permission 0700 - if err := os.Mkdir(filepath.Join(src, "d1"), 0700); err != nil { - t.Fatal(err) - } + err := os.Mkdir(filepath.Join(src, "d1"), 0700) + require.NoError(t, err) - if err := system.Lsetxattr(filepath.Join(src, "d1"), "trusted.overlay.opaque", []byte("y"), 0); err != nil { - t.Fatal(err) - } + err = system.Lsetxattr(filepath.Join(src, "d1"), "trusted.overlay.opaque", []byte("y"), 0) + require.NoError(t, err) - if err := ioutil.WriteFile(filepath.Join(src, "d1", "f1"), []byte{}, 0600); err != nil { - t.Fatal(err) - } + err = ioutil.WriteFile(filepath.Join(src, "d1", "f1"), []byte{}, 0600) + require.NoError(t, err) // Create another opaque directory containing single file but with permission 0750 - if err := os.Mkdir(filepath.Join(src, "d2"), 0750); err != nil { - t.Fatal(err) - } + err = os.Mkdir(filepath.Join(src, "d2"), 0750) + require.NoError(t, err) - if err := system.Lsetxattr(filepath.Join(src, "d2"), "trusted.overlay.opaque", []byte("y"), 0); err != nil { - t.Fatal(err) - } + err = system.Lsetxattr(filepath.Join(src, "d2"), "trusted.overlay.opaque", []byte("y"), 0) + require.NoError(t, err) - if err := ioutil.WriteFile(filepath.Join(src, "d2", "f1"), []byte{}, 0660); err != nil { - t.Fatal(err) - } + err = ioutil.WriteFile(filepath.Join(src, "d2", "f1"), []byte{}, 0660) + require.NoError(t, err) // Create regular directory with deleted file - if err := os.Mkdir(filepath.Join(src, "d3"), 0700); err != nil { - t.Fatal(err) - } + err = os.Mkdir(filepath.Join(src, "d3"), 0700) + require.NoError(t, err) - if err := system.Mknod(filepath.Join(src, "d3", "f1"), unix.S_IFCHR, 0); err != nil { - t.Fatal(err) - } + err = system.Mknod(filepath.Join(src, "d3", "f1"), unix.S_IFCHR, 0) + require.NoError(t, err) } func checkOpaqueness(t *testing.T, path string, opaque string) { xattrOpaque, err := system.Lgetxattr(path, "trusted.overlay.opaque") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + if string(xattrOpaque) != opaque { t.Fatalf("Unexpected opaque value: %q, expected %q", string(xattrOpaque), opaque) } @@ -70,9 +62,8 @@ func checkOpaqueness(t *testing.T, path string, opaque string) { func checkOverlayWhiteout(t *testing.T, path string) { stat, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + statT, ok := stat.Sys().(*syscall.Stat_t) if !ok { t.Fatalf("Unexpected type: %t, expected *syscall.Stat_t", stat.Sys()) @@ -84,9 +75,8 @@ func checkOverlayWhiteout(t *testing.T, path string) { func checkFileMode(t *testing.T, path string, perm os.FileMode) { stat, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + if stat.Mode() != perm { t.Fatalf("Unexpected file mode for %s: %o, expected %o", path, stat.Mode(), perm) } @@ -94,23 +84,17 @@ func checkFileMode(t *testing.T, path string, perm os.FileMode) { func TestOverlayTarUntar(t *testing.T) { oldmask, err := system.Umask(0) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer system.Umask(oldmask) src, err := ioutil.TempDir("", "docker-test-overlay-tar-src") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(src) setupOverlayTestDir(t, src) dst, err := ioutil.TempDir("", "docker-test-overlay-tar-dst") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(dst) options := &TarOptions{ @@ -118,14 +102,11 @@ func TestOverlayTarUntar(t *testing.T) { WhiteoutFormat: OverlayWhiteoutFormat, } archive, err := TarWithOptions(src, options) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer archive.Close() - if err := Untar(archive, dst, options); err != nil { - t.Fatal(err) - } + err = Untar(archive, dst, options) + require.NoError(t, err) checkFileMode(t, filepath.Join(dst, "d1"), 0700|os.ModeDir) checkFileMode(t, filepath.Join(dst, "d2"), 0750|os.ModeDir) @@ -142,40 +123,31 @@ func TestOverlayTarUntar(t *testing.T) { func TestOverlayTarAUFSUntar(t *testing.T) { oldmask, err := system.Umask(0) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer system.Umask(oldmask) src, err := ioutil.TempDir("", "docker-test-overlay-tar-src") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(src) setupOverlayTestDir(t, src) dst, err := ioutil.TempDir("", "docker-test-overlay-tar-dst") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(dst) archive, err := TarWithOptions(src, &TarOptions{ Compression: Uncompressed, WhiteoutFormat: OverlayWhiteoutFormat, }) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer archive.Close() - if err := Untar(archive, dst, &TarOptions{ + err = Untar(archive, dst, &TarOptions{ Compression: Uncompressed, WhiteoutFormat: AUFSWhiteoutFormat, - }); err != nil { - t.Fatal(err) - } + }) + require.NoError(t, err) checkFileMode(t, filepath.Join(dst, "d1"), 0700|os.ModeDir) checkFileMode(t, filepath.Join(dst, "d1", WhiteoutOpaqueDir), 0700) diff --git a/components/engine/pkg/archive/archive_test.go b/components/engine/pkg/archive/archive_test.go index 1371b8ab12..3e66a537e3 100644 --- a/components/engine/pkg/archive/archive_test.go +++ b/components/engine/pkg/archive/archive_test.go @@ -263,9 +263,7 @@ func TestCmdStreamGood(t *testing.T) { func TestUntarPathWithInvalidDest(t *testing.T) { tempFolder, err := ioutil.TempDir("", "docker-archive-test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(tempFolder) invalidDestFolder := filepath.Join(tempFolder, "invalidDest") // Create a src file @@ -284,9 +282,7 @@ func TestUntarPathWithInvalidDest(t *testing.T) { cmd := exec.Command("sh", "-c", "tar cf "+tarFileU+" "+srcFileU) _, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) err = defaultUntarPath(tarFile, invalidDestFolder) if err == nil { @@ -308,9 +304,7 @@ func TestUntarPathWithInvalidSrc(t *testing.T) { func TestUntarPath(t *testing.T) { tmpFolder, err := ioutil.TempDir("", "docker-archive-test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(tmpFolder) srcFile := filepath.Join(tmpFolder, "src") tarFile := filepath.Join(tmpFolder, "src.tar") @@ -331,9 +325,7 @@ func TestUntarPath(t *testing.T) { } cmd := exec.Command("sh", "-c", "tar cf "+tarFileU+" "+srcFileU) _, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) err = defaultUntarPath(tarFile, destFolder) if err != nil { diff --git a/components/engine/pkg/archive/archive_unix_test.go b/components/engine/pkg/archive/archive_unix_test.go index 90f8adaa5d..2a628f4695 100644 --- a/components/engine/pkg/archive/archive_unix_test.go +++ b/components/engine/pkg/archive/archive_unix_test.go @@ -13,6 +13,8 @@ import ( "testing" "github.com/docker/docker/pkg/system" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/sys/unix" ) @@ -70,60 +72,89 @@ func TestChmodTarEntry(t *testing.T) { func TestTarWithHardLink(t *testing.T) { origin, err := ioutil.TempDir("", "docker-test-tar-hardlink") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(origin) - if err := ioutil.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0700); err != nil { - t.Fatal(err) - } - if err := os.Link(filepath.Join(origin, "1"), filepath.Join(origin, "2")); err != nil { - t.Fatal(err) - } + + err = ioutil.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0700) + require.NoError(t, err) + + err = os.Link(filepath.Join(origin, "1"), filepath.Join(origin, "2")) + require.NoError(t, err) var i1, i2 uint64 - if i1, err = getNlink(filepath.Join(origin, "1")); err != nil { - t.Fatal(err) - } + i1, err = getNlink(filepath.Join(origin, "1")) + require.NoError(t, err) + // sanity check that we can hardlink if i1 != 2 { t.Skipf("skipping since hardlinks don't work here; expected 2 links, got %d", i1) } dest, err := ioutil.TempDir("", "docker-test-tar-hardlink-dest") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(dest) // we'll do this in two steps to separate failure fh, err := Tar(origin, Uncompressed) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // ensure we can read the whole thing with no error, before writing back out buf, err := ioutil.ReadAll(fh) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) bRdr := bytes.NewReader(buf) err = Untar(bRdr, dest, &TarOptions{Compression: Uncompressed}) - if err != nil { - t.Fatal(err) + require.NoError(t, err) + + i1, err = getInode(filepath.Join(dest, "1")) + require.NoError(t, err) + + i2, err = getInode(filepath.Join(dest, "2")) + require.NoError(t, err) + + assert.Equal(t, i1, i2) +} + +func TestTarWithHardLinkAndRebase(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "docker-test-tar-hardlink-rebase") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + origin := filepath.Join(tmpDir, "origin") + err = os.Mkdir(origin, 0700) + require.NoError(t, err) + + err = ioutil.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0700) + require.NoError(t, err) + + err = os.Link(filepath.Join(origin, "1"), filepath.Join(origin, "2")) + require.NoError(t, err) + + var i1, i2 uint64 + i1, err = getNlink(filepath.Join(origin, "1")) + require.NoError(t, err) + + // sanity check that we can hardlink + if i1 != 2 { + t.Skipf("skipping since hardlinks don't work here; expected 2 links, got %d", i1) } - if i1, err = getInode(filepath.Join(dest, "1")); err != nil { - t.Fatal(err) - } - if i2, err = getInode(filepath.Join(dest, "2")); err != nil { - t.Fatal(err) - } + dest := filepath.Join(tmpDir, "dest") + bRdr, err := TarResourceRebase(origin, "origin") + require.NoError(t, err) - if i1 != i2 { - t.Errorf("expected matching inodes, but got %d and %d", i1, i2) - } + dstDir, srcBase := SplitPathDirEntry(origin) + _, dstBase := SplitPathDirEntry(dest) + content := RebaseArchiveEntries(bRdr, srcBase, dstBase) + err = Untar(content, dstDir, &TarOptions{Compression: Uncompressed, NoLchown: true, NoOverwriteDirNonDir: true}) + require.NoError(t, err) + + i1, err = getInode(filepath.Join(dest, "1")) + require.NoError(t, err) + i2, err = getInode(filepath.Join(dest, "2")) + require.NoError(t, err) + + assert.Equal(t, i1, i2) } func getNlink(path string) (uint64, error) { @@ -153,51 +184,38 @@ func getInode(path string) (uint64, error) { func TestTarWithBlockCharFifo(t *testing.T) { origin, err := ioutil.TempDir("", "docker-test-tar-hardlink") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + defer os.RemoveAll(origin) - if err := ioutil.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0700); err != nil { - t.Fatal(err) - } - if err := system.Mknod(filepath.Join(origin, "2"), unix.S_IFBLK, int(system.Mkdev(int64(12), int64(5)))); err != nil { - t.Fatal(err) - } - if err := system.Mknod(filepath.Join(origin, "3"), unix.S_IFCHR, int(system.Mkdev(int64(12), int64(5)))); err != nil { - t.Fatal(err) - } - if err := system.Mknod(filepath.Join(origin, "4"), unix.S_IFIFO, int(system.Mkdev(int64(12), int64(5)))); err != nil { - t.Fatal(err) - } + err = ioutil.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0700) + require.NoError(t, err) + + err = system.Mknod(filepath.Join(origin, "2"), unix.S_IFBLK, int(system.Mkdev(int64(12), int64(5)))) + require.NoError(t, err) + err = system.Mknod(filepath.Join(origin, "3"), unix.S_IFCHR, int(system.Mkdev(int64(12), int64(5)))) + require.NoError(t, err) + err = system.Mknod(filepath.Join(origin, "4"), unix.S_IFIFO, int(system.Mkdev(int64(12), int64(5)))) + require.NoError(t, err) dest, err := ioutil.TempDir("", "docker-test-tar-hardlink-dest") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(dest) // we'll do this in two steps to separate failure fh, err := Tar(origin, Uncompressed) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // ensure we can read the whole thing with no error, before writing back out buf, err := ioutil.ReadAll(fh) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) bRdr := bytes.NewReader(buf) err = Untar(bRdr, dest, &TarOptions{Compression: Uncompressed}) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) changes, err := ChangesDirs(origin, dest) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + if len(changes) > 0 { t.Fatalf("Tar with special device (block, char, fifo) should keep them (recreate them when untar) : %v", changes) } @@ -209,22 +227,17 @@ func TestTarUntarWithXattr(t *testing.T) { t.Skip() } origin, err := ioutil.TempDir("", "docker-test-untar-origin") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(origin) - if err := ioutil.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0700); err != nil { - t.Fatal(err) - } - if err := ioutil.WriteFile(filepath.Join(origin, "2"), []byte("welcome!"), 0700); err != nil { - t.Fatal(err) - } - if err := ioutil.WriteFile(filepath.Join(origin, "3"), []byte("will be ignored"), 0700); err != nil { - t.Fatal(err) - } - if err := system.Lsetxattr(filepath.Join(origin, "2"), "security.capability", []byte{0x00}, 0); err != nil { - t.Fatal(err) - } + err = ioutil.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0700) + require.NoError(t, err) + + err = ioutil.WriteFile(filepath.Join(origin, "2"), []byte("welcome!"), 0700) + require.NoError(t, err) + err = ioutil.WriteFile(filepath.Join(origin, "3"), []byte("will be ignored"), 0700) + require.NoError(t, err) + err = system.Lsetxattr(filepath.Join(origin, "2"), "security.capability", []byte{0x00}, 0) + require.NoError(t, err) for _, c := range []Compression{ Uncompressed, diff --git a/components/engine/pkg/archive/changes.go b/components/engine/pkg/archive/changes.go index 5ca39b7215..341c66dea7 100644 --- a/components/engine/pkg/archive/changes.go +++ b/components/engine/pkg/archive/changes.go @@ -13,10 +13,10 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/system" + "github.com/sirupsen/logrus" ) // ChangeType represents the change type. diff --git a/components/engine/pkg/archive/changes_test.go b/components/engine/pkg/archive/changes_test.go index c5d1629e72..8c14a867ae 100644 --- a/components/engine/pkg/archive/changes_test.go +++ b/components/engine/pkg/archive/changes_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/docker/docker/pkg/system" + "github.com/stretchr/testify/require" ) func max(x, y int) int { @@ -79,24 +80,20 @@ func createSampleDir(t *testing.T, root string) { for _, info := range files { p := path.Join(root, info.path) if info.filetype == Dir { - if err := os.MkdirAll(p, info.permissions); err != nil { - t.Fatal(err) - } + err := os.MkdirAll(p, info.permissions) + require.NoError(t, err) } else if info.filetype == Regular { - if err := ioutil.WriteFile(p, []byte(info.contents), info.permissions); err != nil { - t.Fatal(err) - } + err := ioutil.WriteFile(p, []byte(info.contents), info.permissions) + require.NoError(t, err) } else if info.filetype == Symlink { - if err := os.Symlink(info.contents, p); err != nil { - t.Fatal(err) - } + err := os.Symlink(info.contents, p) + require.NoError(t, err) } if info.filetype != Symlink { // Set a consistent ctime, atime for all files and dirs - if err := system.Chtimes(p, now, now); err != nil { - t.Fatal(err) - } + err := system.Chtimes(p, now, now) + require.NoError(t, err) } } } @@ -126,20 +123,14 @@ func TestChangesWithNoChanges(t *testing.T) { t.Skip("symlinks on Windows") } rwLayer, err := ioutil.TempDir("", "docker-changes-test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(rwLayer) layer, err := ioutil.TempDir("", "docker-changes-test-layer") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(layer) createSampleDir(t, layer) changes, err := Changes([]string{layer}, rwLayer) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) if len(changes) != 0 { t.Fatalf("Changes with no difference should have detect no changes, but detected %d", len(changes)) } @@ -153,18 +144,14 @@ func TestChangesWithChanges(t *testing.T) { } // Mock the readonly layer layer, err := ioutil.TempDir("", "docker-changes-test-layer") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(layer) createSampleDir(t, layer) os.MkdirAll(path.Join(layer, "dir1/subfolder"), 0740) // Mock the RW layer rwLayer, err := ioutil.TempDir("", "docker-changes-test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(rwLayer) // Create a folder in RW layer @@ -181,9 +168,7 @@ func TestChangesWithChanges(t *testing.T) { ioutil.WriteFile(newFile, []byte{}, 0740) changes, err := Changes([]string{layer}, rwLayer) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) expectedChanges := []Change{ {"/dir1", ChangeModify}, @@ -224,9 +209,7 @@ func TestChangesWithChangesGH13590(t *testing.T) { ioutil.WriteFile(file, []byte("bye"), 0666) changes, err := Changes([]string{baseLayer}, layer) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) expectedChanges := []Change{ {"/dir1/dir2/dir3", ChangeModify}, @@ -246,9 +229,7 @@ func TestChangesWithChangesGH13590(t *testing.T) { ioutil.WriteFile(file, []byte("bye"), 0666) changes, err = Changes([]string{baseLayer}, layer) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) expectedChanges = []Change{ {"/dir1/dir2/dir3/file.txt", ChangeModify}, @@ -265,20 +246,15 @@ func TestChangesDirsEmpty(t *testing.T) { t.Skip("symlinks on Windows; gcp failure on Solaris") } src, err := ioutil.TempDir("", "docker-changes-test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(src) createSampleDir(t, src) dst := src + "-copy" - if err := copyDir(src, dst); err != nil { - t.Fatal(err) - } + err = copyDir(src, dst) + require.NoError(t, err) defer os.RemoveAll(dst) changes, err := ChangesDirs(dst, src) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) if len(changes) != 0 { t.Fatalf("Reported changes for identical dirs: %v", changes) @@ -289,81 +265,65 @@ func TestChangesDirsEmpty(t *testing.T) { func mutateSampleDir(t *testing.T, root string) { // Remove a regular file - if err := os.RemoveAll(path.Join(root, "file1")); err != nil { - t.Fatal(err) - } + err := os.RemoveAll(path.Join(root, "file1")) + require.NoError(t, err) // Remove a directory - if err := os.RemoveAll(path.Join(root, "dir1")); err != nil { - t.Fatal(err) - } + err = os.RemoveAll(path.Join(root, "dir1")) + require.NoError(t, err) // Remove a symlink - if err := os.RemoveAll(path.Join(root, "symlink1")); err != nil { - t.Fatal(err) - } + err = os.RemoveAll(path.Join(root, "symlink1")) + require.NoError(t, err) // Rewrite a file - if err := ioutil.WriteFile(path.Join(root, "file2"), []byte("fileNN\n"), 0777); err != nil { - t.Fatal(err) - } + err = ioutil.WriteFile(path.Join(root, "file2"), []byte("fileNN\n"), 0777) + require.NoError(t, err) // Replace a file - if err := os.RemoveAll(path.Join(root, "file3")); err != nil { - t.Fatal(err) - } - if err := ioutil.WriteFile(path.Join(root, "file3"), []byte("fileMM\n"), 0404); err != nil { - t.Fatal(err) - } + err = os.RemoveAll(path.Join(root, "file3")) + require.NoError(t, err) + err = ioutil.WriteFile(path.Join(root, "file3"), []byte("fileMM\n"), 0404) + require.NoError(t, err) // Touch file - if err := system.Chtimes(path.Join(root, "file4"), time.Now().Add(time.Second), time.Now().Add(time.Second)); err != nil { - t.Fatal(err) - } + err = system.Chtimes(path.Join(root, "file4"), time.Now().Add(time.Second), time.Now().Add(time.Second)) + require.NoError(t, err) // Replace file with dir - if err := os.RemoveAll(path.Join(root, "file5")); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(path.Join(root, "file5"), 0666); err != nil { - t.Fatal(err) - } + err = os.RemoveAll(path.Join(root, "file5")) + require.NoError(t, err) + err = os.MkdirAll(path.Join(root, "file5"), 0666) + require.NoError(t, err) // Create new file - if err := ioutil.WriteFile(path.Join(root, "filenew"), []byte("filenew\n"), 0777); err != nil { - t.Fatal(err) - } + err = ioutil.WriteFile(path.Join(root, "filenew"), []byte("filenew\n"), 0777) + require.NoError(t, err) // Create new dir - if err := os.MkdirAll(path.Join(root, "dirnew"), 0766); err != nil { - t.Fatal(err) - } + err = os.MkdirAll(path.Join(root, "dirnew"), 0766) + require.NoError(t, err) // Create a new symlink - if err := os.Symlink("targetnew", path.Join(root, "symlinknew")); err != nil { - t.Fatal(err) - } + err = os.Symlink("targetnew", path.Join(root, "symlinknew")) + require.NoError(t, err) // Change a symlink - if err := os.RemoveAll(path.Join(root, "symlink2")); err != nil { - t.Fatal(err) - } - if err := os.Symlink("target2change", path.Join(root, "symlink2")); err != nil { - t.Fatal(err) - } + err = os.RemoveAll(path.Join(root, "symlink2")) + require.NoError(t, err) + + err = os.Symlink("target2change", path.Join(root, "symlink2")) + require.NoError(t, err) // Replace dir with file - if err := os.RemoveAll(path.Join(root, "dir2")); err != nil { - t.Fatal(err) - } - if err := ioutil.WriteFile(path.Join(root, "dir2"), []byte("dir2\n"), 0777); err != nil { - t.Fatal(err) - } + err = os.RemoveAll(path.Join(root, "dir2")) + require.NoError(t, err) + err = ioutil.WriteFile(path.Join(root, "dir2"), []byte("dir2\n"), 0777) + require.NoError(t, err) // Touch dir - if err := system.Chtimes(path.Join(root, "dir3"), time.Now().Add(time.Second), time.Now().Add(time.Second)); err != nil { - t.Fatal(err) - } + err = system.Chtimes(path.Join(root, "dir3"), time.Now().Add(time.Second), time.Now().Add(time.Second)) + require.NoError(t, err) } func TestChangesDirsMutated(t *testing.T) { @@ -374,23 +334,18 @@ func TestChangesDirsMutated(t *testing.T) { t.Skip("symlinks on Windows; gcp failures on Solaris") } src, err := ioutil.TempDir("", "docker-changes-test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) createSampleDir(t, src) dst := src + "-copy" - if err := copyDir(src, dst); err != nil { - t.Fatal(err) - } + err = copyDir(src, dst) + require.NoError(t, err) defer os.RemoveAll(src) defer os.RemoveAll(dst) mutateSampleDir(t, dst) changes, err := ChangesDirs(dst, src) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) sort.Sort(changesByPath(changes)) @@ -436,41 +391,29 @@ func TestApplyLayer(t *testing.T) { t.Skip("symlinks on Windows; gcp failures on Solaris") } src, err := ioutil.TempDir("", "docker-changes-test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) createSampleDir(t, src) defer os.RemoveAll(src) dst := src + "-copy" - if err := copyDir(src, dst); err != nil { - t.Fatal(err) - } + err = copyDir(src, dst) + require.NoError(t, err) mutateSampleDir(t, dst) defer os.RemoveAll(dst) changes, err := ChangesDirs(dst, src) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) layer, err := ExportChanges(dst, changes, nil, nil) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) layerCopy, err := NewTempArchive(layer, "") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) - if _, err := ApplyLayer(src, layerCopy); err != nil { - t.Fatal(err) - } + _, err = ApplyLayer(src, layerCopy) + require.NoError(t, err) changes2, err := ChangesDirs(src, dst) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) if len(changes2) != 0 { t.Fatalf("Unexpected differences after reapplying mutation: %v", changes2) @@ -484,26 +427,18 @@ func TestChangesSizeWithHardlinks(t *testing.T) { t.Skip("hardlinks on Windows") } srcDir, err := ioutil.TempDir("", "docker-test-srcDir") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(srcDir) destDir, err := ioutil.TempDir("", "docker-test-destDir") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(destDir) creationSize, err := prepareUntarSourceDirectory(100, destDir, true) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) changes, err := ChangesDirs(destDir, srcDir) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) got := ChangesSize(destDir, changes) if got != int64(creationSize) { @@ -532,13 +467,12 @@ func TestChangesSize(t *testing.T) { parentPath, err := ioutil.TempDir("", "docker-changes-test") defer os.RemoveAll(parentPath) addition := path.Join(parentPath, "addition") - if err := ioutil.WriteFile(addition, []byte{0x01, 0x01, 0x01}, 0744); err != nil { - t.Fatal(err) - } + err = ioutil.WriteFile(addition, []byte{0x01, 0x01, 0x01}, 0744) + require.NoError(t, err) modification := path.Join(parentPath, "modification") - if err = ioutil.WriteFile(modification, []byte{0x01, 0x01, 0x01}, 0744); err != nil { - t.Fatal(err) - } + err = ioutil.WriteFile(modification, []byte{0x01, 0x01, 0x01}, 0744) + require.NoError(t, err) + changes := []Change{ {Path: "addition", Kind: ChangeAdd}, {Path: "modification", Kind: ChangeModify}, diff --git a/components/engine/pkg/archive/copy.go b/components/engine/pkg/archive/copy.go index 5281e29d18..3adf8a275c 100644 --- a/components/engine/pkg/archive/copy.go +++ b/components/engine/pkg/archive/copy.go @@ -9,8 +9,8 @@ import ( "path/filepath" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/system" + "github.com/sirupsen/logrus" ) // Errors used or returned by this file. diff --git a/components/engine/pkg/archive/copy_unix_test.go b/components/engine/pkg/archive/copy_unix_test.go index 4d5ae79cdb..e08bcb4916 100644 --- a/components/engine/pkg/archive/copy_unix_test.go +++ b/components/engine/pkg/archive/copy_unix_test.go @@ -15,6 +15,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/stretchr/testify/require" ) func removeAllPaths(paths ...string) { @@ -26,13 +28,11 @@ func removeAllPaths(paths ...string) { func getTestTempDirs(t *testing.T) (tmpDirA, tmpDirB string) { var err error - if tmpDirA, err = ioutil.TempDir("", "archive-copy-test"); err != nil { - t.Fatal(err) - } + tmpDirA, err = ioutil.TempDir("", "archive-copy-test") + require.NoError(t, err) - if tmpDirB, err = ioutil.TempDir("", "archive-copy-test"); err != nil { - t.Fatal(err) - } + tmpDirB, err = ioutil.TempDir("", "archive-copy-test") + require.NoError(t, err) return } @@ -118,9 +118,8 @@ func logDirContents(t *testing.T, dirPath string) { t.Logf("logging directory contents: %q", dirPath) - if err := filepath.Walk(dirPath, logWalkedPaths); err != nil { - t.Fatal(err) - } + err := filepath.Walk(dirPath, logWalkedPaths) + require.NoError(t, err) } func testCopyHelper(t *testing.T, srcPath, dstPath string) (err error) { @@ -293,9 +292,8 @@ func TestCopyCaseA(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = fileContentsEqual(t, srcPath, dstPath); err != nil { - t.Fatal(err) - } + err = fileContentsEqual(t, srcPath, dstPath) + require.NoError(t, err) os.Remove(dstPath) symlinkPath := filepath.Join(tmpDirA, "symlink3") @@ -306,17 +304,15 @@ func TestCopyCaseA(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = fileContentsEqual(t, linkTarget, dstPath); err != nil { - t.Fatal(err) - } + err = fileContentsEqual(t, linkTarget, dstPath) + require.NoError(t, err) os.Remove(dstPath) if err = testCopyHelperFSym(t, symlinkPath1, dstPath); err != nil { t.Fatalf("unexpected error %T: %s", err, err) } - if err = fileContentsEqual(t, linkTarget, dstPath); err != nil { - t.Fatal(err) - } + err = fileContentsEqual(t, linkTarget, dstPath) + require.NoError(t, err) } // B. SRC specifies a file and DST (with trailing path separator) doesn't @@ -377,9 +373,8 @@ func TestCopyCaseC(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = fileContentsEqual(t, srcPath, dstPath); err != nil { - t.Fatal(err) - } + err = fileContentsEqual(t, srcPath, dstPath) + require.NoError(t, err) } // C. Symbol link following version: @@ -415,9 +410,8 @@ func TestCopyCaseCFSym(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = fileContentsEqual(t, linkTarget, dstPath); err != nil { - t.Fatal(err) - } + err = fileContentsEqual(t, linkTarget, dstPath) + require.NoError(t, err) } // D. SRC specifies a file and DST exists as a directory. This should place @@ -446,9 +440,8 @@ func TestCopyCaseD(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = fileContentsEqual(t, srcPath, dstPath); err != nil { - t.Fatal(err) - } + err = fileContentsEqual(t, srcPath, dstPath) + require.NoError(t, err) // Now try again but using a trailing path separator for dstDir. @@ -466,9 +459,8 @@ func TestCopyCaseD(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = fileContentsEqual(t, srcPath, dstPath); err != nil { - t.Fatal(err) - } + err = fileContentsEqual(t, srcPath, dstPath) + require.NoError(t, err) } // D. Symbol link following version: @@ -499,9 +491,8 @@ func TestCopyCaseDFSym(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = fileContentsEqual(t, linkTarget, dstPath); err != nil { - t.Fatal(err) - } + err = fileContentsEqual(t, linkTarget, dstPath) + require.NoError(t, err) // Now try again but using a trailing path separator for dstDir. @@ -519,9 +510,8 @@ func TestCopyCaseDFSym(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = fileContentsEqual(t, linkTarget, dstPath); err != nil { - t.Fatal(err) - } + err = fileContentsEqual(t, linkTarget, dstPath) + require.NoError(t, err) } // E. SRC specifies a directory and DST does not exist. This should create a @@ -563,9 +553,8 @@ func TestCopyCaseE(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = dirContentsEqual(t, dstDir, srcDir); err != nil { - t.Fatal(err) - } + err = dirContentsEqual(t, dstDir, srcDir) + require.NoError(t, err) } // E. Symbol link following version: @@ -609,9 +598,8 @@ func TestCopyCaseEFSym(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = dirContentsEqual(t, dstDir, linkTarget); err != nil { - t.Fatal(err) - } + err = dirContentsEqual(t, dstDir, linkTarget) + require.NoError(t, err) } // F. SRC specifies a directory and DST exists as a file. This should cause an @@ -669,9 +657,8 @@ func TestCopyCaseG(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = dirContentsEqual(t, resultDir, srcDir); err != nil { - t.Fatal(err) - } + err = dirContentsEqual(t, resultDir, srcDir) + require.NoError(t, err) // Now try again but using a trailing path separator for dstDir. @@ -689,9 +676,8 @@ func TestCopyCaseG(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = dirContentsEqual(t, resultDir, srcDir); err != nil { - t.Fatal(err) - } + err = dirContentsEqual(t, resultDir, srcDir) + require.NoError(t, err) } // G. Symbol link version: @@ -717,9 +703,8 @@ func TestCopyCaseGFSym(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = dirContentsEqual(t, resultDir, linkTarget); err != nil { - t.Fatal(err) - } + err = dirContentsEqual(t, resultDir, linkTarget) + require.NoError(t, err) // Now try again but using a trailing path separator for dstDir. @@ -737,9 +722,8 @@ func TestCopyCaseGFSym(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = dirContentsEqual(t, resultDir, linkTarget); err != nil { - t.Fatal(err) - } + err = dirContentsEqual(t, resultDir, linkTarget) + require.NoError(t, err) } // H. SRC specifies a directory's contents only and DST does not exist. This @@ -899,9 +883,8 @@ func TestCopyCaseJ(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = dirContentsEqual(t, dstDir, srcDir); err != nil { - t.Fatal(err) - } + err = dirContentsEqual(t, dstDir, srcDir) + require.NoError(t, err) // Now try again but using a trailing path separator for dstDir. @@ -919,9 +902,8 @@ func TestCopyCaseJ(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = dirContentsEqual(t, dstDir, srcDir); err != nil { - t.Fatal(err) - } + err = dirContentsEqual(t, dstDir, srcDir) + require.NoError(t, err) } // J. Symbol link following version: @@ -952,9 +934,8 @@ func TestCopyCaseJFSym(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = dirContentsEqual(t, dstDir, linkTarget); err != nil { - t.Fatal(err) - } + err = dirContentsEqual(t, dstDir, linkTarget) + require.NoError(t, err) // Now try again but using a trailing path separator for dstDir. @@ -972,7 +953,6 @@ func TestCopyCaseJFSym(t *testing.T) { t.Fatalf("unexpected error %T: %s", err, err) } - if err = dirContentsEqual(t, dstDir, linkTarget); err != nil { - t.Fatal(err) - } + err = dirContentsEqual(t, dstDir, linkTarget) + require.NoError(t, err) } diff --git a/components/engine/pkg/archive/diff.go b/components/engine/pkg/archive/diff.go index a2766b5928..019facd383 100644 --- a/components/engine/pkg/archive/diff.go +++ b/components/engine/pkg/archive/diff.go @@ -10,10 +10,10 @@ import ( "runtime" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/system" + "github.com/sirupsen/logrus" ) // UnpackLayer unpack `layer` to a `dest`. The stream `layer` can be diff --git a/components/engine/pkg/archive/example_changes.go b/components/engine/pkg/archive/example_changes.go index cedd46a408..495db809e9 100644 --- a/components/engine/pkg/archive/example_changes.go +++ b/components/engine/pkg/archive/example_changes.go @@ -13,8 +13,8 @@ import ( "os" "path" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/archive" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/pkg/archive/wrap_test.go b/components/engine/pkg/archive/wrap_test.go index 46ab36697a..bd26bda3a2 100644 --- a/components/engine/pkg/archive/wrap_test.go +++ b/components/engine/pkg/archive/wrap_test.go @@ -5,13 +5,13 @@ import ( "bytes" "io" "testing" + + "github.com/stretchr/testify/require" ) func TestGenerateEmptyFile(t *testing.T) { archive, err := Generate("emptyFile") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) if archive == nil { t.Fatal("The generated archive should not be nil.") } @@ -28,9 +28,7 @@ func TestGenerateEmptyFile(t *testing.T) { if err == io.EOF { break } - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) buf := new(bytes.Buffer) buf.ReadFrom(tr) content := buf.String() @@ -54,9 +52,7 @@ func TestGenerateEmptyFile(t *testing.T) { func TestGenerateWithContent(t *testing.T) { archive, err := Generate("file", "content") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) if archive == nil { t.Fatal("The generated archive should not be nil.") } @@ -73,9 +69,7 @@ func TestGenerateWithContent(t *testing.T) { if err == io.EOF { break } - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) buf := new(bytes.Buffer) buf.ReadFrom(tr) content := buf.String() diff --git a/components/engine/pkg/authorization/authz.go b/components/engine/pkg/authorization/authz.go index dc9a9ae56f..b52446add6 100644 --- a/components/engine/pkg/authorization/authz.go +++ b/components/engine/pkg/authorization/authz.go @@ -8,8 +8,8 @@ import ( "net/http" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/ioutils" + "github.com/sirupsen/logrus" ) const maxBodySize = 1048576 // 1MB diff --git a/components/engine/pkg/authorization/middleware.go b/components/engine/pkg/authorization/middleware.go index 7789a758df..24a457ae1d 100644 --- a/components/engine/pkg/authorization/middleware.go +++ b/components/engine/pkg/authorization/middleware.go @@ -4,8 +4,8 @@ import ( "net/http" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/plugingetter" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/pkg/authorization/response.go b/components/engine/pkg/authorization/response.go index 129bf2f417..cea64f2e80 100644 --- a/components/engine/pkg/authorization/response.go +++ b/components/engine/pkg/authorization/response.go @@ -8,7 +8,7 @@ import ( "net" "net/http" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // ResponseModifier allows authorization plugins to read and modify the content of the http.response diff --git a/components/engine/pkg/devicemapper/devmapper.go b/components/engine/pkg/devicemapper/devmapper.go index f331970802..2254a6b530 100644 --- a/components/engine/pkg/devicemapper/devmapper.go +++ b/components/engine/pkg/devicemapper/devmapper.go @@ -9,10 +9,11 @@ import ( "runtime" "unsafe" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) +// Same as DM_DEVICE_* enum values from libdevmapper.h const ( deviceCreate TaskType = iota deviceReload diff --git a/components/engine/pkg/devicemapper/devmapper_log.go b/components/engine/pkg/devicemapper/devmapper_log.go index 098d2405ed..d2c0d4c76a 100644 --- a/components/engine/pkg/devicemapper/devmapper_log.go +++ b/components/engine/pkg/devicemapper/devmapper_log.go @@ -8,7 +8,7 @@ import ( "fmt" "strings" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // DevmapperLogger defines methods required to register as a callback for diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper.go b/components/engine/pkg/devicemapper/devmapper_wrapper.go index da3b43f796..64e82dfd04 100644 --- a/components/engine/pkg/devicemapper/devmapper_wrapper.go +++ b/components/engine/pkg/devicemapper/devmapper_wrapper.go @@ -3,7 +3,6 @@ package devicemapper /* -#cgo LDFLAGS: -L. -ldevmapper #define _GNU_SOURCE #include #include // FIXME: present only for BLKGETSIZE64, maybe we can remove it? diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper_deferred_remove.go b/components/engine/pkg/devicemapper/devmapper_wrapper_deferred_remove.go index 5bdd97d9c0..7f793c2708 100644 --- a/components/engine/pkg/devicemapper/devmapper_wrapper_deferred_remove.go +++ b/components/engine/pkg/devicemapper/devmapper_wrapper_deferred_remove.go @@ -2,13 +2,10 @@ package devicemapper -/* -#cgo LDFLAGS: -L. -ldevmapper -#include -*/ +// #include import "C" -// LibraryDeferredRemovalSupport is supported when statically linked. +// LibraryDeferredRemovalSupport tells if the feature is enabled in the build const LibraryDeferredRemovalSupport = true func dmTaskDeferredRemoveFct(task *cdmTask) int { diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic.go b/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic.go new file mode 100644 index 0000000000..7d84508982 --- /dev/null +++ b/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic.go @@ -0,0 +1,6 @@ +// +build linux,cgo,!static_build + +package devicemapper + +// #cgo pkg-config: devmapper +import "C" diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go b/components/engine/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go index 968b2ce0c0..a880fec8c4 100644 --- a/components/engine/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go +++ b/components/engine/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go @@ -2,7 +2,7 @@ package devicemapper -// LibraryDeferredRemovalSupport is not supported when statically linked. +// LibraryDeferredRemovalSupport tells if the feature is enabled in the build const LibraryDeferredRemovalSupport = false func dmTaskDeferredRemoveFct(task *cdmTask) int { diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper_static.go b/components/engine/pkg/devicemapper/devmapper_wrapper_static.go new file mode 100644 index 0000000000..cf7f26a4c6 --- /dev/null +++ b/components/engine/pkg/devicemapper/devmapper_wrapper_static.go @@ -0,0 +1,6 @@ +// +build linux,cgo,static_build + +package devicemapper + +// #cgo pkg-config: --static devmapper +import "C" diff --git a/components/engine/pkg/discovery/backends.go b/components/engine/pkg/discovery/backends.go index 2eab550e29..a25b86c3e6 100644 --- a/components/engine/pkg/discovery/backends.go +++ b/components/engine/pkg/discovery/backends.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/pkg/discovery/kv/kv.go b/components/engine/pkg/discovery/kv/kv.go index 77eee7d454..e860b369e7 100644 --- a/components/engine/pkg/discovery/kv/kv.go +++ b/components/engine/pkg/discovery/kv/kv.go @@ -6,7 +6,6 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/discovery" "github.com/docker/go-connections/tlsconfig" "github.com/docker/libkv" @@ -14,6 +13,7 @@ import ( "github.com/docker/libkv/store/consul" "github.com/docker/libkv/store/etcd" "github.com/docker/libkv/store/zookeeper" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/pkg/filenotify/poller.go b/components/engine/pkg/filenotify/poller.go index b90111bb4f..1dc9482e02 100644 --- a/components/engine/pkg/filenotify/poller.go +++ b/components/engine/pkg/filenotify/poller.go @@ -7,7 +7,7 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/fsnotify/fsnotify" ) diff --git a/components/engine/pkg/fileutils/fileutils.go b/components/engine/pkg/fileutils/fileutils.go index 57cc087340..a129e654ea 100644 --- a/components/engine/pkg/fileutils/fileutils.go +++ b/components/engine/pkg/fileutils/fileutils.go @@ -10,7 +10,7 @@ import ( "strings" "text/scanner" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // PatternMatcher allows checking paths agaist a list of patterns diff --git a/components/engine/pkg/fileutils/fileutils_unix.go b/components/engine/pkg/fileutils/fileutils_unix.go index d5c3abf568..9e0e97bd64 100644 --- a/components/engine/pkg/fileutils/fileutils_unix.go +++ b/components/engine/pkg/fileutils/fileutils_unix.go @@ -7,7 +7,7 @@ import ( "io/ioutil" "os" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // GetTotalUsedFds Returns the number of used File Descriptors by diff --git a/components/engine/pkg/idtools/idtools_unix_test.go b/components/engine/pkg/idtools/idtools_unix_test.go index 31522a547d..2463342a65 100644 --- a/components/engine/pkg/idtools/idtools_unix_test.go +++ b/components/engine/pkg/idtools/idtools_unix_test.go @@ -7,10 +7,10 @@ import ( "io/ioutil" "os" "path/filepath" - "syscall" "testing" "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" ) type node struct { @@ -187,8 +187,8 @@ func readTree(base, root string) (map[string]node, error) { } for _, info := range dirInfos { - s := &syscall.Stat_t{} - if err := syscall.Stat(filepath.Join(base, info.Name()), s); err != nil { + s := &unix.Stat_t{} + if err := unix.Stat(filepath.Join(base, info.Name()), s); err != nil { return nil, fmt.Errorf("Can't stat file %q: %v", filepath.Join(base, info.Name()), err) } tree[filepath.Join(root, info.Name())] = node{int(s.Uid), int(s.Gid)} diff --git a/components/engine/pkg/loopback/attach_loopback.go b/components/engine/pkg/loopback/attach_loopback.go index 6ea9a309d3..27326d7fe5 100644 --- a/components/engine/pkg/loopback/attach_loopback.go +++ b/components/engine/pkg/loopback/attach_loopback.go @@ -7,7 +7,7 @@ import ( "fmt" "os" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/pkg/loopback/loopback.go b/components/engine/pkg/loopback/loopback.go index c2d91da6f0..c620c63336 100644 --- a/components/engine/pkg/loopback/loopback.go +++ b/components/engine/pkg/loopback/loopback.go @@ -5,9 +5,9 @@ package loopback import ( "fmt" "os" - "syscall" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) func getLoopbackBackingFile(file *os.File) (uint64, uint64, error) { @@ -31,12 +31,13 @@ func SetCapacity(file *os.File) error { // FindLoopDeviceFor returns a loopback device file for the specified file which // is backing file of a loop back device. func FindLoopDeviceFor(file *os.File) *os.File { - stat, err := file.Stat() + var stat unix.Stat_t + err := unix.Stat(file.Name(), &stat) if err != nil { return nil } - targetInode := stat.Sys().(*syscall.Stat_t).Ino - targetDevice := stat.Sys().(*syscall.Stat_t).Dev + targetInode := stat.Ino + targetDevice := stat.Dev for i := 0; true; i++ { path := fmt.Sprintf("/dev/loop%d", i) diff --git a/components/engine/pkg/parsers/kernel/kernel_unix.go b/components/engine/pkg/parsers/kernel/kernel_unix.go index bd137dfb6f..76e1e499f3 100644 --- a/components/engine/pkg/parsers/kernel/kernel_unix.go +++ b/components/engine/pkg/parsers/kernel/kernel_unix.go @@ -7,7 +7,7 @@ package kernel import ( "bytes" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // GetKernelVersion gets the current kernel version. diff --git a/components/engine/pkg/platform/platform.go b/components/engine/pkg/platform/platform.go index e4b03122f4..2845c3c197 100644 --- a/components/engine/pkg/platform/platform.go +++ b/components/engine/pkg/platform/platform.go @@ -3,7 +3,7 @@ package platform import ( "runtime" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/pkg/plugins/client.go b/components/engine/pkg/plugins/client.go index f221a46fcf..9ee1f89976 100644 --- a/components/engine/pkg/plugins/client.go +++ b/components/engine/pkg/plugins/client.go @@ -9,10 +9,10 @@ import ( "net/url" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/plugins/transport" "github.com/docker/go-connections/sockets" "github.com/docker/go-connections/tlsconfig" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/pkg/plugins/plugins.go b/components/engine/pkg/plugins/plugins.go index c0059cba75..f9033ed1bc 100644 --- a/components/engine/pkg/plugins/plugins.go +++ b/components/engine/pkg/plugins/plugins.go @@ -27,8 +27,8 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/go-connections/tlsconfig" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/pkg/signal/trap.go b/components/engine/pkg/signal/trap.go index 638a1ab66c..2884dfee32 100644 --- a/components/engine/pkg/signal/trap.go +++ b/components/engine/pkg/signal/trap.go @@ -11,7 +11,6 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" "github.com/pkg/errors" ) @@ -27,7 +26,9 @@ import ( // the docker daemon is not restarted and also running under systemd. // Fixes https://github.com/docker/docker/issues/19728 // -func Trap(cleanup func()) { +func Trap(cleanup func(), logger interface { + Info(args ...interface{}) +}) { c := make(chan os.Signal, 1) // we will handle INT, TERM, QUIT, SIGPIPE here signals := []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGPIPE} @@ -40,7 +41,7 @@ func Trap(cleanup func()) { } go func(sig os.Signal) { - logrus.Infof("Processing signal '%v'", sig) + logger.Info(fmt.Sprintf("Processing signal '%v'", sig)) switch sig { case os.Interrupt, syscall.SIGTERM: if atomic.LoadUint32(&interruptCount) < 3 { @@ -54,11 +55,11 @@ func Trap(cleanup func()) { } } else { // 3 SIGTERM/INT signals received; force exit without cleanup - logrus.Info("Forcing docker daemon shutdown without cleanup; 3 interrupts received") + logger.Info("Forcing docker daemon shutdown without cleanup; 3 interrupts received") } case syscall.SIGQUIT: DumpStacks("") - logrus.Info("Forcing docker daemon shutdown without cleanup on SIGQUIT") + logger.Info("Forcing docker daemon shutdown without cleanup on SIGQUIT") } //for the SIGINT/TERM, and SIGQUIT non-clean shutdown case, exit with 128 + signal # os.Exit(128 + int(sig.(syscall.Signal))) diff --git a/components/engine/pkg/sysinfo/sysinfo_linux.go b/components/engine/pkg/sysinfo/sysinfo_linux.go index 50ae265bb6..471f786a70 100644 --- a/components/engine/pkg/sysinfo/sysinfo_linux.go +++ b/components/engine/pkg/sysinfo/sysinfo_linux.go @@ -7,8 +7,8 @@ import ( "path" "strings" - "github.com/Sirupsen/logrus" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/pkg/system/syscall_windows.go b/components/engine/pkg/system/syscall_windows.go index eded233b9c..23e9b207c7 100644 --- a/components/engine/pkg/system/syscall_windows.go +++ b/components/engine/pkg/system/syscall_windows.go @@ -3,7 +3,7 @@ package system import ( "unsafe" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) diff --git a/components/engine/pkg/templates/templates_test.go b/components/engine/pkg/templates/templates_test.go deleted file mode 100644 index 296bcb7107..0000000000 --- a/components/engine/pkg/templates/templates_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package templates - -import ( - "bytes" - "testing" - - "github.com/stretchr/testify/assert" -) - -// Github #32120 -func TestParseJSONFunctions(t *testing.T) { - tm, err := Parse(`{{json .Ports}}`) - assert.NoError(t, err) - - var b bytes.Buffer - assert.NoError(t, tm.Execute(&b, map[string]string{"Ports": "0.0.0.0:2->8/udp"})) - want := "\"0.0.0.0:2->8/udp\"" - assert.Equal(t, want, b.String()) -} - -func TestParseStringFunctions(t *testing.T) { - tm, err := Parse(`{{join (split . ":") "/"}}`) - assert.NoError(t, err) - - var b bytes.Buffer - assert.NoError(t, tm.Execute(&b, "text:with:colon")) - want := "text/with/colon" - assert.Equal(t, want, b.String()) -} - -func TestNewParse(t *testing.T) { - tm, err := NewParse("foo", "this is a {{ . }}") - assert.NoError(t, err) - - var b bytes.Buffer - assert.NoError(t, tm.Execute(&b, "string")) - want := "this is a string" - assert.Equal(t, want, b.String()) -} - -func TestParseTruncateFunction(t *testing.T) { - source := "tupx5xzf6hvsrhnruz5cr8gwp" - - testCases := []struct { - template string - expected string - }{ - { - template: `{{truncate . 5}}`, - expected: "tupx5", - }, - { - template: `{{truncate . 25}}`, - expected: "tupx5xzf6hvsrhnruz5cr8gwp", - }, - { - template: `{{truncate . 30}}`, - expected: "tupx5xzf6hvsrhnruz5cr8gwp", - }, - { - template: `{{pad . 3 3}}`, - expected: " tupx5xzf6hvsrhnruz5cr8gwp ", - }, - } - - for _, testCase := range testCases { - tm, err := Parse(testCase.template) - assert.NoError(t, err) - - t.Run("Non Empty Source Test with template: "+testCase.template, func(t *testing.T) { - var b bytes.Buffer - assert.NoError(t, tm.Execute(&b, source)) - assert.Equal(t, testCase.expected, b.String()) - }) - - t.Run("Empty Source Test with template: "+testCase.template, func(t *testing.T) { - var c bytes.Buffer - assert.NoError(t, tm.Execute(&c, "")) - assert.Equal(t, "", c.String()) - }) - - t.Run("Nil Source Test with template: "+testCase.template, func(t *testing.T) { - var c bytes.Buffer - assert.Error(t, tm.Execute(&c, nil)) - assert.Equal(t, "", c.String()) - }) - } -} diff --git a/components/engine/pkg/term/term_linux_test.go b/components/engine/pkg/term/term_linux_test.go index a1628c4c6d..f907ff53a0 100644 --- a/components/engine/pkg/term/term_linux_test.go +++ b/components/engine/pkg/term/term_linux_test.go @@ -3,29 +3,20 @@ package term import ( - "flag" "io/ioutil" "os" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -var rootEnabled bool - -func init() { - flag.BoolVar(&rootEnabled, "test.root", false, "enable tests that require root") -} - // RequiresRoot skips tests that require root, unless the test.root flag has // been set func RequiresRoot(t *testing.T) { - if !rootEnabled { + if os.Getuid() != 0 { t.Skip("skipping test that requires root") return } - assert.Equal(t, 0, os.Getuid(), "This test must be run as root.") } func newTtyForTest(t *testing.T) (*os.File, error) { @@ -113,6 +104,7 @@ func TestDisableEcho(t *testing.T) { defer tty.Close() require.NoError(t, err) state, err := SetRawTerminal(tty.Fd()) + defer RestoreTerminal(tty.Fd(), state) require.NoError(t, err) require.NotNil(t, state) err = DisableEcho(tty.Fd(), state) diff --git a/components/engine/pkg/term/termios_linux.go b/components/engine/pkg/term/termios_linux.go index 3e25eb7a41..0f21abcc2f 100644 --- a/components/engine/pkg/term/termios_linux.go +++ b/components/engine/pkg/term/termios_linux.go @@ -29,6 +29,8 @@ func MakeRaw(fd uintptr) (*State, error) { termios.Lflag &^= (unix.ECHO | unix.ECHONL | unix.ICANON | unix.ISIG | unix.IEXTEN) termios.Cflag &^= (unix.CSIZE | unix.PARENB) termios.Cflag |= unix.CS8 + termios.Cc[unix.VMIN] = 1 + termios.Cc[unix.VTIME] = 0 if err := unix.IoctlSetTermios(int(fd), setTermios, termios); err != nil { return nil, err diff --git a/components/engine/pkg/term/windows/windows.go b/components/engine/pkg/term/windows/windows.go index d67021e45a..c02a93a03f 100644 --- a/components/engine/pkg/term/windows/windows.go +++ b/components/engine/pkg/term/windows/windows.go @@ -10,7 +10,7 @@ import ( "sync" ansiterm "github.com/Azure/go-ansiterm" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) var logger *logrus.Logger diff --git a/components/engine/plugin/backend_linux.go b/components/engine/plugin/backend_linux.go index 055b8e3107..727a26862a 100644 --- a/components/engine/plugin/backend_linux.go +++ b/components/engine/plugin/backend_linux.go @@ -15,7 +15,6 @@ import ( "path/filepath" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" @@ -36,6 +35,7 @@ import ( refstore "github.com/docker/docker/reference" "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/plugin/blobstore.go b/components/engine/plugin/blobstore.go index 2b79a44270..c1259b6781 100644 --- a/components/engine/plugin/blobstore.go +++ b/components/engine/plugin/blobstore.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" - "github.com/Sirupsen/logrus" "github.com/docker/docker/distribution/xfer" "github.com/docker/docker/image" "github.com/docker/docker/layer" @@ -16,6 +15,7 @@ import ( "github.com/docker/docker/pkg/progress" "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/plugin/manager.go b/components/engine/plugin/manager.go index fada0d667a..2281dfdd6c 100644 --- a/components/engine/plugin/manager.go +++ b/components/engine/plugin/manager.go @@ -13,7 +13,6 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/image" @@ -28,6 +27,7 @@ import ( "github.com/docker/docker/registry" "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) const configFileName = "config.json" @@ -105,6 +105,11 @@ func NewManager(config ManagerConfig) (*Manager, error) { if err := os.MkdirAll(manager.tmpDir(), 0700); err != nil { return nil, errors.Wrapf(err, "failed to mkdir %v", manager.tmpDir()) } + + if err := setupRoot(manager.config.Root); err != nil { + return nil, err + } + var err error manager.containerdClient, err = config.Executor.Client(manager) // todo: move to another struct if err != nil { diff --git a/components/engine/plugin/manager_linux.go b/components/engine/plugin/manager_linux.go index 9b84af68dc..301b12c814 100644 --- a/components/engine/plugin/manager_linux.go +++ b/components/engine/plugin/manager_linux.go @@ -10,7 +10,6 @@ import ( "path/filepath" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" "github.com/docker/docker/daemon/initlayer" "github.com/docker/docker/libcontainerd" @@ -22,6 +21,7 @@ import ( "github.com/opencontainers/go-digest" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -162,6 +162,13 @@ func shutdownPlugin(p *v2.Plugin, c *controller, containerdClient libcontainerd. } } +func setupRoot(root string) error { + if err := mount.MakePrivate(root); err != nil { + return errors.Wrap(err, "error setting plugin manager root to private") + } + return nil +} + func (pm *Manager) disable(p *v2.Plugin, c *controller) error { if !p.IsEnabled() { return fmt.Errorf("plugin %s is already disabled", p.Name()) @@ -190,6 +197,7 @@ func (pm *Manager) Shutdown() { shutdownPlugin(p, c, pm.containerdClient) } } + mount.Unmount(pm.config.Root) } func (pm *Manager) upgradePlugin(p *v2.Plugin, configDigest digest.Digest, blobsums []digest.Digest, tmpRootFSDir string, privileges *types.PluginPrivileges) (err error) { diff --git a/components/engine/plugin/manager_solaris.go b/components/engine/plugin/manager_solaris.go index 72ccae72d3..ac03d6e639 100644 --- a/components/engine/plugin/manager_solaris.go +++ b/components/engine/plugin/manager_solaris.go @@ -26,3 +26,5 @@ func (pm *Manager) restore(p *v2.Plugin) error { // Shutdown plugins func (pm *Manager) Shutdown() { } + +func setupRoot(root string) error { return nil } diff --git a/components/engine/plugin/manager_windows.go b/components/engine/plugin/manager_windows.go index 4469a671f7..56a7ee3ece 100644 --- a/components/engine/plugin/manager_windows.go +++ b/components/engine/plugin/manager_windows.go @@ -28,3 +28,5 @@ func (pm *Manager) restore(p *v2.Plugin) error { // Shutdown plugins func (pm *Manager) Shutdown() { } + +func setupRoot(root string) error { return nil } diff --git a/components/engine/plugin/store.go b/components/engine/plugin/store.go index 7f6e954bf8..acb92d2e5a 100644 --- a/components/engine/plugin/store.go +++ b/components/engine/plugin/store.go @@ -4,12 +4,12 @@ import ( "fmt" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/docker/pkg/plugingetter" "github.com/docker/docker/pkg/plugins" "github.com/docker/docker/plugin/v2" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) /* allowV1PluginsFallback determines daemon's support for V1 plugins. diff --git a/components/engine/profiles/apparmor/apparmor.go b/components/engine/profiles/apparmor/apparmor.go index 48b41c5b2d..e957727d52 100644 --- a/components/engine/profiles/apparmor/apparmor.go +++ b/components/engine/profiles/apparmor/apparmor.go @@ -9,9 +9,9 @@ import ( "os" "path" "strings" + "text/template" "github.com/docker/docker/pkg/aaparser" - "github.com/docker/docker/pkg/templates" ) var ( @@ -33,7 +33,7 @@ type profileData struct { // generateDefault creates an apparmor profile from ProfileData. func (p *profileData) generateDefault(out io.Writer) error { - compiled, err := templates.NewParse("apparmor_profile", baseTemplate) + compiled, err := template.New("apparmor_profile").Parse(baseTemplate) if err != nil { return err } diff --git a/components/engine/profiles/seccomp/default.json b/components/engine/profiles/seccomp/default.json index b71a8718a4..26ef2b604d 100755 --- a/components/engine/profiles/seccomp/default.json +++ b/components/engine/profiles/seccomp/default.json @@ -557,6 +557,7 @@ "mount", "name_to_handle_at", "perf_event_open", + "quotactl", "setdomainname", "sethostname", "setns", diff --git a/components/engine/profiles/seccomp/seccomp_default.go b/components/engine/profiles/seccomp/seccomp_default.go index 1e6ea90e34..83a437521e 100644 --- a/components/engine/profiles/seccomp/seccomp_default.go +++ b/components/engine/profiles/seccomp/seccomp_default.go @@ -488,6 +488,7 @@ func DefaultProfile() *types.Seccomp { "mount", "name_to_handle_at", "perf_event_open", + "quotactl", "setdomainname", "sethostname", "setns", diff --git a/components/engine/profiles/seccomp/seccomp_unsupported.go b/components/engine/profiles/seccomp/seccomp_unsupported.go index 3c1d68b1fe..0130effa61 100644 --- a/components/engine/profiles/seccomp/seccomp_unsupported.go +++ b/components/engine/profiles/seccomp/seccomp_unsupported.go @@ -4,7 +4,6 @@ package seccomp import ( "github.com/docker/docker/api/types" - "github.com/opencontainers/runtime-spec/specs-go" ) // DefaultProfile returns a nil pointer on unsupported systems. diff --git a/components/engine/project/ARM.md b/components/engine/project/ARM.md index e61e3b1847..c876231d1e 100644 --- a/components/engine/project/ARM.md +++ b/components/engine/project/ARM.md @@ -17,7 +17,7 @@ From the root of the Docker/Docker repo one can use make to execute the followin - make default - make shell - make test-unit -- make test-integration-cli +- make test-integration - make The Makefile does include logic to determine on which OS and architecture the Docker Development Image is built. diff --git a/components/engine/registry/auth.go b/components/engine/registry/auth.go index 8cadd51ba0..f7a8fdc936 100644 --- a/components/engine/registry/auth.go +++ b/components/engine/registry/auth.go @@ -8,12 +8,12 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/client/auth" "github.com/docker/distribution/registry/client/auth/challenge" "github.com/docker/distribution/registry/client/transport" "github.com/docker/docker/api/types" registrytypes "github.com/docker/docker/api/types/registry" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/registry/config.go b/components/engine/registry/config.go index 182599e38d..f4b3bff945 100644 --- a/components/engine/registry/config.go +++ b/components/engine/registry/config.go @@ -8,11 +8,11 @@ import ( "strconv" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/opts" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/spf13/pflag" ) diff --git a/components/engine/registry/endpoint_v1.go b/components/engine/registry/endpoint_v1.go index c5ca961dd4..59fd72e938 100644 --- a/components/engine/registry/endpoint_v1.go +++ b/components/engine/registry/endpoint_v1.go @@ -9,9 +9,9 @@ import ( "net/url" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/client/transport" registrytypes "github.com/docker/docker/api/types/registry" + "github.com/sirupsen/logrus" ) // V1Endpoint stores basic information about a V1 registry endpoint. diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 17fa97ce3d..a676d63450 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -13,10 +13,10 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/client/transport" "github.com/docker/go-connections/sockets" "github.com/docker/go-connections/tlsconfig" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/registry/registry_mock_test.go b/components/engine/registry/registry_mock_test.go index 58b05d3849..204a98344e 100644 --- a/components/engine/registry/registry_mock_test.go +++ b/components/engine/registry/registry_mock_test.go @@ -21,7 +21,7 @@ import ( registrytypes "github.com/docker/docker/api/types/registry" "github.com/gorilla/mux" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/registry/resumable/resumablerequestreader.go b/components/engine/registry/resumable/resumablerequestreader.go index 5403c76847..12075e552c 100644 --- a/components/engine/registry/resumable/resumablerequestreader.go +++ b/components/engine/registry/resumable/resumablerequestreader.go @@ -6,7 +6,7 @@ import ( "net/http" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) type requestReader struct { diff --git a/components/engine/registry/service.go b/components/engine/registry/service.go index 34e8a13f9e..b4f747380a 100644 --- a/components/engine/registry/service.go +++ b/components/engine/registry/service.go @@ -10,11 +10,11 @@ import ( "golang.org/x/net/context" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/client/auth" "github.com/docker/docker/api/types" registrytypes "github.com/docker/docker/api/types/registry" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/registry/session.go b/components/engine/registry/session.go index 9d7f32193f..bc4a244032 100644 --- a/components/engine/registry/session.go +++ b/components/engine/registry/session.go @@ -18,7 +18,6 @@ import ( "strconv" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/api/errcode" "github.com/docker/docker/api/types" @@ -28,6 +27,7 @@ import ( "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/registry/resumable" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/reports/builder/2017-07-17.md b/components/engine/reports/builder/2017-07-17.md new file mode 100644 index 0000000000..96cc8d1849 --- /dev/null +++ b/components/engine/reports/builder/2017-07-17.md @@ -0,0 +1,79 @@ +# Development Report for July 17, 2017 + + +### BuildKit + +[Repo](https://github.com/moby/buildkit) +[Proposal](https://github.com/moby/moby/issues/32925) + +Following features were added last week: + +#### Git source + +Source code from git repositories can now be accessed directly, similarly for images can be accessed, without the need to execute `git clone`. This has many performance and caching advantages. It accesses the remote repository using shallow fetches to only pull the required data and a uses a shared bare repository for intermediate cache between build invocations. The instruction cache for the git source is based on a commit hash and not string arguments. This means that you can always be sure that you are building the correct source and that you never build the same source twice. + +#### Containerd exporter + +Exporters are used for getting build artifacts out of buildkit. The first exporter that was implemented allows exposing the image to containerd so it can be run and pushed with `ctr` tool. `buildctl` has `--exporter` flag for specifying the exporter and `--exporter-opt` for custom values passed to the exporter. In the case of image exporter an image name can be specified. + +For example: + +``` +go run ./examples/buildkit2/buildkit.go | buildctl build --exporter image --exporter-opt name=docker.io/moby/buildkit:dev +``` + +Accessing from ctr/dist: + +``` +ctr --namespace buildkit images ls +ctr --namespace buildkit rootfs unpack +ctr --namespace buildkit run -t docker.io/moby/buildkit:dev id ash +``` + +#### Local source + +Buildkit now supports building from local sources. Snapshot of the local source files is created similarly to `docker build` build context. The implementation is based on the [incremental context send](https://github.com/moby/moby/pull/32677) feature in `docker-v17.07`. To use in `buildctl` the source definition needs to define a name for local endpoint, and `buildctl build` command provides a mapping from this name to a local directory with a `--local` flag. + +``` +go run ./examples/buildkit3/buildkit.go --local | buildctl build --local buildkit-src=. +``` + +### Typed Dockerfile parsing + +[PR](https://github.com/moby/moby/pull/33492) + +Didn't manage to merge this PR yet. Still in code-review. + + +### Feedback for `RUN --mount` / `COPY --chown` + +There was some new discussion around [`RUN --mount`](https://github.com/moby/moby/issues/32507) or [`COPY --chown`](https://github.com/moby/moby/issues/30110) feature. Currently, it seems that it may be best to try the shared cache capabilities described in `RUN --mount` in https://github.com/moby/buildkit first(it already supports the generic mounting capabilities). So to unblock the people waiting only on the file owner change features it may make sense to implement `COPY --chown` first. Another related candidate for `v17.08` release is https://github.com/moby/moby/issues/32816. + + +### Proposals for new Dockerfile features that need design feedback: + +[Add IMPORT/EXPORT commands to Dockerfile](https://github.com/moby/moby/issues/32100) + +[Add `DOCKEROS/DOCKERARCH` default ARG to Dockerfile](https://github.com/moby/moby/issues/32487) + +[Add support for `RUN --mount`](https://github.com/moby/moby/issues/32507) + +[DAG image builder](https://github.com/moby/moby/issues/32550) + +[Option to export the hash of the build context](https://github.com/moby/moby/issues/32963) (new) + +[Allow --cache-from=*](https://github.com/moby/moby/issues/33002#issuecomment-299041162) (new) + +[Provide advanced .dockeringore use-cases](https://github.com/moby/moby/issues/12886) [2](https://github.com/moby/moby/issues/12886#issuecomment-306247989) + +New: [RFC: Distributed BuildKit](https://github.com/moby/buildkit/issues/62) + +If you are interested in implementing any of them, leave a comment on the specific issues. + +### Builder features currently in code-review: + +[Fix shallow git clone in docker-build](https://github.com/moby/moby/pull/33704) + +### Backlog + +[Build secrets](https://github.com/moby/moby/issues/33343) has not got much traction. If you want this feature to become a reality, please make yourself heard. \ No newline at end of file diff --git a/components/engine/runconfig/hostconfig_test.go b/components/engine/runconfig/hostconfig_test.go index a6a3eef7cd..ec9846ab16 100644 --- a/components/engine/runconfig/hostconfig_test.go +++ b/components/engine/runconfig/hostconfig_test.go @@ -10,6 +10,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/pkg/sysinfo" + "github.com/stretchr/testify/assert" ) // TODO Windows: This will need addressing for a Windows daemon. @@ -61,43 +62,33 @@ func TestNetworkModeTest(t *testing.T) { } func TestIpcModeTest(t *testing.T) { - ipcModes := map[container.IpcMode][]bool{ - // private, host, container, valid - "": {true, false, false, true}, - "something:weird": {true, false, false, false}, - ":weird": {true, false, false, true}, - "host": {false, true, false, true}, - "container:name": {false, false, true, true}, - "container:name:something": {false, false, true, false}, - "container:": {false, false, true, false}, + ipcModes := map[container.IpcMode]struct { + private bool + host bool + container bool + shareable bool + valid bool + ctrName string + }{ + "": {valid: true}, + "private": {private: true, valid: true}, + "something:weird": {}, + ":weird": {}, + "host": {host: true, valid: true}, + "container": {}, + "container:": {container: true, valid: true, ctrName: ""}, + "container:name": {container: true, valid: true, ctrName: "name"}, + "container:name1:name2": {container: true, valid: true, ctrName: "name1:name2"}, + "shareable": {shareable: true, valid: true}, } + for ipcMode, state := range ipcModes { - if ipcMode.IsPrivate() != state[0] { - t.Fatalf("IpcMode.IsPrivate for %v should have been %v but was %v", ipcMode, state[0], ipcMode.IsPrivate()) - } - if ipcMode.IsHost() != state[1] { - t.Fatalf("IpcMode.IsHost for %v should have been %v but was %v", ipcMode, state[1], ipcMode.IsHost()) - } - if ipcMode.IsContainer() != state[2] { - t.Fatalf("IpcMode.IsContainer for %v should have been %v but was %v", ipcMode, state[2], ipcMode.IsContainer()) - } - if ipcMode.Valid() != state[3] { - t.Fatalf("IpcMode.Valid for %v should have been %v but was %v", ipcMode, state[3], ipcMode.Valid()) - } - } - containerIpcModes := map[container.IpcMode]string{ - "": "", - "something": "", - "something:weird": "weird", - "container": "", - "container:": "", - "container:name": "name", - "container:name1:name2": "name1:name2", - } - for ipcMode, container := range containerIpcModes { - if ipcMode.Container() != container { - t.Fatalf("Expected %v for %v but was %v", container, ipcMode, ipcMode.Container()) - } + assert.Equal(t, state.private, ipcMode.IsPrivate(), "IpcMode.IsPrivate() parsing failed for %q", ipcMode) + assert.Equal(t, state.host, ipcMode.IsHost(), "IpcMode.IsHost() parsing failed for %q", ipcMode) + assert.Equal(t, state.container, ipcMode.IsContainer(), "IpcMode.IsContainer() parsing failed for %q", ipcMode) + assert.Equal(t, state.shareable, ipcMode.IsShareable(), "IpcMode.IsShareable() parsing failed for %q", ipcMode) + assert.Equal(t, state.valid, ipcMode.Valid(), "IpcMode.Valid() parsing failed for %q", ipcMode) + assert.Equal(t, state.ctrName, ipcMode.Container(), "IpcMode.Container() parsing failed for %q", ipcMode) } } diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 46934bae28..97c4e46732 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -1,16 +1,17 @@ # the following lines are in sorted order, FYI -github.com/Azure/go-ansiterm 388960b655244e76e24c75f48631564eaefade62 -github.com/Microsoft/hcsshim v0.5.25 -github.com/Microsoft/go-winio v0.4.2 -github.com/Sirupsen/logrus v0.11.0 +github.com/Azure/go-ansiterm 19f72df4d05d31cbe1c56bfc8045c96babff6c7e +github.com/Microsoft/hcsshim v0.6.3 +github.com/Microsoft/go-winio v0.4.5 +github.com/moby/buildkit da2b9dc7dab99e824b2b1067ad7d0523e32dd2d9 https://github.com/dmcgowan/buildkit.git github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git github.com/gorilla/context v1.1 github.com/gorilla/mux v1.1 -github.com/jhowardmsft/opengcs v0.0.9 +github.com/Microsoft/opengcs v0.3.2 github.com/kr/pty 5cf931ef8f github.com/mattn/go-shellwords v1.0.3 +github.com/sirupsen/logrus v1.0.1 github.com/tchap/go-patricia v2.2.6 github.com/vdemeester/shakers 24d7f1d6a71aa5d9cbe7390e4afb66b7eef9e1b3 golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 @@ -26,8 +27,8 @@ github.com/imdario/mergo 0.2.1 golang.org/x/sync de49d9dcd27d4f764488181bea099dfe6179bcf0 #get libnetwork packages -github.com/docker/libnetwork 6426d1e66f33c0b0c8bb135b7ee547447f54d043 -github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894 +github.com/docker/libnetwork 24bb72a8dcfe0b58958414890c8f4138b644b96a +github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec github.com/hashicorp/go-msgpack 71c2886f5a673a35f909803f38ece5810165097b @@ -50,7 +51,7 @@ github.com/boltdb/bolt fff57c100f4dea1905678da7e90d92429dff2904 github.com/miekg/dns 75e6e86cc601825c5dbcd4e0c209eab180997cd7 # get graph and distribution packages -github.com/docker/distribution b38e5838b7b2f2ad48e06ec4b500011976080621 +github.com/docker/distribution edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c github.com/vbatts/tar-split v0.10.1 github.com/opencontainers/go-digest a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb @@ -61,7 +62,7 @@ github.com/pborman/uuid v1.0 google.golang.org/grpc v1.3.0 # When updating, also update RUNC_COMMIT in hack/dockerfile/binaries-commits accordingly -github.com/opencontainers/runc 2d41c047c83e09a6d61d464906feb2a2f3c52aa4 https://github.com/docker/runc +github.com/opencontainers/runc e9325d442f5979c4f79bfa9e09bdf7abb74ba03b https://github.com/dmcgowan/runc.git github.com/opencontainers/image-spec 372ad780f63454fbbbbcc7cf80e5b90245c13e13 github.com/opencontainers/runtime-spec d42f1eb741e6361e858d83fc75aa6893b66292c4 # specs @@ -82,7 +83,7 @@ github.com/philhofer/fwd 98c11a7a6ec829d672b03833c3d69a7fae1ca972 github.com/tinylib/msgp 75ee40d2601edf122ef667e2a07d600d4c44490c # fsnotify -github.com/fsnotify/fsnotify v1.2.11 +github.com/fsnotify/fsnotify v1.4.2 # awslogs deps github.com/aws/aws-sdk-go v1.4.22 @@ -100,13 +101,13 @@ github.com/googleapis/gax-go da06d194a00e19ce00d9011a13931c3f6f6887c7 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 # containerd -github.com/containerd/containerd 3addd840653146c90a254301d6c3a663c7fd6429 +github.com/containerd/containerd fc10004571bb9b26695ccbf2dd4a83213f60b93e https://github.com/dmcgowan/containerd.git github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4 github.com/stevvooe/continuity cd7a8e21e2b6f84799f5dd4b65faf49c8d3ee02d github.com/tonistiigi/fsutil 0ac4c11b053b9c5c7c47558f81f96c7100ce50fb # cluster -github.com/docker/swarmkit 3e2dd3c0a76149b1620b42d28dd6ff48270404e5 +github.com/docker/swarmkit 8bdecc57887ffc598b63d6433f58e0d2852112c3 https://github.com/dmcgowan/swarmkit.git github.com/gogo/protobuf v0.4 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b6506826e @@ -143,4 +144,4 @@ github.com/opencontainers/selinux v1.0.0-rc1 # git --git-dir ./go/.git --work-tree ./go checkout revert-prefix-ignore # cp -a go/src/archive/tar ./vendor/archive/tar # rm -rf ./go -# vndr \ No newline at end of file +# vndr diff --git a/components/engine/vendor/github.com/Azure/go-ansiterm/README.md b/components/engine/vendor/github.com/Azure/go-ansiterm/README.md index e25e382101..261c041e7a 100644 --- a/components/engine/vendor/github.com/Azure/go-ansiterm/README.md +++ b/components/engine/vendor/github.com/Azure/go-ansiterm/README.md @@ -7,3 +7,6 @@ For example the parser might receive "ESC, [, A" as a stream of three characters The parser (parser.go) is a partial implementation of this state machine (http://vt100.net/emu/vt500_parser.png). There are also two event handler implementations, one for tests (test_event_handler.go) to validate that the expected events are being produced and called, the other is a Windows implementation (winterm/win_event_handler.go). See parser_test.go for examples exercising the state machine and generating appropriate function calls. + +----- +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/components/engine/vendor/github.com/Azure/go-ansiterm/parser.go b/components/engine/vendor/github.com/Azure/go-ansiterm/parser.go index 169f68dbef..3286a9cb5e 100644 --- a/components/engine/vendor/github.com/Azure/go-ansiterm/parser.go +++ b/components/engine/vendor/github.com/Azure/go-ansiterm/parser.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "os" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) var logger *logrus.Logger diff --git a/components/engine/vendor/github.com/Azure/go-ansiterm/winterm/win_event_handler.go b/components/engine/vendor/github.com/Azure/go-ansiterm/winterm/win_event_handler.go index 4d858ed611..48998bb051 100644 --- a/components/engine/vendor/github.com/Azure/go-ansiterm/winterm/win_event_handler.go +++ b/components/engine/vendor/github.com/Azure/go-ansiterm/winterm/win_event_handler.go @@ -9,7 +9,7 @@ import ( "strconv" "github.com/Azure/go-ansiterm" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) var logger *logrus.Logger diff --git a/components/engine/vendor/github.com/Microsoft/go-winio/backup.go b/components/engine/vendor/github.com/Microsoft/go-winio/backup.go index 27d6ace0c9..2be34af431 100644 --- a/components/engine/vendor/github.com/Microsoft/go-winio/backup.go +++ b/components/engine/vendor/github.com/Microsoft/go-winio/backup.go @@ -68,10 +68,20 @@ func NewBackupStreamReader(r io.Reader) *BackupStreamReader { return &BackupStreamReader{r, 0} } -// Next returns the next backup stream and prepares for calls to Write(). It skips the remainder of the current stream if +// Next returns the next backup stream and prepares for calls to Read(). It skips the remainder of the current stream if // it was not completely read. func (r *BackupStreamReader) Next() (*BackupHeader, error) { if r.bytesLeft > 0 { + if s, ok := r.r.(io.Seeker); ok { + // Make sure Seek on io.SeekCurrent sometimes succeeds + // before trying the actual seek. + if _, err := s.Seek(0, io.SeekCurrent); err == nil { + if _, err = s.Seek(r.bytesLeft, io.SeekCurrent); err != nil { + return nil, err + } + r.bytesLeft = 0 + } + } if _, err := io.Copy(ioutil.Discard, r); err != nil { return nil, err } @@ -220,7 +230,7 @@ type BackupFileWriter struct { ctx uintptr } -// NewBackupFileWrtier returns a new BackupFileWriter from a file handle. If includeSecurity is true, +// NewBackupFileWriter returns a new BackupFileWriter from a file handle. If includeSecurity is true, // Write() will attempt to restore the security descriptor from the stream. func NewBackupFileWriter(f *os.File, includeSecurity bool) *BackupFileWriter { w := &BackupFileWriter{f, includeSecurity, 0} diff --git a/components/engine/vendor/github.com/Microsoft/go-winio/backuptar/tar.go b/components/engine/vendor/github.com/Microsoft/go-winio/backuptar/tar.go index cfbcedfe04..53da908f14 100644 --- a/components/engine/vendor/github.com/Microsoft/go-winio/backuptar/tar.go +++ b/components/engine/vendor/github.com/Microsoft/go-winio/backuptar/tar.go @@ -36,6 +36,7 @@ const ( hdrSecurityDescriptor = "sd" hdrRawSecurityDescriptor = "rawsd" hdrMountPoint = "mountpoint" + hdrEaPrefix = "xattr." ) func writeZeroes(w io.Writer, count int64) error { @@ -118,6 +119,21 @@ func BasicInfoHeader(name string, size int64, fileInfo *winio.FileBasicInfo) *ta func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size int64, fileInfo *winio.FileBasicInfo) error { name = filepath.ToSlash(name) hdr := BasicInfoHeader(name, size, fileInfo) + + // If r can be seeked, then this function is two-pass: pass 1 collects the + // tar header data, and pass 2 copies the data stream. If r cannot be + // seeked, then some header data (in particular EAs) will be silently lost. + var ( + restartPos int64 + err error + ) + sr, readTwice := r.(io.Seeker) + if readTwice { + if restartPos, err = sr.Seek(0, io.SeekCurrent); err != nil { + readTwice = false + } + } + br := winio.NewBackupStreamReader(r) var dataHdr *winio.BackupHeader for dataHdr == nil { @@ -131,7 +147,9 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size switch bhdr.Id { case winio.BackupData: hdr.Mode |= c_ISREG - dataHdr = bhdr + if !readTwice { + dataHdr = bhdr + } case winio.BackupSecurity: sd, err := ioutil.ReadAll(br) if err != nil { @@ -151,18 +169,54 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size hdr.Winheaders[hdrMountPoint] = "1" } hdr.Linkname = rp.Target - case winio.BackupEaData, winio.BackupLink, winio.BackupPropertyData, winio.BackupObjectId, winio.BackupTxfsData: + + case winio.BackupEaData: + eab, err := ioutil.ReadAll(br) + if err != nil { + return err + } + eas, err := winio.DecodeExtendedAttributes(eab) + if err != nil { + return err + } + for _, ea := range eas { + // Use base64 encoding for the binary value. Note that there + // is no way to encode the EA's flags, since their use doesn't + // make any sense for persisted EAs. + hdr.Winheaders[hdrEaPrefix+ea.Name] = base64.StdEncoding.EncodeToString(ea.Value) + } + + case winio.BackupAlternateData, winio.BackupLink, winio.BackupPropertyData, winio.BackupObjectId, winio.BackupTxfsData: // ignore these streams default: return fmt.Errorf("%s: unknown stream ID %d", name, bhdr.Id) } } - err := t.WriteHeader(hdr) + err = t.WriteHeader(hdr) if err != nil { return err } + if readTwice { + // Get back to the data stream. + if _, err = sr.Seek(restartPos, io.SeekStart); err != nil { + return err + } + for dataHdr == nil { + bhdr, err := br.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + if bhdr.Id == winio.BackupData { + dataHdr = bhdr + } + } + } + if dataHdr != nil { // A data stream was found. Copy the data. if (dataHdr.Attributes & winio.StreamSparseAttributes) == 0 { @@ -293,6 +347,38 @@ func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) ( return nil, err } } + var eas []winio.ExtendedAttribute + for k, v := range hdr.Winheaders { + if !strings.HasPrefix(k, hdrEaPrefix) { + continue + } + data, err := base64.StdEncoding.DecodeString(v) + if err != nil { + return nil, err + } + eas = append(eas, winio.ExtendedAttribute{ + Name: k[len(hdrEaPrefix):], + Value: data, + }) + } + if len(eas) != 0 { + eadata, err := winio.EncodeExtendedAttributes(eas) + if err != nil { + return nil, err + } + bhdr := winio.BackupHeader{ + Id: winio.BackupEaData, + Size: int64(len(eadata)), + } + err = bw.WriteHeader(&bhdr) + if err != nil { + return nil, err + } + _, err = bw.Write(eadata) + if err != nil { + return nil, err + } + } if hdr.Typeflag == tar.TypeSymlink { _, isMountPoint := hdr.Winheaders[hdrMountPoint] rp := winio.ReparsePoint{ diff --git a/components/engine/vendor/github.com/Microsoft/go-winio/ea.go b/components/engine/vendor/github.com/Microsoft/go-winio/ea.go new file mode 100644 index 0000000000..b37e930d6a --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/go-winio/ea.go @@ -0,0 +1,137 @@ +package winio + +import ( + "bytes" + "encoding/binary" + "errors" +) + +type fileFullEaInformation struct { + NextEntryOffset uint32 + Flags uint8 + NameLength uint8 + ValueLength uint16 +} + +var ( + fileFullEaInformationSize = binary.Size(&fileFullEaInformation{}) + + errInvalidEaBuffer = errors.New("invalid extended attribute buffer") + errEaNameTooLarge = errors.New("extended attribute name too large") + errEaValueTooLarge = errors.New("extended attribute value too large") +) + +// ExtendedAttribute represents a single Windows EA. +type ExtendedAttribute struct { + Name string + Value []byte + Flags uint8 +} + +func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) { + var info fileFullEaInformation + err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info) + if err != nil { + err = errInvalidEaBuffer + return + } + + nameOffset := fileFullEaInformationSize + nameLen := int(info.NameLength) + valueOffset := nameOffset + int(info.NameLength) + 1 + valueLen := int(info.ValueLength) + nextOffset := int(info.NextEntryOffset) + if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) { + err = errInvalidEaBuffer + return + } + + ea.Name = string(b[nameOffset : nameOffset+nameLen]) + ea.Value = b[valueOffset : valueOffset+valueLen] + ea.Flags = info.Flags + if info.NextEntryOffset != 0 { + nb = b[info.NextEntryOffset:] + } + return +} + +// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION +// buffer retrieved from BackupRead, ZwQueryEaFile, etc. +func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) { + for len(b) != 0 { + ea, nb, err := parseEa(b) + if err != nil { + return nil, err + } + + eas = append(eas, ea) + b = nb + } + return +} + +func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error { + if int(uint8(len(ea.Name))) != len(ea.Name) { + return errEaNameTooLarge + } + if int(uint16(len(ea.Value))) != len(ea.Value) { + return errEaValueTooLarge + } + entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value)) + withPadding := (entrySize + 3) &^ 3 + nextOffset := uint32(0) + if !last { + nextOffset = withPadding + } + info := fileFullEaInformation{ + NextEntryOffset: nextOffset, + Flags: ea.Flags, + NameLength: uint8(len(ea.Name)), + ValueLength: uint16(len(ea.Value)), + } + + err := binary.Write(buf, binary.LittleEndian, &info) + if err != nil { + return err + } + + _, err = buf.Write([]byte(ea.Name)) + if err != nil { + return err + } + + err = buf.WriteByte(0) + if err != nil { + return err + } + + _, err = buf.Write(ea.Value) + if err != nil { + return err + } + + _, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize]) + if err != nil { + return err + } + + return nil +} + +// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION +// buffer for use with BackupWrite, ZwSetEaFile, etc. +func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) { + var buf bytes.Buffer + for i := range eas { + last := false + if i == len(eas)-1 { + last = true + } + + err := writeEa(&buf, &eas[i], last) + if err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} diff --git a/components/engine/vendor/github.com/Microsoft/go-winio/file.go b/components/engine/vendor/github.com/Microsoft/go-winio/file.go index 613f31b520..57ac3696a9 100644 --- a/components/engine/vendor/github.com/Microsoft/go-winio/file.go +++ b/components/engine/vendor/github.com/Microsoft/go-winio/file.go @@ -23,6 +23,13 @@ type atomicBool int32 func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 } func (b *atomicBool) setFalse() { atomic.StoreInt32((*int32)(b), 0) } func (b *atomicBool) setTrue() { atomic.StoreInt32((*int32)(b), 1) } +func (b *atomicBool) swap(new bool) bool { + var newInt int32 + if new { + newInt = 1 + } + return atomic.SwapInt32((*int32)(b), newInt) == 1 +} const ( cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS = 1 @@ -71,7 +78,8 @@ func initIo() { type win32File struct { handle syscall.Handle wg sync.WaitGroup - closing bool + wgLock sync.RWMutex + closing atomicBool readDeadline deadlineHandler writeDeadline deadlineHandler } @@ -107,14 +115,18 @@ func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) { // closeHandle closes the resources associated with a Win32 handle func (f *win32File) closeHandle() { - if !f.closing { + f.wgLock.Lock() + // Atomically set that we are closing, releasing the resources only once. + if !f.closing.swap(true) { + f.wgLock.Unlock() // cancel all IO and wait for it to complete - f.closing = true cancelIoEx(f.handle, nil) f.wg.Wait() // at this point, no new IO can start syscall.Close(f.handle) f.handle = 0 + } else { + f.wgLock.Unlock() } } @@ -127,10 +139,13 @@ func (f *win32File) Close() error { // prepareIo prepares for a new IO operation. // The caller must call f.wg.Done() when the IO is finished, prior to Close() returning. func (f *win32File) prepareIo() (*ioOperation, error) { - f.wg.Add(1) - if f.closing { + f.wgLock.RLock() + if f.closing.isSet() { + f.wgLock.RUnlock() return nil, ErrFileClosed } + f.wg.Add(1) + f.wgLock.RUnlock() c := &ioOperation{} c.ch = make(chan ioResult) return c, nil @@ -159,7 +174,7 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er return int(bytes), err } - if f.closing { + if f.closing.isSet() { cancelIoEx(f.handle, &c.o) } @@ -175,7 +190,7 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er case r = <-c.ch: err = r.err if err == syscall.ERROR_OPERATION_ABORTED { - if f.closing { + if f.closing.isSet() { err = ErrFileClosed } } diff --git a/components/engine/vendor/github.com/Microsoft/go-winio/pipe.go b/components/engine/vendor/github.com/Microsoft/go-winio/pipe.go index da706cc8a7..44340b8167 100644 --- a/components/engine/vendor/github.com/Microsoft/go-winio/pipe.go +++ b/components/engine/vendor/github.com/Microsoft/go-winio/pipe.go @@ -265,9 +265,9 @@ func (l *win32PipeListener) listenerRoutine() { if err == nil { // Wait for the client to connect. ch := make(chan error) - go func() { + go func(p *win32File) { ch <- connectPipe(p) - }() + }(p) select { case err = <-ch: if err != nil { diff --git a/components/engine/vendor/github.com/Microsoft/go-winio/vhd/vhd.go b/components/engine/vendor/github.com/Microsoft/go-winio/vhd/vhd.go new file mode 100644 index 0000000000..32f0701ea5 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/go-winio/vhd/vhd.go @@ -0,0 +1,82 @@ +// +build windows + +package vhd + +import "syscall" + +//go:generate go run mksyscall_windows.go -output zvhd.go vhd.go + +//sys createVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) [failretval != 0] = VirtDisk.CreateVirtualDisk + +type virtualStorageType struct { + DeviceID uint32 + VendorID [16]byte +} + +const virtualDiskAccessNONE uint32 = 0 +const virtualDiskAccessATTACHRO uint32 = 65536 +const virtualDiskAccessATTACHRW uint32 = 131072 +const virtualDiskAccessDETACH uint32 = 262144 +const virtualDiskAccessGETINFO uint32 = 524288 +const virtualDiskAccessCREATE uint32 = 1048576 +const virtualDiskAccessMETAOPS uint32 = 2097152 +const virtualDiskAccessREAD uint32 = 851968 +const virtualDiskAccessALL uint32 = 4128768 +const virtualDiskAccessWRITABLE uint32 = 3276800 + +const createVirtualDiskFlagNone uint32 = 0 +const createVirtualDiskFlagFullPhysicalAllocation uint32 = 1 +const createVirtualDiskFlagPreventWritesToSourceDisk uint32 = 2 +const createVirtualDiskFlagDoNotCopyMetadataFromParent uint32 = 4 + +type version2 struct { + UniqueID [16]byte // GUID + MaximumSize uint64 + BlockSizeInBytes uint32 + SectorSizeInBytes uint32 + ParentPath *uint16 // string + SourcePath *uint16 // string + OpenFlags uint32 + ParentVirtualStorageType virtualStorageType + SourceVirtualStorageType virtualStorageType + ResiliencyGUID [16]byte // GUID +} + +type createVirtualDiskParameters struct { + Version uint32 // Must always be set to 2 + Version2 version2 +} + +// CreateVhdx will create a simple vhdx file at the given path using default values. +func CreateVhdx(path string, maxSizeInGb, blockSizeInMb uint32) error { + var defaultType virtualStorageType + + parameters := createVirtualDiskParameters{ + Version: 2, + Version2: version2{ + MaximumSize: uint64(maxSizeInGb) * 1024 * 1024 * 1024, + BlockSizeInBytes: blockSizeInMb * 1024 * 1024, + }, + } + + var handle syscall.Handle + + if err := createVirtualDisk( + &defaultType, + path, + virtualDiskAccessNONE, + nil, + createVirtualDiskFlagNone, + 0, + ¶meters, + nil, + &handle); err != nil { + return err + } + + if err := syscall.CloseHandle(handle); err != nil { + return err + } + + return nil +} diff --git a/components/engine/vendor/github.com/Microsoft/go-winio/vhd/zvhd.go b/components/engine/vendor/github.com/Microsoft/go-winio/vhd/zvhd.go new file mode 100644 index 0000000000..c450955ab5 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/go-winio/vhd/zvhd.go @@ -0,0 +1,64 @@ +// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT + +package vhd + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( + modVirtDisk = windows.NewLazySystemDLL("VirtDisk.dll") + + procCreateVirtualDisk = modVirtDisk.NewProc("CreateVirtualDisk") +) + +func createVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) { + var _p0 *uint16 + _p0, err = syscall.UTF16PtrFromString(path) + if err != nil { + return + } + return _createVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, securityDescriptor, flags, providerSpecificFlags, parameters, o, handle) +} + +func _createVirtualDisk(virtualStorageType *virtualStorageType, path *uint16, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) { + r1, _, e1 := syscall.Syscall9(procCreateVirtualDisk.Addr(), 9, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(unsafe.Pointer(securityDescriptor)), uintptr(flags), uintptr(providerSpecificFlags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(o)), uintptr(unsafe.Pointer(handle))) + if r1 != 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/activatelayer.go b/components/engine/vendor/github.com/Microsoft/hcsshim/activatelayer.go index efc4d8029c..6d824d7a79 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/activatelayer.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/activatelayer.go @@ -1,6 +1,6 @@ package hcsshim -import "github.com/Sirupsen/logrus" +import "github.com/sirupsen/logrus" // ActivateLayer will find the layer with the given id and mount it's filesystem. // For a read/write layer, the mounted filesystem will appear as a volume on the diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/container.go b/components/engine/vendor/github.com/Microsoft/hcsshim/container.go index 8bc7d6d2b7..b924d39f46 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/container.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/container.go @@ -8,7 +8,7 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) var ( @@ -16,9 +16,10 @@ var ( ) const ( - pendingUpdatesQuery = `{ "PropertyTypes" : ["PendingUpdates"]}` - statisticsQuery = `{ "PropertyTypes" : ["Statistics"]}` - processListQuery = `{ "PropertyTypes" : ["ProcessList"]}` + pendingUpdatesQuery = `{ "PropertyTypes" : ["PendingUpdates"]}` + statisticsQuery = `{ "PropertyTypes" : ["Statistics"]}` + processListQuery = `{ "PropertyTypes" : ["ProcessList"]}` + mappedVirtualDiskQuery = `{ "PropertyTypes" : ["MappedVirtualDisk"]}` ) type container struct { @@ -30,20 +31,21 @@ type container struct { // ContainerProperties holds the properties for a container and the processes running in that container type ContainerProperties struct { - ID string `json:"Id"` - Name string - SystemType string - Owner string - SiloGUID string `json:"SiloGuid,omitempty"` - RuntimeID string `json:"RuntimeId,omitempty"` - IsRuntimeTemplate bool `json:",omitempty"` - RuntimeImagePath string `json:",omitempty"` - Stopped bool `json:",omitempty"` - ExitType string `json:",omitempty"` - AreUpdatesPending bool `json:",omitempty"` - ObRoot string `json:",omitempty"` - Statistics Statistics `json:",omitempty"` - ProcessList []ProcessListItem `json:",omitempty"` + ID string `json:"Id"` + Name string + SystemType string + Owner string + SiloGUID string `json:"SiloGuid,omitempty"` + RuntimeID string `json:"RuntimeId,omitempty"` + IsRuntimeTemplate bool `json:",omitempty"` + RuntimeImagePath string `json:",omitempty"` + Stopped bool `json:",omitempty"` + ExitType string `json:",omitempty"` + AreUpdatesPending bool `json:",omitempty"` + ObRoot string `json:",omitempty"` + Statistics Statistics `json:",omitempty"` + ProcessList []ProcessListItem `json:",omitempty"` + MappedVirtualDiskControllers map[int]MappedVirtualDiskController `json:",omitempty"` } // MemoryStats holds the memory statistics for a container @@ -103,6 +105,11 @@ type ProcessListItem struct { UserTime100ns uint64 `json:",omitempty"` } +// MappedVirtualDiskController is the structure of an item returned by a MappedVirtualDiskList call on a container +type MappedVirtualDiskController struct { + MappedVirtualDisks map[int]MappedVirtualDisk `json:",omitempty"` +} + // Type of Request Support in ModifySystem type RequestType string @@ -487,6 +494,55 @@ func (container *container) ProcessList() ([]ProcessListItem, error) { return properties.ProcessList, nil } +// MappedVirtualDisks returns a map of the controllers and the disks mapped +// to a container. +// +// Example of JSON returned by the query. +//{ +// "Id":"1126e8d7d279c707a666972a15976371d365eaf622c02cea2c442b84f6f550a3_svm", +// "SystemType":"Container", +// "RuntimeOsType":"Linux", +// "RuntimeId":"00000000-0000-0000-0000-000000000000", +// "State":"Running", +// "MappedVirtualDiskControllers":{ +// "0":{ +// "MappedVirtualDisks":{ +// "2":{ +// "HostPath":"C:\\lcow\\lcow\\scratch\\1126e8d7d279c707a666972a15976371d365eaf622c02cea2c442b84f6f550a3.vhdx", +// "ContainerPath":"/mnt/gcs/LinuxServiceVM/scratch", +// "Lun":2, +// "CreateInUtilityVM":true +// }, +// "3":{ +// "HostPath":"C:\\lcow\\lcow\\1126e8d7d279c707a666972a15976371d365eaf622c02cea2c442b84f6f550a3\\sandbox.vhdx", +// "Lun":3, +// "CreateInUtilityVM":true, +// "AttachOnly":true +// } +// } +// } +// } +//} +func (container *container) MappedVirtualDisks() (map[int]MappedVirtualDiskController, error) { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "MappedVirtualDiskList" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + if container.handle == 0 { + return nil, makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + properties, err := container.properties(mappedVirtualDiskQuery) + if err != nil { + return nil, makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s", container.id) + return properties.MappedVirtualDiskControllers, nil +} + // Pause pauses the execution of the container. This feature is not enabled in TP5. func (container *container) Pause() error { container.handleLock.RLock() diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/createlayer.go b/components/engine/vendor/github.com/Microsoft/hcsshim/createlayer.go index 9ecffb1cb0..035d9c3947 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/createlayer.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/createlayer.go @@ -1,6 +1,6 @@ package hcsshim -import "github.com/Sirupsen/logrus" +import "github.com/sirupsen/logrus" // CreateLayer creates a new, empty, read-only layer on the filesystem based on // the parent layer provided. diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/createsandboxlayer.go b/components/engine/vendor/github.com/Microsoft/hcsshim/createsandboxlayer.go index b69c3da368..7a6a8854cf 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/createsandboxlayer.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/createsandboxlayer.go @@ -1,6 +1,6 @@ package hcsshim -import "github.com/Sirupsen/logrus" +import "github.com/sirupsen/logrus" // CreateSandboxLayer creates and populates new read-write layer for use by a container. // This requires both the id of the direct parent layer, as well as the full list diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/deactivatelayer.go b/components/engine/vendor/github.com/Microsoft/hcsshim/deactivatelayer.go index c02bcb3a0b..fd785030fb 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/deactivatelayer.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/deactivatelayer.go @@ -1,6 +1,6 @@ package hcsshim -import "github.com/Sirupsen/logrus" +import "github.com/sirupsen/logrus" // DeactivateLayer will dismount a layer that was mounted via ActivateLayer. func DeactivateLayer(info DriverInfo, id string) error { diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/destroylayer.go b/components/engine/vendor/github.com/Microsoft/hcsshim/destroylayer.go index 91ed269eef..b1e3b89fc7 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/destroylayer.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/destroylayer.go @@ -1,6 +1,6 @@ package hcsshim -import "github.com/Sirupsen/logrus" +import "github.com/sirupsen/logrus" // DestroyLayer will remove the on-disk files representing the layer with the given // id, including that layer's containing folder, if any. diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/expandsandboxsize.go b/components/engine/vendor/github.com/Microsoft/hcsshim/expandsandboxsize.go index e168921841..6946c6a84f 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/expandsandboxsize.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/expandsandboxsize.go @@ -1,6 +1,6 @@ package hcsshim -import "github.com/Sirupsen/logrus" +import "github.com/sirupsen/logrus" // ExpandSandboxSize expands the size of a layer to at least size bytes. func ExpandSandboxSize(info DriverInfo, layerId string, size uint64) error { diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/exportlayer.go b/components/engine/vendor/github.com/Microsoft/hcsshim/exportlayer.go index 3c9b24ea6b..d7025f20ba 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/exportlayer.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/exportlayer.go @@ -7,7 +7,7 @@ import ( "syscall" "github.com/Microsoft/go-winio" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // ExportLayer will create a folder at exportFolderPath and fill that folder with diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/getlayermountpath.go b/components/engine/vendor/github.com/Microsoft/hcsshim/getlayermountpath.go index 41b5758926..89f8079d0f 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/getlayermountpath.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/getlayermountpath.go @@ -3,7 +3,7 @@ package hcsshim import ( "syscall" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // GetLayerMountPath will look for a mounted layer with the given id and return diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/getsharedbaseimages.go b/components/engine/vendor/github.com/Microsoft/hcsshim/getsharedbaseimages.go index 01ab4da3dd..05d3d9532a 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/getsharedbaseimages.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/getsharedbaseimages.go @@ -1,6 +1,6 @@ package hcsshim -import "github.com/Sirupsen/logrus" +import "github.com/sirupsen/logrus" // GetSharedBaseImages will enumerate the images stored in the common central // image store and return descriptive info about those images for the purpose diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/hcsshim.go b/components/engine/vendor/github.com/Microsoft/hcsshim/hcsshim.go index 3cb3a299ff..236ba1fa30 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/hcsshim.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/hcsshim.go @@ -8,7 +8,7 @@ import ( "syscall" "unsafe" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) //go:generate go run mksyscall_windows.go -output zhcsshim.go hcsshim.go diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go b/components/engine/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go new file mode 100644 index 0000000000..92afc0c249 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go @@ -0,0 +1,318 @@ +package hcsshim + +import ( + "encoding/json" + "fmt" + "net" + + "github.com/sirupsen/logrus" +) + +// HNSEndpoint represents a network endpoint in HNS +type HNSEndpoint struct { + Id string `json:"ID,omitempty"` + Name string `json:",omitempty"` + VirtualNetwork string `json:",omitempty"` + VirtualNetworkName string `json:",omitempty"` + Policies []json.RawMessage `json:",omitempty"` + MacAddress string `json:",omitempty"` + IPAddress net.IP `json:",omitempty"` + DNSSuffix string `json:",omitempty"` + DNSServerList string `json:",omitempty"` + GatewayAddress string `json:",omitempty"` + EnableInternalDNS bool `json:",omitempty"` + DisableICC bool `json:",omitempty"` + PrefixLength uint8 `json:",omitempty"` + IsRemoteEndpoint bool `json:",omitempty"` +} + +//SystemType represents the type of the system on which actions are done +type SystemType string + +// SystemType const +const ( + ContainerType SystemType = "Container" + VirtualMachineType SystemType = "VirtualMachine" + HostType SystemType = "Host" +) + +// EndpointAttachDetachRequest is the structure used to send request to the container to modify the system +// Supported resource types are Network and Request Types are Add/Remove +type EndpointAttachDetachRequest struct { + ContainerID string `json:"ContainerId,omitempty"` + SystemType SystemType `json:"SystemType"` + CompartmentID uint16 `json:"CompartmentId,omitempty"` + VirtualNICName string `json:"VirtualNicName,omitempty"` +} + +// EndpointResquestResponse is object to get the endpoint request response +type EndpointResquestResponse struct { + Success bool + Error string +} + +// HNSEndpointRequest makes a HNS call to modify/query a network endpoint +func HNSEndpointRequest(method, path, request string) (*HNSEndpoint, error) { + endpoint := &HNSEndpoint{} + err := hnsCall(method, "/endpoints/"+path, request, &endpoint) + if err != nil { + return nil, err + } + + return endpoint, nil +} + +// HNSListEndpointRequest makes a HNS call to query the list of available endpoints +func HNSListEndpointRequest() ([]HNSEndpoint, error) { + var endpoint []HNSEndpoint + err := hnsCall("GET", "/endpoints/", "", &endpoint) + if err != nil { + return nil, err + } + + return endpoint, nil +} + +// HotAttachEndpoint makes a HCS Call to attach the endpoint to the container +func HotAttachEndpoint(containerID string, endpointID string) error { + return modifyNetworkEndpoint(containerID, endpointID, Add) +} + +// HotDetachEndpoint makes a HCS Call to detach the endpoint from the container +func HotDetachEndpoint(containerID string, endpointID string) error { + return modifyNetworkEndpoint(containerID, endpointID, Remove) +} + +// ModifyContainer corresponding to the container id, by sending a request +func modifyContainer(id string, request *ResourceModificationRequestResponse) error { + container, err := OpenContainer(id) + if err != nil { + if IsNotExist(err) { + return ErrComputeSystemDoesNotExist + } + return getInnerError(err) + } + defer container.Close() + err = container.Modify(request) + if err != nil { + if IsNotSupported(err) { + return ErrPlatformNotSupported + } + return getInnerError(err) + } + + return nil +} + +func modifyNetworkEndpoint(containerID string, endpointID string, request RequestType) error { + requestMessage := &ResourceModificationRequestResponse{ + Resource: Network, + Request: request, + Data: endpointID, + } + err := modifyContainer(containerID, requestMessage) + + if err != nil { + return err + } + + return nil +} + +// GetHNSEndpointByID get the Endpoint by ID +func GetHNSEndpointByID(endpointID string) (*HNSEndpoint, error) { + return HNSEndpointRequest("GET", endpointID, "") +} + +// GetHNSEndpointByName gets the endpoint filtered by Name +func GetHNSEndpointByName(endpointName string) (*HNSEndpoint, error) { + hnsResponse, err := HNSListEndpointRequest() + if err != nil { + return nil, err + } + for _, hnsEndpoint := range hnsResponse { + if hnsEndpoint.Name == endpointName { + return &hnsEndpoint, nil + } + } + return nil, fmt.Errorf("Endpoint %v not found", endpointName) +} + +// Create Endpoint by sending EndpointRequest to HNS. TODO: Create a separate HNS interface to place all these methods +func (endpoint *HNSEndpoint) Create() (*HNSEndpoint, error) { + operation := "Create" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + + jsonString, err := json.Marshal(endpoint) + if err != nil { + return nil, err + } + return HNSEndpointRequest("POST", "", string(jsonString)) +} + +// Delete Endpoint by sending EndpointRequest to HNS +func (endpoint *HNSEndpoint) Delete() (*HNSEndpoint, error) { + operation := "Delete" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + + return HNSEndpointRequest("DELETE", endpoint.Id, "") +} + +// Update Endpoint +func (endpoint *HNSEndpoint) Update() (*HNSEndpoint, error) { + operation := "Update" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + jsonString, err := json.Marshal(endpoint) + if err != nil { + return nil, err + } + err = hnsCall("POST", "/endpoints/"+endpoint.Id, string(jsonString), &endpoint) + + return endpoint, err +} + +// ContainerHotAttach attaches an endpoint to a running container +func (endpoint *HNSEndpoint) ContainerHotAttach(containerID string) error { + operation := "ContainerHotAttach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s, containerId=%s", endpoint.Id, containerID) + + return modifyNetworkEndpoint(containerID, endpoint.Id, Add) +} + +// ContainerHotDetach detaches an endpoint from a running container +func (endpoint *HNSEndpoint) ContainerHotDetach(containerID string) error { + operation := "ContainerHotDetach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s, containerId=%s", endpoint.Id, containerID) + + return modifyNetworkEndpoint(containerID, endpoint.Id, Remove) +} + +// ApplyACLPolicy applies Acl Policy on the Endpoint +func (endpoint *HNSEndpoint) ApplyACLPolicy(policy *ACLPolicy) error { + operation := "ApplyACLPolicy" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + + jsonString, err := json.Marshal(policy) + if err != nil { + return err + } + endpoint.Policies[0] = jsonString + _, err = endpoint.Update() + return err +} + +// ContainerAttach attaches an endpoint to container +func (endpoint *HNSEndpoint) ContainerAttach(containerID string, compartmentID uint16) error { + operation := "ContainerAttach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + + requestMessage := &EndpointAttachDetachRequest{ + ContainerID: containerID, + CompartmentID: compartmentID, + SystemType: ContainerType, + } + response := &EndpointResquestResponse{} + jsonString, err := json.Marshal(requestMessage) + if err != nil { + return err + } + return hnsCall("POST", "/endpoints/"+endpoint.Id+"/attach", string(jsonString), &response) +} + +// ContainerDetach detaches an endpoint from container +func (endpoint *HNSEndpoint) ContainerDetach(containerID string) error { + operation := "ContainerDetach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + + requestMessage := &EndpointAttachDetachRequest{ + ContainerID: containerID, + SystemType: ContainerType, + } + response := &EndpointResquestResponse{} + + jsonString, err := json.Marshal(requestMessage) + if err != nil { + return err + } + return hnsCall("POST", "/endpoints/"+endpoint.Id+"/detach", string(jsonString), &response) +} + +// HostAttach attaches a nic on the host +func (endpoint *HNSEndpoint) HostAttach(compartmentID uint16) error { + operation := "HostAttach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + requestMessage := &EndpointAttachDetachRequest{ + CompartmentID: compartmentID, + SystemType: HostType, + } + response := &EndpointResquestResponse{} + + jsonString, err := json.Marshal(requestMessage) + if err != nil { + return err + } + return hnsCall("POST", "/endpoints/"+endpoint.Id+"/attach", string(jsonString), &response) + +} + +// HostDetach detaches a nic on the host +func (endpoint *HNSEndpoint) HostDetach() error { + operation := "HostDetach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + requestMessage := &EndpointAttachDetachRequest{ + SystemType: HostType, + } + response := &EndpointResquestResponse{} + + jsonString, err := json.Marshal(requestMessage) + if err != nil { + return err + } + return hnsCall("POST", "/endpoints/"+endpoint.Id+"/detach", string(jsonString), &response) +} + +// VirtualMachineNICAttach attaches a endpoint to a virtual machine +func (endpoint *HNSEndpoint) VirtualMachineNICAttach(virtualMachineNICName string) error { + operation := "VirtualMachineNicAttach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + requestMessage := &EndpointAttachDetachRequest{ + VirtualNICName: virtualMachineNICName, + SystemType: VirtualMachineType, + } + response := &EndpointResquestResponse{} + + jsonString, err := json.Marshal(requestMessage) + if err != nil { + return err + } + return hnsCall("POST", "/endpoints/"+endpoint.Id+"/attach", string(jsonString), &response) +} + +// VirtualMachineNICDetach detaches a endpoint from a virtual machine +func (endpoint *HNSEndpoint) VirtualMachineNICDetach() error { + operation := "VirtualMachineNicDetach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + + requestMessage := &EndpointAttachDetachRequest{ + SystemType: VirtualMachineType, + } + response := &EndpointResquestResponse{} + + jsonString, err := json.Marshal(requestMessage) + if err != nil { + return err + } + return hnsCall("POST", "/endpoints/"+endpoint.Id+"/detach", string(jsonString), &response) +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/hnsfuncs.go b/components/engine/vendor/github.com/Microsoft/hcsshim/hnsfuncs.go index 97ec2e8e0f..2c1b979ae8 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/hnsfuncs.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/hnsfuncs.go @@ -3,99 +3,10 @@ package hcsshim import ( "encoding/json" "fmt" - "net" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) -type NatPolicy struct { - Type string - Protocol string - InternalPort uint16 - ExternalPort uint16 -} - -type QosPolicy struct { - Type string - MaximumOutgoingBandwidthInBytes uint64 -} - -type VlanPolicy struct { - Type string - VLAN uint -} - -type VsidPolicy struct { - Type string - VSID uint -} - -type PaPolicy struct { - Type string - PA string -} - -// Subnet is assoicated with a network and represents a list -// of subnets available to the network -type Subnet struct { - AddressPrefix string `json:",omitempty"` - GatewayAddress string `json:",omitempty"` - Policies []json.RawMessage `json:",omitempty"` -} - -// MacPool is assoicated with a network and represents a list -// of macaddresses available to the network -type MacPool struct { - StartMacAddress string `json:",omitempty"` - EndMacAddress string `json:",omitempty"` -} - -// HNSNetwork represents a network in HNS -type HNSNetwork struct { - Id string `json:"ID,omitempty"` - Name string `json:",omitempty"` - Type string `json:",omitempty"` - NetworkAdapterName string `json:",omitempty"` - SourceMac string `json:",omitempty"` - Policies []json.RawMessage `json:",omitempty"` - MacPools []MacPool `json:",omitempty"` - Subnets []Subnet `json:",omitempty"` - DNSSuffix string `json:",omitempty"` - DNSServerList string `json:",omitempty"` - DNSServerCompartment uint32 `json:",omitempty"` - ManagementIP string `json:",omitempty"` -} - -// HNSEndpoint represents a network endpoint in HNS -type HNSEndpoint struct { - Id string `json:"ID,omitempty"` - Name string `json:",omitempty"` - VirtualNetwork string `json:",omitempty"` - VirtualNetworkName string `json:",omitempty"` - Policies []json.RawMessage `json:",omitempty"` - MacAddress string `json:",omitempty"` - IPAddress net.IP `json:",omitempty"` - DNSSuffix string `json:",omitempty"` - DNSServerList string `json:",omitempty"` - GatewayAddress string `json:",omitempty"` - EnableInternalDNS bool `json:",omitempty"` - DisableICC bool `json:",omitempty"` - PrefixLength uint8 `json:",omitempty"` - IsRemoteEndpoint bool `json:",omitempty"` -} - -type hnsNetworkResponse struct { - Success bool - Error string - Output HNSNetwork -} - -type hnsResponse struct { - Success bool - Error string - Output json.RawMessage -} - func hnsCall(method, path, request string, returnResponse interface{}) error { var responseBuffer *uint16 logrus.Debugf("[%s]=>[%s] Request : %s", method, path, request) @@ -127,145 +38,3 @@ func hnsCall(method, path, request string, returnResponse interface{}) error { return nil } - -// HNSNetworkRequest makes a call into HNS to update/query a single network -func HNSNetworkRequest(method, path, request string) (*HNSNetwork, error) { - var network HNSNetwork - err := hnsCall(method, "/networks/"+path, request, &network) - if err != nil { - return nil, err - } - - return &network, nil -} - -// HNSListNetworkRequest makes a HNS call to query the list of available networks -func HNSListNetworkRequest(method, path, request string) ([]HNSNetwork, error) { - var network []HNSNetwork - err := hnsCall(method, "/networks/"+path, request, &network) - if err != nil { - return nil, err - } - - return network, nil -} - -// HNSEndpointRequest makes a HNS call to modify/query a network endpoint -func HNSEndpointRequest(method, path, request string) (*HNSEndpoint, error) { - endpoint := &HNSEndpoint{} - err := hnsCall(method, "/endpoints/"+path, request, &endpoint) - if err != nil { - return nil, err - } - - return endpoint, nil -} - -// HNSListEndpointRequest makes a HNS call to query the list of available endpoints -func HNSListEndpointRequest() ([]HNSEndpoint, error) { - var endpoint []HNSEndpoint - err := hnsCall("GET", "/endpoints/", "", &endpoint) - if err != nil { - return nil, err - } - - return endpoint, nil -} - -// HotAttachEndpoint makes a HCS Call to attach the endpoint to the container -func HotAttachEndpoint(containerID string, endpointID string) error { - return modifyNetworkEndpoint(containerID, endpointID, Add) -} - -// HotDetachEndpoint makes a HCS Call to detach the endpoint from the container -func HotDetachEndpoint(containerID string, endpointID string) error { - return modifyNetworkEndpoint(containerID, endpointID, Remove) -} - -// ModifyContainer corresponding to the container id, by sending a request -func modifyContainer(id string, request *ResourceModificationRequestResponse) error { - container, err := OpenContainer(id) - if err != nil { - if IsNotExist(err) { - return ErrComputeSystemDoesNotExist - } - return getInnerError(err) - } - defer container.Close() - err = container.Modify(request) - if err != nil { - if IsNotSupported(err) { - return ErrPlatformNotSupported - } - return getInnerError(err) - } - - return nil -} - -func modifyNetworkEndpoint(containerID string, endpointID string, request RequestType) error { - requestMessage := &ResourceModificationRequestResponse{ - Resource: Network, - Request: request, - Data: endpointID, - } - err := modifyContainer(containerID, requestMessage) - - if err != nil { - return err - } - - return nil -} - -// GetHNSNetworkByID -func GetHNSNetworkByID(networkID string) (*HNSNetwork, error) { - return HNSNetworkRequest("GET", networkID, "") -} - -// GetHNSNetworkName filtered by Name -func GetHNSNetworkByName(networkName string) (*HNSNetwork, error) { - hsnnetworks, err := HNSListNetworkRequest("GET", "", "") - if err != nil { - return nil, err - } - for _, hnsnetwork := range hsnnetworks { - if hnsnetwork.Name == networkName { - return &hnsnetwork, nil - } - } - return nil, fmt.Errorf("Network %v not found", networkName) -} - -// Create Endpoint by sending EndpointRequest to HNS. TODO: Create a separate HNS interface to place all these methods -func (endpoint *HNSEndpoint) Create() (*HNSEndpoint, error) { - jsonString, err := json.Marshal(endpoint) - if err != nil { - return nil, err - } - return HNSEndpointRequest("POST", "", string(jsonString)) -} - -// Create Endpoint by sending EndpointRequest to HNS -func (endpoint *HNSEndpoint) Delete() (*HNSEndpoint, error) { - return HNSEndpointRequest("DELETE", endpoint.Id, "") -} - -// GetHNSEndpointByID -func GetHNSEndpointByID(endpointID string) (*HNSEndpoint, error) { - return HNSEndpointRequest("GET", endpointID, "") -} - -// GetHNSNetworkName filtered by Name -func GetHNSEndpointByName(endpointName string) (*HNSEndpoint, error) { - hnsResponse, err := HNSListEndpointRequest() - if err != nil { - return nil, err - } - for _, hnsEndpoint := range hnsResponse { - if hnsEndpoint.Name == endpointName { - return &hnsEndpoint, nil - } - } - return nil, fmt.Errorf("Endpoint %v not found", endpointName) -} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/hnsnetwork.go b/components/engine/vendor/github.com/Microsoft/hcsshim/hnsnetwork.go new file mode 100644 index 0000000000..3345bfa3f2 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/hnsnetwork.go @@ -0,0 +1,142 @@ +package hcsshim + +import ( + "encoding/json" + "fmt" + "net" + + "github.com/sirupsen/logrus" +) + +// Subnet is assoicated with a network and represents a list +// of subnets available to the network +type Subnet struct { + AddressPrefix string `json:",omitempty"` + GatewayAddress string `json:",omitempty"` + Policies []json.RawMessage `json:",omitempty"` +} + +// MacPool is assoicated with a network and represents a list +// of macaddresses available to the network +type MacPool struct { + StartMacAddress string `json:",omitempty"` + EndMacAddress string `json:",omitempty"` +} + +// HNSNetwork represents a network in HNS +type HNSNetwork struct { + Id string `json:"ID,omitempty"` + Name string `json:",omitempty"` + Type string `json:",omitempty"` + NetworkAdapterName string `json:",omitempty"` + SourceMac string `json:",omitempty"` + Policies []json.RawMessage `json:",omitempty"` + MacPools []MacPool `json:",omitempty"` + Subnets []Subnet `json:",omitempty"` + DNSSuffix string `json:",omitempty"` + DNSServerList string `json:",omitempty"` + DNSServerCompartment uint32 `json:",omitempty"` + ManagementIP string `json:",omitempty"` + AutomaticDNS bool `json:",omitempty"` +} + +type hnsNetworkResponse struct { + Success bool + Error string + Output HNSNetwork +} + +type hnsResponse struct { + Success bool + Error string + Output json.RawMessage +} + +// HNSNetworkRequest makes a call into HNS to update/query a single network +func HNSNetworkRequest(method, path, request string) (*HNSNetwork, error) { + var network HNSNetwork + err := hnsCall(method, "/networks/"+path, request, &network) + if err != nil { + return nil, err + } + + return &network, nil +} + +// HNSListNetworkRequest makes a HNS call to query the list of available networks +func HNSListNetworkRequest(method, path, request string) ([]HNSNetwork, error) { + var network []HNSNetwork + err := hnsCall(method, "/networks/"+path, request, &network) + if err != nil { + return nil, err + } + + return network, nil +} + +// GetHNSNetworkByID +func GetHNSNetworkByID(networkID string) (*HNSNetwork, error) { + return HNSNetworkRequest("GET", networkID, "") +} + +// GetHNSNetworkName filtered by Name +func GetHNSNetworkByName(networkName string) (*HNSNetwork, error) { + hsnnetworks, err := HNSListNetworkRequest("GET", "", "") + if err != nil { + return nil, err + } + for _, hnsnetwork := range hsnnetworks { + if hnsnetwork.Name == networkName { + return &hnsnetwork, nil + } + } + return nil, fmt.Errorf("Network %v not found", networkName) +} + +// Create Network by sending NetworkRequest to HNS. +func (network *HNSNetwork) Create() (*HNSNetwork, error) { + operation := "Create" + title := "HCSShim::HNSNetwork::" + operation + logrus.Debugf(title+" id=%s", network.Id) + + jsonString, err := json.Marshal(network) + if err != nil { + return nil, err + } + return HNSNetworkRequest("POST", "", string(jsonString)) +} + +// Delete Network by sending NetworkRequest to HNS +func (network *HNSNetwork) Delete() (*HNSNetwork, error) { + operation := "Delete" + title := "HCSShim::HNSNetwork::" + operation + logrus.Debugf(title+" id=%s", network.Id) + + return HNSNetworkRequest("DELETE", network.Id, "") +} + +// Creates an endpoint on the Network. +func (network *HNSNetwork) NewEndpoint(ipAddress net.IP, macAddress net.HardwareAddr) *HNSEndpoint { + return &HNSEndpoint{ + VirtualNetwork: network.Id, + IPAddress: ipAddress, + MacAddress: string(macAddress), + } +} + +func (network *HNSNetwork) CreateEndpoint(endpoint *HNSEndpoint) (*HNSEndpoint, error) { + operation := "CreateEndpoint" + title := "HCSShim::HNSNetwork::" + operation + logrus.Debugf(title+" id=%s, endpointId=%s", network.Id, endpoint.Id) + + endpoint.VirtualNetwork = network.Id + return endpoint.Create() +} + +func (network *HNSNetwork) CreateRemoteEndpoint(endpoint *HNSEndpoint) (*HNSEndpoint, error) { + operation := "CreateRemoteEndpoint" + title := "HCSShim::HNSNetwork::" + operation + logrus.Debugf(title+" id=%s", network.Id) + endpoint.IsRemoteEndpoint = true + return network.CreateEndpoint(endpoint) +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/hnspolicy.go b/components/engine/vendor/github.com/Microsoft/hcsshim/hnspolicy.go new file mode 100644 index 0000000000..ecfbf0eda3 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/hnspolicy.go @@ -0,0 +1,95 @@ +package hcsshim + +// Type of Request Support in ModifySystem +type PolicyType string + +// RequestType const +const ( + Nat PolicyType = "NAT" + ACL PolicyType = "ACL" + PA PolicyType = "PA" + VLAN PolicyType = "VLAN" + VSID PolicyType = "VSID" + VNet PolicyType = "VNET" + L2Driver PolicyType = "L2Driver" + Isolation PolicyType = "Isolation" + QOS PolicyType = "QOS" + OutboundNat PolicyType = "OutBoundNAT" + ExternalLoadBalancer PolicyType = "ELB" + Route PolicyType = "ROUTE" +) + +type NatPolicy struct { + Type PolicyType `json:"Type"` + Protocol string + InternalPort uint16 + ExternalPort uint16 +} + +type QosPolicy struct { + Type PolicyType `json:"Type"` + MaximumOutgoingBandwidthInBytes uint64 +} + +type IsolationPolicy struct { + Type PolicyType `json:"Type"` + VLAN uint + VSID uint + InDefaultIsolation bool +} + +type VlanPolicy struct { + Type PolicyType `json:"Type"` + VLAN uint +} + +type VsidPolicy struct { + Type PolicyType `json:"Type"` + VSID uint +} + +type PaPolicy struct { + Type PolicyType `json:"Type"` + PA string `json:"PA"` +} + +type OutboundNatPolicy struct { + Policy + VIP string `json:"VIP,omitempty"` + Exceptions []string `json:"ExceptionList,omitempty"` +} + +type ActionType string +type DirectionType string +type RuleType string + +const ( + Allow ActionType = "Allow" + Block ActionType = "Block" + + In DirectionType = "In" + Out DirectionType = "Out" + + Host RuleType = "Host" + Switch RuleType = "Switch" +) + +type ACLPolicy struct { + Type PolicyType `json:"Type"` + Protocol uint16 + InternalPort uint16 + Action ActionType + Direction DirectionType + LocalAddress string + RemoteAddress string + LocalPort uint16 + RemotePort uint16 + RuleType RuleType `json:"RuleType,omitempty"` + + Priority uint16 + ServiceName string +} + +type Policy struct { + Type PolicyType `json:"Type"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/hnspolicylist.go b/components/engine/vendor/github.com/Microsoft/hcsshim/hnspolicylist.go new file mode 100644 index 0000000000..bbd7e1edb0 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/hnspolicylist.go @@ -0,0 +1,200 @@ +package hcsshim + +import ( + "encoding/json" + + "github.com/sirupsen/logrus" +) + +// RoutePolicy is a structure defining schema for Route based Policy +type RoutePolicy struct { + Policy + DestinationPrefix string `json:"DestinationPrefix,omitempty"` + NextHop string `json:"NextHop,omitempty"` + EncapEnabled bool `json:"NeedEncap,omitempty"` +} + +// ELBPolicy is a structure defining schema for ELB LoadBalancing based Policy +type ELBPolicy struct { + LBPolicy + SourceVIP string `json:"SourceVIP,omitempty"` + VIPs []string `json:"VIPs,omitempty"` + ILB bool `json:"ILB,omitempty"` +} + +// LBPolicy is a structure defining schema for LoadBalancing based Policy +type LBPolicy struct { + Policy + Protocol uint16 `json:"Protocol,omitempty"` + InternalPort uint16 + ExternalPort uint16 +} + +// PolicyList is a structure defining schema for Policy list request +type PolicyList struct { + ID string `json:"ID,omitempty"` + EndpointReferences []string `json:"References,omitempty"` + Policies []json.RawMessage `json:"Policies,omitempty"` +} + +// HNSPolicyListRequest makes a call into HNS to update/query a single network +func HNSPolicyListRequest(method, path, request string) (*PolicyList, error) { + var policy PolicyList + err := hnsCall(method, "/policylists/"+path, request, &policy) + if err != nil { + return nil, err + } + + return &policy, nil +} + +// HNSListPolicyListRequest gets all the policy list +func HNSListPolicyListRequest() ([]PolicyList, error) { + var plist []PolicyList + err := hnsCall("GET", "/policylists/", "", &plist) + if err != nil { + return nil, err + } + + return plist, nil +} + +// PolicyListRequest makes a HNS call to modify/query a network policy list +func PolicyListRequest(method, path, request string) (*PolicyList, error) { + policylist := &PolicyList{} + err := hnsCall(method, "/policylists/"+path, request, &policylist) + if err != nil { + return nil, err + } + + return policylist, nil +} + +// GetPolicyListByID get the policy list by ID +func GetPolicyListByID(policyListID string) (*PolicyList, error) { + return PolicyListRequest("GET", policyListID, "") +} + +// Create PolicyList by sending PolicyListRequest to HNS. +func (policylist *PolicyList) Create() (*PolicyList, error) { + operation := "Create" + title := "HCSShim::PolicyList::" + operation + logrus.Debugf(title+" id=%s", policylist.ID) + jsonString, err := json.Marshal(policylist) + if err != nil { + return nil, err + } + return PolicyListRequest("POST", "", string(jsonString)) +} + +// Delete deletes PolicyList +func (policylist *PolicyList) Delete() (*PolicyList, error) { + operation := "Delete" + title := "HCSShim::PolicyList::" + operation + logrus.Debugf(title+" id=%s", policylist.ID) + + return PolicyListRequest("DELETE", policylist.ID, "") +} + +// AddEndpoint add an endpoint to a Policy List +func (policylist *PolicyList) AddEndpoint(endpoint *HNSEndpoint) (*PolicyList, error) { + operation := "AddEndpoint" + title := "HCSShim::PolicyList::" + operation + logrus.Debugf(title+" id=%s, endpointId:%s", policylist.ID, endpoint.Id) + + _, err := policylist.Delete() + if err != nil { + return nil, err + } + + // Add Endpoint to the Existing List + policylist.EndpointReferences = append(policylist.EndpointReferences, "/endpoints/"+endpoint.Id) + + return policylist.Create() +} + +// RemoveEndpoint removes an endpoint from the Policy List +func (policylist *PolicyList) RemoveEndpoint(endpoint *HNSEndpoint) (*PolicyList, error) { + operation := "RemoveEndpoint" + title := "HCSShim::PolicyList::" + operation + logrus.Debugf(title+" id=%s, endpointId:%s", policylist.ID, endpoint.Id) + + _, err := policylist.Delete() + if err != nil { + return nil, err + } + + elementToRemove := "/endpoints/" + endpoint.Id + + var references []string + + for _, endpointReference := range policylist.EndpointReferences { + if endpointReference == elementToRemove { + continue + } + references = append(references, endpointReference) + } + policylist.EndpointReferences = references + return policylist.Create() +} + +// AddLoadBalancer policy list for the specified endpoints +func AddLoadBalancer(endpoints []HNSEndpoint, isILB bool, sourceVIP, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*PolicyList, error) { + operation := "AddLoadBalancer" + title := "HCSShim::PolicyList::" + operation + logrus.Debugf(title+" endpointId=%v, isILB=%v, sourceVIP=%s, vip=%s, protocol=%v, internalPort=%v, externalPort=%v", endpoints, isILB, sourceVIP, vip, protocol, internalPort, externalPort) + + policylist := &PolicyList{} + + elbPolicy := &ELBPolicy{ + SourceVIP: sourceVIP, + ILB: isILB, + } + + if len(vip) > 0 { + elbPolicy.VIPs = []string{vip} + } + elbPolicy.Type = ExternalLoadBalancer + elbPolicy.Protocol = protocol + elbPolicy.InternalPort = internalPort + elbPolicy.ExternalPort = externalPort + + for _, endpoint := range endpoints { + policylist.EndpointReferences = append(policylist.EndpointReferences, "/endpoints/"+endpoint.Id) + } + + jsonString, err := json.Marshal(elbPolicy) + if err != nil { + return nil, err + } + policylist.Policies = append(policylist.Policies, jsonString) + return policylist.Create() +} + +// AddRoute adds route policy list for the specified endpoints +func AddRoute(endpoints []HNSEndpoint, destinationPrefix string, nextHop string, encapEnabled bool) (*PolicyList, error) { + operation := "AddRoute" + title := "HCSShim::PolicyList::" + operation + logrus.Debugf(title+" destinationPrefix:%s", destinationPrefix) + + policylist := &PolicyList{} + + rPolicy := &RoutePolicy{ + DestinationPrefix: destinationPrefix, + NextHop: nextHop, + EncapEnabled: encapEnabled, + } + rPolicy.Type = Route + + for _, endpoint := range endpoints { + policylist.EndpointReferences = append(policylist.EndpointReferences, "/endpoints/"+endpoint.Id) + } + + jsonString, err := json.Marshal(rPolicy) + if err != nil { + return nil, err + } + + policylist.Policies = append(policylist.Policies, jsonString) + return policylist.Create() +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/importlayer.go b/components/engine/vendor/github.com/Microsoft/hcsshim/importlayer.go index 75dcd94777..3aed14376a 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/importlayer.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/importlayer.go @@ -7,7 +7,7 @@ import ( "path/filepath" "github.com/Microsoft/go-winio" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // ImportLayer will take the contents of the folder at importFolderPath and import diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/interface.go b/components/engine/vendor/github.com/Microsoft/hcsshim/interface.go index 5238a641ea..9fc7852e41 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/interface.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/interface.go @@ -37,11 +37,19 @@ type MappedDir struct { IOPSMaximum uint64 } +type MappedPipe struct { + HostPath string + ContainerPipeName string +} + type HvRuntime struct { - ImagePath string `json:",omitempty"` - SkipTemplate bool `json:",omitempty"` - LinuxInitrdFile string `json:",omitempty"` // File under ImagePath on host containing an initrd image for starting a Linux utility VM - LinuxKernelFile string `json:",omitempty"` // File under ImagePath on host containing a kernel for starting a Linux utility VM + ImagePath string `json:",omitempty"` + SkipTemplate bool `json:",omitempty"` + LinuxInitrdFile string `json:",omitempty"` // File under ImagePath on host containing an initrd image for starting a Linux utility VM + LinuxKernelFile string `json:",omitempty"` // File under ImagePath on host containing a kernel for starting a Linux utility VM + LinuxBootParameters string `json:",omitempty"` // Additional boot parameters for starting a Linux Utility VM in initrd mode + BootSource string `json:",omitempty"` // "Vhd" for Linux Utility VM booting from VHD + WritableBootSource bool `json:",omitempty"` // Linux Utility VM booting from VHD } type MappedVirtualDisk struct { @@ -50,6 +58,7 @@ type MappedVirtualDisk struct { CreateInUtilityVM bool `json:",omitempty"` ReadOnly bool `json:",omitempty"` Cache string `json:",omitempty"` // "" (Unspecified); "Disabled"; "Enabled"; "Private"; "PrivateAllowSharing" + AttachOnly bool `json:",omitempty:` } // ContainerConfig is used as both the input of CreateContainer @@ -64,14 +73,15 @@ type ContainerConfig struct { Layers []Layer // List of storage layers. Required for Windows Server and Hyper-V Containers. Format ID=GUID;Path=%root%\windowsfilter\layerID Credentials string `json:",omitempty"` // Credentials information ProcessorCount uint32 `json:",omitempty"` // Number of processors to assign to the container. - ProcessorWeight uint64 `json:",omitempty"` // CPU Shares 0..10000 on Windows; where 0 will be omitted and HCS will default. - ProcessorMaximum int64 `json:",omitempty"` // CPU maximum usage percent 1..100 + ProcessorWeight uint64 `json:",omitempty"` // CPU shares (relative weight to other containers with cpu shares). Range is from 1 to 10000. A value of 0 results in default shares. + ProcessorMaximum int64 `json:",omitempty"` // Specifies the portion of processor cycles that this container can use as a percentage times 100. Range is from 1 to 10000. A value of 0 results in no limit. StorageIOPSMaximum uint64 `json:",omitempty"` // Maximum Storage IOPS StorageBandwidthMaximum uint64 `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second StorageSandboxSize uint64 `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller MemoryMaximumInMB int64 `json:",omitempty"` // Maximum memory available to the container in Megabytes HostName string `json:",omitempty"` // Hostname MappedDirectories []MappedDir `json:",omitempty"` // List of mapped directories (volumes/mounts) + MappedPipes []MappedPipe `json:",omitempty"` // List of mapped Windows named pipes HvPartition bool // True if it a Hyper-V Container NetworkSharedContainerName string `json:",omitempty"` // Name (ID) of the container that we will share the network stack with. EndpointList []string `json:",omitempty"` // List of networking endpoints to be attached to container @@ -124,6 +134,9 @@ type Container interface { // ProcessList returns details for the processes in a container. ProcessList() ([]ProcessListItem, error) + // MappedVirtualDisks returns virtual disks mapped to a utility VM, indexed by controller + MappedVirtualDisks() (map[int]MappedVirtualDiskController, error) + // CreateProcess launches a new process within the container. CreateProcess(c *ProcessConfig) (Process, error) diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/layerexists.go b/components/engine/vendor/github.com/Microsoft/hcsshim/layerexists.go index 522d95cce4..fe46f404c3 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/layerexists.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/layerexists.go @@ -1,6 +1,6 @@ package hcsshim -import "github.com/Sirupsen/logrus" +import "github.com/sirupsen/logrus" // LayerExists will return true if a layer with the given id exists and is known // to the system. diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/layerutils.go b/components/engine/vendor/github.com/Microsoft/hcsshim/layerutils.go index 47229d22e5..c0e5503773 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/layerutils.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/layerutils.go @@ -7,7 +7,7 @@ import ( "path/filepath" "syscall" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) /* To pass into syscall, we need a struct matching the following: diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/legacy.go b/components/engine/vendor/github.com/Microsoft/hcsshim/legacy.go index 85761573ca..c7f6073ac3 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/legacy.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/legacy.go @@ -307,6 +307,16 @@ func (r *legacyLayerReader) Read(b []byte) (int, error) { return r.backupReader.Read(b) } +func (r *legacyLayerReader) Seek(offset int64, whence int) (int64, error) { + if r.backupReader == nil { + if r.currentFile == nil { + return 0, errors.New("no current file") + } + return r.currentFile.Seek(offset, whence) + } + return 0, errors.New("seek not supported on this stream") +} + func (r *legacyLayerReader) Close() error { r.proceed <- false <-r.result diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/nametoguid.go b/components/engine/vendor/github.com/Microsoft/hcsshim/nametoguid.go index 1a522f95e0..b7c6d020c6 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/nametoguid.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/nametoguid.go @@ -1,6 +1,6 @@ package hcsshim -import "github.com/Sirupsen/logrus" +import "github.com/sirupsen/logrus" // NameToGuid converts the given string into a GUID using the algorithm in the // Host Compute Service, ensuring GUIDs generated with the same string are common diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/preparelayer.go b/components/engine/vendor/github.com/Microsoft/hcsshim/preparelayer.go index 2791683467..5c5b618411 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/preparelayer.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/preparelayer.go @@ -3,7 +3,7 @@ package hcsshim import ( "sync" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) var prepareLayerLock sync.Mutex diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/process.go b/components/engine/vendor/github.com/Microsoft/hcsshim/process.go index 4ef0ed3e52..faee2cfeeb 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/process.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/process.go @@ -7,7 +7,7 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // ContainerError is an error encountered in HCS diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/unpreparelayer.go b/components/engine/vendor/github.com/Microsoft/hcsshim/unpreparelayer.go index d0ead0bdda..e8a3b507bf 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/unpreparelayer.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/unpreparelayer.go @@ -1,6 +1,6 @@ package hcsshim -import "github.com/Sirupsen/logrus" +import "github.com/sirupsen/logrus" // UnprepareLayer disables the filesystem filter for the read-write layer with // the given id. diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/waithelper.go b/components/engine/vendor/github.com/Microsoft/hcsshim/waithelper.go index 828d148e58..b7be20ea0c 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/waithelper.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/waithelper.go @@ -3,7 +3,7 @@ package hcsshim import ( "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) func processAsyncHcsResult(err error, resultp *uint16, callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) error { diff --git a/components/engine/vendor/github.com/Microsoft/opengcs/LICENSE b/components/engine/vendor/github.com/Microsoft/opengcs/LICENSE new file mode 100644 index 0000000000..4b1ad51b2f --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/opengcs/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/components/engine/vendor/github.com/Microsoft/opengcs/README.md b/components/engine/vendor/github.com/Microsoft/opengcs/README.md new file mode 100644 index 0000000000..e931fde8f3 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/opengcs/README.md @@ -0,0 +1,14 @@ + +# Open Guest Compute Service (opengcs) [![Build Status](https://travis-ci.org/Microsoft/opengcs.svg?branch=master)](https://travis-ci.org/Microsoft/opengcs) + +Open Guest Compute Service is a Linux open source project to further the development of a production quality implementation of Linux Hyper-V container on Windows (LCOW). It's designed to run inside a custom Linux OS for supporting Linux container payload. + +# Getting Started + + [How to build GCS binaries](./docs/gcsbuildinstructions.md/) + + [How to build custom Linux OS images](./docs/customosbuildinstructions.md/) + +# Contributing + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/components/engine/vendor/github.com/Microsoft/opengcs/client/config.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/config.go new file mode 100644 index 0000000000..12119574a3 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/config.go @@ -0,0 +1,274 @@ +// +build windows + +package client + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/Microsoft/hcsshim" + "github.com/sirupsen/logrus" +) + +// Mode is the operational mode, both requested, and actual after verification +type Mode uint + +const ( + // Constants for the actual mode after validation + + // ModeActualError means an error has occurred during validation + ModeActualError = iota + // ModeActualVhdx means that we are going to use VHDX boot after validation + ModeActualVhdx + // ModeActualKernelInitrd means that we are going to use kernel+initrd for boot after validation + ModeActualKernelInitrd + + // Constants for the requested mode + + // ModeRequestAuto means auto-select the boot mode for a utility VM + ModeRequestAuto = iota // VHDX will be priority over kernel+initrd + // ModeRequestVhdx means request VHDX boot if possible + ModeRequestVhdx + // ModeRequestKernelInitrd means request Kernel+initrd boot if possible + ModeRequestKernelInitrd + + // defaultUvmTimeoutSeconds is the default time to wait for utility VM operations + defaultUvmTimeoutSeconds = 5 * 60 + + // DefaultVhdxSizeGB is the size of the default sandbox & scratch in GB + DefaultVhdxSizeGB = 20 + + // defaultVhdxBlockSizeMB is the block-size for the sandbox/scratch VHDx's this package can create. + defaultVhdxBlockSizeMB = 1 +) + +// Config is the structure used to configuring a utility VM. There are two ways +// of starting. Either supply a VHD, or a Kernel+Initrd. For the latter, both +// must be supplied, and both must be in the same directory. +// +// VHD is the priority. +type Config struct { + Options // Configuration options + Name string // Name of the utility VM + RequestedMode Mode // What mode is preferred when validating + ActualMode Mode // What mode was obtained during validation + UvmTimeoutSeconds int // How long to wait for the utility VM to respond in seconds + Uvm hcsshim.Container // The actual container + MappedVirtualDisks []hcsshim.MappedVirtualDisk // Data-disks to be attached +} + +// Options is the structure used by a client to define configurable options for a utility VM. +type Options struct { + KirdPath string // Path to where kernel/initrd are found (defaults to %PROGRAMFILES%\Linux Containers) + KernelFile string // Kernel for Utility VM (embedded in a UEFI bootloader) - does NOT include full path, just filename + InitrdFile string // Initrd image for Utility VM - does NOT include full path, just filename + Vhdx string // VHD for booting the utility VM - is a full path + TimeoutSeconds int // Requested time for the utility VM to respond in seconds (may be over-ridden by environment) + BootParameters string // Additional boot parameters for initrd booting (not VHDx) +} + +// ParseOptions parses a set of K-V pairs into options used by opengcs. Note +// for consistency with the LCOW graphdriver in docker, we keep the same +// convention of an `lcow.` prefix. +func ParseOptions(options []string) (Options, error) { + rOpts := Options{TimeoutSeconds: 0} + for _, v := range options { + opt := strings.SplitN(v, "=", 2) + if len(opt) == 2 { + switch strings.ToLower(opt[0]) { + case "lcow.kirdpath": + rOpts.KirdPath = opt[1] + case "lcow.kernel": + rOpts.KernelFile = opt[1] + case "lcow.initrd": + rOpts.InitrdFile = opt[1] + case "lcow.vhdx": + rOpts.Vhdx = opt[1] + case "lcow.bootparameters": + rOpts.BootParameters = opt[1] + case "lcow.timeout": + var err error + if rOpts.TimeoutSeconds, err = strconv.Atoi(opt[1]); err != nil { + return rOpts, fmt.Errorf("opengcstimeoutsecs option could not be interpreted as an integer") + } + if rOpts.TimeoutSeconds < 0 { + return rOpts, fmt.Errorf("opengcstimeoutsecs option cannot be negative") + } + } + } + } + + // Set default values if not supplied + if rOpts.KirdPath == "" { + rOpts.KirdPath = filepath.Join(os.Getenv("ProgramFiles"), "Linux Containers") + } + if rOpts.Vhdx == "" { + rOpts.Vhdx = filepath.Join(rOpts.KirdPath, `uvm.vhdx`) + } + if rOpts.KernelFile == "" { + rOpts.KernelFile = `bootx64.efi` + } + if rOpts.InitrdFile == "" { + rOpts.InitrdFile = `initrd.img` + } + + return rOpts, nil +} + +// GenerateDefault generates a default config from a set of options +// If baseDir is not supplied, defaults to $env:ProgramFiles\Linux Containers +func (config *Config) GenerateDefault(options []string) error { + // Parse the options that the user supplied. + var err error + config.Options, err = ParseOptions(options) + if err != nil { + return err + } + + // Get the timeout from the environment + envTimeoutSeconds := 0 + envTimeout := os.Getenv("OPENGCS_UVM_TIMEOUT_SECONDS") + if len(envTimeout) > 0 { + var err error + if envTimeoutSeconds, err = strconv.Atoi(envTimeout); err != nil { + return fmt.Errorf("OPENGCS_UVM_TIMEOUT_SECONDS could not be interpreted as an integer") + } + if envTimeoutSeconds < 0 { + return fmt.Errorf("OPENGCS_UVM_TIMEOUT_SECONDS cannot be negative") + } + } + + // Priority to the requested timeout from the options. + if config.TimeoutSeconds != 0 { + config.UvmTimeoutSeconds = config.TimeoutSeconds + return nil + } + + // Next priority, the environment + if envTimeoutSeconds != 0 { + config.UvmTimeoutSeconds = envTimeoutSeconds + return nil + } + + // Last priority is the default timeout + config.UvmTimeoutSeconds = defaultUvmTimeoutSeconds + + // Set the default requested mode + config.RequestedMode = ModeRequestAuto + + return nil +} + +// Validate validates a Config structure for starting a utility VM. +func (config *Config) Validate() error { + config.ActualMode = ModeActualError + + if config.RequestedMode == ModeRequestVhdx && config.Vhdx == "" { + return fmt.Errorf("VHDx mode must supply a VHDx") + } + if config.RequestedMode == ModeRequestKernelInitrd && (config.KernelFile == "" || config.InitrdFile == "") { + return fmt.Errorf("kernel+initrd mode must supply both kernel and initrd") + } + + // Validate that if VHDX requested or auto, it exists. + if config.RequestedMode == ModeRequestAuto || config.RequestedMode == ModeRequestVhdx { + if _, err := os.Stat(config.Vhdx); os.IsNotExist(err) { + if config.RequestedMode == ModeRequestVhdx { + return fmt.Errorf("VHDx '%s' not found", config.Vhdx) + } + } else { + config.ActualMode = ModeActualVhdx + + // Can't specify boot parameters with VHDx + if config.BootParameters != "" { + return fmt.Errorf("Boot parameters cannot be specified in VHDx mode") + } + return nil + } + } + + // So must be kernel+initrd, or auto where we fallback as the VHDX doesn't exist + if config.InitrdFile == "" || config.KernelFile == "" { + if config.RequestedMode == ModeRequestKernelInitrd { + return fmt.Errorf("initrd and kernel options must be supplied") + } + return fmt.Errorf("opengcs: configuration is invalid") + } + + if _, err := os.Stat(filepath.Join(config.KirdPath, config.KernelFile)); os.IsNotExist(err) { + return fmt.Errorf("kernel '%s' not found", filepath.Join(config.KirdPath, config.KernelFile)) + } + if _, err := os.Stat(filepath.Join(config.KirdPath, config.InitrdFile)); os.IsNotExist(err) { + return fmt.Errorf("initrd '%s' not found", filepath.Join(config.KirdPath, config.InitrdFile)) + } + + config.ActualMode = ModeActualKernelInitrd + + // Ensure all the MappedVirtualDisks exist on the host + for _, mvd := range config.MappedVirtualDisks { + if _, err := os.Stat(mvd.HostPath); err != nil { + return fmt.Errorf("mapped virtual disk '%s' not found", mvd.HostPath) + } + if mvd.ContainerPath == "" { + return fmt.Errorf("mapped virtual disk '%s' requested without a container path", mvd.HostPath) + } + } + + return nil +} + +// StartUtilityVM creates and starts a utility VM from a configuration. +func (config *Config) StartUtilityVM() error { + logrus.Debugf("opengcs: StartUtilityVM: %+v", config) + + if err := config.Validate(); err != nil { + return err + } + + configuration := &hcsshim.ContainerConfig{ + HvPartition: true, + Name: config.Name, + SystemType: "container", + ContainerType: "linux", + TerminateOnLastHandleClosed: true, + MappedVirtualDisks: config.MappedVirtualDisks, + } + + if config.ActualMode == ModeActualVhdx { + configuration.HvRuntime = &hcsshim.HvRuntime{ + ImagePath: config.Vhdx, + BootSource: "Vhd", + WritableBootSource: true, + } + } else { + configuration.HvRuntime = &hcsshim.HvRuntime{ + ImagePath: config.KirdPath, + LinuxInitrdFile: config.InitrdFile, + LinuxKernelFile: config.KernelFile, + LinuxBootParameters: config.BootParameters, + } + } + + configurationS, _ := json.Marshal(configuration) + logrus.Debugf("opengcs: StartUtilityVM: calling HCS with '%s'", string(configurationS)) + uvm, err := hcsshim.CreateContainer(config.Name, configuration) + if err != nil { + return err + } + logrus.Debugf("opengcs: StartUtilityVM: uvm created, starting...") + err = uvm.Start() + if err != nil { + logrus.Debugf("opengcs: StartUtilityVM: uvm failed to start: %s", err) + // Make sure we don't leave it laying around as it's been created in HCS + uvm.Terminate() + return err + } + + config.Uvm = uvm + logrus.Debugf("opengcs StartUtilityVM: uvm %s is running", config.Name) + return nil +} diff --git a/components/engine/vendor/github.com/Microsoft/opengcs/client/createext4vhdx.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/createext4vhdx.go new file mode 100644 index 0000000000..b53ce25149 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/createext4vhdx.go @@ -0,0 +1,165 @@ +// +build windows + +package client + +import ( + "bytes" + "fmt" + "os" + "strings" + "time" + + winio "github.com/Microsoft/go-winio/vhd" + // "github.com/Microsoft/hcsshim" + "github.com/sirupsen/logrus" +) + +// dismount is a simple utility function wrapping a conditional HotRemove. It would +// have been easier if you could cancel a deferred function, but this works just +// as well. +func (config *Config) dismount(file string) error { + logrus.Debugf("opengcs: CreateExt4Vhdx: hot-remove of %s", file) + err := config.HotRemoveVhd(file) + if err != nil { + logrus.Warnf("failed to hot-remove: %s", err) + } + return err +} + +// CreateExt4Vhdx does what it says on the tin. It is the responsibility of the caller to synchronise +// simultaneous attempts to create the cache file. +func (config *Config) CreateExt4Vhdx(destFile string, sizeGB uint32, cacheFile string) error { + // Smallest we can accept is the default sandbox size as we can't size down, only expand. + if sizeGB < DefaultVhdxSizeGB { + sizeGB = DefaultVhdxSizeGB + } + + logrus.Debugf("opengcs: CreateExt4Vhdx: %s size:%dGB cache:%s", destFile, sizeGB, cacheFile) + + // Retrieve from cache if the default size and already on disk + if cacheFile != "" && sizeGB == DefaultVhdxSizeGB { + if _, err := os.Stat(cacheFile); err == nil { + if err := CopyFile(cacheFile, destFile, false); err != nil { + return fmt.Errorf("failed to copy cached file '%s' to '%s': %s", cacheFile, destFile, err) + } + logrus.Debugf("opengcs: CreateExt4Vhdx: %s fulfilled from cache", destFile) + return nil + } + } + + // Must have a utility VM to operate on + if config.Uvm == nil { + return fmt.Errorf("no utility VM") + } + + // Create the VHDX + if err := winio.CreateVhdx(destFile, sizeGB, defaultVhdxBlockSizeMB); err != nil { + return fmt.Errorf("failed to create VHDx %s: %s", destFile, err) + } + + // Attach it to the utility VM, but don't mount it (as there's no filesystem on it) + if err := config.HotAddVhd(destFile, "", false, false); err != nil { + return fmt.Errorf("opengcs: CreateExt4Vhdx: failed to hot-add %s to utility VM: %s", cacheFile, err) + } + + // Get the list of mapped virtual disks to find the controller and LUN IDs + logrus.Debugf("opengcs: CreateExt4Vhdx: %s querying mapped virtual disks", destFile) + mvdControllers, err := config.Uvm.MappedVirtualDisks() + if err != nil { + return fmt.Errorf("failed to get mapped virtual disks: %s", err) + } + + // Find our mapped disk from the list of all currently added. + controller := -1 + lun := -1 + for controllerNumber, controllerElement := range mvdControllers { + for diskNumber, diskElement := range controllerElement.MappedVirtualDisks { + if diskElement.HostPath == destFile { + controller = controllerNumber + lun = diskNumber + break + } + } + } + if controller == -1 || lun == -1 { + config.dismount(destFile) + return fmt.Errorf("failed to find %s in mapped virtual disks after hot-adding", destFile) + } + logrus.Debugf("opengcs: CreateExt4Vhdx: %s at C=%d L=%d", destFile, controller, lun) + + // Validate /sys/bus/scsi/devices/C:0:0:L exists as a directory + testdCommand := fmt.Sprintf(`test -d /sys/bus/scsi/devices/%d:0:0:%d`, controller, lun) + testdProc, err := config.RunProcess(testdCommand, nil, nil, nil) + if err != nil { + config.dismount(destFile) + return fmt.Errorf("failed to `%s` following hot-add %s to utility VM: %s", testdCommand, destFile, err) + } + defer testdProc.Close() + testdProc.WaitTimeout(time.Duration(int(time.Second) * config.UvmTimeoutSeconds)) + testdExitCode, err := testdProc.ExitCode() + if err != nil { + config.dismount(destFile) + return fmt.Errorf("failed to get exit code from `%s` following hot-add %s to utility VM: %s", testdCommand, destFile, err) + } + if testdExitCode != 0 { + config.dismount(destFile) + return fmt.Errorf("`%s` return non-zero exit code (%d) following hot-add %s to utility VM", testdCommand, testdExitCode, destFile) + } + + // Get the device from under the block subdirectory by doing a simple ls. This will come back as (eg) `sda` + lsCommand := fmt.Sprintf(`ls /sys/bus/scsi/devices/%d:0:0:%d/block`, controller, lun) + var lsOutput bytes.Buffer + lsProc, err := config.RunProcess(lsCommand, nil, &lsOutput, nil) + if err != nil { + config.dismount(destFile) + return fmt.Errorf("failed to `%s` following hot-add %s to utility VM: %s", lsCommand, destFile, err) + } + defer lsProc.Close() + lsProc.WaitTimeout(time.Duration(int(time.Second) * config.UvmTimeoutSeconds)) + lsExitCode, err := lsProc.ExitCode() + if err != nil { + config.dismount(destFile) + return fmt.Errorf("failed to get exit code from `%s` following hot-add %s to utility VM: %s", lsCommand, destFile, err) + } + if lsExitCode != 0 { + config.dismount(destFile) + return fmt.Errorf("`%s` return non-zero exit code (%d) following hot-add %s to utility VM", lsCommand, lsExitCode, destFile) + } + device := fmt.Sprintf(`/dev/%s`, strings.TrimSpace(lsOutput.String())) + logrus.Debugf("opengcs: CreateExt4Vhdx: %s: device at %s", destFile, device) + + // Format it ext4 + mkfsCommand := fmt.Sprintf(`mkfs.ext4 -q -E lazy_itable_init=1 -O ^has_journal,sparse_super2,uninit_bg,^resize_inode %s`, device) + var mkfsStderr bytes.Buffer + mkfsProc, err := config.RunProcess(mkfsCommand, nil, nil, &mkfsStderr) + if err != nil { + config.dismount(destFile) + return fmt.Errorf("failed to RunProcess %q following hot-add %s to utility VM: %s", destFile, mkfsCommand, err) + } + defer mkfsProc.Close() + mkfsProc.WaitTimeout(time.Duration(int(time.Second) * config.UvmTimeoutSeconds)) + mkfsExitCode, err := mkfsProc.ExitCode() + if err != nil { + config.dismount(destFile) + return fmt.Errorf("failed to get exit code from `%s` following hot-add %s to utility VM: %s", mkfsCommand, destFile, err) + } + if mkfsExitCode != 0 { + config.dismount(destFile) + return fmt.Errorf("`%s` return non-zero exit code (%d) following hot-add %s to utility VM: %s", mkfsCommand, mkfsExitCode, destFile, strings.TrimSpace(mkfsStderr.String())) + } + + // Dismount before we copy it + if err := config.dismount(destFile); err != nil { + return fmt.Errorf("failed to hot-remove: %s", err) + } + + // Populate the cache. + if cacheFile != "" && (sizeGB == DefaultVhdxSizeGB) { + if err := CopyFile(destFile, cacheFile, true); err != nil { + return fmt.Errorf("failed to seed cache '%s' from '%s': %s", destFile, cacheFile, err) + } + } + + logrus.Debugf("opengcs: CreateExt4Vhdx: %s created (non-cache)", destFile) + return nil +} diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotaddvhd.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/hotaddvhd.go similarity index 82% rename from components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotaddvhd.go rename to components/engine/vendor/github.com/Microsoft/opengcs/client/hotaddvhd.go index c0545925f7..daf7c25f04 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotaddvhd.go +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/hotaddvhd.go @@ -6,14 +6,14 @@ import ( "fmt" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // HotAddVhd hot-adds a VHD to a utility VM. This is used in the global one-utility-VM- // service-VM per host scenario. In order to do a graphdriver `Diff`, we hot-add the // sandbox to /mnt/ so that we can run `exportSandbox` inside the utility VM to // get a tar-stream of the sandboxes contents back to the daemon. -func (config *Config) HotAddVhd(hostPath string, containerPath string) error { +func (config *Config) HotAddVhd(hostPath string, containerPath string, readOnly bool, mount bool) error { logrus.Debugf("opengcs: HotAddVhd: %s: %s", hostPath, containerPath) if config.Uvm == nil { @@ -26,13 +26,14 @@ func (config *Config) HotAddVhd(hostPath string, containerPath string) error { HostPath: hostPath, ContainerPath: containerPath, CreateInUtilityVM: true, - //ReadOnly: true, + ReadOnly: readOnly, + AttachOnly: !mount, }, Request: "Add", } - logrus.Debugf("opengcs: HotAddVhd: %s to %s", hostPath, containerPath) + if err := config.Uvm.Modify(modification); err != nil { - return fmt.Errorf("opengcs: HotAddVhd: failed: %s", err) + return fmt.Errorf("failed to modify utility VM configuration for hot-add: %s", err) } logrus.Debugf("opengcs: HotAddVhd: %s added successfully", hostPath) return nil diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotremovevhd.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/hotremovevhd.go similarity index 87% rename from components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotremovevhd.go rename to components/engine/vendor/github.com/Microsoft/opengcs/client/hotremovevhd.go index 1f3d21aa81..cf1971244d 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotremovevhd.go +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/hotremovevhd.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // HotRemoveVhd hot-removes a VHD from a utility VM. This is used in the global one-utility-VM- @@ -27,7 +27,7 @@ func (config *Config) HotRemoveVhd(hostPath string) error { Request: "Remove", } if err := config.Uvm.Modify(modification); err != nil { - return fmt.Errorf("opengcs: HotRemoveVhd: %s failed: %s", hostPath, err) + return fmt.Errorf("failed modifying utility VM for hot-remove %s: %s", hostPath, err) } logrus.Debugf("opengcs: HotRemoveVhd: %s removed successfully", hostPath) return nil diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/layervhddetails.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/layervhddetails.go similarity index 100% rename from components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/layervhddetails.go rename to components/engine/vendor/github.com/Microsoft/opengcs/client/layervhddetails.go diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/process.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/process.go similarity index 68% rename from components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/process.go rename to components/engine/vendor/github.com/Microsoft/opengcs/client/process.go index 02655227a9..984a95a32e 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/process.go +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/process.go @@ -7,7 +7,7 @@ import ( "io" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // Process is the structure pertaining to a process running in a utility VM. @@ -15,6 +15,7 @@ type process struct { Process hcsshim.Process Stdin io.WriteCloser Stdout io.ReadCloser + Stderr io.ReadCloser } // createUtilsProcess is a convenient wrapper for hcsshim.createUtilsProcess to use when @@ -45,13 +46,13 @@ func (config *Config) createUtilsProcess(commandLine string) (process, error) { } proc.Process, err = config.Uvm.CreateProcess(processConfig) if err != nil { - return process{}, fmt.Errorf("opengcs: createUtilsProcess: CreateProcess %+v failed %s", config, err) + return process{}, fmt.Errorf("failed to create process (%+v) in utility VM: %s", config, err) } - if proc.Stdin, proc.Stdout, _, err = proc.Process.Stdio(); err != nil { + if proc.Stdin, proc.Stdout, proc.Stderr, err = proc.Process.Stdio(); err != nil { proc.Process.Kill() // Should this have a timeout? proc.Process.Close() - return process{}, fmt.Errorf("opengcs: createUtilsProcess: failed to get Stdio pipes %s", err) + return process{}, fmt.Errorf("failed to get stdio pipes for process %+v: %s", config, err) } logrus.Debugf("opengcs: createUtilsProcess success: pid %d", proc.Process.Pid()) @@ -60,41 +61,52 @@ func (config *Config) createUtilsProcess(commandLine string) (process, error) { // RunProcess runs the given command line program in the utilityVM. It takes in // an input to the reader to feed into stdin and returns stdout to output. -func (config *Config) RunProcess(commandLine string, input io.Reader, output io.Writer) error { +// IMPORTANT: It is the responsibility of the caller to call Close() on the returned process. +func (config *Config) RunProcess(commandLine string, stdin io.Reader, stdout io.Writer, stderr io.Writer) (hcsshim.Process, error) { logrus.Debugf("opengcs: RunProcess: %s", commandLine) process, err := config.createUtilsProcess(commandLine) if err != nil { - return err + return nil, err } - defer process.Process.Close() // Send the data into the process's stdin - if input != nil { + if stdin != nil { if _, err = copyWithTimeout(process.Stdin, - input, + stdin, 0, config.UvmTimeoutSeconds, fmt.Sprintf("send to stdin of %s", commandLine)); err != nil { - return err + return nil, err } // Don't need stdin now we've sent everything. This signals GCS that we are finished sending data. if err := process.Process.CloseStdin(); err != nil { - return err + return nil, err } } - if output != nil { + if stdout != nil { // Copy the data over to the writer. - if _, err := copyWithTimeout(output, + if _, err := copyWithTimeout(stdout, process.Stdout, 0, config.UvmTimeoutSeconds, fmt.Sprintf("RunProcess: copy back from %s", commandLine)); err != nil { - return err + return nil, err + } + } + + if stderr != nil { + // Copy the data over to the writer. + if _, err := copyWithTimeout(stderr, + process.Stderr, + 0, + config.UvmTimeoutSeconds, + fmt.Sprintf("RunProcess: copy back from %s", commandLine)); err != nil { + return nil, err } } logrus.Debugf("opengcs: runProcess success: %s", commandLine) - return nil + return process.Process, nil } diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/tartovhd.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/tartovhd.go similarity index 73% rename from components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/tartovhd.go rename to components/engine/vendor/github.com/Microsoft/opengcs/client/tartovhd.go index 3560245b49..9aa6609d48 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/tartovhd.go +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/tartovhd.go @@ -6,7 +6,7 @@ import ( "fmt" "io" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // TarToVhd streams a tarstream contained in an io.Reader to a fixed vhd file @@ -19,24 +19,24 @@ func (config *Config) TarToVhd(targetVHDFile string, reader io.Reader) (int64, e process, err := config.createUtilsProcess("tar2vhd") if err != nil { - return 0, fmt.Errorf("opengcs: TarToVhd: %s: failed to create utils process tar2vhd: %s", targetVHDFile, err) + return 0, fmt.Errorf("failed to start tar2vhd for %s: %s", targetVHDFile, err) } defer process.Process.Close() // Send the tarstream into the `tar2vhd`s stdin if _, err = copyWithTimeout(process.Stdin, reader, 0, config.UvmTimeoutSeconds, fmt.Sprintf("stdin of tar2vhd for generating %s", targetVHDFile)); err != nil { - return 0, fmt.Errorf("opengcs: TarToVhd: %s: failed to send to tar2vhd in uvm: %s", targetVHDFile, err) + return 0, fmt.Errorf("failed sending to tar2vhd for %s: %s", targetVHDFile, err) } // Don't need stdin now we've sent everything. This signals GCS that we are finished sending data. if err := process.Process.CloseStdin(); err != nil { - return 0, fmt.Errorf("opengcs: TarToVhd: %s: failed closing stdin handle: %s", targetVHDFile, err) + return 0, fmt.Errorf("failed closing stdin handle for %s: %s", targetVHDFile, err) } // Write stdout contents of `tar2vhd` to the VHD file payloadSize, err := writeFileFromReader(targetVHDFile, process.Stdout, config.UvmTimeoutSeconds, fmt.Sprintf("stdout of tar2vhd to %s", targetVHDFile)) if err != nil { - return 0, fmt.Errorf("opengcs: TarToVhd: %s: failed writing VHD file: %s", targetVHDFile, err) + return 0, fmt.Errorf("failed to write %s during tar2vhd: %s", targetVHDFile, err) } logrus.Debugf("opengcs: TarToVhd: %s created, %d bytes", targetVHDFile, payloadSize) diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/unsupported.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/unsupported.go similarity index 100% rename from components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/unsupported.go rename to components/engine/vendor/github.com/Microsoft/opengcs/client/unsupported.go diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/utilities.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/utilities.go similarity index 98% rename from components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/utilities.go rename to components/engine/vendor/github.com/Microsoft/opengcs/client/utilities.go index 8976dc0af9..cd779f15c4 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/utilities.go +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/utilities.go @@ -10,7 +10,7 @@ import ( "time" "unsafe" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/vhdtotar.go b/components/engine/vendor/github.com/Microsoft/opengcs/client/vhdtotar.go similarity index 98% rename from components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/vhdtotar.go rename to components/engine/vendor/github.com/Microsoft/opengcs/client/vhdtotar.go index bdbd381929..27225a71e7 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/vhdtotar.go +++ b/components/engine/vendor/github.com/Microsoft/opengcs/client/vhdtotar.go @@ -7,7 +7,7 @@ import ( "io" "os" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // VhdToTar does what is says - it exports a VHD in a specified diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/json_formatter.go b/components/engine/vendor/github.com/Sirupsen/logrus/json_formatter.go deleted file mode 100644 index 2ad6dc5cf4..0000000000 --- a/components/engine/vendor/github.com/Sirupsen/logrus/json_formatter.go +++ /dev/null @@ -1,41 +0,0 @@ -package logrus - -import ( - "encoding/json" - "fmt" -) - -type JSONFormatter struct { - // TimestampFormat sets the format used for marshaling timestamps. - TimestampFormat string -} - -func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) { - data := make(Fields, len(entry.Data)+3) - for k, v := range entry.Data { - switch v := v.(type) { - case error: - // Otherwise errors are ignored by `encoding/json` - // https://github.com/Sirupsen/logrus/issues/137 - data[k] = v.Error() - default: - data[k] = v - } - } - prefixFieldClashes(data) - - timestampFormat := f.TimestampFormat - if timestampFormat == "" { - timestampFormat = DefaultTimestampFormat - } - - data["time"] = entry.Time.Format(timestampFormat) - data["msg"] = entry.Message - data["level"] = entry.Level.String() - - serialized, err := json.Marshal(data) - if err != nil { - return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) - } - return append(serialized, '\n'), nil -} diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/terminal_solaris.go b/components/engine/vendor/github.com/Sirupsen/logrus/terminal_solaris.go deleted file mode 100644 index a3c6f6e7df..0000000000 --- a/components/engine/vendor/github.com/Sirupsen/logrus/terminal_solaris.go +++ /dev/null @@ -1,15 +0,0 @@ -// +build solaris,!appengine - -package logrus - -import ( - "os" - - "golang.org/x/sys/unix" -) - -// IsTerminal returns true if the given file descriptor is a terminal. -func IsTerminal() bool { - _, err := unix.IoctlGetTermios(int(os.Stdout.Fd()), unix.TCGETA) - return err == nil -} diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/terminal_windows.go b/components/engine/vendor/github.com/Sirupsen/logrus/terminal_windows.go deleted file mode 100644 index 3727e8adfb..0000000000 --- a/components/engine/vendor/github.com/Sirupsen/logrus/terminal_windows.go +++ /dev/null @@ -1,27 +0,0 @@ -// Based on ssh/terminal: -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows,!appengine - -package logrus - -import ( - "syscall" - "unsafe" -) - -var kernel32 = syscall.NewLazyDLL("kernel32.dll") - -var ( - procGetConsoleMode = kernel32.NewProc("GetConsoleMode") -) - -// IsTerminal returns true if stderr's file descriptor is a terminal. -func IsTerminal() bool { - fd := syscall.Stderr - var st uint32 - r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0) - return r != 0 && e == 0 -} diff --git a/components/engine/vendor/github.com/containerd/containerd/runtime/container.go b/components/engine/vendor/github.com/containerd/containerd/runtime/container.go index 43351cab17..fe5fb3b28a 100644 --- a/components/engine/vendor/github.com/containerd/containerd/runtime/container.go +++ b/components/engine/vendor/github.com/containerd/containerd/runtime/container.go @@ -12,9 +12,9 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" "github.com/containerd/containerd/specs" ocs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "golang.org/x/sys/unix" ) diff --git a/components/engine/vendor/github.com/containerd/containerd/runtime/process.go b/components/engine/vendor/github.com/containerd/containerd/runtime/process.go index 22d0192780..ea6ec0c37a 100644 --- a/components/engine/vendor/github.com/containerd/containerd/runtime/process.go +++ b/components/engine/vendor/github.com/containerd/containerd/runtime/process.go @@ -14,9 +14,9 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" "github.com/containerd/containerd/osutils" "github.com/containerd/containerd/specs" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) diff --git a/components/engine/vendor/github.com/docker/distribution/README.md b/components/engine/vendor/github.com/docker/distribution/README.md index a6e8db0fb7..998878850c 100644 --- a/components/engine/vendor/github.com/docker/distribution/README.md +++ b/components/engine/vendor/github.com/docker/distribution/README.md @@ -76,8 +76,7 @@ may be the better choice. For those who have previously deployed their own registry based on the Registry 1.0 implementation and wish to deploy a Registry 2.0 while retaining images, data migration is required. A tool to assist with migration efforts has been -created. For more information see [docker/migrator] -(https://github.com/docker/migrator). +created. For more information see [docker/migrator](https://github.com/docker/migrator). ## Contribute diff --git a/components/engine/vendor/github.com/docker/distribution/blobs.go b/components/engine/vendor/github.com/docker/distribution/blobs.go index 79c5fb33be..01d309029f 100644 --- a/components/engine/vendor/github.com/docker/distribution/blobs.go +++ b/components/engine/vendor/github.com/docker/distribution/blobs.go @@ -152,7 +152,7 @@ type BlobProvider interface { // BlobServer can serve blobs via http. type BlobServer interface { - // ServeBlob attempts to serve the blob, identifed by dgst, via http. The + // ServeBlob attempts to serve the blob, identified by dgst, via http. The // service may decide to redirect the client elsewhere or serve the data // directly. // diff --git a/components/engine/vendor/github.com/docker/distribution/context/doc.go b/components/engine/vendor/github.com/docker/distribution/context/doc.go index 3b4ab8882f..9b623074eb 100644 --- a/components/engine/vendor/github.com/docker/distribution/context/doc.go +++ b/components/engine/vendor/github.com/docker/distribution/context/doc.go @@ -64,7 +64,7 @@ // Note that this only affects the new context, the previous context, with the // version field, can be used independently. Put another way, the new logger, // added to the request context, is unique to that context and can have -// request scoped varaibles. +// request scoped variables. // // HTTP Requests // diff --git a/components/engine/vendor/github.com/docker/distribution/context/http.go b/components/engine/vendor/github.com/docker/distribution/context/http.go index 7fe9b8ab05..7d65c8524e 100644 --- a/components/engine/vendor/github.com/docker/distribution/context/http.go +++ b/components/engine/vendor/github.com/docker/distribution/context/http.go @@ -8,9 +8,9 @@ import ( "sync" "time" - log "github.com/Sirupsen/logrus" "github.com/docker/distribution/uuid" "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" ) // Common errors used with this package. diff --git a/components/engine/vendor/github.com/docker/distribution/context/logger.go b/components/engine/vendor/github.com/docker/distribution/context/logger.go index fbb6a0511f..86c5964e49 100644 --- a/components/engine/vendor/github.com/docker/distribution/context/logger.go +++ b/components/engine/vendor/github.com/docker/distribution/context/logger.go @@ -3,7 +3,7 @@ package context import ( "fmt" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "runtime" ) diff --git a/components/engine/vendor/github.com/docker/distribution/errors.go b/components/engine/vendor/github.com/docker/distribution/errors.go index 2062a06fbd..020d33258b 100644 --- a/components/engine/vendor/github.com/docker/distribution/errors.go +++ b/components/engine/vendor/github.com/docker/distribution/errors.go @@ -77,7 +77,7 @@ func (err ErrManifestUnknownRevision) Error() string { type ErrManifestUnverified struct{} func (ErrManifestUnverified) Error() string { - return fmt.Sprintf("unverified manifest") + return "unverified manifest" } // ErrManifestVerification provides a type to collect errors encountered diff --git a/components/engine/vendor/github.com/docker/distribution/manifest/manifestlist/manifestlist.go b/components/engine/vendor/github.com/docker/distribution/manifest/manifestlist/manifestlist.go index 7a8cabbdba..3aa0662d9f 100644 --- a/components/engine/vendor/github.com/docker/distribution/manifest/manifestlist/manifestlist.go +++ b/components/engine/vendor/github.com/docker/distribution/manifest/manifestlist/manifestlist.go @@ -81,7 +81,7 @@ type ManifestList struct { Manifests []ManifestDescriptor `json:"manifests"` } -// References returnes the distribution descriptors for the referenced image +// References returns the distribution descriptors for the referenced image // manifests. func (m ManifestList) References() []distribution.Descriptor { dependencies := make([]distribution.Descriptor, len(m.Manifests)) diff --git a/components/engine/vendor/github.com/docker/distribution/manifest/schema1/verify.go b/components/engine/vendor/github.com/docker/distribution/manifest/schema1/verify.go index fa8daa56f5..ef59065cd6 100644 --- a/components/engine/vendor/github.com/docker/distribution/manifest/schema1/verify.go +++ b/components/engine/vendor/github.com/docker/distribution/manifest/schema1/verify.go @@ -3,8 +3,8 @@ package schema1 import ( "crypto/x509" - "github.com/Sirupsen/logrus" "github.com/docker/libtrust" + "github.com/sirupsen/logrus" ) // Verify verifies the signature of the signed manifest returning the public diff --git a/components/engine/vendor/github.com/docker/distribution/reference/reference.go b/components/engine/vendor/github.com/docker/distribution/reference/reference.go index fd3510e9ee..2f66cca87a 100644 --- a/components/engine/vendor/github.com/docker/distribution/reference/reference.go +++ b/components/engine/vendor/github.com/docker/distribution/reference/reference.go @@ -15,7 +15,7 @@ // tag := /[\w][\w.-]{0,127}/ // // digest := digest-algorithm ":" digest-hex -// digest-algorithm := digest-algorithm-component [ digest-algorithm-separator digest-algorithm-component ] +// digest-algorithm := digest-algorithm-component [ digest-algorithm-separator digest-algorithm-component ]* // digest-algorithm-separator := /[+.-_]/ // digest-algorithm-component := /[A-Za-z][A-Za-z0-9]*/ // digest-hex := /[0-9a-fA-F]{32,}/ ; At least 128 bit digest value diff --git a/components/engine/vendor/github.com/docker/distribution/registry/api/v2/urls.go b/components/engine/vendor/github.com/docker/distribution/registry/api/v2/urls.go index e2e242eab0..1337bdb127 100644 --- a/components/engine/vendor/github.com/docker/distribution/registry/api/v2/urls.go +++ b/components/engine/vendor/github.com/docker/distribution/registry/api/v2/urls.go @@ -1,10 +1,9 @@ package v2 import ( - "net" + "fmt" "net/http" "net/url" - "strconv" "strings" "github.com/docker/distribution/reference" @@ -48,66 +47,42 @@ func NewURLBuilderFromString(root string, relative bool) (*URLBuilder, error) { // NewURLBuilderFromRequest uses information from an *http.Request to // construct the root url. func NewURLBuilderFromRequest(r *http.Request, relative bool) *URLBuilder { - var scheme string - - forwardedProto := r.Header.Get("X-Forwarded-Proto") - // TODO: log the error - forwardedHeader, _, _ := parseForwardedHeader(r.Header.Get("Forwarded")) - - switch { - case len(forwardedProto) > 0: - scheme = forwardedProto - case len(forwardedHeader["proto"]) > 0: - scheme = forwardedHeader["proto"] - case r.TLS != nil: - scheme = "https" - case len(r.URL.Scheme) > 0: - scheme = r.URL.Scheme - default: + var ( scheme = "http" + host = r.Host + ) + + if r.TLS != nil { + scheme = "https" + } else if len(r.URL.Scheme) > 0 { + scheme = r.URL.Scheme } - host := r.Host - - if forwardedHost := r.Header.Get("X-Forwarded-Host"); len(forwardedHost) > 0 { - // According to the Apache mod_proxy docs, X-Forwarded-Host can be a - // comma-separated list of hosts, to which each proxy appends the - // requested host. We want to grab the first from this comma-separated - // list. - hosts := strings.SplitN(forwardedHost, ",", 2) - host = strings.TrimSpace(hosts[0]) - } else if addr, exists := forwardedHeader["for"]; exists { - host = addr - } else if h, exists := forwardedHeader["host"]; exists { - host = h - } - - portLessHost, port := host, "" - if !isIPv6Address(portLessHost) { - // with go 1.6, this would treat the last part of IPv6 address as a port - portLessHost, port, _ = net.SplitHostPort(host) - } - if forwardedPort := r.Header.Get("X-Forwarded-Port"); len(port) == 0 && len(forwardedPort) > 0 { - ports := strings.SplitN(forwardedPort, ",", 2) - forwardedPort = strings.TrimSpace(ports[0]) - if _, err := strconv.ParseInt(forwardedPort, 10, 32); err == nil { - port = forwardedPort + // Handle fowarded headers + // Prefer "Forwarded" header as defined by rfc7239 if given + // see https://tools.ietf.org/html/rfc7239 + if forwarded := r.Header.Get("Forwarded"); len(forwarded) > 0 { + forwardedHeader, _, err := parseForwardedHeader(forwarded) + if err == nil { + if fproto := forwardedHeader["proto"]; len(fproto) > 0 { + scheme = fproto + } + if fhost := forwardedHeader["host"]; len(fhost) > 0 { + host = fhost + } } - } - - if len(portLessHost) > 0 { - host = portLessHost - } - if len(port) > 0 { - // remove enclosing brackets of ipv6 address otherwise they will be duplicated - if len(host) > 1 && host[0] == '[' && host[len(host)-1] == ']' { - host = host[1 : len(host)-1] + } else { + if forwardedProto := r.Header.Get("X-Forwarded-Proto"); len(forwardedProto) > 0 { + scheme = forwardedProto + } + if forwardedHost := r.Header.Get("X-Forwarded-Host"); len(forwardedHost) > 0 { + // According to the Apache mod_proxy docs, X-Forwarded-Host can be a + // comma-separated list of hosts, to which each proxy appends the + // requested host. We want to grab the first from this comma-separated + // list. + hosts := strings.SplitN(forwardedHost, ",", 2) + host = strings.TrimSpace(hosts[0]) } - // JoinHostPort properly encloses ipv6 addresses in square brackets - host = net.JoinHostPort(host, port) - } else if isIPv6Address(host) && host[0] != '[' { - // ipv6 needs to be enclosed in square brackets in urls - host = "[" + host + "]" } basePath := routeDescriptorsMap[RouteNameBase].Path @@ -175,6 +150,8 @@ func (ub *URLBuilder) BuildManifestURL(ref reference.Named) (string, error) { tagOrDigest = v.Tag() case reference.Digested: tagOrDigest = v.Digest().String() + default: + return "", fmt.Errorf("reference must have a tag or digest") } manifestURL, err := route.URL("name", ref.Name(), "reference", tagOrDigest) @@ -287,28 +264,3 @@ func appendValues(u string, values ...url.Values) string { return appendValuesURL(up, values...).String() } - -// isIPv6Address returns true if given string is a valid IPv6 address. No port is allowed. The address may be -// enclosed in square brackets. -func isIPv6Address(host string) bool { - if len(host) > 1 && host[0] == '[' && host[len(host)-1] == ']' { - host = host[1 : len(host)-1] - } - // The IPv6 scoped addressing zone identifier starts after the last percent sign. - if i := strings.LastIndexByte(host, '%'); i > 0 { - host = host[:i] - } - ip := net.ParseIP(host) - if ip == nil { - return false - } - if ip.To16() == nil { - return false - } - if ip.To4() == nil { - return true - } - // dot can be present in ipv4-mapped address, it needs to come after a colon though - i := strings.IndexAny(host, ":.") - return i >= 0 && host[i] == ':' -} diff --git a/components/engine/vendor/github.com/docker/distribution/registry/client/auth/session.go b/components/engine/vendor/github.com/docker/distribution/registry/client/auth/session.go index 3ca5e8b3e7..be474d825f 100644 --- a/components/engine/vendor/github.com/docker/distribution/registry/client/auth/session.go +++ b/components/engine/vendor/github.com/docker/distribution/registry/client/auth/session.go @@ -10,10 +10,10 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client/auth/challenge" "github.com/docker/distribution/registry/client/transport" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/vendor/github.com/docker/distribution/vendor.conf b/components/engine/vendor/github.com/docker/distribution/vendor.conf index 443b4bcfe4..d67edd779e 100644 --- a/components/engine/vendor/github.com/docker/distribution/vendor.conf +++ b/components/engine/vendor/github.com/docker/distribution/vendor.conf @@ -1,19 +1,21 @@ -github.com/Azure/azure-sdk-for-go c6f0533defaaaa26ea4dff3c9774e36033088112 -github.com/Sirupsen/logrus d26492970760ca5d33129d2d799e34be5c4782eb +github.com/Azure/azure-sdk-for-go 088007b3b08cc02b27f2eadfdcd870958460ce7e +github.com/Azure/go-autorest ec5f4903f77ed9927ac95b19ab8e44ada64c1356 +github.com/sirupsen/logrus 3d4380f53a34dcdc95f0c1db702615992b38d9a4 github.com/aws/aws-sdk-go c6fc52983ea2375810aa38ddb5370e9cdf611716 -github.com/bshuster-repo/logrus-logstash-hook 5f729f2fb50a301153cae84ff5c58981d51c095a +github.com/bshuster-repo/logrus-logstash-hook d2c0ecc1836d91814e15e23bb5dc309c3ef51f4a github.com/bugsnag/bugsnag-go b1d153021fcd90ca3f080db36bec96dc690fb274 github.com/bugsnag/osext 0dd3f918b21bec95ace9dc86c7e70266cfc5c702 github.com/bugsnag/panicwrap e2c28503fcd0675329da73bf48b33404db873782 github.com/denverdino/aliyungo afedced274aa9a7fcdd47ac97018f0f8db4e5de2 +github.com/dgrijalva/jwt-go a601269ab70c205d26370c16f7c81e9017c14e04 github.com/docker/goamz f0a21f5b2e12f83a505ecf79b633bb2035cf6f85 github.com/docker/libtrust fa567046d9b14f6aa788882a950d69651d230b21 github.com/garyburd/redigo 535138d7bcd717d6531c701ef5933d98b1866257 github.com/go-ini/ini 2ba15ac2dc9cdf88c110ec2dc0ced7fa45f5678c -github.com/golang/protobuf/proto 8d92cf5fc15a4382f8964b08e1f42a75c0591aa3 +github.com/golang/protobuf 8d92cf5fc15a4382f8964b08e1f42a75c0591aa3 github.com/gorilla/context 14f550f51af52180c2eefed15e5fd18d63c0a64a github.com/gorilla/handlers 60c7bfde3e33c201519a200a4507a158cc03a17b -github.com/gorilla/mux e444e69cbd2e2e3e0749a2f3c717cec491552bbf +github.com/gorilla/mux 599cba5e7b6137d46ddf58fb1765f5d928e69604 github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 github.com/jmespath/go-jmespath bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d github.com/miekg/dns 271c58e0c14f552178ea321a545ff9af38930f39 @@ -21,15 +23,15 @@ github.com/mitchellh/mapstructure 482a9fd5fa83e8c4e7817413b80f3eb8feec03ef github.com/ncw/swift b964f2ca856aac39885e258ad25aec08d5f64ee6 github.com/spf13/cobra 312092086bed4968099259622145a0c9ae280064 github.com/spf13/pflag 5644820622454e71517561946e3d94b9f9db6842 -github.com/stevvooe/resumable 51ad44105773cafcbe91927f70ac68e1bf78f8b4 -github.com/xenolf/lego/acme a9d8cec0e6563575e5868a005359ac97911b5985 +github.com/stevvooe/resumable 2aaf90b2ceea5072cb503ef2a620b08ff3119870 +github.com/xenolf/lego a9d8cec0e6563575e5868a005359ac97911b5985 github.com/yvasiyarov/go-metrics 57bccd1ccd43f94bb17fdd8bf3007059b802f85e github.com/yvasiyarov/gorelic a9bba5b9ab508a086f9a12b8c51fab68478e2128 github.com/yvasiyarov/newrelic_platform_go b21fdbd4370f3717f3bbd2bf41c223bc273068e6 golang.org/x/crypto c10c31b5e94b6f7a0283272dc2bb27163dcea24b golang.org/x/net 4876518f9e71663000c348837735820161a42df7 golang.org/x/oauth2 045497edb6234273d67dbc25da3f2ddbc4c4cacf -golang.org/x/time/rate a4bde12657593d5e90d0533a3e4fd95e635124cb +golang.org/x/time a4bde12657593d5e90d0533a3e4fd95e635124cb google.golang.org/api 9bf6e6e569ff057f75d9604a46c52928f17d2b54 google.golang.org/appengine 12d5545dc1cfa6047a286d5e853841b6471f4c19 google.golang.org/cloud 975617b05ea8a58727e6c1a06b6161ff4185a9f2 diff --git a/components/engine/vendor/github.com/docker/go-events/broadcast.go b/components/engine/vendor/github.com/docker/go-events/broadcast.go index ee0a894ad8..5120078dfb 100644 --- a/components/engine/vendor/github.com/docker/go-events/broadcast.go +++ b/components/engine/vendor/github.com/docker/go-events/broadcast.go @@ -4,7 +4,7 @@ import ( "fmt" "sync" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // Broadcaster sends events to multiple, reliable Sinks. The goal of this @@ -160,7 +160,7 @@ func (b *Broadcaster) run() { } } -func (b Broadcaster) String() string { +func (b *Broadcaster) String() string { // Serialize copy of this broadcaster without the sync.Once, to avoid // a data race. diff --git a/components/engine/vendor/github.com/docker/go-events/channel.go b/components/engine/vendor/github.com/docker/go-events/channel.go index de95fa4017..802cf51ffe 100644 --- a/components/engine/vendor/github.com/docker/go-events/channel.go +++ b/components/engine/vendor/github.com/docker/go-events/channel.go @@ -50,7 +50,7 @@ func (ch *Channel) Close() error { return nil } -func (ch Channel) String() string { +func (ch *Channel) String() string { // Serialize a copy of the Channel that doesn't contain the sync.Once, // to avoid a data race. ch2 := map[string]interface{}{ diff --git a/components/engine/vendor/github.com/docker/go-events/queue.go b/components/engine/vendor/github.com/docker/go-events/queue.go index cbd657cff4..4bb770afc2 100644 --- a/components/engine/vendor/github.com/docker/go-events/queue.go +++ b/components/engine/vendor/github.com/docker/go-events/queue.go @@ -4,7 +4,7 @@ import ( "container/list" "sync" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // Queue accepts all messages into a queue for asynchronous consumption @@ -31,7 +31,7 @@ func NewQueue(dst Sink) *Queue { } // Write accepts the events into the queue, only failing if the queue has -// beend closed. +// been closed. func (eq *Queue) Write(event Event) error { eq.mu.Lock() defer eq.mu.Unlock() diff --git a/components/engine/vendor/github.com/docker/go-events/retry.go b/components/engine/vendor/github.com/docker/go-events/retry.go index 55d40dce9e..2df55d2160 100644 --- a/components/engine/vendor/github.com/docker/go-events/retry.go +++ b/components/engine/vendor/github.com/docker/go-events/retry.go @@ -7,7 +7,7 @@ import ( "sync/atomic" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // RetryingSink retries the write until success or an ErrSinkClosed is @@ -90,7 +90,7 @@ func (rs *RetryingSink) Close() error { return nil } -func (rs RetryingSink) String() string { +func (rs *RetryingSink) String() string { // Serialize a copy of the RetryingSink without the sync.Once, to avoid // a data race. rs2 := map[string]interface{}{ diff --git a/components/engine/vendor/github.com/docker/libnetwork/agent.go b/components/engine/vendor/github.com/docker/libnetwork/agent.go index a45a569500..1328f0bade 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/agent.go +++ b/components/engine/vendor/github.com/docker/libnetwork/agent.go @@ -10,7 +10,6 @@ import ( "sort" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/stringid" "github.com/docker/go-events" "github.com/docker/libnetwork/cluster" @@ -20,6 +19,7 @@ import ( "github.com/docker/libnetwork/networkdb" "github.com/docker/libnetwork/types" "github.com/gogo/protobuf/proto" + "github.com/sirupsen/logrus" ) const ( @@ -214,8 +214,8 @@ func (c *controller) agentSetup(clusterProvider cluster.Provider) error { listen := clusterProvider.GetListenAddress() listenAddr, _, _ := net.SplitHostPort(listen) - logrus.Infof("Initializing Libnetwork Agent Listen-Addr=%s Local-addr=%s Adv-addr=%s Data-addr=%s Remote-addr-list=%v", - listenAddr, bindAddr, advAddr, dataAddr, remoteAddrList) + logrus.Infof("Initializing Libnetwork Agent Listen-Addr=%s Local-addr=%s Adv-addr=%s Data-addr=%s Remote-addr-list=%v MTU=%d", + listenAddr, bindAddr, advAddr, dataAddr, remoteAddrList, c.Config().Daemon.NetworkControlPlaneMTU) if advAddr != "" && agent == nil { if err := c.agentInit(listenAddr, bindAddr, advAddr, dataAddr); err != nil { logrus.Errorf("error in agentInit: %v", err) @@ -286,12 +286,19 @@ func (c *controller) agentInit(listenAddr, bindAddrOrInterface, advertiseAddr, d nodeName := hostname + "-" + stringid.TruncateID(stringid.GenerateRandomID()) logrus.Info("Gossip cluster hostname ", nodeName) - nDB, err := networkdb.New(&networkdb.Config{ - BindAddr: listenAddr, - AdvertiseAddr: advertiseAddr, - NodeName: nodeName, - Keys: keys, - }) + netDBConf := networkdb.DefaultConfig() + netDBConf.NodeName = nodeName + netDBConf.BindAddr = listenAddr + netDBConf.AdvertiseAddr = advertiseAddr + netDBConf.Keys = keys + if c.Config().Daemon.NetworkControlPlaneMTU != 0 { + // Consider the MTU remove the IP hdr (IPv4 or IPv6) and the TCP/UDP hdr. + // To be on the safe side let's cut 100 bytes + netDBConf.PacketBufferSize = (c.Config().Daemon.NetworkControlPlaneMTU - 100) + logrus.Debugf("Control plane MTU: %d will initialize NetworkDB with: %d", + c.Config().Daemon.NetworkControlPlaneMTU, netDBConf.PacketBufferSize) + } + nDB, err := networkdb.New(netDBConf) if err != nil { return err @@ -383,15 +390,11 @@ func (c *controller) agentClose() { agent.Lock() for _, cancelFuncs := range agent.driverCancelFuncs { - for _, cancel := range cancelFuncs { - cancelList = append(cancelList, cancel) - } + cancelList = append(cancelList, cancelFuncs...) } // Add also the cancel functions for the network db - for _, cancel := range agent.coreCancelFuncs { - cancelList = append(cancelList, cancel) - } + cancelList = append(cancelList, agent.coreCancelFuncs...) agent.Unlock() for _, cancel := range cancelList { @@ -738,11 +741,12 @@ func (n *network) addDriverWatches() { return } - agent.networkDB.WalkTable(table.name, func(nid, key string, value []byte) bool { - if nid == n.ID() { + agent.networkDB.WalkTable(table.name, func(nid, key string, value []byte, deleted bool) bool { + // skip the entries that are mark for deletion, this is safe because this function is + // called at initialization time so there is no state to delete + if nid == n.ID() && !deleted { d.EventNotify(driverapi.Create, nid, table.name, key, value) } - return false }) } diff --git a/components/engine/vendor/github.com/docker/libnetwork/bitseq/sequence.go b/components/engine/vendor/github.com/docker/libnetwork/bitseq/sequence.go index 86cf69b34f..3946473d8b 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/bitseq/sequence.go +++ b/components/engine/vendor/github.com/docker/libnetwork/bitseq/sequence.go @@ -10,9 +10,9 @@ import ( "fmt" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) // block sequence constants @@ -497,7 +497,10 @@ func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) { // Derive the this sequence offsets byteOffset := byteStart - inBlockBytePos bitOffset := inBlockBytePos*8 + bitStart - + var firstOffset uint64 + if current == head { + firstOffset = byteOffset + } for current != nil { if current.block != blockMAX { bytePos, bitPos, err := current.getAvailableBit(bitOffset) @@ -505,7 +508,8 @@ func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) { } // Moving to next block: Reset bit offset. bitOffset = 0 - byteOffset += current.count * blockBytes + byteOffset += (current.count * blockBytes) - firstOffset + firstOffset = 0 current = current.next } return invalidPos, invalidPos, ErrNoBitAvailable diff --git a/components/engine/vendor/github.com/docker/libnetwork/common/caller.go b/components/engine/vendor/github.com/docker/libnetwork/common/caller.go new file mode 100644 index 0000000000..0dec3bc0bc --- /dev/null +++ b/components/engine/vendor/github.com/docker/libnetwork/common/caller.go @@ -0,0 +1,29 @@ +package common + +import ( + "runtime" + "strings" +) + +func callerInfo(i int) string { + ptr, _, _, ok := runtime.Caller(i) + fName := "unknown" + if ok { + f := runtime.FuncForPC(ptr) + if f != nil { + // f.Name() is like: github.com/docker/libnetwork/common.MethodName + tmp := strings.Split(f.Name(), ".") + if len(tmp) > 0 { + fName = tmp[len(tmp)-1] + } + } + } + + return fName +} + +// CallerName returns the name of the function at the specified level +// level == 0 means current method name +func CallerName(level int) string { + return callerInfo(2 + level) +} diff --git a/components/engine/vendor/github.com/docker/libnetwork/config/config.go b/components/engine/vendor/github.com/docker/libnetwork/config/config.go index 3acb4320c4..96a157a1fb 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/config/config.go +++ b/components/engine/vendor/github.com/docker/libnetwork/config/config.go @@ -4,7 +4,6 @@ import ( "strings" "github.com/BurntSushi/toml" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/discovery" "github.com/docker/docker/pkg/plugingetter" "github.com/docker/go-connections/tlsconfig" @@ -13,6 +12,7 @@ import ( "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/osl" + "github.com/sirupsen/logrus" ) // Config encapsulates configurations of various Libnetwork components @@ -26,14 +26,15 @@ type Config struct { // DaemonCfg represents libnetwork core configuration type DaemonCfg struct { - Debug bool - Experimental bool - DataDir string - DefaultNetwork string - DefaultDriver string - Labels []string - DriverCfg map[string]interface{} - ClusterProvider cluster.Provider + Debug bool + Experimental bool + DataDir string + DefaultNetwork string + DefaultDriver string + Labels []string + DriverCfg map[string]interface{} + ClusterProvider cluster.Provider + NetworkControlPlaneMTU int } // ClusterCfg represents cluster configuration @@ -221,6 +222,18 @@ func OptionExperimental(exp bool) Option { } } +// OptionNetworkControlPlaneMTU function returns an option setter for control plane MTU +func OptionNetworkControlPlaneMTU(exp int) Option { + return func(c *Config) { + logrus.Debugf("Network Control Plane MTU: %d", exp) + if exp < 1500 { + // if exp == 0 the value won't be used + logrus.Warnf("Received a MTU of %d, this value is very low, the network control plane can misbehave", exp) + } + c.Daemon.NetworkControlPlaneMTU = exp + } +} + // ProcessOptions processes options and stores it in config func (c *Config) ProcessOptions(options ...Option) { for _, opt := range options { @@ -232,10 +245,7 @@ func (c *Config) ProcessOptions(options ...Option) { // IsValidName validates configuration objects supported by libnetwork func IsValidName(name string) bool { - if strings.TrimSpace(name) == "" { - return false - } - return true + return strings.TrimSpace(name) != "" } // OptionLocalKVProvider function returns an option setter for kvstore provider diff --git a/components/engine/vendor/github.com/docker/libnetwork/controller.go b/components/engine/vendor/github.com/docker/libnetwork/controller.go index 1696e07067..801097ab99 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/controller.go +++ b/components/engine/vendor/github.com/docker/libnetwork/controller.go @@ -52,7 +52,6 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/discovery" "github.com/docker/docker/pkg/locker" "github.com/docker/docker/pkg/plugingetter" @@ -69,6 +68,7 @@ import ( "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) // NetworkController provides the interface for controller instance which manages diff --git a/components/engine/vendor/github.com/docker/libnetwork/default_gateway.go b/components/engine/vendor/github.com/docker/libnetwork/default_gateway.go index bf1592435f..9a60fd6758 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/default_gateway.go +++ b/components/engine/vendor/github.com/docker/libnetwork/default_gateway.go @@ -4,9 +4,9 @@ import ( "fmt" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/diagnose/diagnose.go b/components/engine/vendor/github.com/docker/libnetwork/diagnose/diagnose.go new file mode 100644 index 0000000000..2849397486 --- /dev/null +++ b/components/engine/vendor/github.com/docker/libnetwork/diagnose/diagnose.go @@ -0,0 +1,133 @@ +package diagnose + +import ( + "fmt" + "net" + "net/http" + "sync" + + "github.com/sirupsen/logrus" +) + +// HTTPHandlerFunc TODO +type HTTPHandlerFunc func(interface{}, http.ResponseWriter, *http.Request) + +type httpHandlerCustom struct { + ctx interface{} + F func(interface{}, http.ResponseWriter, *http.Request) +} + +// ServeHTTP TODO +func (h httpHandlerCustom) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.F(h.ctx, w, r) +} + +var diagPaths2Func = map[string]HTTPHandlerFunc{ + "/": notImplemented, + "/help": help, + "/ready": ready, +} + +// Server when the debug is enabled exposes a +// This data structure is protected by the Agent mutex so does not require and additional mutex here +type Server struct { + sk net.Listener + port int + mux *http.ServeMux + registeredHanders []string + sync.Mutex +} + +// Init TODO +func (n *Server) Init() { + n.mux = http.NewServeMux() + + // Register local handlers + n.RegisterHandler(n, diagPaths2Func) +} + +// RegisterHandler TODO +func (n *Server) RegisterHandler(ctx interface{}, hdlrs map[string]HTTPHandlerFunc) { + n.Lock() + defer n.Unlock() + for path, fun := range hdlrs { + n.mux.Handle(path, httpHandlerCustom{ctx, fun}) + n.registeredHanders = append(n.registeredHanders, path) + } +} + +// EnableDebug opens a TCP socket to debug the passed network DB +func (n *Server) EnableDebug(ip string, port int) { + n.Lock() + defer n.Unlock() + + n.port = port + logrus.SetLevel(logrus.DebugLevel) + + if n.sk != nil { + logrus.Infof("The server is already up and running") + return + } + + logrus.Infof("Starting the server listening on %d for commands", port) + + // // Create the socket + // var err error + // n.sk, err = net.Listen("tcp", listeningAddr) + // if err != nil { + // log.Fatal(err) + // } + // + // go func() { + // http.Serve(n.sk, n.mux) + // }() + http.ListenAndServe(":8000", n.mux) +} + +// DisableDebug stop the dubug and closes the tcp socket +func (n *Server) DisableDebug() { + n.Lock() + defer n.Unlock() + n.sk.Close() + n.sk = nil +} + +// IsDebugEnable returns true when the debug is enabled +func (n *Server) IsDebugEnable() bool { + n.Lock() + defer n.Unlock() + return n.sk != nil +} + +func notImplemented(ctx interface{}, w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "URL path: %s no method implemented check /help\n", r.URL.Path) +} + +func help(ctx interface{}, w http.ResponseWriter, r *http.Request) { + n, ok := ctx.(*Server) + if ok { + for _, path := range n.registeredHanders { + fmt.Fprintf(w, "%s\n", path) + } + } +} + +func ready(ctx interface{}, w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "OK\n") +} + +// DebugHTTPForm TODO +func DebugHTTPForm(r *http.Request) { + r.ParseForm() + for k, v := range r.Form { + logrus.Debugf("Form[%q] = %q\n", k, v) + } +} + +// HTTPReplyError TODO +func HTTPReplyError(w http.ResponseWriter, message, usage string) { + fmt.Fprintf(w, "%s\n", message) + if usage != "" { + fmt.Fprintf(w, "Usage: %s\n", usage) + } +} diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go index dd79f04d91..64a2743dcf 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go @@ -12,7 +12,6 @@ import ( "sync" "syscall" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" @@ -24,6 +23,7 @@ import ( "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/portmapper" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge_store.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge_store.go index c7c83d8369..b0e4ff02b5 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge_store.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge_store.go @@ -5,11 +5,11 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/interface.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/interface.go index 9b20900416..c9f3e8dfb7 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/interface.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/interface.go @@ -4,7 +4,7 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/link.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/link.go index 53e9eeef99..d364516f1a 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/link.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/link.go @@ -4,9 +4,9 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/iptables" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) type link struct { diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/port_mapping.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/port_mapping.go index 965cc9a039..48010e9c59 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/port_mapping.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/port_mapping.go @@ -6,8 +6,8 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_bridgenetfiltering.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_bridgenetfiltering.go index 884c7115ec..9b90acfac2 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_bridgenetfiltering.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_bridgenetfiltering.go @@ -7,7 +7,7 @@ import ( "os" "syscall" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // Enumeration type saying which versions of IP protocol to process. diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_device.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_device.go index 0961bea55d..a9dfd06771 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_device.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_device.go @@ -3,9 +3,9 @@ package bridge import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/libnetwork/netutils" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ip_forwarding.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ip_forwarding.go index d46f8ddf83..355a14d996 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ip_forwarding.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ip_forwarding.go @@ -4,8 +4,8 @@ import ( "fmt" "io/ioutil" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/iptables" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go index 769debcb80..f01b08dea0 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ip_tables.go @@ -5,8 +5,8 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/iptables" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ipv4.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ipv4.go index 7f8707266d..671bd3302e 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ipv4.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ipv4.go @@ -7,8 +7,8 @@ import ( "net" "path/filepath" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ipv6.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ipv6.go index ee3d753ac1..b944be081e 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ipv6.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_ipv6.go @@ -6,8 +6,8 @@ import ( "net" "os" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_verify.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_verify.go index 330a5b4560..de77c38a66 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_verify.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_verify.go @@ -4,9 +4,9 @@ import ( "fmt" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_endpoint.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_endpoint.go index d30aeac9b5..1fe44f2e71 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_endpoint.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_endpoint.go @@ -3,12 +3,12 @@ package ipvlan import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) // CreateEndpoint assigns the mac, ip and endpoint id for the new container diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_joinleave.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_joinleave.go index 0c08dfce5d..9d229a245e 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_joinleave.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_joinleave.go @@ -4,12 +4,12 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/netutils" "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) type staticRoute struct { diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_network.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_network.go index 801fc20a7e..a9544b5a95 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_network.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_network.go @@ -3,7 +3,6 @@ package ipvlan import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/stringid" "github.com/docker/libnetwork/driverapi" @@ -12,6 +11,7 @@ import ( "github.com/docker/libnetwork/options" "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) // CreateNetwork the network for the specified driver type diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_setup.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_setup.go index 75f08210fb..0c39ec9d27 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_setup.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_setup.go @@ -5,8 +5,8 @@ import ( "strconv" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/ns" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_state.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_state.go index 2d8cb2d8f6..2a4ad25b40 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_state.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_state.go @@ -3,9 +3,9 @@ package ipvlan import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) func (d *driver) network(nid string) *network { diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_store.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_store.go index de994fa7f8..197e29966a 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_store.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan_store.go @@ -5,11 +5,11 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_endpoint.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_endpoint.go index 3e5ccb29f8..d9fae57185 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_endpoint.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_endpoint.go @@ -3,13 +3,13 @@ package macvlan import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/netutils" "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) // CreateEndpoint assigns the mac, ip and endpoint id for the new container diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_joinleave.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_joinleave.go index cf5c2a4bf9..778613db77 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_joinleave.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_joinleave.go @@ -4,11 +4,11 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/netutils" "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/osl" + "github.com/sirupsen/logrus" ) // Join method is invoked when a Sandbox is attached to an endpoint. diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_network.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_network.go index c455b9124d..914c6cdf64 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_network.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_network.go @@ -3,7 +3,6 @@ package macvlan import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/stringid" "github.com/docker/libnetwork/driverapi" @@ -12,6 +11,7 @@ import ( "github.com/docker/libnetwork/options" "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) // CreateNetwork the network for the specified driver type diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_setup.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_setup.go index b5b4be3499..843a2e73bf 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_setup.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_setup.go @@ -5,8 +5,8 @@ import ( "strconv" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/ns" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_state.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_state.go index 8ac3d282f9..8fd1a9e4dc 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_state.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_state.go @@ -3,9 +3,9 @@ package macvlan import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) func (d *driver) network(nid string) *network { diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_store.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_store.go index 3fd92786bf..655a49c08b 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_store.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan_store.go @@ -5,11 +5,11 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go index 1d59f238b0..f12d7a8c67 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go @@ -12,10 +12,10 @@ import ( "strconv" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/iptables" "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/filter.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/filter.go index 40cd7d9f28..1601803aa0 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/filter.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/filter.go @@ -4,8 +4,8 @@ import ( "fmt" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/iptables" + "github.com/sirupsen/logrus" ) const globalChain = "DOCKER-OVERLAY" diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/joinleave.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/joinleave.go index cdbb428281..39c2056a9b 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/joinleave.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/joinleave.go @@ -5,11 +5,11 @@ import ( "net" "syscall" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/types" "github.com/gogo/protobuf/proto" + "github.com/sirupsen/logrus" ) // Join method is invoked when a Sandbox is attached to an endpoint. @@ -120,8 +120,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, } } - d.peerDbAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, - net.ParseIP(d.advertiseAddress), true) + d.peerAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true, false, false, true) if err := d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil { logrus.Warn(err) @@ -205,7 +204,7 @@ func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key stri return } - d.peerAdd(nid, eid, addr.IP, addr.Mask, mac, vtep, true, false, false) + d.peerAdd(nid, eid, addr.IP, addr.Mask, mac, vtep, true, false, false, false) } // Leave method is invoked when a Sandbox detaches from an endpoint. diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ostweaks_linux.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ostweaks_linux.go index 85840cf8c4..c00243ad2c 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ostweaks_linux.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ostweaks_linux.go @@ -5,7 +5,7 @@ import ( "path" "strings" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) var sysctlConf = map[string]string{ diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_endpoint.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_endpoint.go index ebcab51501..d0e63f6b7e 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_endpoint.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_endpoint.go @@ -5,12 +5,12 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/netutils" "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) type endpointTable map[string]*endpoint diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go index 3a4ea41bfc..d12dfc72f7 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go @@ -12,8 +12,8 @@ import ( "strings" "sync" "syscall" + "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/reexec" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/driverapi" @@ -23,6 +23,7 @@ import ( "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/resolvconf" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "github.com/vishvananda/netlink/nl" "github.com/vishvananda/netns" @@ -682,10 +683,12 @@ func (n *network) initSandbox(restore bool) error { return fmt.Errorf("could not get network sandbox (oper %t): %v", restore, err) } + // this is needed to let the peerAdd configure the sandbox n.setSandbox(sbox) if !restore { - n.driver.peerDbUpdateSandbox(n.id) + // Initialize the sandbox with all the peers previously received from networkdb + n.driver.initSandboxPeerDB(n.id) } var nlSock *nl.NetlinkSocket @@ -705,6 +708,7 @@ func (n *network) initSandbox(restore bool) error { } func (n *network) watchMiss(nlSock *nl.NetlinkSocket) { + t := time.Now() for { msgs, err := nlSock.Receive() if err != nil { @@ -757,23 +761,52 @@ func (n *network) watchMiss(nlSock *nl.NetlinkSocket) { continue } - if !n.driver.isSerfAlive() { - continue - } - - mac, IPmask, vtep, err := n.driver.resolvePeer(n.id, ip) - if err != nil { - logrus.Errorf("could not resolve peer %q: %v", ip, err) - continue - } - - if err := n.driver.peerAdd(n.id, "dummy", ip, IPmask, mac, vtep, true, l2Miss, l3Miss); err != nil { - logrus.Errorf("could not add neighbor entry for missed peer %q: %v", ip, err) + if n.driver.isSerfAlive() { + mac, IPmask, vtep, err := n.driver.resolvePeer(n.id, ip) + if err != nil { + logrus.Errorf("could not resolve peer %q: %v", ip, err) + continue + } + n.driver.peerAdd(n.id, "dummy", ip, IPmask, mac, vtep, true, l2Miss, l3Miss, false) + } else { + // If the gc_thresh values are lower kernel might knock off the neighor entries. + // When we get a L3 miss check if its a valid peer and reprogram the neighbor + // entry again. Rate limit it to once attempt every 500ms, just in case a faulty + // container sends a flood of packets to invalid peers + if !l3Miss { + continue + } + if time.Since(t) > 500*time.Millisecond { + t = time.Now() + n.programNeighbor(ip) + } } } } } +func (n *network) programNeighbor(ip net.IP) { + peerMac, _, _, err := n.driver.peerDbSearch(n.id, ip) + if err != nil { + logrus.Errorf("Reprogramming on L3 miss failed for %s, no peer entry", ip) + return + } + s := n.getSubnetforIPAddr(ip) + if s == nil { + logrus.Errorf("Reprogramming on L3 miss failed for %s, not a valid subnet", ip) + return + } + sbox := n.sandbox() + if sbox == nil { + logrus.Errorf("Reprogramming on L3 miss failed for %s, overlay sandbox missing", ip) + return + } + if err := sbox.AddNeighbor(ip, peerMac, true, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil { + logrus.Errorf("Reprogramming on L3 miss failed for %s: %v", ip, err) + return + } +} + func (d *driver) addNetwork(n *network) { d.Lock() d.networks[n.id] = n @@ -1052,6 +1085,15 @@ func (n *network) contains(ip net.IP) bool { return false } +func (n *network) getSubnetforIPAddr(ip net.IP) *subnet { + for _, s := range n.subnets { + if s.subnetIP.Contains(ip) { + return s + } + } + return nil +} + // getSubnetforIP returns the subnet to which the given IP belongs func (n *network) getSubnetforIP(ip *net.IPNet) *subnet { for _, s := range n.subnets { diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_serf.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_serf.go index 9002bce6b7..5b96c515c9 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_serf.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_serf.go @@ -6,8 +6,8 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/hashicorp/serf/serf" + "github.com/sirupsen/logrus" ) type ovNotify struct { @@ -120,15 +120,10 @@ func (d *driver) processEvent(u serf.UserEvent) { switch action { case "join": - if err := d.peerAdd(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, - net.ParseIP(vtepStr), true, false, false); err != nil { - logrus.Errorf("Peer add failed in the driver: %v\n", err) - } + d.peerAdd(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr), + true, false, false, false) case "leave": - if err := d.peerDelete(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, - net.ParseIP(vtepStr), true); err != nil { - logrus.Errorf("Peer delete failed in the driver: %v\n", err) - } + d.peerDelete(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr), true) } } diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_utils.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_utils.go index 8a01914fa4..27f57c1fe2 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_utils.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_utils.go @@ -5,10 +5,10 @@ import ( "strings" "syscall" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/netutils" "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/osl" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/overlay.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/overlay.go index 8d19b2e1d4..0b22439ca7 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/overlay.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/overlay.go @@ -3,11 +3,11 @@ package overlay //go:generate protoc -I.:../../Godeps/_workspace/src/github.com/gogo/protobuf --gogo_out=import_path=github.com/docker/libnetwork/drivers/overlay,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. overlay.proto import ( + "context" "fmt" "net" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" @@ -16,6 +16,7 @@ import ( "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" "github.com/hashicorp/serf/serf" + "github.com/sirupsen/logrus" ) const ( @@ -50,6 +51,8 @@ type driver struct { joinOnce sync.Once localJoinOnce sync.Once keys []*key + peerOpCh chan *peerOperation + peerOpCancel context.CancelFunc sync.Mutex } @@ -64,10 +67,16 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { peerDb: peerNetworkMap{ mp: map[string]*peerMap{}, }, - secMap: &encrMap{nodes: map[string][]*spi{}}, - config: config, + secMap: &encrMap{nodes: map[string][]*spi{}}, + config: config, + peerOpCh: make(chan *peerOperation), } + // Launch the go routine for processing peer operations + ctx, cancel := context.WithCancel(context.Background()) + d.peerOpCancel = cancel + go d.peerOpRoutine(ctx, d.peerOpCh) + if data, ok := config[netlabel.GlobalKVClient]; ok { var err error dsc, ok := data.(discoverapi.DatastoreConfigData) @@ -161,7 +170,7 @@ func (d *driver) restoreEndpoints() error { } n.incEndpointCount() - d.peerDbAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true) + d.peerAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true, false, false, true) } return nil } @@ -170,6 +179,11 @@ func (d *driver) restoreEndpoints() error { func Fini(drv driverapi.Driver) { d := drv.(*driver) + // Notify the peer go routine to return + if d.peerOpCancel != nil { + d.peerOpCancel() + } + if d.exitCh != nil { waitCh := make(chan struct{}) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go index 3c3c0bb257..a80f335892 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go @@ -7,13 +7,13 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/idm" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/peerdb.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/peerdb.go index 21cd1fbe3d..f8f8493743 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/peerdb.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/peerdb.go @@ -1,12 +1,14 @@ package overlay import ( + "context" "fmt" "net" "sync" "syscall" - "github.com/Sirupsen/logrus" + "github.com/docker/libnetwork/common" + "github.com/sirupsen/logrus" ) const ovPeerTable = "overlay_peer_table" @@ -59,8 +61,6 @@ func (pKey *peerKey) Scan(state fmt.ScanState, verb rune) error { return nil } -var peerDbWg sync.WaitGroup - func (d *driver) peerDbWalk(f func(string, *peerKey, *peerEntry) bool) error { d.peerDb.Lock() nids := []string{} @@ -141,8 +141,6 @@ func (d *driver) peerDbSearch(nid string, peerIP net.IP) (net.HardwareAddr, net. func (d *driver) peerDbAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, peerMac net.HardwareAddr, vtep net.IP, isLocal bool) { - peerDbWg.Wait() - d.peerDb.Lock() pMap, ok := d.peerDb.mp[nid] if !ok { @@ -173,7 +171,6 @@ func (d *driver) peerDbAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask func (d *driver) peerDbDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, peerMac net.HardwareAddr, vtep net.IP) peerEntry { - peerDbWg.Wait() d.peerDb.Lock() pMap, ok := d.peerDb.mp[nid] @@ -206,55 +203,109 @@ func (d *driver) peerDbDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPM return pEntry } -func (d *driver) peerDbUpdateSandbox(nid string) { - d.peerDb.Lock() - pMap, ok := d.peerDb.mp[nid] - if !ok { - d.peerDb.Unlock() - return - } - d.peerDb.Unlock() +// The overlay uses a lazy initialization approach, this means that when a network is created +// and the driver registered the overlay does not allocate resources till the moment that a +// sandbox is actually created. +// At the moment of this call, that happens when a sandbox is initialized, is possible that +// networkDB has already delivered some events of peers already available on remote nodes, +// these peers are saved into the peerDB and this function is used to properly configure +// the network sandbox with all those peers that got previously notified. +// Note also that this method sends a single message on the channel and the go routine on the +// other side, will atomically loop on the whole table of peers and will program their state +// in one single atomic operation. This is fundamental to guarantee consistency, and avoid that +// new peerAdd or peerDelete gets reordered during the sandbox init. +func (d *driver) initSandboxPeerDB(nid string) { + d.peerInit(nid) +} - peerDbWg.Add(1) +type peerOperationType int32 - var peerOps []func() - pMap.Lock() - for pKeyStr, pEntry := range pMap.mp { - var pKey peerKey - if _, err := fmt.Sscan(pKeyStr, &pKey); err != nil { - logrus.Errorf("peer key scan failed: %v", err) - } +const ( + peerOperationINIT peerOperationType = iota + peerOperationADD + peerOperationDELETE +) - if pEntry.isLocal { - continue - } +type peerOperation struct { + opType peerOperationType + networkID string + endpointID string + peerIP net.IP + peerIPMask net.IPMask + peerMac net.HardwareAddr + vtepIP net.IP + updateDB bool + l2Miss bool + l3Miss bool + localPeer bool + callerName string +} - // Go captures variables by reference. The pEntry could be - // pointing to the same memory location for every iteration. Make - // a copy of pEntry before capturing it in the following closure. - entry := pEntry - op := func() { - if err := d.peerAdd(nid, entry.eid, pKey.peerIP, entry.peerIPMask, - pKey.peerMac, entry.vtep, - false, false, false); err != nil { - logrus.Errorf("peerdbupdate in sandbox failed for ip %s and mac %s: %v", - pKey.peerIP, pKey.peerMac, err) +func (d *driver) peerOpRoutine(ctx context.Context, ch chan *peerOperation) { + var err error + for { + select { + case <-ctx.Done(): + return + case op := <-ch: + switch op.opType { + case peerOperationINIT: + err = d.peerInitOp(op.networkID) + case peerOperationADD: + err = d.peerAddOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.updateDB, op.l2Miss, op.l3Miss, op.localPeer) + case peerOperationDELETE: + err = d.peerDeleteOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.localPeer) + } + if err != nil { + logrus.Warnf("Peer operation failed:%s op:%v", err, op) } } - - peerOps = append(peerOps, op) } - pMap.Unlock() +} - for _, op := range peerOps { - op() +func (d *driver) peerInit(nid string) { + callerName := common.CallerName(1) + d.peerOpCh <- &peerOperation{ + opType: peerOperationINIT, + networkID: nid, + callerName: callerName, } +} - peerDbWg.Done() +func (d *driver) peerInitOp(nid string) error { + return d.peerDbNetworkWalk(nid, func(pKey *peerKey, pEntry *peerEntry) bool { + // Local entries do not need to be added + if pEntry.isLocal { + return false + } + + d.peerAddOp(nid, pEntry.eid, pKey.peerIP, pEntry.peerIPMask, pKey.peerMac, pEntry.vtep, false, false, false, false) + // return false to loop on all entries + return false + }) } func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, - peerMac net.HardwareAddr, vtep net.IP, updateDb, l2Miss, l3Miss bool) error { + peerMac net.HardwareAddr, vtep net.IP, updateDb, l2Miss, l3Miss, localPeer bool) { + callerName := common.CallerName(1) + d.peerOpCh <- &peerOperation{ + opType: peerOperationADD, + networkID: nid, + endpointID: eid, + peerIP: peerIP, + peerIPMask: peerIPMask, + peerMac: peerMac, + vtepIP: vtep, + updateDB: updateDb, + l2Miss: l2Miss, + l3Miss: l3Miss, + localPeer: localPeer, + callerName: callerName, + } +} + +func (d *driver) peerAddOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, + peerMac net.HardwareAddr, vtep net.IP, updateDb, l2Miss, l3Miss, updateOnlyDB bool) error { if err := validateID(nid, eid); err != nil { return err @@ -262,6 +313,9 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, if updateDb { d.peerDbAdd(nid, eid, peerIP, peerIPMask, peerMac, vtep, false) + if updateOnlyDB { + return nil + } } n := d.network(nid) @@ -271,6 +325,9 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, sbox := n.sandbox() if sbox == nil { + // We are hitting this case for all the events that are arriving before that the sandbox + // is being created. The peer got already added into the database and the sanbox init will + // call the peerDbUpdateSandbox that will configure all these peers from the database return nil } @@ -311,6 +368,22 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, } func (d *driver) peerDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, + peerMac net.HardwareAddr, vtep net.IP, updateDb bool) { + callerName := common.CallerName(1) + d.peerOpCh <- &peerOperation{ + opType: peerOperationDELETE, + networkID: nid, + endpointID: eid, + peerIP: peerIP, + peerIPMask: peerIPMask, + peerMac: peerMac, + vtepIP: vtep, + updateDB: updateDb, + callerName: callerName, + } +} + +func (d *driver) peerDeleteOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, peerMac net.HardwareAddr, vtep net.IP, updateDb bool) error { if err := validateID(nid, eid); err != nil { diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/remote/driver.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/remote/driver.go index ffe0730c95..b52163025a 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/remote/driver.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/remote/driver.go @@ -5,13 +5,13 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/plugins" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/drivers/remote/api" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) type driver struct { diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/bridge/bridge.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/bridge/bridge.go index 2547016a4b..558157abe4 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/bridge/bridge.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/bridge/bridge.go @@ -13,7 +13,6 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" @@ -23,6 +22,7 @@ import ( "github.com/docker/libnetwork/options" "github.com/docker/libnetwork/portmapper" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/bridge/bridge_store.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/bridge/bridge_store.go index 956e82902f..6f5db4f5b9 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/bridge/bridge_store.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/bridge/bridge_store.go @@ -7,11 +7,11 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/bridge/port_mapping.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/bridge/port_mapping.go index 1cf22c13c9..3b67db388a 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/bridge/port_mapping.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/bridge/port_mapping.go @@ -10,8 +10,8 @@ import ( "os" "os/exec" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/encryption.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/encryption.go index ca7b60b4c7..0af3474b93 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/encryption.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/encryption.go @@ -9,8 +9,8 @@ import ( "net" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/joinleave.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/joinleave.go index fd411988d1..ff03f3ccb6 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/joinleave.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/joinleave.go @@ -4,10 +4,10 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/types" "github.com/gogo/protobuf/proto" + "github.com/sirupsen/logrus" ) // Join method is invoked when a Sandbox is attached to an endpoint. diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_endpoint.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_endpoint.go index ca0c477dcd..21b33ffe85 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_endpoint.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_endpoint.go @@ -5,11 +5,11 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/netutils" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) type endpointTable map[string]*endpoint diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_network.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_network.go index b0567261c7..b545bc8903 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_network.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_network.go @@ -10,7 +10,6 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/netlabel" @@ -18,6 +17,7 @@ import ( "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/resolvconf" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_serf.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_serf.go index 53c59b4df2..ddc0509064 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_serf.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_serf.go @@ -6,8 +6,8 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/hashicorp/serf/serf" + "github.com/sirupsen/logrus" ) type ovNotify struct { diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/overlay.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/overlay.go index 0a5a1bfdce..45b62b1835 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/overlay.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/overlay.go @@ -7,7 +7,6 @@ import ( "net" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" @@ -16,6 +15,7 @@ import ( "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" "github.com/hashicorp/serf/serf" + "github.com/sirupsen/logrus" ) // XXX OVERLAY_SOLARIS diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/peerdb.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/peerdb.go index d1499e2132..23d9a9729b 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/peerdb.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/peerdb.go @@ -5,7 +5,7 @@ import ( "net" "sync" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) const ovPeerTable = "overlay_peer_table" diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/labels.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/labels.go index b32c6ffb61..6cb077cb4f 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/labels.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/labels.go @@ -28,6 +28,9 @@ const ( // DNSServers of the network DNSServers = "com.docker.network.windowsshim.dnsservers" + // MacPool of the network + MacPool = "com.docker.network.windowsshim.macpool" + // SourceMac of the network SourceMac = "com.docker.network.windowsshim.sourcemac" diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/joinleave_windows.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/joinleave_windows.go index 91dcf28b32..cded48af64 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/joinleave_windows.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/joinleave_windows.go @@ -4,10 +4,10 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/types" "github.com/gogo/protobuf/proto" + "github.com/sirupsen/logrus" ) // Join method is invoked when a Sandbox is attached to an endpoint. diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/ov_endpoint_windows.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/ov_endpoint_windows.go index 3e45115df7..47af64cb9a 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/ov_endpoint_windows.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/ov_endpoint_windows.go @@ -6,8 +6,8 @@ import ( "net" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/driverapi" + "github.com/sirupsen/logrus" ) type endpointTable map[string]*endpoint diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/ov_network_windows.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/ov_network_windows.go index 65b7e38d3b..70c4f02eda 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/ov_network_windows.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/ov_network_windows.go @@ -9,10 +9,10 @@ import ( "sync" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/overlay_windows.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/overlay_windows.go index 111e517dbd..d415bebcbd 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/overlay_windows.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/overlay_windows.go @@ -8,12 +8,12 @@ import ( "sync" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/peerdb_windows.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/peerdb_windows.go index 04581f784a..0abc232432 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/peerdb_windows.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/overlay/peerdb_windows.go @@ -6,8 +6,8 @@ import ( "encoding/json" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" "github.com/Microsoft/hcsshim" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows.go index eb1522a74a..45f835ee07 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows.go @@ -20,12 +20,12 @@ import ( "sync" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) // networkConfiguration for network specific configuration @@ -38,6 +38,7 @@ type networkConfiguration struct { VLAN uint VSID uint DNSServers string + MacPools []hcsshim.MacPool DNSSuffix string SourceMac string NetworkAdapterName string @@ -168,6 +169,18 @@ func (d *driver) parseNetworkOptions(id string, genericOptions map[string]string config.DNSSuffix = value case DNSServers: config.DNSServers = value + case MacPool: + config.MacPools = make([]hcsshim.MacPool, 0) + s := strings.Split(value, ",") + if len(s)%2 != 0 { + return nil, types.BadRequestErrorf("Invalid mac pool. You must specify both a start range and an end range") + } + for i := 0; i < len(s)-1; i += 2 { + config.MacPools = append(config.MacPools, hcsshim.MacPool{ + StartMacAddress: s[i], + EndMacAddress: s[i+1], + }) + } case VLAN: vlan, err := strconv.ParseUint(value, 10, 32) if err != nil { @@ -274,6 +287,7 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d Subnets: subnets, DNSServerList: config.DNSServers, DNSSuffix: config.DNSSuffix, + MacPools: config.MacPools, SourceMac: config.SourceMac, NetworkAdapterName: config.NetworkAdapterName, } diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows_store.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows_store.go index bbed5aeaa2..caa93c68fb 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows_store.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows_store.go @@ -7,11 +7,11 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers_windows.go b/components/engine/vendor/github.com/docker/libnetwork/drivers_windows.go index 384d855cb8..a037c16efb 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers_windows.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers_windows.go @@ -16,5 +16,6 @@ func getInitializers(experimental bool) []initializer { {windows.GetInit("l2bridge"), "l2bridge"}, {windows.GetInit("l2tunnel"), "l2tunnel"}, {windows.GetInit("nat"), "nat"}, + {windows.GetInit("ics"), "ics"}, } } diff --git a/components/engine/vendor/github.com/docker/libnetwork/endpoint.go b/components/engine/vendor/github.com/docker/libnetwork/endpoint.go index 111b747352..724d0e5315 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/endpoint.go +++ b/components/engine/vendor/github.com/docker/libnetwork/endpoint.go @@ -8,12 +8,12 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/ipamapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/options" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) // Endpoint represents a logical connection between a network and a sandbox. diff --git a/components/engine/vendor/github.com/docker/libnetwork/endpoint_info.go b/components/engine/vendor/github.com/docker/libnetwork/endpoint_info.go index 202c27b308..47bff432cb 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/endpoint_info.go +++ b/components/engine/vendor/github.com/docker/libnetwork/endpoint_info.go @@ -154,9 +154,7 @@ func (epi *endpointInterface) CopyTo(dstEpi *endpointInterface) error { dstEpi.v6PoolID = epi.v6PoolID if len(epi.llAddrs) != 0 { dstEpi.llAddrs = make([]*net.IPNet, 0, len(epi.llAddrs)) - for _, ll := range epi.llAddrs { - dstEpi.llAddrs = append(dstEpi.llAddrs, ll) - } + dstEpi.llAddrs = append(dstEpi.llAddrs, epi.llAddrs...) } for _, route := range epi.routes { @@ -415,7 +413,7 @@ func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error { return err } if v, ok := epMap["gw"]; ok { - epj.gw6 = net.ParseIP(v.(string)) + epj.gw = net.ParseIP(v.(string)) } if v, ok := epMap["gw6"]; ok { epj.gw6 = net.ParseIP(v.(string)) @@ -444,6 +442,6 @@ func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error { dstEpj.driverTableEntries = make([]*tableEntry, len(epj.driverTableEntries)) copy(dstEpj.driverTableEntries, epj.driverTableEntries) dstEpj.gw = types.GetIPCopy(epj.gw) - dstEpj.gw = types.GetIPCopy(epj.gw6) + dstEpj.gw6 = types.GetIPCopy(epj.gw6) return nil } diff --git a/components/engine/vendor/github.com/docker/libnetwork/firewall_linux.go b/components/engine/vendor/github.com/docker/libnetwork/firewall_linux.go index 8c27c1dd25..b2232ac7a5 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/firewall_linux.go +++ b/components/engine/vendor/github.com/docker/libnetwork/firewall_linux.go @@ -1,8 +1,8 @@ package libnetwork import ( - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/iptables" + "github.com/sirupsen/logrus" ) const userChain = "DOCKER-USER" diff --git a/components/engine/vendor/github.com/docker/libnetwork/hostdiscovery/hostdiscovery.go b/components/engine/vendor/github.com/docker/libnetwork/hostdiscovery/hostdiscovery.go index a45ecb60a3..452b5628c1 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/hostdiscovery/hostdiscovery.go +++ b/components/engine/vendor/github.com/docker/libnetwork/hostdiscovery/hostdiscovery.go @@ -4,7 +4,7 @@ import ( "net" "sync" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" mapset "github.com/deckarep/golang-set" "github.com/docker/docker/pkg/discovery" diff --git a/components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go b/components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go index b3876ffded..71c9f39531 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go @@ -6,13 +6,13 @@ import ( "sort" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/bitseq" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/ipamapi" "github.com/docker/libnetwork/ipamutils" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/ipam/store.go b/components/engine/vendor/github.com/docker/libnetwork/ipam/store.go index d05e1b70b9..124d585518 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ipam/store.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ipam/store.go @@ -3,9 +3,9 @@ package ipam import ( "encoding/json" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) // Key provides the Key to be used in KV Store diff --git a/components/engine/vendor/github.com/docker/libnetwork/ipams/remote/remote.go b/components/engine/vendor/github.com/docker/libnetwork/ipams/remote/remote.go index d2e2b4f3c8..9f2f1d5fed 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ipams/remote/remote.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ipams/remote/remote.go @@ -4,12 +4,12 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/plugins" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/ipamapi" "github.com/docker/libnetwork/ipams/remote/api" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) type allocator struct { diff --git a/components/engine/vendor/github.com/docker/libnetwork/ipams/windowsipam/windowsipam.go b/components/engine/vendor/github.com/docker/libnetwork/ipams/windowsipam/windowsipam.go index 6b124d40d2..9cf99d42b1 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ipams/windowsipam/windowsipam.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ipams/windowsipam/windowsipam.go @@ -3,11 +3,11 @@ package windowsipam import ( "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/ipamapi" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/iptables/conntrack.go b/components/engine/vendor/github.com/docker/libnetwork/iptables/conntrack.go index 5731c53c04..08317c33ee 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/iptables/conntrack.go +++ b/components/engine/vendor/github.com/docker/libnetwork/iptables/conntrack.go @@ -5,7 +5,7 @@ import ( "net" "syscall" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/iptables/firewalld.go b/components/engine/vendor/github.com/docker/libnetwork/iptables/firewalld.go index 7dc5127fa4..c9838d5b65 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/iptables/firewalld.go +++ b/components/engine/vendor/github.com/docker/libnetwork/iptables/firewalld.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" - "github.com/Sirupsen/logrus" "github.com/godbus/dbus" + "github.com/sirupsen/logrus" ) // IPV defines the table string diff --git a/components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go b/components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go index 20edb9b5d6..ecce0d3b2a 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go +++ b/components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go @@ -10,7 +10,7 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // Action signifies the iptable action. @@ -151,11 +151,11 @@ func ProgramChain(c *ChainInfo, bridgeName string, hairpinMode, enable bool) err "-j", c.Name} if !Exists(Nat, "PREROUTING", preroute...) && enable { if err := c.Prerouting(Append, preroute...); err != nil { - return fmt.Errorf("Failed to inject docker in PREROUTING chain: %s", err) + return fmt.Errorf("Failed to inject %s in PREROUTING chain: %s", c.Name, err) } } else if Exists(Nat, "PREROUTING", preroute...) && !enable { if err := c.Prerouting(Delete, preroute...); err != nil { - return fmt.Errorf("Failed to remove docker in PREROUTING chain: %s", err) + return fmt.Errorf("Failed to remove %s in PREROUTING chain: %s", c.Name, err) } } output := []string{ @@ -167,11 +167,11 @@ func ProgramChain(c *ChainInfo, bridgeName string, hairpinMode, enable bool) err } if !Exists(Nat, "OUTPUT", output...) && enable { if err := c.Output(Append, output...); err != nil { - return fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err) + return fmt.Errorf("Failed to inject %s in OUTPUT chain: %s", c.Name, err) } } else if Exists(Nat, "OUTPUT", output...) && !enable { if err := c.Output(Delete, output...); err != nil { - return fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err) + return fmt.Errorf("Failed to inject %s in OUTPUT chain: %s", c.Name, err) } } case Filter: diff --git a/components/engine/vendor/github.com/docker/libnetwork/ipvs/netlink.go b/components/engine/vendor/github.com/docker/libnetwork/ipvs/netlink.go index 5450679c3b..b8d33dcdc4 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ipvs/netlink.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ipvs/netlink.go @@ -14,7 +14,7 @@ import ( "syscall" "unsafe" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink/nl" "github.com/vishvananda/netns" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/network.go b/components/engine/vendor/github.com/docker/libnetwork/network.go index 9f99064e11..3f44553b9d 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/network.go +++ b/components/engine/vendor/github.com/docker/libnetwork/network.go @@ -8,7 +8,6 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/stringid" "github.com/docker/libnetwork/common" "github.com/docker/libnetwork/config" @@ -21,6 +20,7 @@ import ( "github.com/docker/libnetwork/networkdb" "github.com/docker/libnetwork/options" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) // A Network represents a logical connectivity zone that containers may @@ -434,15 +434,11 @@ func (n *network) applyConfigurationTo(to *network) error { } if len(n.ipamV4Config) > 0 { to.ipamV4Config = make([]*IpamConf, 0, len(n.ipamV4Config)) - for _, v4conf := range n.ipamV4Config { - to.ipamV4Config = append(to.ipamV4Config, v4conf) - } + to.ipamV4Config = append(to.ipamV4Config, n.ipamV4Config...) } if len(n.ipamV6Config) > 0 { to.ipamV6Config = make([]*IpamConf, 0, len(n.ipamV6Config)) - for _, v6conf := range n.ipamV6Config { - to.ipamV6Config = append(to.ipamV6Config, v6conf) - } + to.ipamV6Config = append(to.ipamV6Config, n.ipamV6Config...) } if len(n.generic) > 0 { to.generic = options.Generic{} @@ -873,8 +869,7 @@ func (n *network) resolveDriver(name string, load bool) (driverapi.Driver, *driv d, cap := c.drvRegistry.Driver(name) if d == nil { if load { - var err error - err = c.loadDriver(name) + err := c.loadDriver(name) if err != nil { return nil, nil, err } @@ -1451,11 +1446,7 @@ func (n *network) ipamAllocate() error { } err = n.ipamAllocateVersion(6, ipam) - if err != nil { - return err - } - - return nil + return err } func (n *network) requestPoolHelper(ipam ipamapi.Ipam, addressSpace, preferredPool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error) { @@ -1654,9 +1645,7 @@ func (n *network) getIPInfo(ipVer int) []*IpamInfo { } l := make([]*IpamInfo, 0, len(info)) n.Lock() - for _, d := range info { - l = append(l, d) - } + l = append(l, info...) n.Unlock() return l } @@ -1870,7 +1859,7 @@ func (n *network) ResolveName(req string, ipType int) ([]net.IP, bool) { // the docker network domain. If the network is not v6 enabled // set ipv6Miss to filter the DNS query from going to external // resolvers. - if ok && n.enableIPv6 == false { + if ok && !n.enableIPv6 { ipv6Miss = true } ipSet, ok = sr.svcIPv6Map.Get(req) diff --git a/components/engine/vendor/github.com/docker/libnetwork/network_windows.go b/components/engine/vendor/github.com/docker/libnetwork/network_windows.go index 07a1c1d424..388b811421 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/network_windows.go +++ b/components/engine/vendor/github.com/docker/libnetwork/network_windows.go @@ -7,10 +7,10 @@ import ( "time" "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/drivers/windows" "github.com/docker/libnetwork/ipamapi" "github.com/docker/libnetwork/ipams/windowsipam" + "github.com/sirupsen/logrus" ) func executeInCompartment(compartmentID uint32, x func()) { @@ -29,7 +29,7 @@ func executeInCompartment(compartmentID uint32, x func()) { func (n *network) startResolver() { n.resolverOnce.Do(func() { - logrus.Debugf("Launching DNS server for network", n.Name()) + logrus.Debugf("Launching DNS server for network %q", n.Name()) options := n.Info().DriverOptions() hnsid := options[windows.HNSID] diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/broadcast.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/broadcast.go index 3fe9f6271a..52e96ec639 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/broadcast.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/broadcast.go @@ -114,7 +114,8 @@ type tableEventMessage struct { } func (m *tableEventMessage) Invalidates(other memberlist.Broadcast) bool { - return false + otherm := other.(*tableEventMessage) + return m.tname == otherm.tname && m.id == otherm.id && m.key == otherm.key } func (m *tableEventMessage) Message() []byte { diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/cluster.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/cluster.go index 9156d0da68..d15a5767ff 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/cluster.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/cluster.go @@ -12,8 +12,8 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/hashicorp/memberlist" + "github.com/sirupsen/logrus" ) const ( @@ -98,10 +98,14 @@ func (nDB *NetworkDB) RemoveKey(key []byte) { } func (nDB *NetworkDB) clusterInit() error { + nDB.lastStatsTimestamp = time.Now() + nDB.lastHealthTimestamp = nDB.lastStatsTimestamp + config := memberlist.DefaultLANConfig() config.Name = nDB.config.NodeName config.BindAddr = nDB.config.BindAddr config.AdvertiseAddr = nDB.config.AdvertiseAddr + config.UDPBufferSize = nDB.config.PacketBufferSize if nDB.config.BindPort != 0 { config.BindPort = nDB.config.BindPort @@ -199,9 +203,8 @@ func (nDB *NetworkDB) clusterJoin(members []string) error { mlist := nDB.memberlist if _, err := mlist.Join(members); err != nil { - // Incase of failure, keep retrying join until it succeeds or the cluster is shutdown. + // In case of failure, keep retrying join until it succeeds or the cluster is shutdown. go nDB.retryJoin(members, nDB.stopCh) - return fmt.Errorf("could not join node to memberlist: %v", err) } @@ -287,13 +290,6 @@ func (nDB *NetworkDB) reconnectNode() { return } - // Update all the local table state to a new time to - // force update on the node we are trying to rejoin, just in - // case that node has these in deleting state still. This is - // facilitate fast convergence after recovering from a gossip - // failure. - nDB.updateLocalTableTime() - logrus.Debugf("Initiating bulk sync with node %s after reconnect", node.Name) nDB.bulkSync([]string{node.Name}, true) } @@ -310,12 +306,11 @@ func (nDB *NetworkDB) reapState() { func (nDB *NetworkDB) reapNetworks() { nDB.Lock() - for name, nn := range nDB.networks { + for _, nn := range nDB.networks { for id, n := range nn { if n.leaving { if n.reapTime <= 0 { delete(nn, id) - nDB.deleteNetworkNode(id, name) continue } n.reapTime -= reapPeriod @@ -373,11 +368,21 @@ func (nDB *NetworkDB) gossip() { networkNodes[nid] = nDB.networkNodes[nid] } + printStats := time.Since(nDB.lastStatsTimestamp) >= nDB.config.StatsPrintPeriod + printHealth := time.Since(nDB.lastHealthTimestamp) >= nDB.config.HealthPrintPeriod nDB.RUnlock() + if printHealth { + healthScore := nDB.memberlist.GetHealthScore() + if healthScore != 0 { + logrus.Warnf("NetworkDB stats - healthscore:%d (connectivity issues)", healthScore) + } + nDB.lastHealthTimestamp = time.Now() + } + for nid, nodes := range networkNodes { mNodes := nDB.mRandomNodes(3, nodes) - bytesAvail := udpSendBuf - compoundHeaderOverhead + bytesAvail := nDB.config.PacketBufferSize - compoundHeaderOverhead nDB.RLock() network, ok := thisNodeNetworks[nid] @@ -398,6 +403,14 @@ func (nDB *NetworkDB) gossip() { } msgs := broadcastQ.GetBroadcasts(compoundOverhead, bytesAvail) + // Collect stats and print the queue info, note this code is here also to have a view of the queues empty + network.qMessagesSent += len(msgs) + if printStats { + logrus.Infof("NetworkDB stats - Queue net:%s qLen:%d netPeers:%d netMsg/s:%d", + nid, broadcastQ.NumQueued(), broadcastQ.NumNodes(), network.qMessagesSent/int((nDB.config.StatsPrintPeriod/time.Second))) + network.qMessagesSent = 0 + } + if len(msgs) == 0 { continue } @@ -415,11 +428,15 @@ func (nDB *NetworkDB) gossip() { } // Send the compound message - if err := nDB.memberlist.SendToUDP(&mnode.Node, compound); err != nil { + if err := nDB.memberlist.SendBestEffort(&mnode.Node, compound); err != nil { logrus.Errorf("Failed to send gossip to %s: %s", mnode.Addr, err) } } } + // Reset the stats + if printStats { + nDB.lastStatsTimestamp = time.Now() + } } func (nDB *NetworkDB) bulkSyncTables() { @@ -590,7 +607,7 @@ func (nDB *NetworkDB) bulkSyncNode(networks []string, node string, unsolicited b nDB.bulkSyncAckTbl[node] = ch nDB.Unlock() - err = nDB.memberlist.SendToTCP(&mnode.Node, buf) + err = nDB.memberlist.SendReliable(&mnode.Node, buf) if err != nil { nDB.Lock() delete(nDB.bulkSyncAckTbl, node) @@ -607,7 +624,7 @@ func (nDB *NetworkDB) bulkSyncNode(networks []string, node string, unsolicited b case <-t.C: logrus.Errorf("Bulk sync to node %s timed out", node) case <-ch: - logrus.Debugf("%s: Bulk sync to node %s took %s", nDB.config.NodeName, node, time.Now().Sub(startTime)) + logrus.Debugf("%s: Bulk sync to node %s took %s", nDB.config.NodeName, node, time.Since(startTime)) } t.Stop() } diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/delegate.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/delegate.go index 6df358382f..ffaf94e8c8 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/delegate.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/delegate.go @@ -5,8 +5,8 @@ import ( "net" "strings" - "github.com/Sirupsen/logrus" "github.com/gogo/protobuf/proto" + "github.com/sirupsen/logrus" ) type delegate struct { @@ -104,6 +104,9 @@ func (nDB *NetworkDB) handleNodeEvent(nEvent *NodeEvent) bool { } n = nDB.checkAndGetNode(nEvent) + if n == nil { + return false + } nDB.purgeSameNode(n) n.ltime = nEvent.LTime @@ -111,9 +114,12 @@ func (nDB *NetworkDB) handleNodeEvent(nEvent *NodeEvent) bool { switch nEvent.Type { case NodeEventTypeJoin: nDB.Lock() + _, found := nDB.nodes[n.Name] nDB.nodes[n.Name] = n nDB.Unlock() - logrus.Infof("Node join event for %s/%s", n.Name, n.Addr) + if !found { + logrus.Infof("Node join event for %s/%s", n.Name, n.Addr) + } return true case NodeEventTypeLeave: nDB.Lock() @@ -127,25 +133,12 @@ func (nDB *NetworkDB) handleNodeEvent(nEvent *NodeEvent) bool { } func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool { - var flushEntries bool // Update our local clock if the received messages has newer // time. nDB.networkClock.Witness(nEvent.LTime) nDB.Lock() - defer func() { - nDB.Unlock() - // When a node leaves a network on the last task removal cleanup the - // local entries for this network & node combination. When the tasks - // on a network are removed we could have missed the gossip updates. - // Not doing this cleanup can leave stale entries because bulksyncs - // from the node will no longer include this network state. - // - // deleteNodeNetworkEntries takes nDB lock. - if flushEntries { - nDB.deleteNodeNetworkEntries(nEvent.NetworkID, nEvent.NodeName) - } - }() + defer nDB.Unlock() if nEvent.NodeName == nDB.config.NodeName { return false @@ -173,10 +166,20 @@ func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool { n.leaving = nEvent.Type == NetworkEventTypeLeave if n.leaving { n.reapTime = reapInterval - flushEntries = true + + // The remote node is leaving the network, but not the gossip cluster. + // Mark all its entries in deleted state, this will guarantee that + // if some node bulk sync with us, the deleted state of + // these entries will be propagated. + nDB.deleteNodeNetworkEntries(nEvent.NetworkID, nEvent.NodeName) + } + + if nEvent.Type == NetworkEventTypeLeave { + nDB.deleteNetworkNode(nEvent.NetworkID, nEvent.NodeName) + } else { + nDB.addNetworkNode(nEvent.NetworkID, nEvent.NodeName) } - nDB.addNetworkNode(nEvent.NetworkID, nEvent.NodeName) return true } @@ -203,17 +206,22 @@ func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool { nDB.RLock() networks := nDB.networks[nDB.config.NodeName] network, ok := networks[tEvent.NetworkID] - nDB.RUnlock() - if !ok || network.leaving { - return true + // Check if the owner of the event is still part of the network + nodes := nDB.networkNodes[tEvent.NetworkID] + var nodePresent bool + for _, node := range nodes { + if node == tEvent.NodeName { + nodePresent = true + break + } } - - e, err := nDB.getEntry(tEvent.TableName, tEvent.NetworkID, tEvent.Key) - if err != nil && tEvent.Type == TableEventTypeDelete { - // If it is a delete event and we don't have the entry here nothing to do. + nDB.RUnlock() + if !ok || network.leaving || !nodePresent { + // I'm out of the network OR the event owner is not anymore part of the network so do not propagate return false } + e, err := nDB.getEntry(tEvent.TableName, tEvent.NetworkID, tEvent.Key) if err == nil { // We have the latest state. Ignore the event // since it is stale. @@ -238,6 +246,11 @@ func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool { nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", tEvent.NetworkID, tEvent.TableName, tEvent.Key), e) nDB.Unlock() + if err != nil && tEvent.Type == TableEventTypeDelete { + // If it is a delete event and we didn't have the entry here don't repropagate + return true + } + var op opType switch tEvent.Type { case TableEventTypeCreate: @@ -278,8 +291,7 @@ func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) { return } - // Do not rebroadcast a bulk sync - if rebroadcast := nDB.handleTableEvent(&tEvent); rebroadcast && !isBulkSync { + if rebroadcast := nDB.handleTableEvent(&tEvent); rebroadcast { var err error buf, err = encodeRawMessage(MessageTypeTableEvent, buf) if err != nil { diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/event_delegate.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/event_delegate.go index 23e16832e7..74aa465294 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/event_delegate.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/event_delegate.go @@ -4,8 +4,8 @@ import ( "encoding/json" "net" - "github.com/Sirupsen/logrus" "github.com/hashicorp/memberlist" + "github.com/sirupsen/logrus" ) type eventDelegate struct { @@ -45,9 +45,12 @@ func (e *eventDelegate) NotifyLeave(mn *memberlist.Node) { var failed bool logrus.Infof("Node %s/%s, left gossip cluster", mn.Name, mn.Addr) e.broadcastNodeEvent(mn.Addr, opDelete) - e.nDB.deleteNodeTableEntries(mn.Name) - e.nDB.deleteNetworkEntriesForNode(mn.Name) + // The node left or failed, delete all the entries created by it. + // If the node was temporary down, deleting the entries will guarantee that the CREATE events will be accepted + // If the node instead left because was going down, then it makes sense to just delete all its state e.nDB.Lock() + e.nDB.deleteNetworkEntriesForNode(mn.Name) + e.nDB.deleteNodeTableEntries(mn.Name) if n, ok := e.nDB.nodes[mn.Name]; ok { delete(e.nDB.nodes, mn.Name) @@ -61,7 +64,6 @@ func (e *eventDelegate) NotifyLeave(mn *memberlist.Node) { if failed { logrus.Infof("Node %s/%s, added to failed nodes list", mn.Name, mn.Addr) } - } func (e *eventDelegate) NotifyUpdate(n *memberlist.Node) { diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/message.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/message.go index a861752bd4..81a6d832a6 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/message.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/message.go @@ -3,10 +3,6 @@ package networkdb import "github.com/gogo/protobuf/proto" const ( - // Max udp message size chosen to avoid network packet - // fragmentation. - udpSendBuf = 1400 - // Compound message header overhead 1 byte(message type) + 4 // bytes (num messages) compoundHeaderOverhead = 5 diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go index b93a90d019..73dd999097 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go @@ -1,20 +1,21 @@ package networkdb -//go:generate protoc -I.:../Godeps/_workspace/src/github.com/gogo/protobuf --gogo_out=import_path=github.com/docker/libnetwork/networkdb,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. networkdb.proto +//go:generate protoc -I.:../vendor/github.com/gogo/protobuf --gogo_out=import_path=github.com/docker/libnetwork/networkdb,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. networkdb.proto import ( "fmt" "net" + "os" "strings" "sync" "time" - "github.com/Sirupsen/logrus" "github.com/armon/go-radix" "github.com/docker/go-events" "github.com/docker/libnetwork/types" "github.com/hashicorp/memberlist" "github.com/hashicorp/serf/serf" + "github.com/sirupsen/logrus" ) const ( @@ -93,6 +94,12 @@ type NetworkDB struct { // bootStrapIP is the list of IPs that can be used to bootstrap // the gossip. bootStrapIP []net.IP + + // lastStatsTimestamp is the last timestamp when the stats got printed + lastStatsTimestamp time.Time + + // lastHealthTimestamp is the last timestamp when the health score got printed + lastHealthTimestamp time.Time } // PeerInfo represents the peer (gossip cluster) nodes of a network @@ -101,6 +108,11 @@ type PeerInfo struct { IP string } +// PeerClusterInfo represents the peer (gossip cluster) nodes +type PeerClusterInfo struct { + PeerInfo +} + type node struct { memberlist.Node ltime serf.LamportTime @@ -126,6 +138,9 @@ type network struct { // The broadcast queue for table event gossip. This is only // initialized for this node's network attachment entries. tableBroadcasts *memberlist.TransmitLimitedQueue + + // Number of gossip messages sent related to this network during the last stats collection period + qMessagesSent int } // Config represents the configuration of the networdb instance and @@ -149,6 +164,21 @@ type Config struct { // Keys to be added to the Keyring of the memberlist. Key at index // 0 is the primary key Keys [][]byte + + // PacketBufferSize is the maximum number of bytes that memberlist will + // put in a packet (this will be for UDP packets by default with a NetTransport). + // A safe value for this is typically 1400 bytes (which is the default). However, + // depending on your network's MTU (Maximum Transmission Unit) you may + // be able to increase this to get more content into each gossip packet. + PacketBufferSize int + + // StatsPrintPeriod the period to use to print queue stats + // Default is 5min + StatsPrintPeriod time.Duration + + // HealthPrintPeriod the period to use to print the health score + // Default is 1min + HealthPrintPeriod time.Duration } // entry defines a table entry @@ -171,6 +201,18 @@ type entry struct { reapTime time.Duration } +// DefaultConfig returns a NetworkDB config with default values +func DefaultConfig() *Config { + hostname, _ := os.Hostname() + return &Config{ + NodeName: hostname, + BindAddr: "0.0.0.0", + PacketBufferSize: 1400, + StatsPrintPeriod: 5 * time.Minute, + HealthPrintPeriod: 1 * time.Minute, + } +} + // New creates a new instance of NetworkDB using the Config passed by // the caller. func New(c *Config) (*NetworkDB, error) { @@ -200,6 +242,7 @@ func New(c *Config) (*NetworkDB, error) { // instances passed by the caller in the form of addr:port func (nDB *NetworkDB) Join(members []string) error { nDB.Lock() + nDB.bootStrapIP = make([]net.IP, 0, len(members)) for _, m := range members { nDB.bootStrapIP = append(nDB.bootStrapIP, net.ParseIP(m)) } @@ -215,6 +258,20 @@ func (nDB *NetworkDB) Close() { } } +// ClusterPeers returns all the gossip cluster peers. +func (nDB *NetworkDB) ClusterPeers() []PeerInfo { + nDB.RLock() + defer nDB.RUnlock() + peers := make([]PeerInfo, 0, len(nDB.nodes)) + for _, node := range nDB.nodes { + peers = append(peers, PeerInfo{ + Name: node.Name, + IP: node.Node.Addr.String(), + }) + } + return peers +} + // Peers returns the gossip peers for a given network. func (nDB *NetworkDB) Peers(nid string) []PeerInfo { nDB.RLock() @@ -361,7 +418,6 @@ func (nDB *NetworkDB) DeleteEntry(tname, nid, key string) error { } func (nDB *NetworkDB) deleteNetworkEntriesForNode(deletedNode string) { - nDB.Lock() for nid, nodes := range nDB.networkNodes { updatedNodes := make([]string, 0, len(nodes)) for _, node := range nodes { @@ -376,11 +432,25 @@ func (nDB *NetworkDB) deleteNetworkEntriesForNode(deletedNode string) { } delete(nDB.networks, deletedNode) - nDB.Unlock() } +// deleteNodeNetworkEntries is called in 2 conditions with 2 different outcomes: +// 1) when a notification is coming of a node leaving the network +// - Walk all the network entries and mark the leaving node's entries for deletion +// These will be garbage collected when the reap timer will expire +// 2) when the local node is leaving the network +// - Walk all the network entries: +// A) if the entry is owned by the local node +// then we will mark it for deletion. This will ensure that if a node did not +// yet received the notification that the local node is leaving, will be aware +// of the entries to be deleted. +// B) if the entry is owned by a remote node, then we can safely delete it. This +// ensures that if we join back this network as we receive the CREATE event for +// entries owned by remote nodes, we will accept them and we notify the application func (nDB *NetworkDB) deleteNodeNetworkEntries(nid, node string) { - nDB.Lock() + // Indicates if the delete is triggered for the local node + isNodeLocal := node == nDB.config.NodeName + nDB.indexes[byNetwork].WalkPrefix(fmt.Sprintf("/%s", nid), func(path string, v interface{}) bool { oldEntry := v.(*entry) @@ -389,7 +459,15 @@ func (nDB *NetworkDB) deleteNodeNetworkEntries(nid, node string) { tname := params[1] key := params[2] - if oldEntry.node != node { + // If the entry is owned by a remote node and this node is not leaving the network + if oldEntry.node != node && !isNodeLocal { + // Don't do anything because the event is triggered for a node that does not own this entry + return false + } + + // If this entry is already marked for deletion and this node is not leaving the network + if oldEntry.deleting && !isNodeLocal { + // Don't do anything this entry will be already garbage collected using the old reapTime return false } @@ -401,17 +479,29 @@ func (nDB *NetworkDB) deleteNodeNetworkEntries(nid, node string) { reapTime: reapInterval, } - nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry) - nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry) + // we arrived at this point in 2 cases: + // 1) this entry is owned by the node that is leaving the network + // 2) the local node is leaving the network + if oldEntry.node == node { + if isNodeLocal { + // TODO fcrisciani: this can be removed if there is no way to leave the network + // without doing a delete of all the objects + entry.ltime++ + } + nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry) + nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry) + } else { + // the local node is leaving the network, all the entries of remote nodes can be safely removed + nDB.indexes[byTable].Delete(fmt.Sprintf("/%s/%s/%s", tname, nid, key)) + nDB.indexes[byNetwork].Delete(fmt.Sprintf("/%s/%s/%s", nid, tname, key)) + } nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, entry.value)) return false }) - nDB.Unlock() } func (nDB *NetworkDB) deleteNodeTableEntries(node string) { - nDB.Lock() nDB.indexes[byTable].Walk(func(path string, v interface{}) bool { oldEntry := v.(*entry) if oldEntry.node != node { @@ -423,27 +513,18 @@ func (nDB *NetworkDB) deleteNodeTableEntries(node string) { nid := params[1] key := params[2] - entry := &entry{ - ltime: oldEntry.ltime, - node: node, - value: oldEntry.value, - deleting: true, - reapTime: reapInterval, - } + nDB.indexes[byTable].Delete(fmt.Sprintf("/%s/%s/%s", tname, nid, key)) + nDB.indexes[byNetwork].Delete(fmt.Sprintf("/%s/%s/%s", nid, tname, key)) - nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry) - nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry) - - nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, entry.value)) + nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, oldEntry.value)) return false }) - nDB.Unlock() } // WalkTable walks a single table in NetworkDB and invokes the passed // function for each entry in the table passing the network, key, // value. The walk stops if the passed function returns a true. -func (nDB *NetworkDB) WalkTable(tname string, fn func(string, string, []byte) bool) error { +func (nDB *NetworkDB) WalkTable(tname string, fn func(string, string, []byte, bool) bool) error { nDB.RLock() values := make(map[string]interface{}) nDB.indexes[byTable].WalkPrefix(fmt.Sprintf("/%s", tname), func(path string, v interface{}) bool { @@ -456,7 +537,7 @@ func (nDB *NetworkDB) WalkTable(tname string, fn func(string, string, []byte) bo params := strings.Split(k[1:], "/") nid := params[1] key := params[2] - if fn(nid, key, v.(*entry).value) { + if fn(nid, key, v.(*entry).value, v.(*entry).deleting) { return nil } } @@ -481,13 +562,12 @@ func (nDB *NetworkDB) JoinNetwork(nid string) error { nodeNetworks[nid].tableBroadcasts = &memberlist.TransmitLimitedQueue{ NumNodes: func() int { nDB.RLock() - num := len(nDB.networkNodes[nid]) - nDB.RUnlock() - return num + defer nDB.RUnlock() + return len(nDB.networkNodes[nid]) }, RetransmitMult: 4, } - nDB.networkNodes[nid] = append(nDB.networkNodes[nid], nDB.config.NodeName) + nDB.addNetworkNode(nid, nDB.config.NodeName) networkNodes := nDB.networkNodes[nid] nDB.Unlock() @@ -517,35 +597,12 @@ func (nDB *NetworkDB) LeaveNetwork(nid string) error { nDB.Lock() defer nDB.Unlock() - var ( - paths []string - entries []*entry - ) - nwWalker := func(path string, v interface{}) bool { - entry, ok := v.(*entry) - if !ok { - return false - } - paths = append(paths, path) - entries = append(entries, entry) - return false - } + // Remove myself from the list of the nodes participating to the network + nDB.deleteNetworkNode(nid, nDB.config.NodeName) - nDB.indexes[byNetwork].WalkPrefix(fmt.Sprintf("/%s", nid), nwWalker) - for _, path := range paths { - params := strings.Split(path[1:], "/") - tname := params[1] - key := params[2] - - if _, ok := nDB.indexes[byTable].Delete(fmt.Sprintf("/%s/%s/%s", tname, nid, key)); !ok { - logrus.Errorf("Could not delete entry in table %s with network id %s and key %s as it does not exist", tname, nid, key) - } - - if _, ok := nDB.indexes[byNetwork].Delete(fmt.Sprintf("/%s/%s/%s", nid, tname, key)); !ok { - logrus.Errorf("Could not delete entry in network %s with table name %s and key %s as it does not exist", nid, tname, key) - } - } + // Update all the local entries marking them for deletion and delete all the remote entries + nDB.deleteNodeNetworkEntries(nid, nDB.config.NodeName) nodeNetworks, ok := nDB.networks[nDB.config.NodeName] if !ok { @@ -558,6 +615,7 @@ func (nDB *NetworkDB) LeaveNetwork(nid string) error { } n.ltime = ltime + n.reapTime = reapInterval n.leaving = true return nil } @@ -580,7 +638,10 @@ func (nDB *NetworkDB) addNetworkNode(nid string, nodeName string) { // passed network. Caller should hold the NetworkDB lock while calling // this func (nDB *NetworkDB) deleteNetworkNode(nid string, nodeName string) { - nodes := nDB.networkNodes[nid] + nodes, ok := nDB.networkNodes[nid] + if !ok || len(nodes) == 0 { + return + } newNodes := make([]string, 0, len(nodes)-1) for _, name := range nodes { if name == nodeName { @@ -618,27 +679,3 @@ func (nDB *NetworkDB) updateLocalNetworkTime() { n.ltime = ltime } } - -func (nDB *NetworkDB) updateLocalTableTime() { - nDB.Lock() - defer nDB.Unlock() - - ltime := nDB.tableClock.Increment() - nDB.indexes[byTable].Walk(func(path string, v interface{}) bool { - entry := v.(*entry) - if entry.node != nDB.config.NodeName { - return false - } - - params := strings.Split(path[1:], "/") - tname := params[0] - nid := params[1] - key := params[2] - entry.ltime = ltime - - nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry) - nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry) - - return false - }) -} diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdbdiagnose.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdbdiagnose.go new file mode 100644 index 0000000000..d70cec7bb6 --- /dev/null +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdbdiagnose.go @@ -0,0 +1,242 @@ +package networkdb + +import ( + "fmt" + "net/http" + "strings" + + "github.com/docker/libnetwork/diagnose" +) + +const ( + missingParameter = "missing parameter" +) + +// NetDbPaths2Func TODO +var NetDbPaths2Func = map[string]diagnose.HTTPHandlerFunc{ + "/join": dbJoin, + "/networkpeers": dbPeers, + "/clusterpeers": dbClusterPeers, + "/joinnetwork": dbJoinNetwork, + "/leavenetwork": dbLeaveNetwork, + "/createentry": dbCreateEntry, + "/updateentry": dbUpdateEntry, + "/deleteentry": dbDeleteEntry, + "/getentry": dbGetEntry, + "/gettable": dbGetTable, +} + +func dbJoin(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + diagnose.DebugHTTPForm(r) + if len(r.Form["members"]) < 1 { + diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?members=ip1,ip2,...", r.URL.Path)) + return + } + + nDB, ok := ctx.(*NetworkDB) + if ok { + err := nDB.Join(strings.Split(r.Form["members"][0], ",")) + if err != nil { + fmt.Fprintf(w, "%s error in the DB join %s\n", r.URL.Path, err) + return + } + + fmt.Fprintf(w, "OK\n") + } +} + +func dbPeers(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + diagnose.DebugHTTPForm(r) + if len(r.Form["nid"]) < 1 { + diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?nid=test", r.URL.Path)) + return + } + + nDB, ok := ctx.(*NetworkDB) + if ok { + peers := nDB.Peers(r.Form["nid"][0]) + fmt.Fprintf(w, "Network:%s Total peers: %d\n", r.Form["nid"], len(peers)) + for i, peerInfo := range peers { + fmt.Fprintf(w, "%d) %s -> %s\n", i, peerInfo.Name, peerInfo.IP) + } + } +} + +func dbClusterPeers(ctx interface{}, w http.ResponseWriter, r *http.Request) { + nDB, ok := ctx.(*NetworkDB) + if ok { + peers := nDB.ClusterPeers() + fmt.Fprintf(w, "Total peers: %d\n", len(peers)) + for i, peerInfo := range peers { + fmt.Fprintf(w, "%d) %s -> %s\n", i, peerInfo.Name, peerInfo.IP) + } + } +} + +func dbCreateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + diagnose.DebugHTTPForm(r) + if len(r.Form["tname"]) < 1 || + len(r.Form["nid"]) < 1 || + len(r.Form["key"]) < 1 || + len(r.Form["value"]) < 1 { + diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path)) + return + } + + tname := r.Form["tname"][0] + nid := r.Form["nid"][0] + key := r.Form["key"][0] + value := r.Form["value"][0] + + nDB, ok := ctx.(*NetworkDB) + if ok { + if err := nDB.CreateEntry(tname, nid, key, []byte(value)); err != nil { + diagnose.HTTPReplyError(w, err.Error(), "") + return + } + fmt.Fprintf(w, "OK\n") + } +} + +func dbUpdateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + diagnose.DebugHTTPForm(r) + if len(r.Form["tname"]) < 1 || + len(r.Form["nid"]) < 1 || + len(r.Form["key"]) < 1 || + len(r.Form["value"]) < 1 { + diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path)) + return + } + + tname := r.Form["tname"][0] + nid := r.Form["nid"][0] + key := r.Form["key"][0] + value := r.Form["value"][0] + + nDB, ok := ctx.(*NetworkDB) + if ok { + if err := nDB.UpdateEntry(tname, nid, key, []byte(value)); err != nil { + diagnose.HTTPReplyError(w, err.Error(), "") + return + } + fmt.Fprintf(w, "OK\n") + } +} + +func dbDeleteEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + diagnose.DebugHTTPForm(r) + if len(r.Form["tname"]) < 1 || + len(r.Form["nid"]) < 1 || + len(r.Form["key"]) < 1 { + diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path)) + return + } + + tname := r.Form["tname"][0] + nid := r.Form["nid"][0] + key := r.Form["key"][0] + + nDB, ok := ctx.(*NetworkDB) + if ok { + err := nDB.DeleteEntry(tname, nid, key) + if err != nil { + diagnose.HTTPReplyError(w, err.Error(), "") + return + } + fmt.Fprintf(w, "OK\n") + } +} + +func dbGetEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + diagnose.DebugHTTPForm(r) + if len(r.Form["tname"]) < 1 || + len(r.Form["nid"]) < 1 || + len(r.Form["key"]) < 1 { + diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path)) + return + } + + tname := r.Form["tname"][0] + nid := r.Form["nid"][0] + key := r.Form["key"][0] + + nDB, ok := ctx.(*NetworkDB) + if ok { + value, err := nDB.GetEntry(tname, nid, key) + if err != nil { + diagnose.HTTPReplyError(w, err.Error(), "") + return + } + fmt.Fprintf(w, "key:`%s` value:`%s`\n", key, string(value)) + } +} + +func dbJoinNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + diagnose.DebugHTTPForm(r) + if len(r.Form["nid"]) < 1 { + diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path)) + return + } + + nid := r.Form["nid"][0] + + nDB, ok := ctx.(*NetworkDB) + if ok { + if err := nDB.JoinNetwork(nid); err != nil { + diagnose.HTTPReplyError(w, err.Error(), "") + return + } + fmt.Fprintf(w, "OK\n") + } +} + +func dbLeaveNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + diagnose.DebugHTTPForm(r) + if len(r.Form["nid"]) < 1 { + diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path)) + return + } + + nid := r.Form["nid"][0] + + nDB, ok := ctx.(*NetworkDB) + if ok { + if err := nDB.LeaveNetwork(nid); err != nil { + diagnose.HTTPReplyError(w, err.Error(), "") + return + } + fmt.Fprintf(w, "OK\n") + } +} + +func dbGetTable(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + diagnose.DebugHTTPForm(r) + if len(r.Form["tname"]) < 1 || + len(r.Form["nid"]) < 1 { + diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id", r.URL.Path)) + return + } + + tname := r.Form["tname"][0] + nid := r.Form["nid"][0] + + nDB, ok := ctx.(*NetworkDB) + if ok { + table := nDB.GetTableByNetwork(tname, nid) + fmt.Fprintf(w, "total elements: %d\n", len(table)) + i := 0 + for k, v := range table { + fmt.Fprintf(w, "%d) k:`%s` -> v:`%s`\n", i, k, string(v.([]byte))) + i++ + } + } +} diff --git a/components/engine/vendor/github.com/docker/libnetwork/ns/init_linux.go b/components/engine/vendor/github.com/docker/libnetwork/ns/init_linux.go index 84d4950750..567a6242ac 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ns/init_linux.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ns/init_linux.go @@ -9,7 +9,7 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/osl/interface_linux.go b/components/engine/vendor/github.com/docker/libnetwork/osl/interface_linux.go index 8e8a830ce9..0ecda09f6e 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/osl/interface_linux.go +++ b/components/engine/vendor/github.com/docker/libnetwork/osl/interface_linux.go @@ -8,9 +8,9 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" ) @@ -26,7 +26,6 @@ type nwIface struct { mac net.HardwareAddr address *net.IPNet addressIPv6 *net.IPNet - ipAliases []*net.IPNet llAddrs []*net.IPNet routes []*net.IPNet bridge bool @@ -97,13 +96,6 @@ func (i *nwIface) LinkLocalAddresses() []*net.IPNet { return i.llAddrs } -func (i *nwIface) IPAliases() []*net.IPNet { - i.Lock() - defer i.Unlock() - - return i.ipAliases -} - func (i *nwIface) Routes() []*net.IPNet { i.Lock() defer i.Unlock() @@ -337,7 +329,6 @@ func configureInterface(nlh *netlink.Handle, iface netlink.Link, i *nwIface) err {setInterfaceIPv6, fmt.Sprintf("error setting interface %q IPv6 to %v", ifaceName, i.AddressIPv6())}, {setInterfaceMaster, fmt.Sprintf("error setting interface %q master to %q", ifaceName, i.DstMaster())}, {setInterfaceLinkLocalIPs, fmt.Sprintf("error setting interface %q link local IPs to %v", ifaceName, i.LinkLocalAddresses())}, - {setInterfaceIPAliases, fmt.Sprintf("error setting interface %q IP Aliases to %v", ifaceName, i.IPAliases())}, } for _, config := range ifaceConfigurators { @@ -399,16 +390,6 @@ func setInterfaceLinkLocalIPs(nlh *netlink.Handle, iface netlink.Link, i *nwIfac return nil } -func setInterfaceIPAliases(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error { - for _, si := range i.IPAliases() { - ipAddr := &netlink.Addr{IPNet: si} - if err := nlh.AddrAdd(iface, ipAddr); err != nil { - return err - } - } - return nil -} - func setInterfaceName(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error { return nlh.LinkSetName(iface, i.DstName()) } diff --git a/components/engine/vendor/github.com/docker/libnetwork/osl/namespace_linux.go b/components/engine/vendor/github.com/docker/libnetwork/osl/namespace_linux.go index d6b9d78234..6ddf7f16ad 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/osl/namespace_linux.go +++ b/components/engine/vendor/github.com/docker/libnetwork/osl/namespace_linux.go @@ -14,10 +14,10 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/reexec" "github.com/docker/libnetwork/ns" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" ) @@ -356,6 +356,22 @@ func (n *networkNamespace) loopbackUp() error { return n.nlHandle.LinkSetUp(iface) } +func (n *networkNamespace) AddLoopbackAliasIP(ip *net.IPNet) error { + iface, err := n.nlHandle.LinkByName("lo") + if err != nil { + return err + } + return n.nlHandle.AddrAdd(iface, &netlink.Addr{IPNet: ip}) +} + +func (n *networkNamespace) RemoveLoopbackAliasIP(ip *net.IPNet) error { + iface, err := n.nlHandle.LinkByName("lo") + if err != nil { + return err + } + return n.nlHandle.AddrDel(iface, &netlink.Addr{IPNet: ip}) +} + func (n *networkNamespace) InvokeFunc(f func()) error { return nsInvoke(n.nsPath(), func(nsFD int) error { return nil }, func(callerFD int) error { f() diff --git a/components/engine/vendor/github.com/docker/libnetwork/osl/neigh_linux.go b/components/engine/vendor/github.com/docker/libnetwork/osl/neigh_linux.go index 161ffa7beb..4e479489fa 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/osl/neigh_linux.go +++ b/components/engine/vendor/github.com/docker/libnetwork/osl/neigh_linux.go @@ -5,7 +5,7 @@ import ( "fmt" "net" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) @@ -91,9 +91,7 @@ func (n *networkNamespace) DeleteNeighbor(dstIP net.IP, dstMac net.HardwareAddr, if nh.linkDst != "" { nlnh.LinkIndex = iface.Attrs().Index } - if err := nlh.NeighDel(nlnh); err != nil { - logrus.Warnf("Deleting bridge mac mac %s failed, %v", dstMac, err) - } + nlh.NeighDel(nlnh) } } diff --git a/components/engine/vendor/github.com/docker/libnetwork/osl/options_linux.go b/components/engine/vendor/github.com/docker/libnetwork/osl/options_linux.go index 64309d0506..818669647f 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/osl/options_linux.go +++ b/components/engine/vendor/github.com/docker/libnetwork/osl/options_linux.go @@ -66,12 +66,6 @@ func (n *networkNamespace) LinkLocalAddresses(list []*net.IPNet) IfaceOption { } } -func (n *networkNamespace) IPAliases(list []*net.IPNet) IfaceOption { - return func(i *nwIface) { - i.ipAliases = list - } -} - func (n *networkNamespace) Routes(routes []*net.IPNet) IfaceOption { return func(i *nwIface) { i.routes = routes diff --git a/components/engine/vendor/github.com/docker/libnetwork/osl/sandbox.go b/components/engine/vendor/github.com/docker/libnetwork/osl/sandbox.go index 64288f9307..6ffc46775c 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/osl/sandbox.go +++ b/components/engine/vendor/github.com/docker/libnetwork/osl/sandbox.go @@ -32,6 +32,12 @@ type Sandbox interface { // Unset the previously set default IPv6 gateway in the sandbox UnsetGatewayIPv6() error + // AddLoopbackAliasIP adds the passed IP address to the sandbox loopback interface + AddLoopbackAliasIP(ip *net.IPNet) error + + // RemoveLoopbackAliasIP removes the passed IP address from the sandbox loopback interface + RemoveLoopbackAliasIP(ip *net.IPNet) error + // Add a static route to the sandbox. AddStaticRoute(*types.StaticRoute) error @@ -91,9 +97,6 @@ type IfaceOptionSetter interface { // LinkLocalAddresses returns an option setter to set the link-local IP addresses. LinkLocalAddresses([]*net.IPNet) IfaceOption - // IPAliases returns an option setter to set IP address Aliases - IPAliases([]*net.IPNet) IfaceOption - // Master returns an option setter to set the master interface if any for this // interface. The master interface name should refer to the srcname of a // previously added interface of type bridge. @@ -150,9 +153,6 @@ type Interface interface { // LinkLocalAddresses returns the link-local IP addresses assigned to the interface. LinkLocalAddresses() []*net.IPNet - // IPAliases returns the IP address aliases assigned to the interface. - IPAliases() []*net.IPNet - // IP routes for the interface. Routes() []*net.IPNet diff --git a/components/engine/vendor/github.com/docker/libnetwork/portmapper/mapper.go b/components/engine/vendor/github.com/docker/libnetwork/portmapper/mapper.go index 7f2a67c89f..f480edae03 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/portmapper/mapper.go +++ b/components/engine/vendor/github.com/docker/libnetwork/portmapper/mapper.go @@ -6,9 +6,9 @@ import ( "net" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/iptables" "github.com/docker/libnetwork/portallocator" + "github.com/sirupsen/logrus" ) type mapping struct { diff --git a/components/engine/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go b/components/engine/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go index 9da6922e2b..5cb251b131 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go +++ b/components/engine/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go @@ -8,10 +8,10 @@ import ( "strings" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/ioutils" "github.com/docker/libnetwork/resolvconf/dns" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/resolver.go b/components/engine/vendor/github.com/docker/libnetwork/resolver.go index cc7692621f..83382b645b 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/resolver.go +++ b/components/engine/vendor/github.com/docker/libnetwork/resolver.go @@ -8,9 +8,9 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/types" "github.com/miekg/dns" + "github.com/sirupsen/logrus" ) // Resolver represents the embedded DNS server in Docker. It operates @@ -231,7 +231,7 @@ func (r *resolver) handleIPQuery(name string, query *dns.Msg, ipType int) (*dns. if addr == nil && ipv6Miss { // Send a reply without any Answer sections - logrus.Debugf("Lookup name %s present without IPv6 address", name) + logrus.Debugf("[resolver] lookup name %s present without IPv6 address", name) resp := createRespMsg(query) return resp, nil } @@ -239,7 +239,7 @@ func (r *resolver) handleIPQuery(name string, query *dns.Msg, ipType int) (*dns. return nil, nil } - logrus.Debugf("Lookup for %s: IP %v", name, addr) + logrus.Debugf("[resolver] lookup for %s: IP %v", name, addr) resp := createRespMsg(query) if len(addr) > 1 { @@ -280,7 +280,7 @@ func (r *resolver) handlePTRQuery(ptr string, query *dns.Msg) (*dns.Msg, error) return nil, nil } - logrus.Debugf("Lookup for IP %s: name %s", parts[0], host) + logrus.Debugf("[resolver] lookup for IP %s: name %s", parts[0], host) fqdn := dns.Fqdn(host) resp := new(dns.Msg) @@ -431,10 +431,12 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) { } } if err != nil { - logrus.Warnf("Connect failed: %s", err) + logrus.Warnf("[resolver] connect failed: %s", err) continue } - logrus.Debugf("Query %s[%d] from %s, forwarding to %s:%s", name, query.Question[0].Qtype, + + queryType := dns.TypeToString[query.Question[0].Qtype] + logrus.Debugf("[resolver] query %s (%s) from %s, forwarding to %s:%s", name, queryType, extConn.LocalAddr().String(), proto, extDNS.IPStr) // Timeout has to be set for every IO operation. @@ -446,11 +448,11 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) { defer co.Close() // limits the number of outstanding concurrent queries. - if r.forwardQueryStart() == false { + if !r.forwardQueryStart() { old := r.tStamp r.tStamp = time.Now() if r.tStamp.Sub(old) > logInterval { - logrus.Errorf("More than %v concurrent queries from %s", maxConcurrent, extConn.LocalAddr().String()) + logrus.Errorf("[resolver] more than %v concurrent queries from %s", maxConcurrent, extConn.LocalAddr().String()) } continue } @@ -458,7 +460,7 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) { err = co.WriteMsg(query) if err != nil { r.forwardQueryEnd() - logrus.Debugf("Send to DNS server failed, %s", err) + logrus.Debugf("[resolver] send to DNS server failed, %s", err) continue } @@ -467,22 +469,32 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) { // client can retry over TCP if err != nil && err != dns.ErrTruncated { r.forwardQueryEnd() - logrus.Debugf("Read from DNS server failed, %s", err) + logrus.Debugf("[resolver] read from DNS server failed, %s", err) continue } r.forwardQueryEnd() if resp != nil { + answers := 0 for _, rr := range resp.Answer { h := rr.Header() switch h.Rrtype { case dns.TypeA: + answers++ ip := rr.(*dns.A).A + logrus.Debugf("[resolver] received A record %q for %q from %s:%s", ip, h.Name, proto, extDNS.IPStr) r.backend.HandleQueryResp(h.Name, ip) case dns.TypeAAAA: + answers++ ip := rr.(*dns.AAAA).AAAA + logrus.Debugf("[resolver] received AAAA record %q for %q from %s:%s", ip, h.Name, proto, extDNS.IPStr) r.backend.HandleQueryResp(h.Name, ip) } } + if resp.Answer == nil || answers == 0 { + logrus.Debugf("[resolver] external DNS %s:%s did not return any %s records for %q", proto, extDNS.IPStr, queryType, name) + } + } else { + logrus.Debugf("[resolver] external DNS %s:%s returned empty response for %q", proto, extDNS.IPStr, name) } resp.Compress = true break @@ -493,7 +505,7 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) { } if err = w.WriteMsg(resp); err != nil { - logrus.Errorf("error writing resolver resp, %s", err) + logrus.Errorf("[resolver] error writing resolver resp, %s", err) } } @@ -514,7 +526,7 @@ func (r *resolver) forwardQueryEnd() { defer r.queryLock.Unlock() if r.count == 0 { - logrus.Error("Invalid concurrent query count") + logrus.Error("[resolver] invalid concurrent query count") } else { r.count-- } diff --git a/components/engine/vendor/github.com/docker/libnetwork/resolver_unix.go b/components/engine/vendor/github.com/docker/libnetwork/resolver_unix.go index 5fcc6b9fa9..b35009ee74 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/resolver_unix.go +++ b/components/engine/vendor/github.com/docker/libnetwork/resolver_unix.go @@ -9,9 +9,9 @@ import ( "os/exec" "runtime" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/reexec" "github.com/docker/libnetwork/iptables" + "github.com/sirupsen/logrus" "github.com/vishvananda/netns" ) diff --git a/components/engine/vendor/github.com/docker/libnetwork/sandbox.go b/components/engine/vendor/github.com/docker/libnetwork/sandbox.go index 9454c5c286..6f4c2508b2 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/sandbox.go +++ b/components/engine/vendor/github.com/docker/libnetwork/sandbox.go @@ -9,11 +9,11 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/etchosts" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) // Sandbox provides the control over the network container entity. It is a one to one mapping with the container. @@ -621,7 +621,7 @@ func (sb *sandbox) resolveName(req string, networkName string, epList []*endpoin func (sb *sandbox) SetKey(basePath string) error { start := time.Now() defer func() { - logrus.Debugf("sandbox set key processing took %s for container %s", time.Now().Sub(start), sb.ContainerID()) + logrus.Debugf("sandbox set key processing took %s for container %s", time.Since(start), sb.ContainerID()) }() if basePath == "" { @@ -709,8 +709,15 @@ func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint) { ep.Lock() joinInfo := ep.joinInfo + vip := ep.virtualIP ep.Unlock() + if len(vip) != 0 { + if err := osSbox.RemoveLoopbackAliasIP(&net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)}); err != nil { + logrus.Warnf("Remove virtual IP %v failed: %v", vip, err) + } + } + if joinInfo == nil { return } @@ -767,15 +774,9 @@ func (sb *sandbox) restoreOslSandbox() error { if len(i.llAddrs) != 0 { ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs)) } - if len(ep.virtualIP) != 0 { - vipAlias := &net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)} - ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().IPAliases([]*net.IPNet{vipAlias})) - } Ifaces[fmt.Sprintf("%s+%s", i.srcName, i.dstPrefix)] = ifaceOptions if joinInfo != nil { - for _, r := range joinInfo.StaticRoutes { - routes = append(routes, r) - } + routes = append(routes, joinInfo.StaticRoutes...) } if ep.needResolver() { sb.startResolver(true) @@ -789,11 +790,7 @@ func (sb *sandbox) restoreOslSandbox() error { // restore osl sandbox err := sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6) - if err != nil { - return err - } - - return nil + return err } func (sb *sandbox) populateNetworkResources(ep *endpoint) error { @@ -824,10 +821,6 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error { if len(i.llAddrs) != 0 { ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs)) } - if len(ep.virtualIP) != 0 { - vipAlias := &net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)} - ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().IPAliases([]*net.IPNet{vipAlias})) - } if i.mac != nil { ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac)) } @@ -837,6 +830,13 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error { } } + if len(ep.virtualIP) != 0 { + err := sb.osSbox.AddLoopbackAliasIP(&net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)}) + if err != nil { + return fmt.Errorf("failed to add virtual IP %v: %v", ep.virtualIP, err) + } + } + if joinInfo != nil { // Set up non-interface routes. for _, r := range joinInfo.StaticRoutes { @@ -958,9 +958,7 @@ func (sb *sandbox) joinLeaveStart() { joinLeaveDone := sb.joinLeaveDone sb.Unlock() - select { - case <-joinLeaveDone: - } + <-joinLeaveDone sb.Lock() } diff --git a/components/engine/vendor/github.com/docker/libnetwork/sandbox_dns_unix.go b/components/engine/vendor/github.com/docker/libnetwork/sandbox_dns_unix.go index 867cc8abb4..f18f0b3ee6 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/sandbox_dns_unix.go +++ b/components/engine/vendor/github.com/docker/libnetwork/sandbox_dns_unix.go @@ -11,11 +11,11 @@ import ( "strconv" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/etchosts" "github.com/docker/libnetwork/resolvconf" "github.com/docker/libnetwork/resolvconf/dns" "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/vendor/github.com/docker/libnetwork/sandbox_externalkey_unix.go b/components/engine/vendor/github.com/docker/libnetwork/sandbox_externalkey_unix.go index 4a7ac06b59..f4c4276848 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/sandbox_externalkey_unix.go +++ b/components/engine/vendor/github.com/docker/libnetwork/sandbox_externalkey_unix.go @@ -10,9 +10,9 @@ import ( "net" "os" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/types" "github.com/opencontainers/runc/libcontainer/configs" + "github.com/sirupsen/logrus" ) const udsBase = "/run/docker/libnetwork/" @@ -52,7 +52,6 @@ func processSetKeyReexec() { controllerID := os.Args[2] err = SetExternalKey(controllerID, containerID, fmt.Sprintf("/proc/%d/ns/net", state.Pid)) - return } // SetExternalKey provides a convenient way to set an External key to a sandbox diff --git a/components/engine/vendor/github.com/docker/libnetwork/sandbox_store.go b/components/engine/vendor/github.com/docker/libnetwork/sandbox_store.go index 38b2bd7e8b..a083644598 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/sandbox_store.go +++ b/components/engine/vendor/github.com/docker/libnetwork/sandbox_store.go @@ -5,9 +5,9 @@ import ( "encoding/json" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/osl" + "github.com/sirupsen/logrus" ) const ( @@ -115,9 +115,7 @@ func (sbs *sbState) CopyTo(o datastore.KVObject) error { dstSbs.dbExists = sbs.dbExists dstSbs.EpPriority = sbs.EpPriority - for _, eps := range sbs.Eps { - dstSbs.Eps = append(dstSbs.Eps, eps) - } + dstSbs.Eps = append(dstSbs.Eps, sbs.Eps...) if len(sbs.ExtDNS2) > 0 { for _, dns := range sbs.ExtDNS2 { diff --git a/components/engine/vendor/github.com/docker/libnetwork/service_common.go b/components/engine/vendor/github.com/docker/libnetwork/service_common.go index 7e3367c71c..64d283e541 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/service_common.go +++ b/components/engine/vendor/github.com/docker/libnetwork/service_common.go @@ -5,8 +5,8 @@ package libnetwork import ( "net" - "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/common" + "github.com/sirupsen/logrus" ) func (c *controller) addEndpointNameResolution(svcName, svcID, nID, eID, containerName string, vip net.IP, serviceAliases, taskAliases []string, ip net.IP, addService bool, method string) error { diff --git a/components/engine/vendor/github.com/docker/libnetwork/service_linux.go b/components/engine/vendor/github.com/docker/libnetwork/service_linux.go index 1cf7ee91aa..00665941ed 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/service_linux.go +++ b/components/engine/vendor/github.com/docker/libnetwork/service_linux.go @@ -14,12 +14,12 @@ import ( "sync" "syscall" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/reexec" "github.com/docker/libnetwork/iptables" "github.com/docker/libnetwork/ipvs" "github.com/docker/libnetwork/ns" "github.com/gogo/protobuf/proto" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink/nl" "github.com/vishvananda/netns" ) @@ -372,6 +372,7 @@ func programIngress(gwIP net.IP, ingressPorts []*PortConfig, isDelete bool) erro if err := iptables.RawCombinedOutput("-I", "FORWARD", "-j", ingressChain); err != nil { return fmt.Errorf("failed to add jump rule to %s in filter table forward chain: %v", ingressChain, err) } + arrangeUserFilterRule() } oifName, err := findOIFName(gwIP) @@ -438,7 +439,9 @@ func programIngress(gwIP net.IP, ingressPorts []*PortConfig, isDelete bool) erro return nil } -// In the filter table FORWARD chain first rule should be to jump to INGRESS-CHAIN +// In the filter table FORWARD chain the first rule should be to jump to +// DOCKER-USER so the user is able to filter packet first. +// The second rule should be jump to INGRESS-CHAIN. // This chain has the rules to allow access to the published ports for swarm tasks // from local bridge networks and docker_gwbridge (ie:taks on other swarm netwroks) func arrangeIngressFilterRule() { diff --git a/components/engine/vendor/github.com/docker/libnetwork/store.go b/components/engine/vendor/github.com/docker/libnetwork/store.go index 398794bfef..1a897bfe01 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/store.go +++ b/components/engine/vendor/github.com/docker/libnetwork/store.go @@ -4,12 +4,12 @@ import ( "fmt" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/libkv/store/boltdb" "github.com/docker/libkv/store/consul" "github.com/docker/libkv/store/etcd" "github.com/docker/libkv/store/zookeeper" "github.com/docker/libnetwork/datastore" + "github.com/sirupsen/logrus" ) func registerKVStores() { diff --git a/components/engine/vendor/github.com/docker/libnetwork/vendor.conf b/components/engine/vendor/github.com/docker/libnetwork/vendor.conf index 203e1cebf5..6751cba47e 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/vendor.conf +++ b/components/engine/vendor/github.com/docker/libnetwork/vendor.conf @@ -1,8 +1,7 @@ -github.com/Azure/go-ansiterm 04b7f292a41fcb5da32dda536c0807fc13e8351c +github.com/Azure/go-ansiterm 19f72df4d05d31cbe1c56bfc8045c96babff6c7e github.com/BurntSushi/toml f706d00e3de6abe700c994cdd545a1a4915af060 github.com/Microsoft/go-winio ce2922f643c8fd76b46cadc7f404a06282678b34 -github.com/Microsoft/hcsshim e439b7d2b63f036d3a50c93a9e0b154a0d50e788 -github.com/Sirupsen/logrus 4b6ea7319e214d98c938f12692336f7ca9348d6b +github.com/Microsoft/hcsshim v0.6.1 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 github.com/boltdb/bolt c6ba97b89e0454fec9aa92e1d33a4e2c5fc1f631 @@ -11,9 +10,9 @@ github.com/coreos/etcd 925d1d74cec8c3b169c52fd4b2dc234a35934fce github.com/coreos/go-systemd b4a58d95188dd092ae20072bac14cece0e67c388 github.com/deckarep/golang-set ef32fa3046d9f249d399f98ebaf9be944430fd1d -github.com/docker/docker 9c96768eae4b3a65147b47a55c850c103ab8972d -github.com/docker/go-connections 34b5052da6b11e27f5f2e357b38b571ddddd3928 -github.com/docker/go-events 2e7d352816128aa84f4d29b2a21d400133701a0d +github.com/docker/docker 2cac43e3573893cf8fd816e0ad5615426acb87f4 https://github.com/dmcgowan/docker.git +github.com/docker/go-connections 3ede32e2033de7505e6500d6c868c2b9ed9f169d +github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/docker/go-units 8e2d4523730c73120e10d4652f36ad6010998f4e github.com/docker/libkv 1d8431073ae03cdaedb198a89722f3aab6d418ef @@ -31,9 +30,10 @@ github.com/hashicorp/go-sockaddr acd314c5781ea706c710d9ea70069fd2e110d61d github.com/hashicorp/serf 598c54895cc5a7b1a24a398d635e8c0ea0959870 github.com/mattn/go-shellwords 525bedee691b5a8df547cb5cf9f86b7fb1883e24 github.com/miekg/dns d27455715200c7d3e321a1e5cadb27c9ee0b0f02 -github.com/opencontainers/runc ba1568de399395774ad84c2ace65937814c542ed +github.com/opencontainers/runc 8694d576ea3ce3c9e2c804b7f91b4e1e9a575d1c https://github.com/dmcgowan/runc.git github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374 github.com/seccomp/libseccomp-golang 1b506fc7c24eec5a3693cdcbed40d9c226cfc6a1 +github.com/sirupsen/logrus v1.0.1 github.com/stretchr/testify dab07ac62d4905d3e48d17dc549c684ac3b7c15a github.com/syndtr/gocapability 2c00daeb6c3b45114c80ac44119e7b8801fdd852 github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065 diff --git a/components/engine/vendor/github.com/docker/swarmkit/agent/exec/controller.go b/components/engine/vendor/github.com/docker/swarmkit/agent/exec/controller.go index 1cafb47fd1..85110ba971 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/agent/exec/controller.go +++ b/components/engine/vendor/github.com/docker/swarmkit/agent/exec/controller.go @@ -4,12 +4,12 @@ import ( "fmt" "time" - "github.com/Sirupsen/logrus" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/api/equality" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/protobuf/ptypes" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/vendor/github.com/docker/swarmkit/agent/session.go b/components/engine/vendor/github.com/docker/swarmkit/agent/session.go index faa1fa8cac..0c00c4f1bf 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/agent/session.go +++ b/components/engine/vendor/github.com/docker/swarmkit/agent/session.go @@ -5,10 +5,10 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/connectionbroker" "github.com/docker/swarmkit/log" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/components/engine/vendor/github.com/docker/swarmkit/agent/worker.go b/components/engine/vendor/github.com/docker/swarmkit/agent/worker.go index ff138c2b74..6b500059a8 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/agent/worker.go +++ b/components/engine/vendor/github.com/docker/swarmkit/agent/worker.go @@ -3,12 +3,12 @@ package agent import ( "sync" - "github.com/Sirupsen/logrus" "github.com/boltdb/bolt" "github.com/docker/swarmkit/agent/exec" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/watch" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/vendor/github.com/docker/swarmkit/ca/auth.go b/components/engine/vendor/github.com/docker/swarmkit/ca/auth.go index 10cb2c76a1..488d34dd33 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/ca/auth.go +++ b/components/engine/vendor/github.com/docker/swarmkit/ca/auth.go @@ -5,7 +5,7 @@ import ( "crypto/x509/pkix" "strings" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" diff --git a/components/engine/vendor/github.com/docker/swarmkit/ca/config.go b/components/engine/vendor/github.com/docker/swarmkit/ca/config.go index cfaccd00b9..3731df1c49 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/ca/config.go +++ b/components/engine/vendor/github.com/docker/swarmkit/ca/config.go @@ -12,7 +12,6 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" cfconfig "github.com/cloudflare/cfssl/config" events "github.com/docker/go-events" "github.com/docker/swarmkit/api" @@ -22,6 +21,7 @@ import ( "github.com/docker/swarmkit/watch" "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "google.golang.org/grpc/credentials" "golang.org/x/net/context" diff --git a/components/engine/vendor/github.com/docker/swarmkit/ca/external.go b/components/engine/vendor/github.com/docker/swarmkit/ca/external.go index ef76e2fbe5..47c8dd8f80 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/ca/external.go +++ b/components/engine/vendor/github.com/docker/swarmkit/ca/external.go @@ -14,13 +14,13 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/cloudflare/cfssl/api" "github.com/cloudflare/cfssl/config" "github.com/cloudflare/cfssl/csr" "github.com/cloudflare/cfssl/signer" "github.com/docker/swarmkit/log" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "golang.org/x/net/context/ctxhttp" ) diff --git a/components/engine/vendor/github.com/docker/swarmkit/ca/renewer.go b/components/engine/vendor/github.com/docker/swarmkit/ca/renewer.go index 6d0229bd1d..2a2fae7808 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/ca/renewer.go +++ b/components/engine/vendor/github.com/docker/swarmkit/ca/renewer.go @@ -4,11 +4,11 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/go-events" "github.com/docker/swarmkit/connectionbroker" "github.com/docker/swarmkit/log" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/vendor/github.com/docker/swarmkit/ca/server.go b/components/engine/vendor/github.com/docker/swarmkit/ca/server.go index cb3b6acdc1..33a6367695 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/ca/server.go +++ b/components/engine/vendor/github.com/docker/swarmkit/ca/server.go @@ -7,7 +7,6 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/api/equality" "github.com/docker/swarmkit/identity" @@ -15,6 +14,7 @@ import ( "github.com/docker/swarmkit/manager/state/store" gogotypes "github.com/gogo/protobuf/types" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/components/engine/vendor/github.com/docker/swarmkit/log/context.go b/components/engine/vendor/github.com/docker/swarmkit/log/context.go index ce7da930ba..ac4f848806 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/log/context.go +++ b/components/engine/vendor/github.com/docker/swarmkit/log/context.go @@ -3,7 +3,7 @@ package log import ( "path" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/config.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/config.go index 3a6f6a19ee..d0fe8a56c1 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/config.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/config.go @@ -4,11 +4,11 @@ import ( "bytes" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/state/store" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go index 23708146d6..2f8d2559cf 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go @@ -4,12 +4,12 @@ import ( "crypto/subtle" "strings" - "github.com/Sirupsen/logrus" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/api/validation" "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/state/store" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/assignments.go b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/assignments.go index 3f17a6d20f..4373e99d59 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/assignments.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/assignments.go @@ -3,12 +3,12 @@ package dispatcher import ( "fmt" - "github.com/Sirupsen/logrus" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/api/equality" "github.com/docker/swarmkit/api/validation" "github.com/docker/swarmkit/manager/drivers" "github.com/docker/swarmkit/manager/state/store" + "github.com/sirupsen/logrus" ) // Used as a key in tasksUsingDependency and changes. Only using the diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go index 8ec3ae27df..cb630047ba 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go @@ -11,7 +11,6 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/transport" - "github.com/Sirupsen/logrus" "github.com/docker/go-events" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/api/equality" @@ -23,6 +22,7 @@ import ( "github.com/docker/swarmkit/watch" gogotypes "github.com/gogo/protobuf/types" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go b/components/engine/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go index f5ec2b30bd..7f064bb91c 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go @@ -9,7 +9,6 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" - "github.com/Sirupsen/logrus" "github.com/docker/go-events" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" @@ -17,6 +16,7 @@ import ( "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/watch" + "github.com/sirupsen/logrus" "golang.org/x/net/context" ) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go b/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go index 4771c9dcbe..0b64a9a69c 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go @@ -13,7 +13,6 @@ import ( "syscall" "time" - "github.com/Sirupsen/logrus" "github.com/cloudflare/cfssl/helpers" "github.com/docker/docker/pkg/plugingetter" "github.com/docker/go-events" @@ -45,6 +44,7 @@ import ( gogotypes "github.com/gogo/protobuf/types" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/credentials" diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go index b793374095..b37953829a 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go @@ -9,7 +9,6 @@ import ( "sync/atomic" "time" - "github.com/Sirupsen/logrus" "github.com/coreos/etcd/pkg/idutil" "github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft/raftpb" @@ -28,6 +27,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pivotal-golang/clock" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "golang.org/x/time/rate" "google.golang.org/grpc" diff --git a/components/engine/vendor/github.com/docker/swarmkit/node/node.go b/components/engine/vendor/github.com/docker/swarmkit/node/node.go index 77fe5b3d75..a6a747216b 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/node/node.go +++ b/components/engine/vendor/github.com/docker/swarmkit/node/node.go @@ -14,7 +14,6 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/boltdb/bolt" "github.com/docker/docker/pkg/plugingetter" metrics "github.com/docker/go-metrics" @@ -31,6 +30,7 @@ import ( "github.com/docker/swarmkit/xnet" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/components/engine/vendor/github.com/docker/swarmkit/watch/queue/queue.go b/components/engine/vendor/github.com/docker/swarmkit/watch/queue/queue.go index 10bdb92080..bb6f92da37 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/watch/queue/queue.go +++ b/components/engine/vendor/github.com/docker/swarmkit/watch/queue/queue.go @@ -5,8 +5,8 @@ import ( "fmt" "sync" - "github.com/Sirupsen/logrus" "github.com/docker/go-events" + "github.com/sirupsen/logrus" ) // ErrQueueFull is returned by a Write operation when that Write causes the diff --git a/components/engine/vendor/github.com/fsnotify/fsnotify/README.md b/components/engine/vendor/github.com/fsnotify/fsnotify/README.md index fee2a36ba4..3c891e349b 100644 --- a/components/engine/vendor/github.com/fsnotify/fsnotify/README.md +++ b/components/engine/vendor/github.com/fsnotify/fsnotify/README.md @@ -1,8 +1,12 @@ # File system notifications for Go -[![GoDoc](https://godoc.org/github.com/fsnotify/fsnotify?status.svg)](https://godoc.org/github.com/fsnotify/fsnotify) [![Go Report Card](https://goreportcard.com/badge/github.com/fsnotify/fsnotify)](https://goreportcard.com/report/github.com/fsnotify/fsnotify) [![Coverage](http://gocover.io/_badge/github.com/fsnotify/fsnotify)](http://gocover.io/github.com/fsnotify/fsnotify) +[![GoDoc](https://godoc.org/github.com/fsnotify/fsnotify?status.svg)](https://godoc.org/github.com/fsnotify/fsnotify) [![Go Report Card](https://goreportcard.com/badge/github.com/fsnotify/fsnotify)](https://goreportcard.com/report/github.com/fsnotify/fsnotify) -Go 1.3+ required. +fsnotify utilizes [golang.org/x/sys](https://godoc.org/golang.org/x/sys) rather than `syscall` from the standard library. Ensure you have the latest version installed by running: + +```console +go get -u golang.org/x/sys/... +``` Cross platform: Windows, Linux, BSD and OS X. @@ -27,7 +31,7 @@ fsnotify is a fork of [howeyc/fsnotify](https://godoc.org/github.com/howeyc/fsno All [releases](https://github.com/fsnotify/fsnotify/releases) are tagged based on [Semantic Versioning](http://semver.org/). Further API changes are [planned](https://github.com/fsnotify/fsnotify/milestones), and will be tagged with a new major revision number. -Go 1.6 supports dependencies located in the `vendor/` folder. Unless you are creating a library, it is recommended that you copy fsnotify into `vendor/github.com/fsnotify/fsnotify` within your project. +Go 1.6 supports dependencies located in the `vendor/` folder. Unless you are creating a library, it is recommended that you copy fsnotify into `vendor/github.com/fsnotify/fsnotify` within your project, and likewise for `golang.org/x/sys`. ## Contributing diff --git a/components/engine/vendor/github.com/fsnotify/fsnotify/fsnotify.go b/components/engine/vendor/github.com/fsnotify/fsnotify/fsnotify.go index d1d39a0ebd..e7f55fee7a 100644 --- a/components/engine/vendor/github.com/fsnotify/fsnotify/fsnotify.go +++ b/components/engine/vendor/github.com/fsnotify/fsnotify/fsnotify.go @@ -30,33 +30,33 @@ const ( Chmod ) -// String returns a string representation of the event in the form -// "file: REMOVE|WRITE|..." -func (e Event) String() string { +func (op Op) String() string { // Use a buffer for efficient string concatenation var buffer bytes.Buffer - if e.Op&Create == Create { + if op&Create == Create { buffer.WriteString("|CREATE") } - if e.Op&Remove == Remove { + if op&Remove == Remove { buffer.WriteString("|REMOVE") } - if e.Op&Write == Write { + if op&Write == Write { buffer.WriteString("|WRITE") } - if e.Op&Rename == Rename { + if op&Rename == Rename { buffer.WriteString("|RENAME") } - if e.Op&Chmod == Chmod { + if op&Chmod == Chmod { buffer.WriteString("|CHMOD") } - - // If buffer remains empty, return no event names if buffer.Len() == 0 { - return fmt.Sprintf("%q: ", e.Name) + return "" } - - // Return a list of event names, with leading pipe character stripped - return fmt.Sprintf("%q: %s", e.Name, buffer.String()[1:]) + return buffer.String()[1:] // Strip leading pipe +} + +// String returns a string representation of the event in the form +// "file: REMOVE|WRITE|..." +func (e Event) String() string { + return fmt.Sprintf("%q: %s", e.Name, e.Op.String()) } diff --git a/components/engine/vendor/github.com/fsnotify/fsnotify/inotify.go b/components/engine/vendor/github.com/fsnotify/fsnotify/inotify.go index 780b2a005d..f3b74c51f0 100644 --- a/components/engine/vendor/github.com/fsnotify/fsnotify/inotify.go +++ b/components/engine/vendor/github.com/fsnotify/fsnotify/inotify.go @@ -14,8 +14,9 @@ import ( "path/filepath" "strings" "sync" - "syscall" "unsafe" + + "golang.org/x/sys/unix" ) // Watcher watches a set of files, delivering events to a channel. @@ -35,14 +36,14 @@ type Watcher struct { // NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. func NewWatcher() (*Watcher, error) { // Create inotify fd - fd, errno := syscall.InotifyInit() + fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC) if fd == -1 { return nil, errno } // Create epoll poller, err := newFdPoller(fd) if err != nil { - syscall.Close(fd) + unix.Close(fd) return nil, err } w := &Watcher{ @@ -95,9 +96,9 @@ func (w *Watcher) Add(name string) error { return errors.New("inotify instance already closed") } - const agnosticEvents = syscall.IN_MOVED_TO | syscall.IN_MOVED_FROM | - syscall.IN_CREATE | syscall.IN_ATTRIB | syscall.IN_MODIFY | - syscall.IN_MOVE_SELF | syscall.IN_DELETE | syscall.IN_DELETE_SELF + const agnosticEvents = unix.IN_MOVED_TO | unix.IN_MOVED_FROM | + unix.IN_CREATE | unix.IN_ATTRIB | unix.IN_MODIFY | + unix.IN_MOVE_SELF | unix.IN_DELETE | unix.IN_DELETE_SELF var flags uint32 = agnosticEvents @@ -106,9 +107,9 @@ func (w *Watcher) Add(name string) error { w.mu.Unlock() if found { watchEntry.flags |= flags - flags |= syscall.IN_MASK_ADD + flags |= unix.IN_MASK_ADD } - wd, errno := syscall.InotifyAddWatch(w.fd, name, flags) + wd, errno := unix.InotifyAddWatch(w.fd, name, flags) if wd == -1 { return errno } @@ -140,7 +141,7 @@ func (w *Watcher) Remove(name string) error { // by calling inotify_rm_watch() below. e.g. readEvents() goroutine receives IN_IGNORE // so that EINVAL means that the wd is being rm_watch()ed or its file removed // by another thread and we have not received IN_IGNORE event. - success, errno := syscall.InotifyRmWatch(w.fd, watch.wd) + success, errno := unix.InotifyRmWatch(w.fd, watch.wd) if success == -1 { // TODO: Perhaps it's not helpful to return an error here in every case. // the only two possible errors are: @@ -170,16 +171,16 @@ type watch struct { // received events into Event objects and sends them via the Events channel func (w *Watcher) readEvents() { var ( - buf [syscall.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events - n int // Number of bytes read with read() - errno error // Syscall errno - ok bool // For poller.wait + buf [unix.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events + n int // Number of bytes read with read() + errno error // Syscall errno + ok bool // For poller.wait ) defer close(w.doneResp) defer close(w.Errors) defer close(w.Events) - defer syscall.Close(w.fd) + defer unix.Close(w.fd) defer w.poller.close() for { @@ -202,20 +203,20 @@ func (w *Watcher) readEvents() { continue } - n, errno = syscall.Read(w.fd, buf[:]) + n, errno = unix.Read(w.fd, buf[:]) // If a signal interrupted execution, see if we've been asked to close, and try again. // http://man7.org/linux/man-pages/man7/signal.7.html : // "Before Linux 3.8, reads from an inotify(7) file descriptor were not restartable" - if errno == syscall.EINTR { + if errno == unix.EINTR { continue } - // syscall.Read might have been woken up by Close. If so, we're done. + // unix.Read might have been woken up by Close. If so, we're done. if w.isClosed() { return } - if n < syscall.SizeofInotifyEvent { + if n < unix.SizeofInotifyEvent { var err error if n == 0 { // If EOF is received. This should really never happen. @@ -238,9 +239,9 @@ func (w *Watcher) readEvents() { var offset uint32 // We don't know how many events we just read into the buffer // While the offset points to at least one whole event... - for offset <= uint32(n-syscall.SizeofInotifyEvent) { + for offset <= uint32(n-unix.SizeofInotifyEvent) { // Point "raw" to the event in the buffer - raw := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[offset])) + raw := (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset])) mask := uint32(raw.Mask) nameLen := uint32(raw.Len) @@ -253,7 +254,7 @@ func (w *Watcher) readEvents() { w.mu.Unlock() if nameLen > 0 { // Point "bytes" at the first byte of the filename - bytes := (*[syscall.PathMax]byte)(unsafe.Pointer(&buf[offset+syscall.SizeofInotifyEvent])) + bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent])) // The filename is padded with NULL bytes. TrimRight() gets rid of those. name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") } @@ -270,7 +271,7 @@ func (w *Watcher) readEvents() { } // Move to the next event in the buffer - offset += syscall.SizeofInotifyEvent + nameLen + offset += unix.SizeofInotifyEvent + nameLen } } } @@ -280,7 +281,7 @@ func (w *Watcher) readEvents() { // against files that do not exist. func (e *Event) ignoreLinux(w *Watcher, wd int32, mask uint32) bool { // Ignore anything the inotify API says to ignore - if mask&syscall.IN_IGNORED == syscall.IN_IGNORED { + if mask&unix.IN_IGNORED == unix.IN_IGNORED { w.mu.Lock() defer w.mu.Unlock() name := w.paths[int(wd)] @@ -305,19 +306,19 @@ func (e *Event) ignoreLinux(w *Watcher, wd int32, mask uint32) bool { // newEvent returns an platform-independent Event based on an inotify mask. func newEvent(name string, mask uint32) Event { e := Event{Name: name} - if mask&syscall.IN_CREATE == syscall.IN_CREATE || mask&syscall.IN_MOVED_TO == syscall.IN_MOVED_TO { + if mask&unix.IN_CREATE == unix.IN_CREATE || mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { e.Op |= Create } - if mask&syscall.IN_DELETE_SELF == syscall.IN_DELETE_SELF || mask&syscall.IN_DELETE == syscall.IN_DELETE { + if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF || mask&unix.IN_DELETE == unix.IN_DELETE { e.Op |= Remove } - if mask&syscall.IN_MODIFY == syscall.IN_MODIFY { + if mask&unix.IN_MODIFY == unix.IN_MODIFY { e.Op |= Write } - if mask&syscall.IN_MOVE_SELF == syscall.IN_MOVE_SELF || mask&syscall.IN_MOVED_FROM == syscall.IN_MOVED_FROM { + if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF || mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { e.Op |= Rename } - if mask&syscall.IN_ATTRIB == syscall.IN_ATTRIB { + if mask&unix.IN_ATTRIB == unix.IN_ATTRIB { e.Op |= Chmod } return e diff --git a/components/engine/vendor/github.com/fsnotify/fsnotify/inotify_poller.go b/components/engine/vendor/github.com/fsnotify/fsnotify/inotify_poller.go index 23a5ca1460..cc7db4b22e 100644 --- a/components/engine/vendor/github.com/fsnotify/fsnotify/inotify_poller.go +++ b/components/engine/vendor/github.com/fsnotify/fsnotify/inotify_poller.go @@ -8,7 +8,8 @@ package fsnotify import ( "errors" - "syscall" + + "golang.org/x/sys/unix" ) type fdPoller struct { @@ -39,32 +40,32 @@ func newFdPoller(fd int) (*fdPoller, error) { poller.fd = fd // Create epoll fd - poller.epfd, errno = syscall.EpollCreate1(0) + poller.epfd, errno = unix.EpollCreate1(0) if poller.epfd == -1 { return nil, errno } // Create pipe; pipe[0] is the read end, pipe[1] the write end. - errno = syscall.Pipe2(poller.pipe[:], syscall.O_NONBLOCK) + errno = unix.Pipe2(poller.pipe[:], unix.O_NONBLOCK) if errno != nil { return nil, errno } // Register inotify fd with epoll - event := syscall.EpollEvent{ + event := unix.EpollEvent{ Fd: int32(poller.fd), - Events: syscall.EPOLLIN, + Events: unix.EPOLLIN, } - errno = syscall.EpollCtl(poller.epfd, syscall.EPOLL_CTL_ADD, poller.fd, &event) + errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.fd, &event) if errno != nil { return nil, errno } // Register pipe fd with epoll - event = syscall.EpollEvent{ + event = unix.EpollEvent{ Fd: int32(poller.pipe[0]), - Events: syscall.EPOLLIN, + Events: unix.EPOLLIN, } - errno = syscall.EpollCtl(poller.epfd, syscall.EPOLL_CTL_ADD, poller.pipe[0], &event) + errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.pipe[0], &event) if errno != nil { return nil, errno } @@ -80,11 +81,11 @@ func (poller *fdPoller) wait() (bool, error) { // I don't know whether epoll_wait returns the number of events returned, // or the total number of events ready. // I decided to catch both by making the buffer one larger than the maximum. - events := make([]syscall.EpollEvent, 7) + events := make([]unix.EpollEvent, 7) for { - n, errno := syscall.EpollWait(poller.epfd, events, -1) + n, errno := unix.EpollWait(poller.epfd, events, -1) if n == -1 { - if errno == syscall.EINTR { + if errno == unix.EINTR { continue } return false, errno @@ -103,31 +104,31 @@ func (poller *fdPoller) wait() (bool, error) { epollin := false for _, event := range ready { if event.Fd == int32(poller.fd) { - if event.Events&syscall.EPOLLHUP != 0 { + if event.Events&unix.EPOLLHUP != 0 { // This should not happen, but if it does, treat it as a wakeup. epollhup = true } - if event.Events&syscall.EPOLLERR != 0 { + if event.Events&unix.EPOLLERR != 0 { // If an error is waiting on the file descriptor, we should pretend - // something is ready to read, and let syscall.Read pick up the error. + // something is ready to read, and let unix.Read pick up the error. epollerr = true } - if event.Events&syscall.EPOLLIN != 0 { + if event.Events&unix.EPOLLIN != 0 { // There is data to read. epollin = true } } if event.Fd == int32(poller.pipe[0]) { - if event.Events&syscall.EPOLLHUP != 0 { + if event.Events&unix.EPOLLHUP != 0 { // Write pipe descriptor was closed, by us. This means we're closing down the // watcher, and we should wake up. } - if event.Events&syscall.EPOLLERR != 0 { + if event.Events&unix.EPOLLERR != 0 { // If an error is waiting on the pipe file descriptor. // This is an absolute mystery, and should never ever happen. return false, errors.New("Error on the pipe descriptor.") } - if event.Events&syscall.EPOLLIN != 0 { + if event.Events&unix.EPOLLIN != 0 { // This is a regular wakeup, so we have to clear the buffer. err := poller.clearWake() if err != nil { @@ -147,9 +148,9 @@ func (poller *fdPoller) wait() (bool, error) { // Close the write end of the poller. func (poller *fdPoller) wake() error { buf := make([]byte, 1) - n, errno := syscall.Write(poller.pipe[1], buf) + n, errno := unix.Write(poller.pipe[1], buf) if n == -1 { - if errno == syscall.EAGAIN { + if errno == unix.EAGAIN { // Buffer is full, poller will wake. return nil } @@ -161,9 +162,9 @@ func (poller *fdPoller) wake() error { func (poller *fdPoller) clearWake() error { // You have to be woken up a LOT in order to get to 100! buf := make([]byte, 100) - n, errno := syscall.Read(poller.pipe[0], buf) + n, errno := unix.Read(poller.pipe[0], buf) if n == -1 { - if errno == syscall.EAGAIN { + if errno == unix.EAGAIN { // Buffer is empty, someone else cleared our wake. return nil } @@ -175,12 +176,12 @@ func (poller *fdPoller) clearWake() error { // Close all poller file descriptors, but not the one passed to it. func (poller *fdPoller) close() { if poller.pipe[1] != -1 { - syscall.Close(poller.pipe[1]) + unix.Close(poller.pipe[1]) } if poller.pipe[0] != -1 { - syscall.Close(poller.pipe[0]) + unix.Close(poller.pipe[0]) } if poller.epfd != -1 { - syscall.Close(poller.epfd) + unix.Close(poller.epfd) } } diff --git a/components/engine/vendor/github.com/fsnotify/fsnotify/kqueue.go b/components/engine/vendor/github.com/fsnotify/fsnotify/kqueue.go index b8ea30846d..c2b4acb18d 100644 --- a/components/engine/vendor/github.com/fsnotify/fsnotify/kqueue.go +++ b/components/engine/vendor/github.com/fsnotify/fsnotify/kqueue.go @@ -13,8 +13,9 @@ import ( "os" "path/filepath" "sync" - "syscall" "time" + + "golang.org/x/sys/unix" ) // Watcher watches a set of files, delivering events to a channel. @@ -113,12 +114,12 @@ func (w *Watcher) Remove(name string) error { return fmt.Errorf("can't remove non-existent kevent watch for: %s", name) } - const registerRemove = syscall.EV_DELETE + const registerRemove = unix.EV_DELETE if err := register(w.kq, []int{watchfd}, registerRemove, 0); err != nil { return err } - syscall.Close(watchfd) + unix.Close(watchfd) w.mu.Lock() isDir := w.paths[watchfd].isDir @@ -152,7 +153,7 @@ func (w *Watcher) Remove(name string) error { } // Watch all events (except NOTE_EXTEND, NOTE_LINK, NOTE_REVOKE) -const noteAllEvents = syscall.NOTE_DELETE | syscall.NOTE_WRITE | syscall.NOTE_ATTRIB | syscall.NOTE_RENAME +const noteAllEvents = unix.NOTE_DELETE | unix.NOTE_WRITE | unix.NOTE_ATTRIB | unix.NOTE_RENAME // keventWaitTime to block on each read from kevent var keventWaitTime = durationToTimespec(100 * time.Millisecond) @@ -219,7 +220,7 @@ func (w *Watcher) addWatch(name string, flags uint32) (string, error) { } } - watchfd, err = syscall.Open(name, openMode, 0700) + watchfd, err = unix.Open(name, openMode, 0700) if watchfd == -1 { return "", err } @@ -227,9 +228,9 @@ func (w *Watcher) addWatch(name string, flags uint32) (string, error) { isDir = fi.IsDir() } - const registerAdd = syscall.EV_ADD | syscall.EV_CLEAR | syscall.EV_ENABLE + const registerAdd = unix.EV_ADD | unix.EV_CLEAR | unix.EV_ENABLE if err := register(w.kq, []int{watchfd}, registerAdd, flags); err != nil { - syscall.Close(watchfd) + unix.Close(watchfd) return "", err } @@ -245,8 +246,8 @@ func (w *Watcher) addWatch(name string, flags uint32) (string, error) { // or if it was watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles) w.mu.Lock() - watchDir := (flags&syscall.NOTE_WRITE) == syscall.NOTE_WRITE && - (!alreadyWatching || (w.dirFlags[name]&syscall.NOTE_WRITE) != syscall.NOTE_WRITE) + watchDir := (flags&unix.NOTE_WRITE) == unix.NOTE_WRITE && + (!alreadyWatching || (w.dirFlags[name]&unix.NOTE_WRITE) != unix.NOTE_WRITE) // Store flags so this watch can be updated later w.dirFlags[name] = flags w.mu.Unlock() @@ -263,13 +264,13 @@ func (w *Watcher) addWatch(name string, flags uint32) (string, error) { // readEvents reads from kqueue and converts the received kevents into // Event values that it sends down the Events channel. func (w *Watcher) readEvents() { - eventBuffer := make([]syscall.Kevent_t, 10) + eventBuffer := make([]unix.Kevent_t, 10) for { // See if there is a message on the "done" channel select { case <-w.done: - err := syscall.Close(w.kq) + err := unix.Close(w.kq) if err != nil { w.Errors <- err } @@ -282,7 +283,7 @@ func (w *Watcher) readEvents() { // Get new events kevents, err := read(w.kq, eventBuffer, &keventWaitTime) // EINTR is okay, the syscall was interrupted before timeout expired. - if err != nil && err != syscall.EINTR { + if err != nil && err != unix.EINTR { w.Errors <- err continue } @@ -356,16 +357,16 @@ func (w *Watcher) readEvents() { // newEvent returns an platform-independent Event based on kqueue Fflags. func newEvent(name string, mask uint32) Event { e := Event{Name: name} - if mask&syscall.NOTE_DELETE == syscall.NOTE_DELETE { + if mask&unix.NOTE_DELETE == unix.NOTE_DELETE { e.Op |= Remove } - if mask&syscall.NOTE_WRITE == syscall.NOTE_WRITE { + if mask&unix.NOTE_WRITE == unix.NOTE_WRITE { e.Op |= Write } - if mask&syscall.NOTE_RENAME == syscall.NOTE_RENAME { + if mask&unix.NOTE_RENAME == unix.NOTE_RENAME { e.Op |= Rename } - if mask&syscall.NOTE_ATTRIB == syscall.NOTE_ATTRIB { + if mask&unix.NOTE_ATTRIB == unix.NOTE_ATTRIB { e.Op |= Chmod } return e @@ -451,7 +452,7 @@ func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) (string, erro flags := w.dirFlags[name] w.mu.Unlock() - flags |= syscall.NOTE_DELETE | syscall.NOTE_RENAME + flags |= unix.NOTE_DELETE | unix.NOTE_RENAME return w.addWatch(name, flags) } @@ -461,7 +462,7 @@ func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) (string, erro // kqueue creates a new kernel event queue and returns a descriptor. func kqueue() (kq int, err error) { - kq, err = syscall.Kqueue() + kq, err = unix.Kqueue() if kq == -1 { return kq, err } @@ -470,16 +471,16 @@ func kqueue() (kq int, err error) { // register events with the queue func register(kq int, fds []int, flags int, fflags uint32) error { - changes := make([]syscall.Kevent_t, len(fds)) + changes := make([]unix.Kevent_t, len(fds)) for i, fd := range fds { // SetKevent converts int to the platform-specific types: - syscall.SetKevent(&changes[i], fd, syscall.EVFILT_VNODE, flags) + unix.SetKevent(&changes[i], fd, unix.EVFILT_VNODE, flags) changes[i].Fflags = fflags } // register the events - success, err := syscall.Kevent(kq, changes, nil, nil) + success, err := unix.Kevent(kq, changes, nil, nil) if success == -1 { return err } @@ -488,8 +489,8 @@ func register(kq int, fds []int, flags int, fflags uint32) error { // read retrieves pending events, or waits until an event occurs. // A timeout of nil blocks indefinitely, while 0 polls the queue. -func read(kq int, events []syscall.Kevent_t, timeout *syscall.Timespec) ([]syscall.Kevent_t, error) { - n, err := syscall.Kevent(kq, nil, events, timeout) +func read(kq int, events []unix.Kevent_t, timeout *unix.Timespec) ([]unix.Kevent_t, error) { + n, err := unix.Kevent(kq, nil, events, timeout) if err != nil { return nil, err } @@ -497,6 +498,6 @@ func read(kq int, events []syscall.Kevent_t, timeout *syscall.Timespec) ([]sysca } // durationToTimespec prepares a timeout value -func durationToTimespec(d time.Duration) syscall.Timespec { - return syscall.NsecToTimespec(d.Nanoseconds()) +func durationToTimespec(d time.Duration) unix.Timespec { + return unix.NsecToTimespec(d.Nanoseconds()) } diff --git a/components/engine/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go b/components/engine/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go index c57ccb427b..7d8de14513 100644 --- a/components/engine/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go +++ b/components/engine/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go @@ -6,6 +6,6 @@ package fsnotify -import "syscall" +import "golang.org/x/sys/unix" -const openMode = syscall.O_NONBLOCK | syscall.O_RDONLY +const openMode = unix.O_NONBLOCK | unix.O_RDONLY diff --git a/components/engine/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go b/components/engine/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go index 174b2c331f..9139e17161 100644 --- a/components/engine/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go +++ b/components/engine/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go @@ -6,7 +6,7 @@ package fsnotify -import "syscall" +import "golang.org/x/sys/unix" // note: this constant is not defined on BSD -const openMode = syscall.O_EVTONLY +const openMode = unix.O_EVTONLY diff --git a/components/engine/vendor/github.com/fsnotify/fsnotify/windows.go b/components/engine/vendor/github.com/fsnotify/fsnotify/windows.go index c836bdb3db..09436f31d8 100644 --- a/components/engine/vendor/github.com/fsnotify/fsnotify/windows.go +++ b/components/engine/vendor/github.com/fsnotify/fsnotify/windows.go @@ -306,7 +306,7 @@ func (w *Watcher) remWatch(pathname string) error { watch.mask = 0 } else { name := filepath.Base(pathname) - w.sendEvent(watch.path+"\\"+name, watch.names[name]&sysFSIGNORED) + w.sendEvent(filepath.Join(watch.path, name), watch.names[name]&sysFSIGNORED) delete(watch.names, name) } return w.startRead(watch) @@ -316,7 +316,7 @@ func (w *Watcher) remWatch(pathname string) error { func (w *Watcher) deleteWatch(watch *watch) { for name, mask := range watch.names { if mask&provisional == 0 { - w.sendEvent(watch.path+"\\"+name, mask&sysFSIGNORED) + w.sendEvent(filepath.Join(watch.path, name), mask&sysFSIGNORED) } delete(watch.names, name) } @@ -453,7 +453,7 @@ func (w *Watcher) readEvents() { raw := (*syscall.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) buf := (*[syscall.MAX_PATH]uint16)(unsafe.Pointer(&raw.FileName)) name := syscall.UTF16ToString(buf[:raw.FileNameLength/2]) - fullname := watch.path + "\\" + name + fullname := filepath.Join(watch.path, name) var mask uint64 switch raw.Action { @@ -491,7 +491,7 @@ func (w *Watcher) readEvents() { } } if raw.Action == syscall.FILE_ACTION_RENAMED_NEW_NAME { - fullname = watch.path + "\\" + watch.rename + fullname = filepath.Join(watch.path, watch.rename) sendNameEvent() } diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/LICENSE b/components/engine/vendor/github.com/jhowardmsft/opengcs/LICENSE deleted file mode 100644 index 8739a025ea..0000000000 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 Microsoft - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/config.go b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/config.go deleted file mode 100644 index fedb563abe..0000000000 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/config.go +++ /dev/null @@ -1,245 +0,0 @@ -// +build windows - -package client - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/Microsoft/hcsshim" - "github.com/Sirupsen/logrus" -) - -// Mode is the operational mode, both requested, and actual after verification -type Mode uint - -const ( - // Constants for the actual mode after validation - - // ModeActualError means an error has occurred during validation - ModeActualError = iota - // ModeActualVhdx means that we are going to use VHDX boot after validation - ModeActualVhdx - // ModeActualKernelInitrd means that we are going to use kernel+initrd for boot after validation - ModeActualKernelInitrd - - // Constants for the requested mode - - // ModeRequestAuto means auto-select the boot mode for a utility VM - ModeRequestAuto = iota // VHDX will be priority over kernel+initrd - // ModeRequestVhdx means request VHDX boot if possible - ModeRequestVhdx - // ModeRequestKernelInitrd means request Kernel+initrd boot if possible - ModeRequestKernelInitrd - - // defaultUvmTimeoutSeconds is the default time to wait for utility VM operations - defaultUvmTimeoutSeconds = 5 * 60 - - // DefaultSandboxSizeMB is the size of the default sandbox size in MB - DefaultSandboxSizeMB = 20 * 1024 * 1024 -) - -// Config is the structure used to configuring a utility VM to be used -// as a service VM. There are two ways of starting. Either supply a VHD, -// or a Kernel+Initrd. For the latter, both must be supplied, and both -// must be in the same directory. -// -// VHD is the priority. -type Config struct { - KirdPath string // Path to where kernel/initrd are found (defaults to c:\program files\Linux Containers) - KernelFile string // Kernel for Utility VM (embedded in a UEFI bootloader) - does NOT include full path, just filename - InitrdFile string // Initrd image for Utility VM - does NOT include full path, just filename - Vhdx string // VHD for booting the utility VM - is a full path - Name string // Name of the utility VM - RequestedMode Mode // What mode is preferred when validating - ActualMode Mode // What mode was obtained during validation - UvmTimeoutSeconds int // How long to wait for the utility VM to respond in seconds - Uvm hcsshim.Container // The actual container - MappedVirtualDisks []hcsshim.MappedVirtualDisk // Data-disks to be attached -} - -// GenerateDefault generates a default config from a set of options -// If baseDir is not supplied, defaults to $env:ProgramFiles\Linux Containers -func (config *Config) GenerateDefault(options []string) error { - if config.UvmTimeoutSeconds < 0 { - return fmt.Errorf("opengcs: cannot generate a config when supplied a negative utility VM timeout") - } - - envTimeoutSeconds := 0 - optTimeoutSeconds := 0 - - if config.UvmTimeoutSeconds != 0 { - envTimeout := os.Getenv("OPENGCS_UVM_TIMEOUT_SECONDS") - if len(envTimeout) > 0 { - var err error - if envTimeoutSeconds, err = strconv.Atoi(envTimeout); err != nil { - return fmt.Errorf("opengcs: OPENGCS_UVM_TIMEOUT_SECONDS could not be interpreted as an integer") - } - if envTimeoutSeconds < 0 { - return fmt.Errorf("opengcs: OPENGCS_UVM_TIMEOUT_SECONDS cannot be negative") - } - } - } - - for _, v := range options { - opt := strings.SplitN(v, "=", 2) - if len(opt) == 2 { - switch strings.ToLower(opt[0]) { - case "opengcskirdpath": - config.KirdPath = opt[1] - case "opengcskernel": - config.KernelFile = opt[1] - case "opengcsinitrd": - config.InitrdFile = opt[1] - case "opengcsvhdx": - config.Vhdx = opt[1] - case "opengcstimeoutsecs": - var err error - if optTimeoutSeconds, err = strconv.Atoi(opt[1]); err != nil { - return fmt.Errorf("opengcs: opengcstimeoutsecs option could not be interpreted as an integer") - } - if optTimeoutSeconds < 0 { - return fmt.Errorf("opengcs: opengcstimeoutsecs option cannot be negative") - } - } - } - } - - if config.KirdPath == "" { - config.KirdPath = filepath.Join(os.Getenv("ProgramFiles"), "Linux Containers") - } - - if config.Vhdx == "" { - config.Vhdx = filepath.Join(config.KirdPath, `uvm.vhdx`) - } - if config.KernelFile == "" { - config.KernelFile = `bootx64.efi` - } - if config.InitrdFile == "" { - config.InitrdFile = `initrd.img` - } - - // Which timeout are we going to take? If not through option or environment, - // then use the default constant, otherwise the maximum of the option or - // environment supplied setting. A requested on in the config supplied - // overrides all of this. - if config.UvmTimeoutSeconds == 0 { - config.UvmTimeoutSeconds = defaultUvmTimeoutSeconds - if optTimeoutSeconds != 0 || envTimeoutSeconds != 0 { - config.UvmTimeoutSeconds = optTimeoutSeconds - if envTimeoutSeconds > optTimeoutSeconds { - config.UvmTimeoutSeconds = envTimeoutSeconds - } - } - } - - config.MappedVirtualDisks = nil - - return nil -} - -// validate validates a Config structure for starting a utility VM. -func (config *Config) validate() error { - config.ActualMode = ModeActualError - - if config.RequestedMode == ModeRequestVhdx && config.Vhdx == "" { - return fmt.Errorf("opengcs: config is invalid - request for VHDX mode did not supply a VHDX") - } - if config.RequestedMode == ModeRequestKernelInitrd && (config.KernelFile == "" || config.InitrdFile == "") { - return fmt.Errorf("opengcs: config is invalid - request for Kernel+Initrd mode must supply both kernel and initrd") - } - - // Validate that if VHDX requested or auto, it exists. - if config.RequestedMode == ModeRequestAuto || config.RequestedMode == ModeRequestVhdx { - if _, err := os.Stat(config.Vhdx); os.IsNotExist(err) { - if config.RequestedMode == ModeRequestVhdx { - return fmt.Errorf("opengcs: mode requested was VHDX but '%s' could not be found", config.Vhdx) - } - } else { - config.ActualMode = ModeActualVhdx - return nil - } - } - - // So must be kernel+initrd, or auto where we fallback as the VHDX doesn't exist - if config.InitrdFile == "" || config.KernelFile == "" { - if config.RequestedMode == ModeRequestKernelInitrd { - return fmt.Errorf("opengcs: both initrd and kernel options for utility VM boot must be supplied") - } - return fmt.Errorf("opengcs: configuration is invalid") - } - - if _, err := os.Stat(filepath.Join(config.KirdPath, config.KernelFile)); os.IsNotExist(err) { - return fmt.Errorf("opengcs: kernel '%s' was not found", filepath.Join(config.KirdPath, config.KernelFile)) - } - if _, err := os.Stat(filepath.Join(config.KirdPath, config.InitrdFile)); os.IsNotExist(err) { - return fmt.Errorf("opengcs: initrd '%s' was not found", filepath.Join(config.KirdPath, config.InitrdFile)) - } - - config.ActualMode = ModeActualKernelInitrd - - // Ensure all the MappedVirtualDisks exist on the host - for _, mvd := range config.MappedVirtualDisks { - if _, err := os.Stat(mvd.HostPath); err != nil { - return fmt.Errorf("opengcs: MappedVirtualDisk '%s' was not found", mvd.HostPath) - } - if mvd.ContainerPath == "" { - return fmt.Errorf("opengcs: MappedVirtualDisk '%s' has no container path", mvd.HostPath) - } - } - - return nil -} - -// Create creates a utility VM from a configuration. -func (config *Config) Create() error { - logrus.Debugf("opengcs Create: %+v", config) - - if err := config.validate(); err != nil { - return err - } - - configuration := &hcsshim.ContainerConfig{ - HvPartition: true, - Name: config.Name, - SystemType: "container", - ContainerType: "linux", - TerminateOnLastHandleClosed: true, - MappedVirtualDisks: config.MappedVirtualDisks, - } - - if config.ActualMode == ModeActualVhdx { - configuration.HvRuntime = &hcsshim.HvRuntime{ - ImagePath: config.Vhdx, - } - } else { - configuration.HvRuntime = &hcsshim.HvRuntime{ - ImagePath: config.KirdPath, - LinuxInitrdFile: config.InitrdFile, - LinuxKernelFile: config.KernelFile, - } - } - - configurationS, _ := json.Marshal(configuration) - logrus.Debugf("opengcs Create: calling HCS with '%s'", string(configurationS)) - uvm, err := hcsshim.CreateContainer(config.Name, configuration) - if err != nil { - return err - } - logrus.Debugf("opengcs Create: uvm created, starting...") - err = uvm.Start() - if err != nil { - logrus.Debugf("opengcs Create: uvm failed to start: %s", err) - // Make sure we don't leave it laying around as it's been created in HCS - uvm.Terminate() - return err - } - - config.Uvm = uvm - logrus.Debugf("opengcs Create: uvm %s is running", config.Name) - return nil -} diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/createsandbox.go b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/createsandbox.go deleted file mode 100644 index 65971601a1..0000000000 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/createsandbox.go +++ /dev/null @@ -1,67 +0,0 @@ -// +build windows - -package client - -import ( - "fmt" - "os" - - "github.com/Sirupsen/logrus" -) - -// CreateSandbox does what it says on the tin. This is done by copying a prebuilt-sandbox from the ServiceVM. -// It is the responsibility of the caller to synchronise simultaneous attempts to create the cache file. -// TODO: @jhowardmsft maxSizeInMB isn't hooked up in GCS. Needs a platform change which is in flight. -func (config *Config) CreateSandbox(destFile string, maxSizeInMB uint32, cacheFile string) error { - // Smallest we can accept is the default sandbox size as we can't size down, only expand. - if maxSizeInMB < DefaultSandboxSizeMB { - maxSizeInMB = DefaultSandboxSizeMB - } - - logrus.Debugf("opengcs: CreateSandbox: %s size:%dMB cache:%s", destFile, maxSizeInMB, cacheFile) - - // Retrieve from cache if the default size and already on disk - if cacheFile != "" && maxSizeInMB == DefaultSandboxSizeMB { - if _, err := os.Stat(cacheFile); err == nil { - if err := CopyFile(cacheFile, destFile, false); err != nil { - return fmt.Errorf("opengcs: CreateSandbox: Failed to copy cached sandbox '%s' to '%s': %s", cacheFile, destFile, err) - } - logrus.Debugf("opengcs: CreateSandbox: %s fulfilled from cache", destFile) - return nil - } - } - - if config.Uvm == nil { - return fmt.Errorf("opengcs: CreateSandbox: No utility VM has been created") - } - - // TODO @jhowardmsft - needs a platform change so that can specify size. eg fmt.Sprintf("createSandbox -size %d", maxSizeInMB)) - process, err := config.createUtilsProcess("createSandbox") - if err != nil { - return fmt.Errorf("opengcs: CreateSandbox: %s: failed to create utils process: %s", destFile, err) - } - - defer func() { - process.Process.Close() - }() - - logrus.Debugf("opengcs: CreateSandbox: %s: writing from stdout", destFile) - // Get back the sandbox VHDx stream from the service VM and write it to file - resultSize, err := writeFileFromReader(destFile, process.Stdout, config.UvmTimeoutSeconds, fmt.Sprintf("createSandbox %s", destFile)) - if err != nil { - return fmt.Errorf("opengcs: CreateSandbox: %s: failed writing %d bytes to target file: %s", destFile, resultSize, err) - } - - // Populate the cache - if cacheFile != "" && maxSizeInMB == DefaultSandboxSizeMB { - // It may already exist due to being created on another thread, in which case no copy back needed. - if _, err := os.Stat(cacheFile); os.IsNotExist(err) { - if err := CopyFile(destFile, cacheFile, false); err != nil { - return fmt.Errorf("opengcs: CreateSandbox: Failed to seed sandbox cache '%s' from '%s': %s", destFile, cacheFile, err) - } - } - } - - logrus.Debugf("opengcs: CreateSandbox: %s created (non-cache)", destFile) - return nil -} diff --git a/components/engine/vendor/github.com/moby/buildkit/LICENSE b/components/engine/vendor/github.com/moby/buildkit/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/components/engine/vendor/github.com/moby/buildkit/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/components/engine/vendor/github.com/moby/buildkit/README.md b/components/engine/vendor/github.com/moby/buildkit/README.md new file mode 100644 index 0000000000..ddcbb01ceb --- /dev/null +++ b/components/engine/vendor/github.com/moby/buildkit/README.md @@ -0,0 +1,87 @@ +### Important: This repository is in an early development phase and not suitable for practical workloads. It does not compare with `docker build` features yet. + +[![asciicinema example](https://asciinema.org/a/gPEIEo1NzmDTUu2bEPsUboqmU.png)](https://asciinema.org/a/gPEIEo1NzmDTUu2bEPsUboqmU) + + +## BuildKit + +BuildKit is a toolkit for converting source code to build artifacts in an efficient, expressive and repeatable manner. + +Key features: +- Automatic garbage collection +- Extendable frontend formats +- Concurrent dependency resolution +- Efficient instruction caching +- Build cache import/export +- Nested build job invocations +- Distributable workers +- Multiple output formats +- Pluggable architecture + + +Read the proposal from https://github.com/moby/moby/issues/32925 + +#### Quick start + +BuildKit daemon can be built in two different versions: one that uses [containerd](https://github.com/containerd/containerd) for execution and distribution, and a standalone version that doesn't have other dependencies apart from [runc](https://github.com/opencontainers/runc). We are open for adding more backends. `buildd` is a CLI utility for running the gRPC API. + +```bash +# buildd daemon (choose one) +go build -o buildd-containerd -tags containerd ./cmd/buildd +go build -o buildd-standalone -tags standalone ./cmd/buildd + +# buildctl utility +go build -o buildctl ./cmd/buildctl +``` + +You can also use `make binaries` that prepares all binaries into the `bin/` directory. + +The first thing to test could be to try building BuildKit with BuildKit. BuildKit provides a low-level solver format that could be used by multiple build definitions. Preparation work for making the Dockerfile parser reusable as a frontend is tracked in https://github.com/moby/moby/pull/33492. As no frontends have been integrated yet we currently have to use a client library to generate this low-level definition. + +`examples/buildkit*` directory contains scripts that define how to build different configurations of BuildKit and its dependencies using the `client` package. Running one of these script generates a protobuf definition of a build graph. Note that the script itself does not execute any steps of the build. + +You can use `buildctl debug dump-llb` to see what data is this definition. + +```bash +go run examples/buildkit0/buildkit.go | buildctl debug dump-llb | jq . +``` + +To start building use `buildctl build` command. The script accepts `--target` flag to choose between `containerd` and `standalone` configurations. In standalone mode BuildKit binaries are built together with `runc`. In containerd mode, the `containerd` binary is built as well from the upstream repo. + +```bash +go run examples/buildkit0/buildkit.go | buildctl build +``` + +`buildctl build` will show interactive progress bar by default while the build job is running. It will also show you the path to the trace file that contains all information about the timing of the individual steps and logs. + +Different versions of the example scripts show different ways of describing the build definition for this project to show the capabilities of the library. New versions have been added when new features have become available. + +- `./examples/buildkit0` - uses only exec operations, defines a full stage per component. +- `./examples/buildkit1` - cloning git repositories has been separated for extra concurrency. +- `./examples/buildkit2` - uses git sources directly instead of running `git clone`, allowing better performance and much safer caching. + +#### Supported runc version + +During development buildkit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/3707703a694187c7d08e2f333da6ddd58bcb729d/RUNC.md) for more information. + + +#### Contributing + +Running tests: + +```bash +make test +``` + +Updating vendored dependencies: + +```bash +# update vendor.conf +make vendor +``` + +Validating your updates before submission: + +```bash +make validate-all +``` diff --git a/components/engine/client/session/filesync/diffcopy.go b/components/engine/vendor/github.com/moby/buildkit/session/filesync/diffcopy.go similarity index 95% rename from components/engine/client/session/filesync/diffcopy.go rename to components/engine/vendor/github.com/moby/buildkit/session/filesync/diffcopy.go index 533847acdf..58b29686cc 100644 --- a/components/engine/client/session/filesync/diffcopy.go +++ b/components/engine/vendor/github.com/moby/buildkit/session/filesync/diffcopy.go @@ -5,7 +5,7 @@ import ( "google.golang.org/grpc" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/tonistiigi/fsutil" ) diff --git a/components/engine/client/session/filesync/filesync.go b/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.go similarity index 98% rename from components/engine/client/session/filesync/filesync.go rename to components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.go index 9a2ffc8578..fe4d00a729 100644 --- a/components/engine/client/session/filesync/filesync.go +++ b/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.go @@ -4,7 +4,7 @@ import ( "os" "strings" - "github.com/docker/docker/client/session" + "github.com/moby/buildkit/session" "github.com/pkg/errors" "github.com/tonistiigi/fsutil" "golang.org/x/net/context" diff --git a/components/engine/client/session/filesync/filesync.pb.go b/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.pb.go similarity index 100% rename from components/engine/client/session/filesync/filesync.pb.go rename to components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.pb.go diff --git a/components/engine/client/session/filesync/filesync.proto b/components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.proto similarity index 100% rename from components/engine/client/session/filesync/filesync.proto rename to components/engine/vendor/github.com/moby/buildkit/session/filesync/filesync.proto diff --git a/components/engine/client/session/filesync/generate.go b/components/engine/vendor/github.com/moby/buildkit/session/filesync/generate.go similarity index 100% rename from components/engine/client/session/filesync/generate.go rename to components/engine/vendor/github.com/moby/buildkit/session/filesync/generate.go diff --git a/components/engine/client/session/filesync/tarstream.go b/components/engine/vendor/github.com/moby/buildkit/session/filesync/tarstream.go similarity index 98% rename from components/engine/client/session/filesync/tarstream.go rename to components/engine/vendor/github.com/moby/buildkit/session/filesync/tarstream.go index da139ebf5d..5cab867498 100644 --- a/components/engine/client/session/filesync/tarstream.go +++ b/components/engine/vendor/github.com/moby/buildkit/session/filesync/tarstream.go @@ -3,10 +3,10 @@ package filesync import ( "io" - "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "google.golang.org/grpc" ) diff --git a/components/engine/client/session/grpc.go b/components/engine/vendor/github.com/moby/buildkit/session/grpc.go similarity index 97% rename from components/engine/client/session/grpc.go rename to components/engine/vendor/github.com/moby/buildkit/session/grpc.go index 0f20b15047..71d77bf448 100644 --- a/components/engine/client/session/grpc.go +++ b/components/engine/vendor/github.com/moby/buildkit/session/grpc.go @@ -4,8 +4,8 @@ import ( "net" "time" - "github.com/Sirupsen/logrus" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" "golang.org/x/net/http2" "google.golang.org/grpc" diff --git a/components/engine/client/session/manager.go b/components/engine/vendor/github.com/moby/buildkit/session/manager.go similarity index 100% rename from components/engine/client/session/manager.go rename to components/engine/vendor/github.com/moby/buildkit/session/manager.go diff --git a/components/engine/client/session/session.go b/components/engine/vendor/github.com/moby/buildkit/session/session.go similarity index 100% rename from components/engine/client/session/session.go rename to components/engine/vendor/github.com/moby/buildkit/session/session.go diff --git a/components/engine/vendor/github.com/moby/buildkit/vendor.conf b/components/engine/vendor/github.com/moby/buildkit/vendor.conf new file mode 100644 index 0000000000..b13cfa9675 --- /dev/null +++ b/components/engine/vendor/github.com/moby/buildkit/vendor.conf @@ -0,0 +1,40 @@ +github.com/boltdb/bolt e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd +github.com/pkg/errors c605e284fe17294bda444b34710735b29d1a9d90 + +github.com/stretchr/testify v1.1.4 +github.com/davecgh/go-spew v1.1.0 +github.com/pmezard/go-difflib v1.0.0 +golang.org/x/sys 739734461d1c916b6c72a63d7efda2b27edb369f + +github.com/containerd/containerd 3707703a694187c7d08e2f333da6ddd58bcb729d +golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c +github.com/Sirupsen/logrus v0.11.0 +google.golang.org/grpc v1.3.0 +github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448 +golang.org/x/net 1f9224279e98554b6a6432d4dd998a739f8b2b7c +github.com/gogo/protobuf d2e1ade2d719b78fe5b061b4c18a9f7111b5bdc8 +github.com/golang/protobuf 5a0f697c9ed9d68fef0116532c6e05cfeae00e55 +github.com/containerd/continuity 86cec1535a968310e7532819f699ff2830ed7463 +github.com/opencontainers/image-spec v1.0.0-rc6 +github.com/opencontainers/runc 429a5387123625040bacfbb60d96b1cbd02293ab +github.com/Microsoft/go-winio v0.4.1 +github.com/containerd/fifo 69b99525e472735860a5269b75af1970142b3062 +github.com/opencontainers/runtime-spec 198f23f827eea397d4331d7eb048d9d4c7ff7bee +github.com/containerd/go-runc 2774a2ea124a5c2d0aba13b5c2dd8a5a9a48775d +github.com/containerd/console 7fed77e673ca4abcd0cbd6d4d0e0e22137cbd778 +github.com/Azure/go-ansiterm fa152c58bc15761d0200cb75fe958b89a9d4888e +google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 +golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 +github.com/docker/go-events aa2e3b613fbbfdddbe055a7b9e3ce271cfd83eca + +github.com/urfave/cli d70f47eeca3afd795160003bc6e28b001d60c67c +github.com/docker/go-units 0dadbb0345b35ec7ef35e228dabb8de89a65bf52 +github.com/google/shlex 6f45313302b9c56850fc17f99e40caebce98c716 +golang.org/x/time 8be79e1e0910c292df4e79c241bb7e8f7e725959 + +github.com/BurntSushi/locker 392720b78f44e9d0249fcac6c43b111b47a370b8 +github.com/docker/docker 05c7c311390911daebcf5d9519dee813fc02a887 +github.com/pkg/profile 5b67d428864e92711fcbd2f8629456121a56d91f + +github.com/tonistiigi/fsutil 0ac4c11b053b9c5c7c47558f81f96c7100ce50fb +github.com/stevvooe/continuity 86cec1535a968310e7532819f699ff2830ed7463 diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go index 98f4b8585f..d681ad78c2 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go @@ -7,8 +7,8 @@ import ( "os/exec" "time" - "github.com/Sirupsen/logrus" "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" ) type Rlimit struct { diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/LICENSE b/components/engine/vendor/github.com/sirupsen/logrus/LICENSE similarity index 100% rename from components/engine/vendor/github.com/Sirupsen/logrus/LICENSE rename to components/engine/vendor/github.com/sirupsen/logrus/LICENSE diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/README.md b/components/engine/vendor/github.com/sirupsen/logrus/README.md similarity index 71% rename from components/engine/vendor/github.com/Sirupsen/logrus/README.md rename to components/engine/vendor/github.com/sirupsen/logrus/README.md index 126cd1fc2b..82aeb4eef3 100644 --- a/components/engine/vendor/github.com/Sirupsen/logrus/README.md +++ b/components/engine/vendor/github.com/sirupsen/logrus/README.md @@ -1,11 +1,24 @@ -# Logrus :walrus: [![Build Status](https://travis-ci.org/Sirupsen/logrus.svg?branch=master)](https://travis-ci.org/Sirupsen/logrus) [![GoDoc](https://godoc.org/github.com/Sirupsen/logrus?status.svg)](https://godoc.org/github.com/Sirupsen/logrus) +# Logrus :walrus: [![Build Status](https://travis-ci.org/sirupsen/logrus.svg?branch=master)](https://travis-ci.org/sirupsen/logrus) [![GoDoc](https://godoc.org/github.com/sirupsen/logrus?status.svg)](https://godoc.org/github.com/sirupsen/logrus) Logrus is a structured logger for Go (golang), completely API compatible with -the standard library logger. [Godoc][godoc]. **Please note the Logrus API is not -yet stable (pre 1.0). Logrus itself is completely stable and has been used in -many large deployments. The core API is unlikely to change much but please -version control your Logrus to make sure you aren't fetching latest `master` on -every build.** +the standard library logger. [Godoc][godoc]. + +**Seeing weird case-sensitive problems?** It's in the past been possible to +import Logrus as both upper- and lower-case. Due to the Go package environment, +this caused issues in the community and we needed a standard. Some environments +experienced problems with the upper-case variant, so the lower-case was decided. +Everything using `logrus` will need to use the lower-case: +`github.com/sirupsen/logrus`. Any package that isn't, should be changed. + +To fix Glide, see [these +comments](https://github.com/sirupsen/logrus/issues/553#issuecomment-306591437). +For an in-depth explanation of the casing issue, see [this +comment](https://github.com/sirupsen/logrus/issues/570#issuecomment-313933276). + +**Are you interested in assisting in maintaining Logrus?** Currently I have a +lot of obligations, and I am unable to provide Logrus with the maintainership it +needs. If you'd like to help, please reach out to me at `simon at author's +username dot com`. Nicely color-coded in development (when a TTY is attached, otherwise just plain text): @@ -46,6 +59,12 @@ time="2015-03-26T01:27:38-04:00" level=fatal msg="The ice breaks!" err=&{0x20822 exit status 1 ``` +#### Case-sensitivity + +The organization's name was changed to lower-case--and this will not be changed +back. If you are getting import conflicts due to case sensitivity, please use +the lower-case import: `github.com/sirupsen/logrus`. + #### Example The simplest way to use Logrus is simply the package-level exported logger: @@ -54,7 +73,7 @@ The simplest way to use Logrus is simply the package-level exported logger: package main import ( - log "github.com/Sirupsen/logrus" + log "github.com/sirupsen/logrus" ) func main() { @@ -65,7 +84,7 @@ func main() { ``` Note that it's completely api-compatible with the stdlib logger, so you can -replace your `log` imports everywhere with `log "github.com/Sirupsen/logrus"` +replace your `log` imports everywhere with `log "github.com/sirupsen/logrus"` and you'll now have the flexibility of Logrus. You can customize it all you want: @@ -74,15 +93,16 @@ package main import ( "os" - log "github.com/Sirupsen/logrus" + log "github.com/sirupsen/logrus" ) func init() { // Log as JSON instead of the default ASCII formatter. log.SetFormatter(&log.JSONFormatter{}) - // Output to stderr instead of stdout, could also be a file. - log.SetOutput(os.Stderr) + // Output to stdout instead of the default stderr + // Can be any io.Writer, see below for File example + log.SetOutput(os.Stdout) // Only log the warning severity or above. log.SetLevel(log.WarnLevel) @@ -123,7 +143,8 @@ application, you can also create an instance of the `logrus` Logger: package main import ( - "github.com/Sirupsen/logrus" + "os" + "github.com/sirupsen/logrus" ) // Create a new instance of the logger. You can have any number of instances. @@ -132,7 +153,15 @@ var log = logrus.New() func main() { // The API for setting attributes is a little different than the package level // exported logger. See Godoc. - log.Out = os.Stderr + log.Out = os.Stdout + + // You could set this to any `io.Writer` such as a file + // file, err := os.OpenFile("logrus.log", os.O_CREATE|os.O_WRONLY, 0666) + // if err == nil { + // log.Out = file + // } else { + // log.Info("Failed to log to file, using default stderr") + // } log.WithFields(logrus.Fields{ "animal": "walrus", @@ -143,7 +172,7 @@ func main() { #### Fields -Logrus encourages careful, structured logging though logging fields instead of +Logrus encourages careful, structured logging through logging fields instead of long, unparseable error messages. For example, instead of: `log.Fatalf("Failed to send event %s to topic %s with key %d")`, you should log the much more discoverable: @@ -165,6 +194,20 @@ In general, with Logrus using any of the `printf`-family functions should be seen as a hint you should add a field, however, you can still use the `printf`-family functions with Logrus. +#### Default Fields + +Often it's helpful to have fields _always_ attached to log statements in an +application or parts of one. For example, you may want to always log the +`request_id` and `user_ip` in the context of a request. Instead of writing +`log.WithFields(log.Fields{"request_id": request_id, "user_ip": user_ip})` on +every line, you can create a `logrus.Entry` to pass around instead: + +```go +requestLogger := log.WithFields(log.Fields{"request_id": request_id, "user_ip": user_ip}) +requestLogger.Info("something happened on that request") # will log request_id and user_ip +requestLogger.Warn("something not great happened") +``` + #### Hooks You can add hooks for logging levels. For example to send errors to an exception @@ -176,9 +219,9 @@ Logrus comes with [built-in hooks](hooks/). Add those, or your custom hook, in ```go import ( - log "github.com/Sirupsen/logrus" + log "github.com/sirupsen/logrus" "gopkg.in/gemnasium/logrus-airbrake-hook.v2" // the package is named "aibrake" - logrus_syslog "github.com/Sirupsen/logrus/hooks/syslog" + logrus_syslog "github.com/sirupsen/logrus/hooks/syslog" "log/syslog" ) @@ -200,40 +243,52 @@ Note: Syslog hook also support connecting to local syslog (Ex. "/dev/log" or "/v | Hook | Description | | ----- | ----------- | -| [Airbrake](https://github.com/gemnasium/logrus-airbrake-hook) | Send errors to the Airbrake API V3. Uses the official [`gobrake`](https://github.com/airbrake/gobrake) behind the scenes. | | [Airbrake "legacy"](https://github.com/gemnasium/logrus-airbrake-legacy-hook) | Send errors to an exception tracking service compatible with the Airbrake API V2. Uses [`airbrake-go`](https://github.com/tobi/airbrake-go) behind the scenes. | -| [Papertrail](https://github.com/polds/logrus-papertrail-hook) | Send errors to the [Papertrail](https://papertrailapp.com) hosted logging service via UDP. | -| [Syslog](https://github.com/Sirupsen/logrus/blob/master/hooks/syslog/syslog.go) | Send errors to remote syslog server. Uses standard library `log/syslog` behind the scenes. | -| [Bugsnag](https://github.com/Shopify/logrus-bugsnag/blob/master/bugsnag.go) | Send errors to the Bugsnag exception tracking service. | -| [Sentry](https://github.com/evalphobia/logrus_sentry) | Send errors to the Sentry error logging and aggregation service. | -| [Hiprus](https://github.com/nubo/hiprus) | Send errors to a channel in hipchat. | -| [Logrusly](https://github.com/sebest/logrusly) | Send logs to [Loggly](https://www.loggly.com/) | -| [Slackrus](https://github.com/johntdyer/slackrus) | Hook for Slack chat. | -| [Journalhook](https://github.com/wercker/journalhook) | Hook for logging to `systemd-journald` | -| [Graylog](https://github.com/gemnasium/logrus-graylog-hook) | Hook for logging to [Graylog](http://graylog2.org/) | -| [Raygun](https://github.com/squirkle/logrus-raygun-hook) | Hook for logging to [Raygun.io](http://raygun.io/) | -| [LFShook](https://github.com/rifflock/lfshook) | Hook for logging to the local filesystem | -| [Honeybadger](https://github.com/agonzalezro/logrus_honeybadger) | Hook for sending exceptions to Honeybadger | -| [Mail](https://github.com/zbindenren/logrus_mail) | Hook for sending exceptions via mail | -| [Rollrus](https://github.com/heroku/rollrus) | Hook for sending errors to rollbar | -| [Fluentd](https://github.com/evalphobia/logrus_fluent) | Hook for logging to fluentd | -| [Mongodb](https://github.com/weekface/mgorus) | Hook for logging to mongodb | -| [Influxus] (http://github.com/vlad-doru/influxus) | Hook for concurrently logging to [InfluxDB] (http://influxdata.com/) | -| [InfluxDB](https://github.com/Abramovic/logrus_influxdb) | Hook for logging to influxdb | -| [Octokit](https://github.com/dorajistyle/logrus-octokit-hook) | Hook for logging to github via octokit | -| [DeferPanic](https://github.com/deferpanic/dp-logrus) | Hook for logging to DeferPanic | -| [Redis-Hook](https://github.com/rogierlommers/logrus-redis-hook) | Hook for logging to a ELK stack (through Redis) | +| [Airbrake](https://github.com/gemnasium/logrus-airbrake-hook) | Send errors to the Airbrake API V3. Uses the official [`gobrake`](https://github.com/airbrake/gobrake) behind the scenes. | +| [Amazon Kinesis](https://github.com/evalphobia/logrus_kinesis) | Hook for logging to [Amazon Kinesis](https://aws.amazon.com/kinesis/) | | [Amqp-Hook](https://github.com/vladoatanasov/logrus_amqp) | Hook for logging to Amqp broker (Like RabbitMQ) | -| [KafkaLogrus](https://github.com/goibibo/KafkaLogrus) | Hook for logging to kafka | -| [Typetalk](https://github.com/dragon3/logrus-typetalk-hook) | Hook for logging to [Typetalk](https://www.typetalk.in/) | +| [Bugsnag](https://github.com/Shopify/logrus-bugsnag/blob/master/bugsnag.go) | Send errors to the Bugsnag exception tracking service. | +| [DeferPanic](https://github.com/deferpanic/dp-logrus) | Hook for logging to DeferPanic | +| [Discordrus](https://github.com/kz/discordrus) | Hook for logging to [Discord](https://discordapp.com/) | | [ElasticSearch](https://github.com/sohlich/elogrus) | Hook for logging to ElasticSearch| -| [Sumorus](https://github.com/doublefree/sumorus) | Hook for logging to [SumoLogic](https://www.sumologic.com/)| -| [Scribe](https://github.com/sagar8192/logrus-scribe-hook) | Hook for logging to [Scribe](https://github.com/facebookarchive/scribe)| -| [Logstash](https://github.com/bshuster-repo/logrus-logstash-hook) | Hook for logging to [Logstash](https://www.elastic.co/products/logstash) | -| [logz.io](https://github.com/ripcurld00d/logrus-logzio-hook) | Hook for logging to [logz.io](https://logz.io), a Log as a Service using Logstash | +| [Firehose](https://github.com/beaubrewer/logrus_firehose) | Hook for logging to [Amazon Firehose](https://aws.amazon.com/kinesis/firehose/) +| [Fluentd](https://github.com/evalphobia/logrus_fluent) | Hook for logging to fluentd | +| [Go-Slack](https://github.com/multiplay/go-slack) | Hook for logging to [Slack](https://slack.com) | +| [Graylog](https://github.com/gemnasium/logrus-graylog-hook) | Hook for logging to [Graylog](http://graylog2.org/) | +| [Hiprus](https://github.com/nubo/hiprus) | Send errors to a channel in hipchat. | +| [Honeybadger](https://github.com/agonzalezro/logrus_honeybadger) | Hook for sending exceptions to Honeybadger | +| [InfluxDB](https://github.com/Abramovic/logrus_influxdb) | Hook for logging to influxdb | +| [Influxus](http://github.com/vlad-doru/influxus) | Hook for concurrently logging to [InfluxDB](http://influxdata.com/) | +| [Journalhook](https://github.com/wercker/journalhook) | Hook for logging to `systemd-journald` | +| [KafkaLogrus](https://github.com/goibibo/KafkaLogrus) | Hook for logging to kafka | +| [LFShook](https://github.com/rifflock/lfshook) | Hook for logging to the local filesystem | +| [Logentries](https://github.com/jcftang/logentriesrus) | Hook for logging to [Logentries](https://logentries.com/) | +| [Logentrus](https://github.com/puddingfactory/logentrus) | Hook for logging to [Logentries](https://logentries.com/) | | [Logmatic.io](https://github.com/logmatic/logmatic-go) | Hook for logging to [Logmatic.io](http://logmatic.io/) | +| [Logrusly](https://github.com/sebest/logrusly) | Send logs to [Loggly](https://www.loggly.com/) | +| [Logstash](https://github.com/bshuster-repo/logrus-logstash-hook) | Hook for logging to [Logstash](https://www.elastic.co/products/logstash) | +| [Mail](https://github.com/zbindenren/logrus_mail) | Hook for sending exceptions via mail | +| [Mattermost](https://github.com/shuLhan/mattermost-integration/tree/master/hooks/logrus) | Hook for logging to [Mattermost](https://mattermost.com/) | +| [Mongodb](https://github.com/weekface/mgorus) | Hook for logging to mongodb | +| [NATS-Hook](https://github.com/rybit/nats_logrus_hook) | Hook for logging to [NATS](https://nats.io) | +| [Octokit](https://github.com/dorajistyle/logrus-octokit-hook) | Hook for logging to github via octokit | +| [Papertrail](https://github.com/polds/logrus-papertrail-hook) | Send errors to the [Papertrail](https://papertrailapp.com) hosted logging service via UDP. | +| [PostgreSQL](https://github.com/gemnasium/logrus-postgresql-hook) | Send logs to [PostgreSQL](http://postgresql.org) | | [Pushover](https://github.com/toorop/logrus_pushover) | Send error via [Pushover](https://pushover.net) | - +| [Raygun](https://github.com/squirkle/logrus-raygun-hook) | Hook for logging to [Raygun.io](http://raygun.io/) | +| [Redis-Hook](https://github.com/rogierlommers/logrus-redis-hook) | Hook for logging to a ELK stack (through Redis) | +| [Rollrus](https://github.com/heroku/rollrus) | Hook for sending errors to rollbar | +| [Scribe](https://github.com/sagar8192/logrus-scribe-hook) | Hook for logging to [Scribe](https://github.com/facebookarchive/scribe)| +| [Sentry](https://github.com/evalphobia/logrus_sentry) | Send errors to the Sentry error logging and aggregation service. | +| [Slackrus](https://github.com/johntdyer/slackrus) | Hook for Slack chat. | +| [Stackdriver](https://github.com/knq/sdhook) | Hook for logging to [Google Stackdriver](https://cloud.google.com/logging/) | +| [Sumorus](https://github.com/doublefree/sumorus) | Hook for logging to [SumoLogic](https://www.sumologic.com/)| +| [Syslog](https://github.com/sirupsen/logrus/blob/master/hooks/syslog/syslog.go) | Send errors to remote syslog server. Uses standard library `log/syslog` behind the scenes. | +| [Syslog TLS](https://github.com/shinji62/logrus-syslog-ng) | Send errors to remote syslog server with TLS support. | +| [TraceView](https://github.com/evalphobia/logrus_appneta) | Hook for logging to [AppNeta TraceView](https://www.appneta.com/products/traceview/) | +| [Typetalk](https://github.com/dragon3/logrus-typetalk-hook) | Hook for logging to [Typetalk](https://www.typetalk.in/) | +| [logz.io](https://github.com/ripcurld00d/logrus-logzio-hook) | Hook for logging to [logz.io](https://logz.io), a Log as a Service using Logstash | +| [SQS-Hook](https://github.com/tsarpaul/logrus_sqs) | Hook for logging to [Amazon Simple Queue Service (SQS)](https://aws.amazon.com/sqs/) | #### Level logging @@ -282,7 +337,7 @@ could do: ```go import ( - log "github.com/Sirupsen/logrus" + log "github.com/sirupsen/logrus" ) init() { @@ -309,8 +364,11 @@ The built-in logging formatters are: without colors. * *Note:* to force colored output when there is no TTY, set the `ForceColors` field to `true`. To force no colored output even if there is a TTY set the - `DisableColors` field to `true` + `DisableColors` field to `true`. For Windows, see + [github.com/mattn/go-colorable](https://github.com/mattn/go-colorable). + * All options are listed in the [generated docs](https://godoc.org/github.com/sirupsen/logrus#TextFormatter). * `logrus.JSONFormatter`. Logs fields as JSON. + * All options are listed in the [generated docs](https://godoc.org/github.com/sirupsen/logrus#JSONFormatter). Third party logging formatters: @@ -359,6 +417,18 @@ srv := http.Server{ Each line written to that writer will be printed the usual way, using formatters and hooks. The level for those entries is `info`. +This means that we can override the standard library logger easily: + +```go +logger := logrus.New() +logger.Formatter = &logrus.JSONFormatter{} + +// Use logrus for standard log output +// Note that `log` here references stdlib's log +// Not logrus imported under the name `log`. +log.SetOutput(logger.Writer()) +``` + #### Rotation Log rotation is not provided with Logrus. Log rotation should be done by an @@ -370,7 +440,7 @@ entries. It should not be a feature of the application-level logger. | Tool | Description | | ---- | ----------- | |[Logrus Mate](https://github.com/gogap/logrus_mate)|Logrus mate is a tool for Logrus to manage loggers, you can initial logger's level, hook and formatter by config file, the logger will generated with different config at different environment.| -|[Logrus Viper Helper](https://github.com/heirko/go-contrib/tree/master/logrusHelper)|An Helper arround Logrus to wrap with spf13/Viper to load configuration with fangs! And to simplify Logrus configuration use some behavior of [Logrus Mate](https://github.com/gogap/logrus_mate). [sample](https://github.com/heirko/iris-contrib/blob/master/middleware/logrus-logger/example) | +|[Logrus Viper Helper](https://github.com/heirko/go-contrib/tree/master/logrusHelper)|An Helper around Logrus to wrap with spf13/Viper to load configuration with fangs! And to simplify Logrus configuration use some behavior of [Logrus Mate](https://github.com/gogap/logrus_mate). [sample](https://github.com/heirko/iris-contrib/blob/master/middleware/logrus-logger/example) | #### Testing @@ -380,15 +450,24 @@ Logrus has a built in facility for asserting the presence of log messages. This * a test logger (`test.NewNullLogger`) that just records log messages (and does not output any): ```go -logger, hook := NewNullLogger() -logger.Error("Hello error") +import( + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/assert" + "testing" +) -assert.Equal(1, len(hook.Entries)) -assert.Equal(logrus.ErrorLevel, hook.LastEntry().Level) -assert.Equal("Hello error", hook.LastEntry().Message) +func TestSomething(t*testing.T){ + logger, hook := test.NewNullLogger() + logger.Error("Helloerror") -hook.Reset() -assert.Nil(hook.LastEntry()) + assert.Equal(t, 1, len(hook.Entries)) + assert.Equal(t, logrus.ErrorLevel, hook.LastEntry().Level) + assert.Equal(t, "Helloerror", hook.LastEntry().Message) + + hook.Reset() + assert.Nil(t, hook.LastEntry()) +} ``` #### Fatal handlers @@ -407,7 +486,7 @@ logrus.RegisterExitHandler(handler) ... ``` -#### Thread safty +#### Thread safety By default Logger is protected by mutex for concurrent writes, this mutex is invoked when calling hooks and writing logs. If you are sure such locking is not needed, you can call logger.SetNoLock() to disable the locking. diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/alt_exit.go b/components/engine/vendor/github.com/sirupsen/logrus/alt_exit.go similarity index 96% rename from components/engine/vendor/github.com/Sirupsen/logrus/alt_exit.go rename to components/engine/vendor/github.com/sirupsen/logrus/alt_exit.go index b4c9e84754..8af90637a9 100644 --- a/components/engine/vendor/github.com/Sirupsen/logrus/alt_exit.go +++ b/components/engine/vendor/github.com/sirupsen/logrus/alt_exit.go @@ -1,7 +1,7 @@ package logrus // The following code was sourced and modified from the -// https://bitbucket.org/tebeka/atexit package governed by the following license: +// https://github.com/tebeka/atexit package governed by the following license: // // Copyright (c) 2012 Miki Tebeka . // diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/doc.go b/components/engine/vendor/github.com/sirupsen/logrus/doc.go similarity index 83% rename from components/engine/vendor/github.com/Sirupsen/logrus/doc.go rename to components/engine/vendor/github.com/sirupsen/logrus/doc.go index dddd5f877b..da67aba06d 100644 --- a/components/engine/vendor/github.com/Sirupsen/logrus/doc.go +++ b/components/engine/vendor/github.com/sirupsen/logrus/doc.go @@ -7,7 +7,7 @@ The simplest way to use Logrus is simply the package-level exported logger: package main import ( - log "github.com/Sirupsen/logrus" + log "github.com/sirupsen/logrus" ) func main() { @@ -21,6 +21,6 @@ The simplest way to use Logrus is simply the package-level exported logger: Output: time="2015-09-07T08:48:33Z" level=info msg="A walrus appears" animal=walrus number=1 size=10 -For a full guide visit https://github.com/Sirupsen/logrus +For a full guide visit https://github.com/sirupsen/logrus */ package logrus diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/entry.go b/components/engine/vendor/github.com/sirupsen/logrus/entry.go similarity index 89% rename from components/engine/vendor/github.com/Sirupsen/logrus/entry.go rename to components/engine/vendor/github.com/sirupsen/logrus/entry.go index 4edbe7a2de..320e5d5b8b 100644 --- a/components/engine/vendor/github.com/Sirupsen/logrus/entry.go +++ b/components/engine/vendor/github.com/sirupsen/logrus/entry.go @@ -126,7 +126,7 @@ func (entry Entry) log(level Level, msg string) { } func (entry *Entry) Debug(args ...interface{}) { - if entry.Logger.Level >= DebugLevel { + if entry.Logger.level() >= DebugLevel { entry.log(DebugLevel, fmt.Sprint(args...)) } } @@ -136,13 +136,13 @@ func (entry *Entry) Print(args ...interface{}) { } func (entry *Entry) Info(args ...interface{}) { - if entry.Logger.Level >= InfoLevel { + if entry.Logger.level() >= InfoLevel { entry.log(InfoLevel, fmt.Sprint(args...)) } } func (entry *Entry) Warn(args ...interface{}) { - if entry.Logger.Level >= WarnLevel { + if entry.Logger.level() >= WarnLevel { entry.log(WarnLevel, fmt.Sprint(args...)) } } @@ -152,20 +152,20 @@ func (entry *Entry) Warning(args ...interface{}) { } func (entry *Entry) Error(args ...interface{}) { - if entry.Logger.Level >= ErrorLevel { + if entry.Logger.level() >= ErrorLevel { entry.log(ErrorLevel, fmt.Sprint(args...)) } } func (entry *Entry) Fatal(args ...interface{}) { - if entry.Logger.Level >= FatalLevel { + if entry.Logger.level() >= FatalLevel { entry.log(FatalLevel, fmt.Sprint(args...)) } Exit(1) } func (entry *Entry) Panic(args ...interface{}) { - if entry.Logger.Level >= PanicLevel { + if entry.Logger.level() >= PanicLevel { entry.log(PanicLevel, fmt.Sprint(args...)) } panic(fmt.Sprint(args...)) @@ -174,13 +174,13 @@ func (entry *Entry) Panic(args ...interface{}) { // Entry Printf family functions func (entry *Entry) Debugf(format string, args ...interface{}) { - if entry.Logger.Level >= DebugLevel { + if entry.Logger.level() >= DebugLevel { entry.Debug(fmt.Sprintf(format, args...)) } } func (entry *Entry) Infof(format string, args ...interface{}) { - if entry.Logger.Level >= InfoLevel { + if entry.Logger.level() >= InfoLevel { entry.Info(fmt.Sprintf(format, args...)) } } @@ -190,7 +190,7 @@ func (entry *Entry) Printf(format string, args ...interface{}) { } func (entry *Entry) Warnf(format string, args ...interface{}) { - if entry.Logger.Level >= WarnLevel { + if entry.Logger.level() >= WarnLevel { entry.Warn(fmt.Sprintf(format, args...)) } } @@ -200,20 +200,20 @@ func (entry *Entry) Warningf(format string, args ...interface{}) { } func (entry *Entry) Errorf(format string, args ...interface{}) { - if entry.Logger.Level >= ErrorLevel { + if entry.Logger.level() >= ErrorLevel { entry.Error(fmt.Sprintf(format, args...)) } } func (entry *Entry) Fatalf(format string, args ...interface{}) { - if entry.Logger.Level >= FatalLevel { + if entry.Logger.level() >= FatalLevel { entry.Fatal(fmt.Sprintf(format, args...)) } Exit(1) } func (entry *Entry) Panicf(format string, args ...interface{}) { - if entry.Logger.Level >= PanicLevel { + if entry.Logger.level() >= PanicLevel { entry.Panic(fmt.Sprintf(format, args...)) } } @@ -221,13 +221,13 @@ func (entry *Entry) Panicf(format string, args ...interface{}) { // Entry Println family functions func (entry *Entry) Debugln(args ...interface{}) { - if entry.Logger.Level >= DebugLevel { + if entry.Logger.level() >= DebugLevel { entry.Debug(entry.sprintlnn(args...)) } } func (entry *Entry) Infoln(args ...interface{}) { - if entry.Logger.Level >= InfoLevel { + if entry.Logger.level() >= InfoLevel { entry.Info(entry.sprintlnn(args...)) } } @@ -237,7 +237,7 @@ func (entry *Entry) Println(args ...interface{}) { } func (entry *Entry) Warnln(args ...interface{}) { - if entry.Logger.Level >= WarnLevel { + if entry.Logger.level() >= WarnLevel { entry.Warn(entry.sprintlnn(args...)) } } @@ -247,20 +247,20 @@ func (entry *Entry) Warningln(args ...interface{}) { } func (entry *Entry) Errorln(args ...interface{}) { - if entry.Logger.Level >= ErrorLevel { + if entry.Logger.level() >= ErrorLevel { entry.Error(entry.sprintlnn(args...)) } } func (entry *Entry) Fatalln(args ...interface{}) { - if entry.Logger.Level >= FatalLevel { + if entry.Logger.level() >= FatalLevel { entry.Fatal(entry.sprintlnn(args...)) } Exit(1) } func (entry *Entry) Panicln(args ...interface{}) { - if entry.Logger.Level >= PanicLevel { + if entry.Logger.level() >= PanicLevel { entry.Panic(entry.sprintlnn(args...)) } } diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/exported.go b/components/engine/vendor/github.com/sirupsen/logrus/exported.go similarity index 99% rename from components/engine/vendor/github.com/Sirupsen/logrus/exported.go rename to components/engine/vendor/github.com/sirupsen/logrus/exported.go index 9a0120ac1d..1aeaa90ba2 100644 --- a/components/engine/vendor/github.com/Sirupsen/logrus/exported.go +++ b/components/engine/vendor/github.com/sirupsen/logrus/exported.go @@ -31,14 +31,14 @@ func SetFormatter(formatter Formatter) { func SetLevel(level Level) { std.mu.Lock() defer std.mu.Unlock() - std.Level = level + std.setLevel(level) } // GetLevel returns the standard logger level. func GetLevel() Level { std.mu.Lock() defer std.mu.Unlock() - return std.Level + return std.level() } // AddHook adds a hook to the standard logger hooks. diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/formatter.go b/components/engine/vendor/github.com/sirupsen/logrus/formatter.go similarity index 100% rename from components/engine/vendor/github.com/Sirupsen/logrus/formatter.go rename to components/engine/vendor/github.com/sirupsen/logrus/formatter.go diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/hooks.go b/components/engine/vendor/github.com/sirupsen/logrus/hooks.go similarity index 100% rename from components/engine/vendor/github.com/Sirupsen/logrus/hooks.go rename to components/engine/vendor/github.com/sirupsen/logrus/hooks.go diff --git a/components/engine/vendor/github.com/sirupsen/logrus/json_formatter.go b/components/engine/vendor/github.com/sirupsen/logrus/json_formatter.go new file mode 100644 index 0000000000..e787ea1750 --- /dev/null +++ b/components/engine/vendor/github.com/sirupsen/logrus/json_formatter.go @@ -0,0 +1,74 @@ +package logrus + +import ( + "encoding/json" + "fmt" +) + +type fieldKey string +type FieldMap map[fieldKey]string + +const ( + FieldKeyMsg = "msg" + FieldKeyLevel = "level" + FieldKeyTime = "time" +) + +func (f FieldMap) resolve(key fieldKey) string { + if k, ok := f[key]; ok { + return k + } + + return string(key) +} + +type JSONFormatter struct { + // TimestampFormat sets the format used for marshaling timestamps. + TimestampFormat string + + // DisableTimestamp allows disabling automatic timestamps in output + DisableTimestamp bool + + // FieldMap allows users to customize the names of keys for various fields. + // As an example: + // formatter := &JSONFormatter{ + // FieldMap: FieldMap{ + // FieldKeyTime: "@timestamp", + // FieldKeyLevel: "@level", + // FieldKeyMsg: "@message", + // }, + // } + FieldMap FieldMap +} + +func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) { + data := make(Fields, len(entry.Data)+3) + for k, v := range entry.Data { + switch v := v.(type) { + case error: + // Otherwise errors are ignored by `encoding/json` + // https://github.com/sirupsen/logrus/issues/137 + data[k] = v.Error() + default: + data[k] = v + } + } + prefixFieldClashes(data) + + timestampFormat := f.TimestampFormat + if timestampFormat == "" { + timestampFormat = DefaultTimestampFormat + } + + if !f.DisableTimestamp { + data[f.FieldMap.resolve(FieldKeyTime)] = entry.Time.Format(timestampFormat) + } + data[f.FieldMap.resolve(FieldKeyMsg)] = entry.Message + data[f.FieldMap.resolve(FieldKeyLevel)] = entry.Level.String() + + serialized, err := json.Marshal(data) + if err != nil { + return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) + } + return append(serialized, '\n'), nil +} diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/logger.go b/components/engine/vendor/github.com/sirupsen/logrus/logger.go similarity index 88% rename from components/engine/vendor/github.com/Sirupsen/logrus/logger.go rename to components/engine/vendor/github.com/sirupsen/logrus/logger.go index b769f3d352..370fff5d1b 100644 --- a/components/engine/vendor/github.com/Sirupsen/logrus/logger.go +++ b/components/engine/vendor/github.com/sirupsen/logrus/logger.go @@ -4,6 +4,7 @@ import ( "io" "os" "sync" + "sync/atomic" ) type Logger struct { @@ -112,7 +113,7 @@ func (logger *Logger) WithError(err error) *Entry { } func (logger *Logger) Debugf(format string, args ...interface{}) { - if logger.Level >= DebugLevel { + if logger.level() >= DebugLevel { entry := logger.newEntry() entry.Debugf(format, args...) logger.releaseEntry(entry) @@ -120,7 +121,7 @@ func (logger *Logger) Debugf(format string, args ...interface{}) { } func (logger *Logger) Infof(format string, args ...interface{}) { - if logger.Level >= InfoLevel { + if logger.level() >= InfoLevel { entry := logger.newEntry() entry.Infof(format, args...) logger.releaseEntry(entry) @@ -134,7 +135,7 @@ func (logger *Logger) Printf(format string, args ...interface{}) { } func (logger *Logger) Warnf(format string, args ...interface{}) { - if logger.Level >= WarnLevel { + if logger.level() >= WarnLevel { entry := logger.newEntry() entry.Warnf(format, args...) logger.releaseEntry(entry) @@ -142,7 +143,7 @@ func (logger *Logger) Warnf(format string, args ...interface{}) { } func (logger *Logger) Warningf(format string, args ...interface{}) { - if logger.Level >= WarnLevel { + if logger.level() >= WarnLevel { entry := logger.newEntry() entry.Warnf(format, args...) logger.releaseEntry(entry) @@ -150,7 +151,7 @@ func (logger *Logger) Warningf(format string, args ...interface{}) { } func (logger *Logger) Errorf(format string, args ...interface{}) { - if logger.Level >= ErrorLevel { + if logger.level() >= ErrorLevel { entry := logger.newEntry() entry.Errorf(format, args...) logger.releaseEntry(entry) @@ -158,7 +159,7 @@ func (logger *Logger) Errorf(format string, args ...interface{}) { } func (logger *Logger) Fatalf(format string, args ...interface{}) { - if logger.Level >= FatalLevel { + if logger.level() >= FatalLevel { entry := logger.newEntry() entry.Fatalf(format, args...) logger.releaseEntry(entry) @@ -167,7 +168,7 @@ func (logger *Logger) Fatalf(format string, args ...interface{}) { } func (logger *Logger) Panicf(format string, args ...interface{}) { - if logger.Level >= PanicLevel { + if logger.level() >= PanicLevel { entry := logger.newEntry() entry.Panicf(format, args...) logger.releaseEntry(entry) @@ -175,7 +176,7 @@ func (logger *Logger) Panicf(format string, args ...interface{}) { } func (logger *Logger) Debug(args ...interface{}) { - if logger.Level >= DebugLevel { + if logger.level() >= DebugLevel { entry := logger.newEntry() entry.Debug(args...) logger.releaseEntry(entry) @@ -183,7 +184,7 @@ func (logger *Logger) Debug(args ...interface{}) { } func (logger *Logger) Info(args ...interface{}) { - if logger.Level >= InfoLevel { + if logger.level() >= InfoLevel { entry := logger.newEntry() entry.Info(args...) logger.releaseEntry(entry) @@ -197,7 +198,7 @@ func (logger *Logger) Print(args ...interface{}) { } func (logger *Logger) Warn(args ...interface{}) { - if logger.Level >= WarnLevel { + if logger.level() >= WarnLevel { entry := logger.newEntry() entry.Warn(args...) logger.releaseEntry(entry) @@ -205,7 +206,7 @@ func (logger *Logger) Warn(args ...interface{}) { } func (logger *Logger) Warning(args ...interface{}) { - if logger.Level >= WarnLevel { + if logger.level() >= WarnLevel { entry := logger.newEntry() entry.Warn(args...) logger.releaseEntry(entry) @@ -213,7 +214,7 @@ func (logger *Logger) Warning(args ...interface{}) { } func (logger *Logger) Error(args ...interface{}) { - if logger.Level >= ErrorLevel { + if logger.level() >= ErrorLevel { entry := logger.newEntry() entry.Error(args...) logger.releaseEntry(entry) @@ -221,7 +222,7 @@ func (logger *Logger) Error(args ...interface{}) { } func (logger *Logger) Fatal(args ...interface{}) { - if logger.Level >= FatalLevel { + if logger.level() >= FatalLevel { entry := logger.newEntry() entry.Fatal(args...) logger.releaseEntry(entry) @@ -230,7 +231,7 @@ func (logger *Logger) Fatal(args ...interface{}) { } func (logger *Logger) Panic(args ...interface{}) { - if logger.Level >= PanicLevel { + if logger.level() >= PanicLevel { entry := logger.newEntry() entry.Panic(args...) logger.releaseEntry(entry) @@ -238,7 +239,7 @@ func (logger *Logger) Panic(args ...interface{}) { } func (logger *Logger) Debugln(args ...interface{}) { - if logger.Level >= DebugLevel { + if logger.level() >= DebugLevel { entry := logger.newEntry() entry.Debugln(args...) logger.releaseEntry(entry) @@ -246,7 +247,7 @@ func (logger *Logger) Debugln(args ...interface{}) { } func (logger *Logger) Infoln(args ...interface{}) { - if logger.Level >= InfoLevel { + if logger.level() >= InfoLevel { entry := logger.newEntry() entry.Infoln(args...) logger.releaseEntry(entry) @@ -260,7 +261,7 @@ func (logger *Logger) Println(args ...interface{}) { } func (logger *Logger) Warnln(args ...interface{}) { - if logger.Level >= WarnLevel { + if logger.level() >= WarnLevel { entry := logger.newEntry() entry.Warnln(args...) logger.releaseEntry(entry) @@ -268,7 +269,7 @@ func (logger *Logger) Warnln(args ...interface{}) { } func (logger *Logger) Warningln(args ...interface{}) { - if logger.Level >= WarnLevel { + if logger.level() >= WarnLevel { entry := logger.newEntry() entry.Warnln(args...) logger.releaseEntry(entry) @@ -276,7 +277,7 @@ func (logger *Logger) Warningln(args ...interface{}) { } func (logger *Logger) Errorln(args ...interface{}) { - if logger.Level >= ErrorLevel { + if logger.level() >= ErrorLevel { entry := logger.newEntry() entry.Errorln(args...) logger.releaseEntry(entry) @@ -284,7 +285,7 @@ func (logger *Logger) Errorln(args ...interface{}) { } func (logger *Logger) Fatalln(args ...interface{}) { - if logger.Level >= FatalLevel { + if logger.level() >= FatalLevel { entry := logger.newEntry() entry.Fatalln(args...) logger.releaseEntry(entry) @@ -293,7 +294,7 @@ func (logger *Logger) Fatalln(args ...interface{}) { } func (logger *Logger) Panicln(args ...interface{}) { - if logger.Level >= PanicLevel { + if logger.level() >= PanicLevel { entry := logger.newEntry() entry.Panicln(args...) logger.releaseEntry(entry) @@ -306,3 +307,11 @@ func (logger *Logger) Panicln(args ...interface{}) { func (logger *Logger) SetNoLock() { logger.mu.Disable() } + +func (logger *Logger) level() Level { + return Level(atomic.LoadUint32((*uint32)(&logger.Level))) +} + +func (logger *Logger) setLevel(level Level) { + atomic.StoreUint32((*uint32)(&logger.Level), uint32(level)) +} diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/logrus.go b/components/engine/vendor/github.com/sirupsen/logrus/logrus.go similarity index 99% rename from components/engine/vendor/github.com/Sirupsen/logrus/logrus.go rename to components/engine/vendor/github.com/sirupsen/logrus/logrus.go index e596691116..dd38999741 100644 --- a/components/engine/vendor/github.com/Sirupsen/logrus/logrus.go +++ b/components/engine/vendor/github.com/sirupsen/logrus/logrus.go @@ -10,7 +10,7 @@ import ( type Fields map[string]interface{} // Level type -type Level uint8 +type Level uint32 // Convert the Level to a string. E.g. PanicLevel becomes "panic". func (level Level) String() string { diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/terminal_appengine.go b/components/engine/vendor/github.com/sirupsen/logrus/terminal_appengine.go similarity index 71% rename from components/engine/vendor/github.com/Sirupsen/logrus/terminal_appengine.go rename to components/engine/vendor/github.com/sirupsen/logrus/terminal_appengine.go index 1960169ef2..e011a86945 100644 --- a/components/engine/vendor/github.com/Sirupsen/logrus/terminal_appengine.go +++ b/components/engine/vendor/github.com/sirupsen/logrus/terminal_appengine.go @@ -2,7 +2,9 @@ package logrus +import "io" + // IsTerminal returns true if stderr's file descriptor is a terminal. -func IsTerminal() bool { +func IsTerminal(f io.Writer) bool { return true } diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/terminal_bsd.go b/components/engine/vendor/github.com/sirupsen/logrus/terminal_bsd.go similarity index 100% rename from components/engine/vendor/github.com/Sirupsen/logrus/terminal_bsd.go rename to components/engine/vendor/github.com/sirupsen/logrus/terminal_bsd.go diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/terminal_linux.go b/components/engine/vendor/github.com/sirupsen/logrus/terminal_linux.go similarity index 100% rename from components/engine/vendor/github.com/Sirupsen/logrus/terminal_linux.go rename to components/engine/vendor/github.com/sirupsen/logrus/terminal_linux.go diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/terminal_notwindows.go b/components/engine/vendor/github.com/sirupsen/logrus/terminal_notwindows.go similarity index 60% rename from components/engine/vendor/github.com/Sirupsen/logrus/terminal_notwindows.go rename to components/engine/vendor/github.com/sirupsen/logrus/terminal_notwindows.go index 329038f6ca..190297abf3 100644 --- a/components/engine/vendor/github.com/Sirupsen/logrus/terminal_notwindows.go +++ b/components/engine/vendor/github.com/sirupsen/logrus/terminal_notwindows.go @@ -9,14 +9,20 @@ package logrus import ( + "io" + "os" "syscall" "unsafe" ) // IsTerminal returns true if stderr's file descriptor is a terminal. -func IsTerminal() bool { - fd := syscall.Stderr +func IsTerminal(f io.Writer) bool { var termios Termios - _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) - return err == 0 + switch v := f.(type) { + case *os.File: + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(v.Fd()), ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) + return err == 0 + default: + return false + } } diff --git a/components/engine/vendor/github.com/sirupsen/logrus/terminal_solaris.go b/components/engine/vendor/github.com/sirupsen/logrus/terminal_solaris.go new file mode 100644 index 0000000000..3c86b1abee --- /dev/null +++ b/components/engine/vendor/github.com/sirupsen/logrus/terminal_solaris.go @@ -0,0 +1,21 @@ +// +build solaris,!appengine + +package logrus + +import ( + "io" + "os" + + "golang.org/x/sys/unix" +) + +// IsTerminal returns true if the given file descriptor is a terminal. +func IsTerminal(f io.Writer) bool { + switch v := f.(type) { + case *os.File: + _, err := unix.IoctlGetTermios(int(v.Fd()), unix.TCGETA) + return err == nil + default: + return false + } +} diff --git a/components/engine/vendor/github.com/sirupsen/logrus/terminal_windows.go b/components/engine/vendor/github.com/sirupsen/logrus/terminal_windows.go new file mode 100644 index 0000000000..7a336307e5 --- /dev/null +++ b/components/engine/vendor/github.com/sirupsen/logrus/terminal_windows.go @@ -0,0 +1,82 @@ +// Based on ssh/terminal: +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows,!appengine + +package logrus + +import ( + "bytes" + "errors" + "io" + "os" + "os/exec" + "strconv" + "strings" + "syscall" + "unsafe" +) + +var kernel32 = syscall.NewLazyDLL("kernel32.dll") + +var ( + procGetConsoleMode = kernel32.NewProc("GetConsoleMode") + procSetConsoleMode = kernel32.NewProc("SetConsoleMode") +) + +const ( + enableProcessedOutput = 0x0001 + enableWrapAtEolOutput = 0x0002 + enableVirtualTerminalProcessing = 0x0004 +) + +func getVersion() (float64, error) { + stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{} + cmd := exec.Command("cmd", "ver") + cmd.Stdout = stdout + cmd.Stderr = stderr + err := cmd.Run() + if err != nil { + return -1, err + } + + // The output should be like "Microsoft Windows [Version XX.X.XXXXXX]" + version := strings.Replace(stdout.String(), "\n", "", -1) + version = strings.Replace(version, "\r\n", "", -1) + + x1 := strings.Index(version, "[Version") + + if x1 == -1 || strings.Index(version, "]") == -1 { + return -1, errors.New("Can't determine Windows version") + } + + return strconv.ParseFloat(version[x1+9:x1+13], 64) +} + +func init() { + ver, err := getVersion() + if err != nil { + return + } + + // Activate Virtual Processing for Windows CMD + // Info: https://msdn.microsoft.com/en-us/library/windows/desktop/ms686033(v=vs.85).aspx + if ver >= 10 { + handle := syscall.Handle(os.Stderr.Fd()) + procSetConsoleMode.Call(uintptr(handle), enableProcessedOutput|enableWrapAtEolOutput|enableVirtualTerminalProcessing) + } +} + +// IsTerminal returns true if stderr's file descriptor is a terminal. +func IsTerminal(f io.Writer) bool { + switch v := f.(type) { + case *os.File: + var st uint32 + r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(v.Fd()), uintptr(unsafe.Pointer(&st)), 0) + return r != 0 && e == 0 + default: + return false + } +} diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/text_formatter.go b/components/engine/vendor/github.com/sirupsen/logrus/text_formatter.go similarity index 69% rename from components/engine/vendor/github.com/Sirupsen/logrus/text_formatter.go rename to components/engine/vendor/github.com/sirupsen/logrus/text_formatter.go index 9114b3ca47..26dcc15538 100644 --- a/components/engine/vendor/github.com/Sirupsen/logrus/text_formatter.go +++ b/components/engine/vendor/github.com/sirupsen/logrus/text_formatter.go @@ -3,9 +3,9 @@ package logrus import ( "bytes" "fmt" - "runtime" "sort" "strings" + "sync" "time" ) @@ -20,16 +20,10 @@ const ( var ( baseTimestamp time.Time - isTerminal bool ) func init() { baseTimestamp = time.Now() - isTerminal = IsTerminal() -} - -func miniTS() int { - return int(time.Since(baseTimestamp) / time.Second) } type TextFormatter struct { @@ -54,11 +48,32 @@ type TextFormatter struct { // that log extremely frequently and don't use the JSON formatter this may not // be desired. DisableSorting bool + + // QuoteEmptyFields will wrap empty fields in quotes if true + QuoteEmptyFields bool + + // QuoteCharacter can be set to the override the default quoting character " + // with something else. For example: ', or `. + QuoteCharacter string + + // Whether the logger's out is to a terminal + isTerminal bool + + sync.Once +} + +func (f *TextFormatter) init(entry *Entry) { + if len(f.QuoteCharacter) == 0 { + f.QuoteCharacter = "\"" + } + if entry.Logger != nil { + f.isTerminal = IsTerminal(entry.Logger.Out) + } } func (f *TextFormatter) Format(entry *Entry) ([]byte, error) { var b *bytes.Buffer - var keys []string = make([]string, 0, len(entry.Data)) + keys := make([]string, 0, len(entry.Data)) for k := range entry.Data { keys = append(keys, k) } @@ -74,8 +89,9 @@ func (f *TextFormatter) Format(entry *Entry) ([]byte, error) { prefixFieldClashes(entry.Data) - isColorTerminal := isTerminal && (runtime.GOOS != "windows") - isColored := (f.ForceColors || isColorTerminal) && !f.DisableColors + f.Do(func() { f.init(entry) }) + + isColored := (f.ForceColors || f.isTerminal) && !f.DisableColors timestampFormat := f.TimestampFormat if timestampFormat == "" { @@ -115,8 +131,10 @@ func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []strin levelText := strings.ToUpper(entry.Level.String())[0:4] - if !f.FullTimestamp { - fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%04d] %-44s ", levelColor, levelText, miniTS(), entry.Message) + if f.DisableTimestamp { + fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m %-44s ", levelColor, levelText, entry.Message) + } else if !f.FullTimestamp { + fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%04d] %-44s ", levelColor, levelText, int(entry.Time.Sub(baseTimestamp)/time.Second), entry.Message) } else { fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %-44s ", levelColor, levelText, entry.Time.Format(timestampFormat), entry.Message) } @@ -127,7 +145,10 @@ func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []strin } } -func needsQuoting(text string) bool { +func (f *TextFormatter) needsQuoting(text string) bool { + if f.QuoteEmptyFields && len(text) == 0 { + return true + } for _, ch := range text { if !((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || @@ -150,19 +171,26 @@ func (f *TextFormatter) appendKeyValue(b *bytes.Buffer, key string, value interf func (f *TextFormatter) appendValue(b *bytes.Buffer, value interface{}) { switch value := value.(type) { case string: - if !needsQuoting(value) { + if !f.needsQuoting(value) { b.WriteString(value) } else { - fmt.Fprintf(b, "%q", value) + b.WriteString(f.quoteString(value)) } case error: errmsg := value.Error() - if !needsQuoting(errmsg) { + if !f.needsQuoting(errmsg) { b.WriteString(errmsg) } else { - fmt.Fprintf(b, "%q", errmsg) + b.WriteString(f.quoteString(errmsg)) } default: fmt.Fprint(b, value) } } + +func (f *TextFormatter) quoteString(v string) string { + escapedQuote := fmt.Sprintf("\\%s", f.QuoteCharacter) + escapedValue := strings.Replace(v, f.QuoteCharacter, escapedQuote, -1) + + return fmt.Sprintf("%s%v%s", f.QuoteCharacter, escapedValue, f.QuoteCharacter) +} diff --git a/components/engine/vendor/github.com/Sirupsen/logrus/writer.go b/components/engine/vendor/github.com/sirupsen/logrus/writer.go similarity index 54% rename from components/engine/vendor/github.com/Sirupsen/logrus/writer.go rename to components/engine/vendor/github.com/sirupsen/logrus/writer.go index f74d2aa5fc..7bdebedc60 100644 --- a/components/engine/vendor/github.com/Sirupsen/logrus/writer.go +++ b/components/engine/vendor/github.com/sirupsen/logrus/writer.go @@ -11,39 +11,48 @@ func (logger *Logger) Writer() *io.PipeWriter { } func (logger *Logger) WriterLevel(level Level) *io.PipeWriter { + return NewEntry(logger).WriterLevel(level) +} + +func (entry *Entry) Writer() *io.PipeWriter { + return entry.WriterLevel(InfoLevel) +} + +func (entry *Entry) WriterLevel(level Level) *io.PipeWriter { reader, writer := io.Pipe() var printFunc func(args ...interface{}) + switch level { case DebugLevel: - printFunc = logger.Debug + printFunc = entry.Debug case InfoLevel: - printFunc = logger.Info + printFunc = entry.Info case WarnLevel: - printFunc = logger.Warn + printFunc = entry.Warn case ErrorLevel: - printFunc = logger.Error + printFunc = entry.Error case FatalLevel: - printFunc = logger.Fatal + printFunc = entry.Fatal case PanicLevel: - printFunc = logger.Panic + printFunc = entry.Panic default: - printFunc = logger.Print + printFunc = entry.Print } - go logger.writerScanner(reader, printFunc) + go entry.writerScanner(reader, printFunc) runtime.SetFinalizer(writer, writerFinalizer) return writer } -func (logger *Logger) writerScanner(reader *io.PipeReader, printFunc func(args ...interface{})) { +func (entry *Entry) writerScanner(reader *io.PipeReader, printFunc func(args ...interface{})) { scanner := bufio.NewScanner(reader) for scanner.Scan() { printFunc(scanner.Text()) } if err := scanner.Err(); err != nil { - logger.Errorf("Error while reading from Writer: %s", err) + entry.Errorf("Error while reading from Writer: %s", err) } reader.Close() } diff --git a/components/engine/volume/drivers/adapter.go b/components/engine/volume/drivers/adapter.go index 304c81bc00..54d1ff2630 100644 --- a/components/engine/volume/drivers/adapter.go +++ b/components/engine/volume/drivers/adapter.go @@ -6,8 +6,8 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" "github.com/docker/docker/volume" + "github.com/sirupsen/logrus" ) var ( diff --git a/components/engine/volume/local/local.go b/components/engine/volume/local/local.go index 43ba1e1db7..329febb4d7 100644 --- a/components/engine/volume/local/local.go +++ b/components/engine/volume/local/local.go @@ -15,11 +15,11 @@ import ( "github.com/pkg/errors" - "github.com/Sirupsen/logrus" "github.com/docker/docker/api" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/mount" "github.com/docker/docker/volume" + "github.com/sirupsen/logrus" ) // VolumeDataPathName is the name of the directory where the volume data is stored. diff --git a/components/engine/volume/store/db.go b/components/engine/volume/store/db.go index c5fd1643f5..01f0abb238 100644 --- a/components/engine/volume/store/db.go +++ b/components/engine/volume/store/db.go @@ -3,9 +3,9 @@ package store import ( "encoding/json" - "github.com/Sirupsen/logrus" "github.com/boltdb/bolt" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) var volumeBucketName = []byte("volumes") diff --git a/components/engine/volume/store/restore.go b/components/engine/volume/store/restore.go index c0c5b519bc..e2a72a6551 100644 --- a/components/engine/volume/store/restore.go +++ b/components/engine/volume/store/restore.go @@ -3,10 +3,10 @@ package store import ( "sync" - "github.com/Sirupsen/logrus" "github.com/boltdb/bolt" "github.com/docker/docker/volume" "github.com/docker/docker/volume/drivers" + "github.com/sirupsen/logrus" ) // restore is called when a new volume store is created. diff --git a/components/engine/volume/store/store.go b/components/engine/volume/store/store.go index cded883e6e..b1742061bb 100644 --- a/components/engine/volume/store/store.go +++ b/components/engine/volume/store/store.go @@ -9,11 +9,11 @@ import ( "github.com/pkg/errors" - "github.com/Sirupsen/logrus" "github.com/boltdb/bolt" "github.com/docker/docker/pkg/locker" "github.com/docker/docker/volume" "github.com/docker/docker/volume/drivers" + "github.com/sirupsen/logrus" ) const ( diff --git a/components/engine/volume/validate.go b/components/engine/volume/validate.go index 42396a0dad..5de46198f6 100644 --- a/components/engine/volume/validate.go +++ b/components/engine/volume/validate.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" "os" - "path/filepath" + "runtime" "github.com/docker/docker/api/types/mount" ) @@ -12,8 +12,7 @@ import ( var errBindNotExist = errors.New("bind source path does not exist") type validateOpts struct { - skipBindSourceCheck bool - skipAbsolutePathCheck bool + skipBindSourceCheck bool } func validateMountConfig(mnt *mount.Mount, options ...func(*validateOpts)) error { @@ -30,10 +29,8 @@ func validateMountConfig(mnt *mount.Mount, options ...func(*validateOpts)) error return &errMountConfig{mnt, err} } - if !opts.skipAbsolutePathCheck { - if err := validateAbsolute(mnt.Target); err != nil { - return &errMountConfig{mnt, err} - } + if err := validateAbsolute(mnt.Target); err != nil { + return &errMountConfig{mnt, err} } switch mnt.Type { @@ -97,6 +94,31 @@ func validateMountConfig(mnt *mount.Mount, options ...func(*validateOpts)) error if _, err := ConvertTmpfsOptions(mnt.TmpfsOptions, mnt.ReadOnly); err != nil { return &errMountConfig{mnt, err} } + case mount.TypeNamedPipe: + if runtime.GOOS != "windows" { + return &errMountConfig{mnt, errors.New("named pipe bind mounts are not supported on this OS")} + } + + if len(mnt.Source) == 0 { + return &errMountConfig{mnt, errMissingField("Source")} + } + + if mnt.BindOptions != nil { + return &errMountConfig{mnt, errExtraField("BindOptions")} + } + + if mnt.ReadOnly { + return &errMountConfig{mnt, errExtraField("ReadOnly")} + } + + if detectMountType(mnt.Source) != mount.TypeNamedPipe { + return &errMountConfig{mnt, fmt.Errorf("'%s' is not a valid pipe path", mnt.Source)} + } + + if detectMountType(mnt.Target) != mount.TypeNamedPipe { + return &errMountConfig{mnt, fmt.Errorf("'%s' is not a valid pipe path", mnt.Target)} + } + default: return &errMountConfig{mnt, errors.New("mount type unknown")} } @@ -121,7 +143,7 @@ func errMissingField(name string) error { func validateAbsolute(p string) error { p = convertSlash(p) - if filepath.IsAbs(p) { + if isAbsPath(p) { return nil } return fmt.Errorf("invalid mount path: '%s' mount path must be absolute", p) diff --git a/components/engine/volume/volume.go b/components/engine/volume/volume.go index 8598d4cb8f..7e8d16cc68 100644 --- a/components/engine/volume/volume.go +++ b/components/engine/volume/volume.go @@ -3,7 +3,6 @@ package volume import ( "fmt" "os" - "path/filepath" "strings" "syscall" "time" @@ -284,12 +283,7 @@ func ParseMountRaw(raw, volumeDriver string) (*MountPoint, error) { return nil, errInvalidMode(mode) } - if filepath.IsAbs(spec.Source) { - spec.Type = mounttypes.TypeBind - } else { - spec.Type = mounttypes.TypeVolume - } - + spec.Type = detectMountType(spec.Source) spec.ReadOnly = !ReadWrite(mode) // cannot assume that if a volume driver is passed in that we should set it @@ -350,7 +344,7 @@ func ParseMountSpec(cfg mounttypes.Mount, options ...func(*validateOpts)) (*Moun mp.CopyData = false } } - case mounttypes.TypeBind: + case mounttypes.TypeBind, mounttypes.TypeNamedPipe: mp.Source = clean(convertSlash(cfg.Source)) if cfg.BindOptions != nil && len(cfg.BindOptions.Propagation) > 0 { mp.Propagation = cfg.BindOptions.Propagation diff --git a/components/engine/volume/volume_test.go b/components/engine/volume/volume_test.go index 5c3e0e381b..395f374ff0 100644 --- a/components/engine/volume/volume_test.go +++ b/components/engine/volume/volume_test.go @@ -143,6 +143,7 @@ func TestParseMountRaw(t *testing.T) { type testParseMountRaw struct { bind string driver string + expType mount.Type expDest string expSource string expName string @@ -155,28 +156,31 @@ func TestParseMountRawSplit(t *testing.T) { var cases []testParseMountRaw if runtime.GOOS == "windows" { cases = []testParseMountRaw{ - {`c:\:d:`, "local", `d:`, `c:\`, ``, "", true, false}, - {`c:\:d:\`, "local", `d:\`, `c:\`, ``, "", true, false}, - {`c:\:d:\:ro`, "local", `d:\`, `c:\`, ``, "", false, false}, - {`c:\:d:\:rw`, "local", `d:\`, `c:\`, ``, "", true, false}, - {`c:\:d:\:foo`, "local", `d:\`, `c:\`, ``, "", false, true}, - {`name:d::rw`, "local", `d:`, ``, `name`, "local", true, false}, - {`name:d:`, "local", `d:`, ``, `name`, "local", true, false}, - {`name:d::ro`, "local", `d:`, ``, `name`, "local", false, false}, - {`name:c:`, "", ``, ``, ``, "", true, true}, - {`driver/name:c:`, "", ``, ``, ``, "", true, true}, + {`c:\:d:`, "local", mount.TypeBind, `d:`, `c:\`, ``, "", true, false}, + {`c:\:d:\`, "local", mount.TypeBind, `d:\`, `c:\`, ``, "", true, false}, + {`c:\:d:\:ro`, "local", mount.TypeBind, `d:\`, `c:\`, ``, "", false, false}, + {`c:\:d:\:rw`, "local", mount.TypeBind, `d:\`, `c:\`, ``, "", true, false}, + {`c:\:d:\:foo`, "local", mount.TypeBind, `d:\`, `c:\`, ``, "", false, true}, + {`\\.\pipe\foo:\\.\pipe\bar`, "local", mount.TypeNamedPipe, `\\.\pipe\bar`, `\\.\pipe\foo`, "", "", true, false}, + {`\\.\pipe\foo:c:\foo\bar`, "local", mount.TypeNamedPipe, ``, ``, "", "", true, true}, + {`c:\foo\bar:\\.\pipe\foo`, "local", mount.TypeNamedPipe, ``, ``, "", "", true, true}, + {`name:d::rw`, "local", mount.TypeVolume, `d:`, ``, `name`, "local", true, false}, + {`name:d:`, "local", mount.TypeVolume, `d:`, ``, `name`, "local", true, false}, + {`name:d::ro`, "local", mount.TypeVolume, `d:`, ``, `name`, "local", false, false}, + {`name:c:`, "", mount.TypeVolume, ``, ``, ``, "", true, true}, + {`driver/name:c:`, "", mount.TypeVolume, ``, ``, ``, "", true, true}, } } else { cases = []testParseMountRaw{ - {"/tmp:/tmp1", "", "/tmp1", "/tmp", "", "", true, false}, - {"/tmp:/tmp2:ro", "", "/tmp2", "/tmp", "", "", false, false}, - {"/tmp:/tmp3:rw", "", "/tmp3", "/tmp", "", "", true, false}, - {"/tmp:/tmp4:foo", "", "", "", "", "", false, true}, - {"name:/named1", "", "/named1", "", "name", "", true, false}, - {"name:/named2", "external", "/named2", "", "name", "external", true, false}, - {"name:/named3:ro", "local", "/named3", "", "name", "local", false, false}, - {"local/name:/tmp:rw", "", "/tmp", "", "local/name", "", true, false}, - {"/tmp:tmp", "", "", "", "", "", true, true}, + {"/tmp:/tmp1", "", mount.TypeBind, "/tmp1", "/tmp", "", "", true, false}, + {"/tmp:/tmp2:ro", "", mount.TypeBind, "/tmp2", "/tmp", "", "", false, false}, + {"/tmp:/tmp3:rw", "", mount.TypeBind, "/tmp3", "/tmp", "", "", true, false}, + {"/tmp:/tmp4:foo", "", mount.TypeBind, "", "", "", "", false, true}, + {"name:/named1", "", mount.TypeVolume, "/named1", "", "name", "", true, false}, + {"name:/named2", "external", mount.TypeVolume, "/named2", "", "name", "external", true, false}, + {"name:/named3:ro", "local", mount.TypeVolume, "/named3", "", "name", "local", false, false}, + {"local/name:/tmp:rw", "", mount.TypeVolume, "/tmp", "", "local/name", "", true, false}, + {"/tmp:tmp", "", mount.TypeBind, "", "", "", "", true, true}, } } @@ -195,8 +199,12 @@ func TestParseMountRawSplit(t *testing.T) { continue } + if m.Type != c.expType { + t.Fatalf("Expected type '%s', was '%s', for spec '%s'", c.expType, m.Type, c.bind) + } + if m.Destination != c.expDest { - t.Fatalf("Expected destination '%s, was %s', for spec '%s'", c.expDest, m.Destination, c.bind) + t.Fatalf("Expected destination '%s', was '%s', for spec '%s'", c.expDest, m.Destination, c.bind) } if m.Source != c.expSource { diff --git a/components/engine/volume/volume_unix.go b/components/engine/volume/volume_unix.go index e35b70c03b..5dde82147f 100644 --- a/components/engine/volume/volume_unix.go +++ b/components/engine/volume/volume_unix.go @@ -124,7 +124,12 @@ func validateCopyMode(mode bool) error { } func convertSlash(p string) string { - return filepath.ToSlash(p) + return p +} + +// isAbsPath reports whether the path is absolute. +func isAbsPath(p string) bool { + return filepath.IsAbs(p) } func splitRawSpec(raw string) ([]string, error) { @@ -139,6 +144,13 @@ func splitRawSpec(raw string) ([]string, error) { return arr, nil } +func detectMountType(p string) mounttypes.Type { + if filepath.IsAbs(p) { + return mounttypes.TypeBind + } + return mounttypes.TypeVolume +} + func clean(p string) string { return filepath.Clean(p) } diff --git a/components/engine/volume/volume_windows.go b/components/engine/volume/volume_windows.go index 22f6fc7a14..d792b385f8 100644 --- a/components/engine/volume/volume_windows.go +++ b/components/engine/volume/volume_windows.go @@ -6,6 +6,8 @@ import ( "path/filepath" "regexp" "strings" + + mounttypes "github.com/docker/docker/api/types/mount" ) // read-write modes @@ -18,14 +20,7 @@ var roModes = map[string]bool{ "ro": true, } -var platformRawValidationOpts = []func(*validateOpts){ - // filepath.IsAbs is weird on Windows: - // `c:` is not considered an absolute path - // `c:\` is considered an absolute path - // In any case, the regex matching below ensures absolute paths - // TODO: consider this a bug with filepath.IsAbs (?) - func(o *validateOpts) { o.skipAbsolutePathCheck = true }, -} +var platformRawValidationOpts = []func(*validateOpts){} const ( // Spec should be in the format [source:]destination[:mode] @@ -49,11 +44,13 @@ const ( RXHostDir = `[a-z]:\\(?:[^\\/:*?"<>|\r\n]+\\?)*` // RXName is the second option of a source RXName = `[^\\/:*?"<>|\r\n]+` + // RXPipe is a named path pipe (starts with `\\.\pipe\`, possibly with / instead of \) + RXPipe = `[/\\]{2}.[/\\]pipe[/\\][^:*?"<>|\r\n]+` // RXReservedNames are reserved names not possible on Windows RXReservedNames = `(con)|(prn)|(nul)|(aux)|(com[1-9])|(lpt[1-9])` // RXSource is the combined possibilities for a source - RXSource = `((?P((` + RXHostDir + `)|(` + RXName + `))):)?` + RXSource = `((?P((` + RXHostDir + `)|(` + RXName + `)|(` + RXPipe + `))):)?` // Source. Can be either a host directory, a name, or omitted: // HostDir: @@ -69,8 +66,10 @@ const ( // - And then followed by a colon which is not in the capture group // - And can be optional + // RXDestinationDir is the file path option for the mount destination + RXDestinationDir = `([a-z]):((?:\\[^\\/:*?"<>\r\n]+)*\\?)` // RXDestination is the regex expression for the mount destination - RXDestination = `(?P([a-z]):((?:\\[^\\/:*?"<>\r\n]+)*\\?))` + RXDestination = `(?P(` + RXDestinationDir + `)|(` + RXPipe + `))` // Destination (aka container path): // - Variation on hostdir but can be a drive followed by colon as well // - If a path, must be absolute. Can include spaces @@ -140,6 +139,15 @@ func splitRawSpec(raw string) ([]string, error) { return split, nil } +func detectMountType(p string) mounttypes.Type { + if strings.HasPrefix(filepath.FromSlash(p), `\\.\pipe\`) { + return mounttypes.TypeNamedPipe + } else if filepath.IsAbs(p) { + return mounttypes.TypeBind + } + return mounttypes.TypeVolume +} + // IsVolumeNameValid checks a volume name in a platform specific manner. func IsVolumeNameValid(name string) (bool, error) { nameExp := regexp.MustCompile(`^` + RXName + `$`) @@ -186,8 +194,19 @@ func convertSlash(p string) string { return filepath.FromSlash(p) } +// isAbsPath returns whether a path is absolute for the purposes of mounting into a container +// (absolute paths, drive letter paths such as X:, and paths starting with `\\.\` to support named pipes). +func isAbsPath(p string) bool { + return filepath.IsAbs(p) || + strings.HasPrefix(p, `\\.\`) || + (len(p) == 2 && p[1] == ':' && ((p[0] >= 'a' && p[0] <= 'z') || (p[0] >= 'A' && p[0] <= 'Z'))) +} + +// Do not clean plain drive letters or paths starting with `\\.\`. +var cleanRegexp = regexp.MustCompile(`^([a-z]:|[/\\]{2}\.[/\\].*)$`) + func clean(p string) string { - if match, _ := regexp.MatchString("^[a-z]:$", p); match { + if match := cleanRegexp.MatchString(p); match { return p } return filepath.Clean(p)