diff --git a/components/engine/client/hijack.go b/components/engine/client/hijack.go index 628adfda65..5a6354561b 100644 --- a/components/engine/client/hijack.go +++ b/components/engine/client/hijack.go @@ -188,8 +188,14 @@ func (cli *Client) setupHijackConn(req *http.Request, proto string) (net.Conn, e c, br := clientconn.Hijack() if br.Buffered() > 0 { - // If there is buffered content, wrap the connection - c = &hijackedConn{c, br} + // If there is buffered content, wrap the connection. We return an + // object that implements CloseWrite iff the underlying connection + // implements it. + if _, ok := c.(types.CloseWriter); ok { + c = &hijackedConnCloseWriter{c, br} + } else { + c = &hijackedConn{c, br} + } } else { br.Reset(nil) } @@ -197,6 +203,10 @@ func (cli *Client) setupHijackConn(req *http.Request, proto string) (net.Conn, e return c, nil } +// hijackedConn wraps a net.Conn and is returned by setupHijackConn in the case +// that a) there was already buffered data in the http layer when Hijack() was +// called, and b) the underlying net.Conn does *not* implement CloseWrite(). +// hijackedConn does not implement CloseWrite() either. type hijackedConn struct { net.Conn r *bufio.Reader @@ -205,3 +215,16 @@ type hijackedConn struct { func (c *hijackedConn) Read(b []byte) (int, error) { return c.r.Read(b) } + +// hijackedConnCloseWriter is a hijackedConn which additionally implements +// CloseWrite(). It is returned by setupHijackConn in the case that a) there +// was already buffered data in the http layer when Hijack() was called, and b) +// the underlying net.Conn *does* implement CloseWrite(). +type hijackedConnCloseWriter hijackedConn + +var _ types.CloseWriter = &hijackedConnCloseWriter{} + +func (c *hijackedConnCloseWriter) CloseWrite() error { + conn := c.Conn.(types.CloseWriter) + return conn.CloseWrite() +} diff --git a/components/engine/container/archive.go b/components/engine/container/archive.go index 960d7bf615..ed72c4a405 100644 --- a/components/engine/container/archive.go +++ b/components/engine/container/archive.go @@ -6,6 +6,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/system" + "github.com/pkg/errors" ) // ResolvePath resolves the given path in the container to a resource on the @@ -13,6 +14,9 @@ import ( // the absolute path to the resource relative to the container's rootfs, and // an error if the path points to outside the container's rootfs. func (container *Container) ResolvePath(path string) (resolvedPath, absPath string, err error) { + if container.BaseFS == nil { + return "", "", errors.New("ResolvePath: BaseFS of container " + container.ID + " is unexpectedly nil") + } // Check if a drive letter supplied, it must be the system drive. No-op except on Windows path, err = system.CheckSystemDriveAndRemoveDriveLetter(path, container.BaseFS) if err != nil { @@ -45,6 +49,9 @@ func (container *Container) ResolvePath(path string) (resolvedPath, absPath stri // resolved to a path on the host corresponding to the given absolute path // inside the container. func (container *Container) StatPath(resolvedPath, absPath string) (stat *types.ContainerPathStat, err error) { + if container.BaseFS == nil { + return nil, errors.New("StatPath: BaseFS of container " + container.ID + " is unexpectedly nil") + } driver := container.BaseFS lstat, err := driver.Lstat(resolvedPath) diff --git a/components/engine/container/container.go b/components/engine/container/container.go index 461139b435..a076e80746 100644 --- a/components/engine/container/container.go +++ b/components/engine/container/container.go @@ -311,6 +311,9 @@ func (container *Container) SetupWorkingDirectory(rootIDs idtools.IDPair) error // symlinking to a different path) between using this method and using the // path. See symlink.FollowSymlinkInScope for more details. func (container *Container) GetResourcePath(path string) (string, error) { + if container.BaseFS == nil { + return "", errors.New("GetResourcePath: BaseFS of container " + container.ID + " is unexpectedly nil") + } // IMPORTANT - These are paths on the OS where the daemon is running, hence // any filepath operations must be done in an OS agnostic way. r, e := container.BaseFS.ResolveScopedPath(path, false) diff --git a/components/engine/daemon/export.go b/components/engine/daemon/export.go index 52c23a3c28..737e161edc 100644 --- a/components/engine/daemon/export.go +++ b/components/engine/daemon/export.go @@ -61,12 +61,12 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R } }() - _, err = rwlayer.Mount(container.GetMountLabel()) + basefs, err := rwlayer.Mount(container.GetMountLabel()) if err != nil { return nil, err } - archive, err := archivePath(container.BaseFS, container.BaseFS.Path(), &archive.TarOptions{ + archive, err := archivePath(basefs, basefs.Path(), &archive.TarOptions{ Compression: archive.Uncompressed, UIDMaps: daemon.idMappings.UIDs(), GIDMaps: daemon.idMappings.GIDs(), diff --git a/components/engine/daemon/oci_linux.go b/components/engine/daemon/oci_linux.go index 8d5eebb885..256e1c3e55 100644 --- a/components/engine/daemon/oci_linux.go +++ b/components/engine/daemon/oci_linux.go @@ -705,6 +705,9 @@ func setMounts(daemon *Daemon, s *specs.Spec, c *container.Container, mounts []c } func (daemon *Daemon) populateCommonSpec(s *specs.Spec, c *container.Container) error { + if c.BaseFS == nil { + return errors.New("populateCommonSpec: BaseFS of container " + c.ID + " is unexpectedly nil") + } linkedEnv, err := daemon.setupLinkedContainers(c) if err != nil { return err diff --git a/components/engine/daemon/oci_windows.go b/components/engine/daemon/oci_windows.go index e2e10f9999..6f65c15fb4 100644 --- a/components/engine/daemon/oci_windows.go +++ b/components/engine/daemon/oci_windows.go @@ -221,6 +221,9 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { // Sets the Windows-specific fields of the OCI spec func (daemon *Daemon) createSpecWindowsFields(c *container.Container, s *specs.Spec, isHyperV bool) error { + if c.BaseFS == nil { + return errors.New("createSpecWindowsFields: BaseFS of container " + c.ID + " is unexpectedly nil") + } if len(s.Process.Cwd) == 0 { // We default to C:\ to workaround the oddity of the case that the // default directory for cmd running as LocalSystem (or diff --git a/components/engine/hack/dockerfile/install/proxy.installer b/components/engine/hack/dockerfile/install/proxy.installer index ed9ea7cbce..bc2f92a63a 100755 --- a/components/engine/hack/dockerfile/install/proxy.installer +++ b/components/engine/hack/dockerfile/install/proxy.installer @@ -3,7 +3,7 @@ # LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When # updating the binary version, consider updating github.com/docker/libnetwork # in vendor.conf accordingly -LIBNETWORK_COMMIT=ed2130d117c11c542327b4d5216a5db36770bc65 +LIBNETWORK_COMMIT=1b91bc94094ecfdae41daa465cc0c8df37dfb3dd install_proxy() { case "$1" in diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index b769d4206a..42003972e0 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -26,7 +26,7 @@ github.com/google/go-cmp v0.1.0 github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5 github.com/imdario/mergo 0.2.1 -golang.org/x/sync de49d9dcd27d4f764488181bea099dfe6179bcf0 +golang.org/x/sync fd80eb99c8f653c847d294a001bdf2a3a6f768f5 github.com/moby/buildkit aaff9d591ef128560018433fe61beb802e149de8 github.com/tonistiigi/fsutil dea3a0da73aee887fc02142d995be764106ac5e2 @@ -34,7 +34,7 @@ github.com/tonistiigi/fsutil dea3a0da73aee887fc02142d995be764106ac5e2 #get libnetwork packages # When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly -github.com/docker/libnetwork 3aca383eb555510f3f17696f9505f7bfbd25f0e5 +github.com/docker/libnetwork 1b91bc94094ecfdae41daa465cc0c8df37dfb3dd github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec diff --git a/components/engine/vendor/github.com/docker/libnetwork/bitseq/sequence.go b/components/engine/vendor/github.com/docker/libnetwork/bitseq/sequence.go index a1a9810dc5..0069d495b7 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/bitseq/sequence.go +++ b/components/engine/vendor/github.com/docker/libnetwork/bitseq/sequence.go @@ -108,6 +108,12 @@ func (s *sequence) getAvailableBit(from uint64) (uint64, uint64, error) { bitSel >>= 1 bits++ } + // Check if the loop exited because it could not + // find any available bit int block starting from + // "from". Return invalid pos in that case. + if bitSel == 0 { + return invalidPos, invalidPos, ErrNoBitAvailable + } return bits / 8, bits % 8, nil } @@ -313,14 +319,14 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool, serial curr := uint64(0) h.Lock() store = h.store - h.Unlock() if store != nil { + h.Unlock() // The lock is acquired in the GetObject if err := store.GetObject(datastore.Key(h.Key()...), h); err != nil && err != datastore.ErrKeyNotFound { return ret, err } + h.Lock() // Acquire the lock back } - - h.Lock() + logrus.Debugf("Received set for ordinal %v, start %v, end %v, any %t, release %t, serial:%v curr:%d \n", ordinal, start, end, any, release, serial, h.curr) if serial { curr = h.curr } @@ -346,7 +352,6 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool, serial // Create a private copy of h and work on it nh := h.getCopy() - h.Unlock() nh.head = pushReservation(bytePos, bitPos, nh.head, release) if release { @@ -355,22 +360,25 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool, serial nh.unselected-- } - // Attempt to write private copy to store - if err := nh.writeToStore(); err != nil { - if _, ok := err.(types.RetryError); !ok { - return ret, fmt.Errorf("internal failure while setting the bit: %v", err) + if h.store != nil { + h.Unlock() + // Attempt to write private copy to store + if err := nh.writeToStore(); err != nil { + if _, ok := err.(types.RetryError); !ok { + return ret, fmt.Errorf("internal failure while setting the bit: %v", err) + } + // Retry + continue } - // Retry - continue + h.Lock() } // Previous atomic push was succesfull. Save private copy to local copy - h.Lock() - defer h.Unlock() h.unselected = nh.unselected h.head = nh.head h.dbExists = nh.dbExists h.dbIndex = nh.dbIndex + h.Unlock() return ret, nil } } @@ -498,24 +506,40 @@ func (h *Handle) UnmarshalJSON(data []byte) error { func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) { // Find sequence which contains the start bit byteStart, bitStart := ordinalToPos(start) - current, _, _, inBlockBytePos := findSequence(head, byteStart) - + current, _, precBlocks, inBlockBytePos := findSequence(head, byteStart) // Derive the this sequence offsets byteOffset := byteStart - inBlockBytePos bitOffset := inBlockBytePos*8 + bitStart - var firstOffset uint64 - if current == head { - firstOffset = byteOffset - } for current != nil { if current.block != blockMAX { + // If the current block is not full, check if there is any bit + // from the current bit in the current block. If not, before proceeding to the + // next block node, make sure we check for available bit in the next + // instance of the same block. Due to RLE same block signature will be + // compressed. + retry: bytePos, bitPos, err := current.getAvailableBit(bitOffset) + if err != nil && precBlocks == current.count-1 { + // This is the last instance in the same block node, + // so move to the next block. + goto next + } + if err != nil { + // There are some more instances of the same block, so add the offset + // and be optimistic that you will find the available bit in the next + // instance of the same block. + bitOffset = 0 + byteOffset += blockBytes + precBlocks++ + goto retry + } return byteOffset + bytePos, bitPos, err } // Moving to next block: Reset bit offset. + next: bitOffset = 0 - byteOffset += (current.count * blockBytes) - firstOffset - firstOffset = 0 + byteOffset += (current.count * blockBytes) - (precBlocks * blockBytes) + precBlocks = 0 current = current.next } return invalidPos, invalidPos, ErrNoBitAvailable @@ -526,19 +550,20 @@ func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) { // This can be further optimized to check from start till curr in case of a rollover func getAvailableFromCurrent(head *sequence, start, curr, end uint64) (uint64, uint64, error) { var bytePos, bitPos uint64 + var err error if curr != 0 && curr > start { - bytePos, bitPos, _ = getFirstAvailable(head, curr) + bytePos, bitPos, err = getFirstAvailable(head, curr) ret := posToOrdinal(bytePos, bitPos) - if end < ret { + if end < ret || err != nil { goto begin } return bytePos, bitPos, nil } begin: - bytePos, bitPos, _ = getFirstAvailable(head, start) + bytePos, bitPos, err = getFirstAvailable(head, start) ret := posToOrdinal(bytePos, bitPos) - if end < ret { + if end < ret || err != nil { return invalidPos, invalidPos, ErrNoBitAvailable } return bytePos, bitPos, nil diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows.go index eabf590a9d..5927fd8560 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows.go @@ -365,6 +365,22 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d config.HnsID = hnsresponse.Id genData[HNSID] = config.HnsID + + } else { + // Delete any stale HNS endpoints for this network. + if endpoints, err := hcsshim.HNSListEndpointRequest(); err == nil { + for _, ep := range endpoints { + if ep.VirtualNetwork == config.HnsID { + logrus.Infof("Removing stale HNS endpoint %s", ep.Id) + _, err = hcsshim.HNSEndpointRequest("DELETE", ep.Id, "") + if err != nil { + logrus.Warnf("Error removing HNS endpoint %s", ep.Id) + } + } + } + } else { + logrus.Warnf("Error listing HNS endpoints for network %s", config.HnsID) + } } n, err := d.getNetwork(id) diff --git a/components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go b/components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go index 5beb429dfc..d1a91c077f 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go @@ -402,15 +402,15 @@ func (a *Allocator) getPredefinedPool(as string, ipV6 bool) (*net.IPNet, error) continue } aSpace.Lock() - _, ok := aSpace.subnets[SubnetKey{AddressSpace: as, Subnet: nw.String()}] - aSpace.Unlock() - if ok { + if _, ok := aSpace.subnets[SubnetKey{AddressSpace: as, Subnet: nw.String()}]; ok { + aSpace.Unlock() continue } - if !aSpace.contains(as, nw) { + aSpace.Unlock() return nw, nil } + aSpace.Unlock() } return nil, types.NotFoundErrorf("could not find an available, non-overlapping IPv%d address pool among the defaults to assign to the network", v) diff --git a/components/engine/vendor/github.com/docker/libnetwork/vendor.conf b/components/engine/vendor/github.com/docker/libnetwork/vendor.conf index 73e2a6495b..3e7181564b 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/vendor.conf +++ b/components/engine/vendor/github.com/docker/libnetwork/vendor.conf @@ -50,5 +50,6 @@ github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25 golang.org/x/crypto 558b6879de74bc843225cde5686419267ff707ca golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 golang.org/x/sys 07c182904dbd53199946ba614a412c61d3c548f5 +golang.org/x/sync fd80eb99c8f653c847d294a001bdf2a3a6f768f5 github.com/pkg/errors 839d9e913e063e28dfd0e6c7b7512793e0a48be9 github.com/ishidawataru/sctp 07191f837fedd2f13d1ec7b5f885f0f3ec54b1cb diff --git a/components/engine/vendor/golang.org/x/sync/README b/components/engine/vendor/golang.org/x/sync/README deleted file mode 100644 index 59c9dcb498..0000000000 --- a/components/engine/vendor/golang.org/x/sync/README +++ /dev/null @@ -1,2 +0,0 @@ -This repository provides Go concurrency primitives in addition to the -ones provided by the language and "sync" and "sync/atomic" packages. diff --git a/components/engine/vendor/golang.org/x/sync/README.md b/components/engine/vendor/golang.org/x/sync/README.md new file mode 100644 index 0000000000..1f8436cc9c --- /dev/null +++ b/components/engine/vendor/golang.org/x/sync/README.md @@ -0,0 +1,18 @@ +# Go Sync + +This repository provides Go concurrency primitives in addition to the +ones provided by the language and "sync" and "sync/atomic" packages. + +## Download/Install + +The easiest way to install is to run `go get -u golang.org/x/sync`. You can +also manually git clone the repository to `$GOPATH/src/golang.org/x/sync`. + +## Report Issues / Send Patches + +This repository uses Gerrit for code changes. To learn how to submit changes to +this repository, see https://golang.org/doc/contribute.html. + +The main issue tracker for the sync repository is located at +https://github.com/golang/go/issues. Prefix your issue with "x/sync:" in the +subject line, so it is easy to find.