From 4a29152cb9065d25a23936d0f0d3fe299038d809 Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 9 Oct 2018 11:58:26 -0700 Subject: [PATCH 01/25] Windows:Allow process isolation Signed-off-by: John Howard (cherry picked from commit c907c2486c0850030f8ac1819ac8c87631472c68) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 7184074c0880c656be00645007588a00ec2266cd Component: engine --- components/engine/daemon/daemon_windows.go | 27 ++++++++++++++-------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/components/engine/daemon/daemon_windows.go b/components/engine/daemon/daemon_windows.go index 04d3de9924..e534d7eccc 100644 --- a/components/engine/daemon/daemon_windows.go +++ b/components/engine/daemon/daemon_windows.go @@ -193,12 +193,15 @@ func verifyContainerResources(resources *containertypes.Resources, isHyperv bool // hostconfig and config structures. func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes.HostConfig, config *containertypes.Config, update bool) ([]string, error) { warnings := []string{} - + osv := system.GetOSVersion() hyperv := daemon.runAsHyperVContainer(hostConfig) - if !hyperv && system.IsWindowsClient() && !system.IsIoTCore() { - // @engine maintainers. This block should not be removed. It partially enforces licensing - // restrictions on Windows. Ping @jhowardmsft if there are concerns or PRs to change this. - return warnings, fmt.Errorf("Windows client operating systems only support Hyper-V containers") + + // On RS5, we allow (but don't strictly support) process isolation on Client SKUs. + // Prior to RS5, we don't allow process isolation on Client SKUs. + // @engine maintainers. This block should not be removed. It partially enforces licensing + // restrictions on Windows. Ping @jhowardmsft if there are concerns or PRs to change this. + if !hyperv && system.IsWindowsClient() && osv.Build < 17763 { + return warnings, fmt.Errorf("Windows client operating systems earlier than version 1809 can only run Hyper-V containers") } w, err := verifyContainerResources(&hostConfig.Resources, hyperv) @@ -592,9 +595,12 @@ func (daemon *Daemon) stats(c *container.Container) (*types.StatsJSON, error) { // daemon to run in. This is only applicable on Windows func (daemon *Daemon) setDefaultIsolation() error { daemon.defaultIsolation = containertypes.Isolation("process") - // On client SKUs, default to Hyper-V. Note that IoT reports as a client SKU - // but it should not be treated as such. - if system.IsWindowsClient() && !system.IsIoTCore() { + osv := system.GetOSVersion() + + // On client SKUs, default to Hyper-V. @engine maintainers. This + // should not be removed. Ping @jhowardmsft is there are PRs to + // to change this. + if system.IsWindowsClient() { daemon.defaultIsolation = containertypes.Isolation("hyperv") } for _, option := range daemon.configStore.ExecOptions { @@ -613,10 +619,11 @@ func (daemon *Daemon) setDefaultIsolation() error { daemon.defaultIsolation = containertypes.Isolation("hyperv") } if containertypes.Isolation(val).IsProcess() { - if system.IsWindowsClient() && !system.IsIoTCore() { + if system.IsWindowsClient() && osv.Build < 17763 { + // On RS5, we allow (but don't strictly support) process isolation on Client SKUs. // @engine maintainers. This block should not be removed. It partially enforces licensing // restrictions on Windows. Ping @jhowardmsft if there are concerns or PRs to change this. - return fmt.Errorf("Windows client operating systems only support Hyper-V containers") + return fmt.Errorf("Windows client operating systems earlier than version 1809 can only run Hyper-V containers") } daemon.defaultIsolation = containertypes.Isolation("process") } From b1f3714ba101bce62ff1651a2bf4514e531ebde9 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 11 Oct 2018 12:44:43 +0200 Subject: [PATCH 02/25] Deprecate "devicemapper" storage driver, and add warning The `devicemapper` storage driver is deprecated in favor of `overlay2`, and will be removed in a future release. Users of the `devicemapper` storage driver are recommended to migrate to a different storage driver, such as `overlay2`, which is now the default storage driver. The `devicemapper` storage driver facilitates running Docker on older (3.x) kernels that have no support for other storage drivers (such as overlay2, or AUFS). Now that support for `overlay2` is added to all supported distros (as they are either on kernel 4.x, or have support for multiple lowerdirs backported), there is no reason to continue maintenance of the `devicemapper` storage driver. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 06fcabbaa0d4f25db0580d3604bbed708223fb06) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 734e7a8e55e41b064025273776dc5a76e45bf2e1 Component: engine --- .../engine/daemon/graphdriver/driver.go | 21 +++++++++++++++++++ components/engine/daemon/info.go | 4 ++++ 2 files changed, 25 insertions(+) diff --git a/components/engine/daemon/graphdriver/driver.go b/components/engine/daemon/graphdriver/driver.go index a9e1957393..baf349ac73 100644 --- a/components/engine/daemon/graphdriver/driver.go +++ b/components/engine/daemon/graphdriver/driver.go @@ -195,6 +195,7 @@ type Options struct { func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, error) { if name != "" { logrus.Debugf("[graphdriver] trying provided driver: %s", name) // so the logs show specified driver + logDeprecatedWarning(name) return GetDriver(name, pg, config) } @@ -232,6 +233,7 @@ func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, err } logrus.Infof("[graphdriver] using prior storage driver: %s", name) + logDeprecatedWarning(name) return driver, nil } } @@ -245,6 +247,7 @@ func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, err } return nil, err } + logDeprecatedWarning(name) return driver, nil } @@ -257,6 +260,7 @@ func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, err } return nil, err } + logDeprecatedWarning(name) return driver, nil } return nil, fmt.Errorf("No supported storage backend found") @@ -305,3 +309,20 @@ func isEmptyDir(name string) bool { } return false } + +// isDeprecated checks if a storage-driver is marked "deprecated" +func isDeprecated(name string) bool { + switch name { + // NOTE: when deprecating a driver, update daemon.fillDriverInfo() accordingly + case "devicemapper": + return true + } + return false +} + +// logDeprecatedWarning logs a warning if the given storage-driver is marked "deprecated" +func logDeprecatedWarning(name string) { + if isDeprecated(name) { + logrus.Warnf("[graphdriver] WARNING: the %s storage-driver is deprecated, and will be removed in a future release", name) + } +} diff --git a/components/engine/daemon/info.go b/components/engine/daemon/info.go index bf84342b54..a0fd01eaaf 100644 --- a/components/engine/daemon/info.go +++ b/components/engine/daemon/info.go @@ -131,6 +131,10 @@ func (daemon *Daemon) fillDriverInfo(v *types.Info) { if len(daemon.graphDrivers) > 1 { drivers += fmt.Sprintf(" (%s) ", os) } + switch gd { + case "devicemapper": + v.Warnings = append(v.Warnings, fmt.Sprintf("WARNING: the %s storage-driver is deprecated, and will be removed in a future release.", gd)) + } } drivers = strings.TrimSpace(drivers) From 09de879e7ab912e3efa5576d7df7760c230f47ca Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 11 Oct 2018 12:47:34 +0200 Subject: [PATCH 03/25] Deprecate legacy overlay storage driver, and add warning The `overlay` storage driver is deprecated in favor of the `overlay2` storage driver, which has all the benefits of `overlay`, without its limitations (excessive inode consumption). The legacy `overlay` storage driver will be removed in a future release. Users of the `overlay` storage driver should migrate to the `overlay2` storage driver. The legacy `overlay` storage driver allowed using overlayFS-backed filesystems on pre 4.x kernels. Now that all supported distributions are able to run `overlay2` (as they are either on kernel 4.x, or have support for multiple lowerdirs backported), there is no reason to keep maintaining the `overlay` storage driver. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 31be4e0ba11532e5d6df8401f5d459e292c58f36) Signed-off-by: Sebastiaan van Stijn Upstream-commit: c20e8dffbbb4dd328e4c00d18e04eecb80cf4d4e Component: engine --- components/engine/daemon/graphdriver/driver.go | 2 +- components/engine/daemon/info.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/daemon/graphdriver/driver.go b/components/engine/daemon/graphdriver/driver.go index baf349ac73..2db61aac49 100644 --- a/components/engine/daemon/graphdriver/driver.go +++ b/components/engine/daemon/graphdriver/driver.go @@ -314,7 +314,7 @@ func isEmptyDir(name string) bool { func isDeprecated(name string) bool { switch name { // NOTE: when deprecating a driver, update daemon.fillDriverInfo() accordingly - case "devicemapper": + case "devicemapper", "overlay": return true } return false diff --git a/components/engine/daemon/info.go b/components/engine/daemon/info.go index a0fd01eaaf..603474ab97 100644 --- a/components/engine/daemon/info.go +++ b/components/engine/daemon/info.go @@ -132,7 +132,7 @@ func (daemon *Daemon) fillDriverInfo(v *types.Info) { drivers += fmt.Sprintf(" (%s) ", os) } switch gd { - case "devicemapper": + case "devicemapper", "overlay": v.Warnings = append(v.Warnings, fmt.Sprintf("WARNING: the %s storage-driver is deprecated, and will be removed in a future release.", gd)) } } From cd9d72e1859c999fdbf9191f18d84dc69a9bd189 Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 2 Oct 2018 16:22:08 -0700 Subject: [PATCH 04/25] LCOW: ApplyDiff() use tar2ext4, not SVM Signed-off-by: John Howard This removes the need for an SVM in the LCOW driver to ApplyDiff. This change relates to a fix for https://github.com/moby/moby/issues/36353 However, it found another issue, tracked by https://github.com/moby/moby/issues/37955 (cherry picked from commit bde99960657818efabf313991867c42921882275) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 9cf6464b639aef09df9b16999cc340a1276d0bf8 Component: engine --- .../engine/daemon/graphdriver/lcow/lcow.go | 95 +++++++++++++++---- 1 file changed, 74 insertions(+), 21 deletions(-) diff --git a/components/engine/daemon/graphdriver/lcow/lcow.go b/components/engine/daemon/graphdriver/lcow/lcow.go index 8a89cc12d0..7004a72c10 100644 --- a/components/engine/daemon/graphdriver/lcow/lcow.go +++ b/components/engine/daemon/graphdriver/lcow/lcow.go @@ -71,20 +71,33 @@ import ( "time" "github.com/Microsoft/hcsshim" + "github.com/Microsoft/hcsshim/ext4/tar2ext4" "github.com/Microsoft/opengcs/client" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/containerfs" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/pkg/system" "github.com/sirupsen/logrus" ) +// noreexec controls reexec functionality. Off by default, on for debugging purposes. +var noreexec = false + // init registers this driver to the register. It gets initialised by the // function passed in the second parameter, implemented in this file. func init() { graphdriver.Register("lcow", InitDriver) + // DOCKER_LCOW_NOREEXEC allows for inline processing which makes + // debugging issues in the re-exec codepath significantly easier. + if os.Getenv("DOCKER_LCOW_NOREEXEC") != "" { + logrus.Warnf("LCOW Graphdriver is set to not re-exec. This is intended for debugging purposes only.") + noreexec = true + } else { + reexec.Register("docker-lcow-tar2ext4", tar2ext4Reexec) + } } const ( @@ -846,32 +859,72 @@ func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) { func (d *Driver) ApplyDiff(id, parent string, diff io.Reader) (int64, error) { logrus.Debugf("lcowdriver: applydiff: id %s", id) - svm, err := d.startServiceVMIfNotRunning(id, nil, fmt.Sprintf("applydiff %s", id)) + // Log failures here as it's undiagnosable sometimes, due to a possible panic. + // See https://github.com/moby/moby/issues/37955 for more information. + + dest := filepath.Join(d.dataRoot, id, layerFilename) + if !noreexec { + cmd := reexec.Command([]string{"docker-lcow-tar2ext4", dest}...) + stdout := bytes.NewBuffer(nil) + stderr := bytes.NewBuffer(nil) + cmd.Stdin = diff + cmd.Stdout = stdout + cmd.Stderr = stderr + + if err := cmd.Start(); err != nil { + logrus.Warnf("lcowdriver: applydiff: id %s failed to start re-exec: %s", id, err) + return 0, err + } + + if err := cmd.Wait(); err != nil { + logrus.Warnf("lcowdriver: applydiff: id %s failed %s", id, err) + return 0, fmt.Errorf("re-exec error: %v: stderr: %s", err, stderr) + } + return strconv.ParseInt(stdout.String(), 10, 64) + } + // The inline case + size, err := tar2ext4Actual(dest, diff) + if err != nil { + logrus.Warnf("lcowdriver: applydiff: id %s failed %s", id, err) + } + return size, err +} + +// tar2ext4Reexec is the re-exec entry point for writing a layer from a tar file +func tar2ext4Reexec() { + size, err := tar2ext4Actual(os.Args[1], os.Stdin) + if err != nil { + fmt.Fprint(os.Stderr, err) + os.Exit(1) + } + fmt.Fprint(os.Stdout, size) +} + +// tar2ext4Actual is the implementation of tar2ext to write a layer from a tar file. +// It can be called through re-exec (default), or inline for debugging. +func tar2ext4Actual(dest string, diff io.Reader) (int64, error) { + // maxDiskSize is not relating to the sandbox size - this is the + // maximum possible size a layer VHD generated can be from an EXT4 + // layout perspective. + const maxDiskSize = 128 * 1024 * 1024 * 1024 // 128GB + out, err := os.Create(dest) if err != nil { return 0, err } - defer d.terminateServiceVM(id, fmt.Sprintf("applydiff %s", id), false) - - logrus.Debugf("lcowdriver: applydiff: waiting for svm to finish booting") - err = svm.getStartError() + defer out.Close() + if err := tar2ext4.Convert( + diff, + out, + tar2ext4.AppendVhdFooter, + tar2ext4.ConvertWhiteout, + tar2ext4.MaximumDiskSize(maxDiskSize)); err != nil { + return 0, err + } + fi, err := os.Stat(dest) if err != nil { - return 0, fmt.Errorf("lcowdriver: applydiff: svm failed to boot: %s", err) - } - - // TODO @jhowardmsft - the retries are temporary to overcome platform reliability issues. - // Obviously this will be removed as platform bugs are fixed. - retries := 0 - for { - retries++ - size, err := svm.config.TarToVhd(filepath.Join(d.dataRoot, id, layerFilename), diff) - if err != nil { - if retries <= 10 { - continue - } - return 0, err - } - return size, err + return 0, err } + return fi.Size(), nil } // Changes produces a list of changes between the specified layer From 3d605988c4f24a8031050ad37b84174c065ecb4c Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 2 Oct 2018 15:25:41 -0700 Subject: [PATCH 05/25] Vendor Microsoft/hcsshim @ v0.7.9 Signed-off-by: John Howard (cherry picked from commit d03ab106624ebc30b69080d0092702ad1fe1285c) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 41f3cea42f2f45244051b7f829c0ef9c27383c26 Component: engine --- components/engine/vendor.conf | 2 +- .../ext4/internal/compactext4/compact.go | 1263 +++++++++++++++++ .../hcsshim/ext4/internal/format/format.go | 411 ++++++ .../hcsshim/ext4/tar2ext4/tar2ext4.go | 174 +++ .../hcsshim/ext4/tar2ext4/vhdfooter.go | 76 + .../github.com/Microsoft/hcsshim/version.go | 6 - 6 files changed, 1925 insertions(+), 7 deletions(-) create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/ext4/internal/compactext4/compact.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/ext4/internal/format/format.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/ext4/tar2ext4/tar2ext4.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/ext4/tar2ext4/vhdfooter.go delete mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/version.go diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index b822620bd3..73a11723a6 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -1,6 +1,6 @@ # the following lines are in sorted order, FYI github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109 -github.com/Microsoft/hcsshim v0.7.6 +github.com/Microsoft/hcsshim v0.7.9 github.com/Microsoft/go-winio v0.4.11 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/ext4/internal/compactext4/compact.go b/components/engine/vendor/github.com/Microsoft/hcsshim/ext4/internal/compactext4/compact.go new file mode 100644 index 0000000000..856173751f --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/ext4/internal/compactext4/compact.go @@ -0,0 +1,1263 @@ +package compactext4 + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "path" + "sort" + "strings" + "time" + + "github.com/Microsoft/hcsshim/ext4/internal/format" +) + +// Writer writes a compact ext4 file system. +type Writer struct { + f io.ReadWriteSeeker + bw *bufio.Writer + inodes []*inode + curName string + curInode *inode + pos int64 + dataWritten, dataMax int64 + err error + initialized bool + supportInlineData bool + maxDiskSize int64 + gdBlocks uint32 +} + +// Mode flags for Linux files. +const ( + S_IXOTH = format.S_IXOTH + S_IWOTH = format.S_IWOTH + S_IROTH = format.S_IROTH + S_IXGRP = format.S_IXGRP + S_IWGRP = format.S_IWGRP + S_IRGRP = format.S_IRGRP + S_IXUSR = format.S_IXUSR + S_IWUSR = format.S_IWUSR + S_IRUSR = format.S_IRUSR + S_ISVTX = format.S_ISVTX + S_ISGID = format.S_ISGID + S_ISUID = format.S_ISUID + S_IFIFO = format.S_IFIFO + S_IFCHR = format.S_IFCHR + S_IFDIR = format.S_IFDIR + S_IFBLK = format.S_IFBLK + S_IFREG = format.S_IFREG + S_IFLNK = format.S_IFLNK + S_IFSOCK = format.S_IFSOCK + + TypeMask = format.TypeMask +) + +type inode struct { + Size int64 + Atime, Ctime, Mtime, Crtime uint64 + Number format.InodeNumber + Mode uint16 + Uid, Gid uint32 + LinkCount uint32 + XattrBlock uint32 + BlockCount uint32 + Devmajor, Devminor uint32 + Flags format.InodeFlag + Data []byte + XattrInline []byte + Children directory +} + +func (node *inode) FileType() uint16 { + return node.Mode & format.TypeMask +} + +func (node *inode) IsDir() bool { + return node.FileType() == S_IFDIR +} + +// A File represents a file to be added to an ext4 file system. +type File struct { + Linkname string + Size int64 + Mode uint16 + Uid, Gid uint32 + Atime, Ctime, Mtime, Crtime time.Time + Devmajor, Devminor uint32 + Xattrs map[string][]byte +} + +const ( + inodeFirst = 11 + inodeLostAndFound = inodeFirst + + blockSize = 4096 + blocksPerGroup = blockSize * 8 + inodeSize = 256 + maxInodesPerGroup = blockSize * 8 // Limited by the inode bitmap + inodesPerGroupIncrement = blockSize / inodeSize + + defaultMaxDiskSize = 16 * 1024 * 1024 * 1024 // 16GB + maxMaxDiskSize = 16 * 1024 * 1024 * 1024 * 1024 // 16TB + + groupDescriptorSize = 32 // Use the small group descriptor + groupsPerDescriptorBlock = blockSize / groupDescriptorSize + + maxFileSize = 128 * 1024 * 1024 * 1024 // 128GB file size maximum for now + smallSymlinkSize = 59 // max symlink size that goes directly in the inode + maxBlocksPerExtent = 0x8000 // maximum number of blocks in an extent + inodeDataSize = 60 + inodeUsedSize = 152 // fields through CrtimeExtra + inodeExtraSize = inodeSize - inodeUsedSize + xattrInodeOverhead = 4 + 4 // magic number + empty next entry value + xattrBlockOverhead = 32 + 4 // header + empty next entry value + inlineDataXattrOverhead = xattrInodeOverhead + 16 + 4 // entry + "data" + inlineDataSize = inodeDataSize + inodeExtraSize - inlineDataXattrOverhead +) + +type exceededMaxSizeError struct { + Size int64 +} + +func (err exceededMaxSizeError) Error() string { + return fmt.Sprintf("disk exceeded maximum size of %d bytes", err.Size) +} + +var directoryEntrySize = binary.Size(format.DirectoryEntry{}) +var extraIsize = uint16(inodeUsedSize - 128) + +type directory map[string]*inode + +func splitFirst(p string) (string, string) { + n := strings.IndexByte(p, '/') + if n >= 0 { + return p[:n], p[n+1:] + } + return p, "" +} + +func (w *Writer) findPath(root *inode, p string) *inode { + inode := root + for inode != nil && len(p) != 0 { + name, rest := splitFirst(p) + p = rest + inode = inode.Children[name] + } + return inode +} + +func timeToFsTime(t time.Time) uint64 { + if t.IsZero() { + return 0 + } + s := t.Unix() + if s < -0x80000000 { + return 0x80000000 + } + if s > 0x37fffffff { + return 0x37fffffff + } + return uint64(s) | uint64(t.Nanosecond())<<34 +} + +func fsTimeToTime(t uint64) time.Time { + if t == 0 { + return time.Time{} + } + s := int64(t & 0x3ffffffff) + if s > 0x7fffffff && s < 0x100000000 { + s = int64(int32(uint32(s))) + } + return time.Unix(s, int64(t>>34)) +} + +func (w *Writer) getInode(i format.InodeNumber) *inode { + if i == 0 || int(i) > len(w.inodes) { + return nil + } + return w.inodes[i-1] +} + +var xattrPrefixes = []struct { + Index uint8 + Prefix string +}{ + {2, "system.posix_acl_access"}, + {3, "system.posix_acl_default"}, + {8, "system.richacl"}, + {7, "system."}, + {1, "user."}, + {4, "trusted."}, + {6, "security."}, +} + +func compressXattrName(name string) (uint8, string) { + for _, p := range xattrPrefixes { + if strings.HasPrefix(name, p.Prefix) { + return p.Index, name[len(p.Prefix):] + } + } + return 0, name +} + +func decompressXattrName(index uint8, name string) string { + for _, p := range xattrPrefixes { + if index == p.Index { + return p.Prefix + name + } + } + return name +} + +func hashXattrEntry(name string, value []byte) uint32 { + var hash uint32 + for i := 0; i < len(name); i++ { + hash = (hash << 5) ^ (hash >> 27) ^ uint32(name[i]) + } + + for i := 0; i+3 < len(value); i += 4 { + hash = (hash << 16) ^ (hash >> 16) ^ binary.LittleEndian.Uint32(value[i:i+4]) + } + + if len(value)%4 != 0 { + var last [4]byte + copy(last[:], value[len(value)&^3:]) + hash = (hash << 16) ^ (hash >> 16) ^ binary.LittleEndian.Uint32(last[:]) + } + return hash +} + +type xattr struct { + Name string + Index uint8 + Value []byte +} + +func (x *xattr) EntryLen() int { + return (len(x.Name)+3)&^3 + 16 +} + +func (x *xattr) ValueLen() int { + return (len(x.Value) + 3) &^ 3 +} + +type xattrState struct { + inode, block []xattr + inodeLeft, blockLeft int +} + +func (s *xattrState) init() { + s.inodeLeft = inodeExtraSize - xattrInodeOverhead + s.blockLeft = blockSize - xattrBlockOverhead +} + +func (s *xattrState) addXattr(name string, value []byte) bool { + index, name := compressXattrName(name) + x := xattr{ + Index: index, + Name: name, + Value: value, + } + length := x.EntryLen() + x.ValueLen() + if s.inodeLeft >= length { + s.inode = append(s.inode, x) + s.inodeLeft -= length + } else if s.blockLeft >= length { + s.block = append(s.block, x) + s.blockLeft -= length + } else { + return false + } + return true +} + +func putXattrs(xattrs []xattr, b []byte, offsetDelta uint16) { + offset := uint16(len(b)) + offsetDelta + eb := b + db := b + for _, xattr := range xattrs { + vl := xattr.ValueLen() + offset -= uint16(vl) + eb[0] = uint8(len(xattr.Name)) + eb[1] = xattr.Index + binary.LittleEndian.PutUint16(eb[2:], offset) + binary.LittleEndian.PutUint32(eb[8:], uint32(len(xattr.Value))) + binary.LittleEndian.PutUint32(eb[12:], hashXattrEntry(xattr.Name, xattr.Value)) + copy(eb[16:], xattr.Name) + eb = eb[xattr.EntryLen():] + copy(db[len(db)-vl:], xattr.Value) + db = db[:len(db)-vl] + } +} + +func getXattrs(b []byte, xattrs map[string][]byte, offsetDelta uint16) { + eb := b + for len(eb) != 0 { + nameLen := eb[0] + if nameLen == 0 { + break + } + index := eb[1] + offset := binary.LittleEndian.Uint16(eb[2:]) - offsetDelta + valueLen := binary.LittleEndian.Uint32(eb[8:]) + attr := xattr{ + Index: index, + Name: string(eb[16 : 16+nameLen]), + Value: b[offset : uint32(offset)+valueLen], + } + xattrs[decompressXattrName(index, attr.Name)] = attr.Value + eb = eb[attr.EntryLen():] + } +} + +func (w *Writer) writeXattrs(inode *inode, state *xattrState) error { + // Write the inline attributes. + if len(state.inode) != 0 { + inode.XattrInline = make([]byte, inodeExtraSize) + binary.LittleEndian.PutUint32(inode.XattrInline[0:], format.XAttrHeaderMagic) // Magic + putXattrs(state.inode, inode.XattrInline[4:], 0) + } + + // Write the block attributes. If there was previously an xattr block, then + // rewrite it even if it is now empty. + if len(state.block) != 0 || inode.XattrBlock != 0 { + sort.Slice(state.block, func(i, j int) bool { + return state.block[i].Index < state.block[j].Index || + len(state.block[i].Name) < len(state.block[j].Name) || + state.block[i].Name < state.block[j].Name + }) + + var b [blockSize]byte + binary.LittleEndian.PutUint32(b[0:], format.XAttrHeaderMagic) // Magic + binary.LittleEndian.PutUint32(b[4:], 1) // ReferenceCount + binary.LittleEndian.PutUint32(b[8:], 1) // Blocks + putXattrs(state.block, b[32:], 32) + + orig := w.block() + if inode.XattrBlock == 0 { + inode.XattrBlock = orig + inode.BlockCount++ + } else { + // Reuse the original block. + w.seekBlock(inode.XattrBlock) + defer w.seekBlock(orig) + } + + if _, err := w.write(b[:]); err != nil { + return err + } + } + + return nil +} + +func (w *Writer) write(b []byte) (int, error) { + if w.err != nil { + return 0, w.err + } + if w.pos+int64(len(b)) > w.maxDiskSize { + w.err = exceededMaxSizeError{w.maxDiskSize} + return 0, w.err + } + n, err := w.bw.Write(b) + w.pos += int64(n) + w.err = err + return n, err +} + +func (w *Writer) zero(n int64) (int64, error) { + if w.err != nil { + return 0, w.err + } + if w.pos+int64(n) > w.maxDiskSize { + w.err = exceededMaxSizeError{w.maxDiskSize} + return 0, w.err + } + n, err := io.CopyN(w.bw, zero, n) + w.pos += n + w.err = err + return n, err +} + +func (w *Writer) makeInode(f *File, node *inode) (*inode, error) { + mode := f.Mode + if mode&format.TypeMask == 0 { + mode |= format.S_IFREG + } + typ := mode & format.TypeMask + ino := format.InodeNumber(len(w.inodes) + 1) + if node == nil { + node = &inode{ + Number: ino, + } + if typ == S_IFDIR { + node.Children = make(directory) + node.LinkCount = 1 // A directory is linked to itself. + } + } else if node.Flags&format.InodeFlagExtents != 0 { + // Since we cannot deallocate or reuse blocks, don't allow updates that + // would invalidate data that has already been written. + return nil, errors.New("cannot overwrite file with non-inline data") + } + node.Mode = mode + node.Uid = f.Uid + node.Gid = f.Gid + node.Flags = format.InodeFlagHugeFile + node.Atime = timeToFsTime(f.Atime) + node.Ctime = timeToFsTime(f.Ctime) + node.Mtime = timeToFsTime(f.Mtime) + node.Crtime = timeToFsTime(f.Crtime) + node.Devmajor = f.Devmajor + node.Devminor = f.Devminor + node.Data = nil + node.XattrInline = nil + + var xstate xattrState + xstate.init() + + var size int64 + switch typ { + case format.S_IFREG: + size = f.Size + if f.Size > maxFileSize { + return nil, fmt.Errorf("file too big: %d > %d", f.Size, maxFileSize) + } + if f.Size <= inlineDataSize && w.supportInlineData { + node.Data = make([]byte, f.Size) + extra := 0 + if f.Size > inodeDataSize { + extra = int(f.Size - inodeDataSize) + } + // Add a dummy entry for now. + if !xstate.addXattr("system.data", node.Data[:extra]) { + panic("not enough room for inline data") + } + node.Flags |= format.InodeFlagInlineData + } + case format.S_IFLNK: + node.Mode |= 0777 // Symlinks should appear as ugw rwx + size = int64(len(f.Linkname)) + if size <= smallSymlinkSize { + // Special case: small symlinks go directly in Block without setting + // an inline data flag. + node.Data = make([]byte, len(f.Linkname)) + copy(node.Data, f.Linkname) + } + case format.S_IFDIR, format.S_IFIFO, format.S_IFSOCK, format.S_IFCHR, format.S_IFBLK: + default: + return nil, fmt.Errorf("invalid mode %o", mode) + } + + // Accumulate the extended attributes. + if len(f.Xattrs) != 0 { + // Sort the xattrs to avoid non-determinism in map iteration. + var xattrs []string + for name := range f.Xattrs { + xattrs = append(xattrs, name) + } + sort.Strings(xattrs) + for _, name := range xattrs { + if !xstate.addXattr(name, f.Xattrs[name]) { + return nil, fmt.Errorf("could not fit xattr %s", name) + } + } + } + + if err := w.writeXattrs(node, &xstate); err != nil { + return nil, err + } + + node.Size = size + if typ == format.S_IFLNK && size > smallSymlinkSize { + // Write the link name as data. + w.startInode("", node, size) + if _, err := w.Write([]byte(f.Linkname)); err != nil { + return nil, err + } + if err := w.finishInode(); err != nil { + return nil, err + } + } + + if int(node.Number-1) >= len(w.inodes) { + w.inodes = append(w.inodes, node) + } + return node, nil +} + +func (w *Writer) root() *inode { + return w.getInode(format.InodeRoot) +} + +func (w *Writer) lookup(name string, mustExist bool) (*inode, *inode, string, error) { + root := w.root() + cleanname := path.Clean("/" + name)[1:] + if len(cleanname) == 0 { + return root, root, "", nil + } + dirname, childname := path.Split(cleanname) + if len(childname) == 0 || len(childname) > 0xff { + return nil, nil, "", fmt.Errorf("%s: invalid name", name) + } + dir := w.findPath(root, dirname) + if dir == nil || !dir.IsDir() { + return nil, nil, "", fmt.Errorf("%s: path not found", name) + } + child := dir.Children[childname] + if child == nil && mustExist { + return nil, nil, "", fmt.Errorf("%s: file not found", name) + } + return dir, child, childname, nil +} + +// Create adds a file to the file system. +func (w *Writer) Create(name string, f *File) error { + if err := w.finishInode(); err != nil { + return err + } + dir, existing, childname, err := w.lookup(name, false) + if err != nil { + return err + } + var reuse *inode + if existing != nil { + if existing.IsDir() { + if f.Mode&TypeMask != S_IFDIR { + return fmt.Errorf("%s: cannot replace a directory with a file", name) + } + reuse = existing + } else if f.Mode&TypeMask == S_IFDIR { + return fmt.Errorf("%s: cannot replace a file with a directory", name) + } else if existing.LinkCount < 2 { + reuse = existing + } + } else { + if f.Mode&TypeMask == S_IFDIR && dir.LinkCount >= format.MaxLinks { + return fmt.Errorf("%s: exceeded parent directory maximum link count", name) + } + } + child, err := w.makeInode(f, reuse) + if err != nil { + return fmt.Errorf("%s: %s", name, err) + } + if existing != child { + if existing != nil { + existing.LinkCount-- + } + dir.Children[childname] = child + child.LinkCount++ + if child.IsDir() { + dir.LinkCount++ + } + } + if child.Mode&format.TypeMask == format.S_IFREG { + w.startInode(name, child, f.Size) + } + return nil +} + +// Link adds a hard link to the file system. +func (w *Writer) Link(oldname, newname string) error { + if err := w.finishInode(); err != nil { + return err + } + newdir, existing, newchildname, err := w.lookup(newname, false) + if err != nil { + return err + } + if existing != nil && (existing.IsDir() || existing.LinkCount < 2) { + return fmt.Errorf("%s: cannot orphan existing file or directory", newname) + } + + _, oldfile, _, err := w.lookup(oldname, true) + if err != nil { + return err + } + switch oldfile.Mode & format.TypeMask { + case format.S_IFDIR, format.S_IFLNK: + return fmt.Errorf("%s: link target cannot be a directory or symlink: %s", newname, oldname) + } + + if existing != oldfile && oldfile.LinkCount >= format.MaxLinks { + return fmt.Errorf("%s: link target would exceed maximum link count: %s", newname, oldname) + } + + if existing != nil { + existing.LinkCount-- + } + oldfile.LinkCount++ + newdir.Children[newchildname] = oldfile + return nil +} + +// Stat returns information about a file that has been written. +func (w *Writer) Stat(name string) (*File, error) { + if err := w.finishInode(); err != nil { + return nil, err + } + _, node, _, err := w.lookup(name, true) + if err != nil { + return nil, err + } + f := &File{ + Size: node.Size, + Mode: node.Mode, + Uid: node.Uid, + Gid: node.Gid, + Atime: fsTimeToTime(node.Atime), + Ctime: fsTimeToTime(node.Ctime), + Mtime: fsTimeToTime(node.Mtime), + Crtime: fsTimeToTime(node.Crtime), + Devmajor: node.Devmajor, + Devminor: node.Devminor, + } + f.Xattrs = make(map[string][]byte) + if node.XattrBlock != 0 || len(node.XattrInline) != 0 { + if node.XattrBlock != 0 { + orig := w.block() + w.seekBlock(node.XattrBlock) + if w.err != nil { + return nil, w.err + } + var b [blockSize]byte + _, err := w.f.Read(b[:]) + w.seekBlock(orig) + if err != nil { + return nil, err + } + getXattrs(b[32:], f.Xattrs, 32) + } + if len(node.XattrInline) != 0 { + getXattrs(node.XattrInline[4:], f.Xattrs, 0) + delete(f.Xattrs, "system.data") + } + } + if node.FileType() == S_IFLNK { + if node.Size > smallSymlinkSize { + return nil, fmt.Errorf("%s: cannot retrieve link information", name) + } + f.Linkname = string(node.Data) + } + return f, nil +} + +func (w *Writer) Write(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + if w.dataWritten+int64(len(b)) > w.dataMax { + return 0, fmt.Errorf("%s: wrote too much: %d > %d", w.curName, w.dataWritten+int64(len(b)), w.dataMax) + } + + if w.curInode.Flags&format.InodeFlagInlineData != 0 { + copy(w.curInode.Data[w.dataWritten:], b) + w.dataWritten += int64(len(b)) + return len(b), nil + } + + n, err := w.write(b) + w.dataWritten += int64(n) + return n, err +} + +func (w *Writer) startInode(name string, inode *inode, size int64) { + if w.curInode != nil { + panic("inode already in progress") + } + w.curName = name + w.curInode = inode + w.dataWritten = 0 + w.dataMax = size +} + +func (w *Writer) block() uint32 { + return uint32(w.pos / blockSize) +} + +func (w *Writer) seekBlock(block uint32) { + w.pos = int64(block) * blockSize + if w.err != nil { + return + } + w.err = w.bw.Flush() + if w.err != nil { + return + } + _, w.err = w.f.Seek(w.pos, io.SeekStart) +} + +func (w *Writer) nextBlock() { + if w.pos%blockSize != 0 { + // Simplify callers; w.err is updated on failure. + w.zero(blockSize - w.pos%blockSize) + } +} + +func fillExtents(hdr *format.ExtentHeader, extents []format.ExtentLeafNode, startBlock, offset, inodeSize uint32) { + *hdr = format.ExtentHeader{ + Magic: format.ExtentHeaderMagic, + Entries: uint16(len(extents)), + Max: uint16(cap(extents)), + Depth: 0, + } + for i := range extents { + block := offset + uint32(i)*maxBlocksPerExtent + length := inodeSize - block + if length > maxBlocksPerExtent { + length = maxBlocksPerExtent + } + start := startBlock + block + extents[i] = format.ExtentLeafNode{ + Block: block, + Length: uint16(length), + StartLow: start, + } + } +} + +func (w *Writer) writeExtents(inode *inode) error { + start := w.pos - w.dataWritten + if start%blockSize != 0 { + panic("unaligned") + } + w.nextBlock() + + startBlock := uint32(start / blockSize) + blocks := w.block() - startBlock + usedBlocks := blocks + + const extentNodeSize = 12 + const extentsPerBlock = blockSize/extentNodeSize - 1 + + extents := (blocks + maxBlocksPerExtent - 1) / maxBlocksPerExtent + var b bytes.Buffer + if extents == 0 { + // Nothing to do. + } else if extents <= 4 { + var root struct { + hdr format.ExtentHeader + extents [4]format.ExtentLeafNode + } + fillExtents(&root.hdr, root.extents[:extents], startBlock, 0, blocks) + binary.Write(&b, binary.LittleEndian, root) + } else if extents <= 4*extentsPerBlock { + const extentsPerBlock = blockSize/extentNodeSize - 1 + extentBlocks := extents/extentsPerBlock + 1 + usedBlocks += extentBlocks + var b2 bytes.Buffer + + var root struct { + hdr format.ExtentHeader + nodes [4]format.ExtentIndexNode + } + root.hdr = format.ExtentHeader{ + Magic: format.ExtentHeaderMagic, + Entries: uint16(extentBlocks), + Max: 4, + Depth: 1, + } + for i := uint32(0); i < extentBlocks; i++ { + root.nodes[i] = format.ExtentIndexNode{ + Block: i * extentsPerBlock * maxBlocksPerExtent, + LeafLow: w.block(), + } + extentsInBlock := extents - i*extentBlocks + if extentsInBlock > extentsPerBlock { + extentsInBlock = extentsPerBlock + } + + var node struct { + hdr format.ExtentHeader + extents [extentsPerBlock]format.ExtentLeafNode + _ [blockSize - (extentsPerBlock+1)*extentNodeSize]byte + } + + offset := i * extentsPerBlock * maxBlocksPerExtent + fillExtents(&node.hdr, node.extents[:extentsInBlock], startBlock+offset, offset, blocks) + binary.Write(&b2, binary.LittleEndian, node) + if _, err := w.write(b2.Next(blockSize)); err != nil { + return err + } + } + binary.Write(&b, binary.LittleEndian, root) + } else { + panic("file too big") + } + + inode.Data = b.Bytes() + inode.Flags |= format.InodeFlagExtents + inode.BlockCount += usedBlocks + return w.err +} + +func (w *Writer) finishInode() error { + if !w.initialized { + if err := w.init(); err != nil { + return err + } + } + if w.curInode == nil { + return nil + } + if w.dataWritten != w.dataMax { + return fmt.Errorf("did not write the right amount: %d != %d", w.dataWritten, w.dataMax) + } + + if w.dataMax != 0 && w.curInode.Flags&format.InodeFlagInlineData == 0 { + if err := w.writeExtents(w.curInode); err != nil { + return err + } + } + + w.dataWritten = 0 + w.dataMax = 0 + w.curInode = nil + return w.err +} + +func modeToFileType(mode uint16) format.FileType { + switch mode & format.TypeMask { + default: + return format.FileTypeUnknown + case format.S_IFREG: + return format.FileTypeRegular + case format.S_IFDIR: + return format.FileTypeDirectory + case format.S_IFCHR: + return format.FileTypeCharacter + case format.S_IFBLK: + return format.FileTypeBlock + case format.S_IFIFO: + return format.FileTypeFIFO + case format.S_IFSOCK: + return format.FileTypeSocket + case format.S_IFLNK: + return format.FileTypeSymbolicLink + } +} + +type constReader byte + +var zero = constReader(0) + +func (r constReader) Read(b []byte) (int, error) { + for i := range b { + b[i] = byte(r) + } + return len(b), nil +} + +func (w *Writer) writeDirectory(dir, parent *inode) error { + if err := w.finishInode(); err != nil { + return err + } + + // The size of the directory is not known yet. + w.startInode("", dir, 0x7fffffffffffffff) + left := blockSize + finishBlock := func() error { + if left > 0 { + e := format.DirectoryEntry{ + RecordLength: uint16(left), + } + err := binary.Write(w, binary.LittleEndian, e) + if err != nil { + return err + } + left -= directoryEntrySize + if left < 4 { + panic("not enough space for trailing entry") + } + _, err = io.CopyN(w, zero, int64(left)) + if err != nil { + return err + } + } + left = blockSize + return nil + } + + writeEntry := func(ino format.InodeNumber, name string) error { + rlb := directoryEntrySize + len(name) + rl := (rlb + 3) & ^3 + if left < rl+12 { + if err := finishBlock(); err != nil { + return err + } + } + e := format.DirectoryEntry{ + Inode: ino, + RecordLength: uint16(rl), + NameLength: uint8(len(name)), + FileType: modeToFileType(w.getInode(ino).Mode), + } + err := binary.Write(w, binary.LittleEndian, e) + if err != nil { + return err + } + _, err = w.Write([]byte(name)) + if err != nil { + return err + } + var zero [4]byte + _, err = w.Write(zero[:rl-rlb]) + if err != nil { + return err + } + left -= rl + return nil + } + if err := writeEntry(dir.Number, "."); err != nil { + return err + } + if err := writeEntry(parent.Number, ".."); err != nil { + return err + } + + // Follow e2fsck's convention and sort the children by inode number. + var children []string + for name := range dir.Children { + children = append(children, name) + } + sort.Slice(children, func(i, j int) bool { + return dir.Children[children[i]].Number < dir.Children[children[j]].Number + }) + + for _, name := range children { + child := dir.Children[name] + if err := writeEntry(child.Number, name); err != nil { + return err + } + } + if err := finishBlock(); err != nil { + return err + } + w.curInode.Size = w.dataWritten + w.dataMax = w.dataWritten + return nil +} + +func (w *Writer) writeDirectoryRecursive(dir, parent *inode) error { + if err := w.writeDirectory(dir, parent); err != nil { + return err + } + for _, child := range dir.Children { + if child.IsDir() { + if err := w.writeDirectoryRecursive(child, dir); err != nil { + return err + } + } + } + return nil +} + +func (w *Writer) writeInodeTable(tableSize uint32) error { + var b bytes.Buffer + for _, inode := range w.inodes { + if inode != nil { + binode := format.Inode{ + Mode: inode.Mode, + Uid: uint16(inode.Uid & 0xffff), + Gid: uint16(inode.Gid & 0xffff), + SizeLow: uint32(inode.Size & 0xffffffff), + SizeHigh: uint32(inode.Size >> 32), + LinksCount: uint16(inode.LinkCount), + BlocksLow: inode.BlockCount, + Flags: inode.Flags, + XattrBlockLow: inode.XattrBlock, + UidHigh: uint16(inode.Uid >> 16), + GidHigh: uint16(inode.Gid >> 16), + ExtraIsize: uint16(inodeUsedSize - 128), + Atime: uint32(inode.Atime), + AtimeExtra: uint32(inode.Atime >> 32), + Ctime: uint32(inode.Ctime), + CtimeExtra: uint32(inode.Ctime >> 32), + Mtime: uint32(inode.Mtime), + MtimeExtra: uint32(inode.Mtime >> 32), + Crtime: uint32(inode.Crtime), + CrtimeExtra: uint32(inode.Crtime >> 32), + } + switch inode.Mode & format.TypeMask { + case format.S_IFDIR, format.S_IFREG, format.S_IFLNK: + n := copy(binode.Block[:], inode.Data) + if n < len(inode.Data) { + // Rewrite the first xattr with the data. + xattr := [1]xattr{{ + Name: "data", + Index: 7, // "system." + Value: inode.Data[n:], + }} + putXattrs(xattr[:], inode.XattrInline[4:], 0) + } + case format.S_IFBLK, format.S_IFCHR: + dev := inode.Devminor&0xff | inode.Devmajor<<8 | (inode.Devminor&0xffffff00)<<12 + binary.LittleEndian.PutUint32(binode.Block[4:], dev) + } + + binary.Write(&b, binary.LittleEndian, binode) + b.Truncate(inodeUsedSize) + n, _ := b.Write(inode.XattrInline) + io.CopyN(&b, zero, int64(inodeExtraSize-n)) + } else { + io.CopyN(&b, zero, inodeSize) + } + if _, err := w.write(b.Next(inodeSize)); err != nil { + return err + } + } + rest := tableSize - uint32(len(w.inodes)*inodeSize) + if _, err := w.zero(int64(rest)); err != nil { + return err + } + return nil +} + +// NewWriter returns a Writer that writes an ext4 file system to the provided +// WriteSeeker. +func NewWriter(f io.ReadWriteSeeker, opts ...Option) *Writer { + w := &Writer{ + f: f, + bw: bufio.NewWriterSize(f, 65536*8), + maxDiskSize: defaultMaxDiskSize, + } + for _, opt := range opts { + opt(w) + } + return w +} + +// An Option provides extra options to NewWriter. +type Option func(*Writer) + +// InlineData instructs the Writer to write small files into the inode +// structures directly. This creates smaller images but currently is not +// compatible with DAX. +func InlineData(w *Writer) { + w.supportInlineData = true +} + +// MaximumDiskSize instructs the writer to reserve enough metadata space for the +// specified disk size. If not provided, then 16GB is the default. +func MaximumDiskSize(size int64) Option { + return func(w *Writer) { + if size < 0 || size > maxMaxDiskSize { + w.maxDiskSize = maxMaxDiskSize + } else if size == 0 { + w.maxDiskSize = defaultMaxDiskSize + } else { + w.maxDiskSize = (size + blockSize - 1) &^ (blockSize - 1) + } + } +} + +func (w *Writer) init() error { + // Skip the defective block inode. + w.inodes = make([]*inode, 1, 32) + // Create the root directory. + root, _ := w.makeInode(&File{ + Mode: format.S_IFDIR | 0755, + }, nil) + root.LinkCount++ // The root is linked to itself. + // Skip until the first non-reserved inode. + w.inodes = append(w.inodes, make([]*inode, inodeFirst-len(w.inodes)-1)...) + maxBlocks := (w.maxDiskSize-1)/blockSize + 1 + maxGroups := (maxBlocks-1)/blocksPerGroup + 1 + w.gdBlocks = uint32((maxGroups-1)/groupsPerDescriptorBlock + 1) + + // Skip past the superblock and block descriptor table. + w.seekBlock(1 + w.gdBlocks) + w.initialized = true + + // The lost+found directory is required to exist for e2fsck to pass. + if err := w.Create("lost+found", &File{Mode: format.S_IFDIR | 0700}); err != nil { + return err + } + return w.err +} + +func groupCount(blocks uint32, inodes uint32, inodesPerGroup uint32) uint32 { + inodeBlocksPerGroup := inodesPerGroup * inodeSize / blockSize + dataBlocksPerGroup := blocksPerGroup - inodeBlocksPerGroup - 2 // save room for the bitmaps + + // Increase the block count to ensure there are enough groups for all the + // inodes. + minBlocks := (inodes-1)/inodesPerGroup*dataBlocksPerGroup + 1 + if blocks < minBlocks { + blocks = minBlocks + } + + return (blocks + dataBlocksPerGroup - 1) / dataBlocksPerGroup +} + +func bestGroupCount(blocks uint32, inodes uint32) (groups uint32, inodesPerGroup uint32) { + groups = 0xffffffff + for ipg := uint32(inodesPerGroupIncrement); ipg <= maxInodesPerGroup; ipg += inodesPerGroupIncrement { + g := groupCount(blocks, inodes, ipg) + if g < groups { + groups = g + inodesPerGroup = ipg + } + } + return +} + +func (w *Writer) Close() error { + if err := w.finishInode(); err != nil { + return err + } + root := w.root() + if err := w.writeDirectoryRecursive(root, root); err != nil { + return err + } + // Finish the last inode (probably a directory). + if err := w.finishInode(); err != nil { + return err + } + + // Write the inode table + inodeTableOffset := w.block() + groups, inodesPerGroup := bestGroupCount(inodeTableOffset, uint32(len(w.inodes))) + err := w.writeInodeTable(groups * inodesPerGroup * inodeSize) + if err != nil { + return err + } + + // Write the bitmaps. + bitmapOffset := w.block() + bitmapSize := groups * 2 + validDataSize := bitmapOffset + bitmapSize + diskSize := validDataSize + minSize := (groups-1)*blocksPerGroup + 1 + if diskSize < minSize { + diskSize = minSize + } + + usedGdBlocks := (groups-1)/groupDescriptorSize + 1 + if usedGdBlocks > w.gdBlocks { + return exceededMaxSizeError{w.maxDiskSize} + } + + gds := make([]format.GroupDescriptor, w.gdBlocks*groupsPerDescriptorBlock) + inodeTableSizePerGroup := inodesPerGroup * inodeSize / blockSize + var totalUsedBlocks, totalUsedInodes uint32 + for g := uint32(0); g < groups; g++ { + var b [blockSize * 2]byte + var dirCount, usedInodeCount, usedBlockCount uint16 + + // Block bitmap + if (g+1)*blocksPerGroup <= validDataSize { + // This group is fully allocated. + for j := range b[:blockSize] { + b[j] = 0xff + } + usedBlockCount = blocksPerGroup + } else if g*blocksPerGroup < validDataSize { + for j := uint32(0); j < validDataSize-g*blocksPerGroup; j++ { + b[j/8] |= 1 << (j % 8) + usedBlockCount++ + } + } + if g == 0 { + // Unused group descriptor blocks should be cleared. + for j := 1 + usedGdBlocks; j < 1+w.gdBlocks; j++ { + b[j/8] &^= 1 << (j % 8) + usedBlockCount-- + } + } + if g == groups-1 && diskSize%blocksPerGroup != 0 { + // Blocks that aren't present in the disk should be marked as + // allocated. + for j := diskSize % blocksPerGroup; j < blocksPerGroup; j++ { + b[j/8] |= 1 << (j % 8) + usedBlockCount++ + } + } + // Inode bitmap + for j := uint32(0); j < inodesPerGroup; j++ { + ino := format.InodeNumber(1 + g*inodesPerGroup + j) + inode := w.getInode(ino) + if ino < inodeFirst || inode != nil { + b[blockSize+j/8] |= 1 << (j % 8) + usedInodeCount++ + } + if inode != nil && inode.Mode&format.TypeMask == format.S_IFDIR { + dirCount++ + } + } + _, err := w.write(b[:]) + if err != nil { + return err + } + gds[g] = format.GroupDescriptor{ + BlockBitmapLow: bitmapOffset + 2*g, + InodeBitmapLow: bitmapOffset + 2*g + 1, + InodeTableLow: inodeTableOffset + g*inodeTableSizePerGroup, + UsedDirsCountLow: dirCount, + FreeInodesCountLow: uint16(inodesPerGroup) - usedInodeCount, + FreeBlocksCountLow: blocksPerGroup - usedBlockCount, + } + + totalUsedBlocks += uint32(usedBlockCount) + totalUsedInodes += uint32(usedInodeCount) + } + + // Zero up to the disk size. + _, err = w.zero(int64(diskSize-bitmapOffset-bitmapSize) * blockSize) + if err != nil { + return err + } + + // Write the block descriptors + w.seekBlock(1) + if w.err != nil { + return w.err + } + err = binary.Write(w.bw, binary.LittleEndian, gds) + if err != nil { + return err + } + + // Write the super block + var blk [blockSize]byte + b := bytes.NewBuffer(blk[:1024]) + sb := &format.SuperBlock{ + InodesCount: inodesPerGroup * groups, + BlocksCountLow: diskSize, + FreeBlocksCountLow: blocksPerGroup*groups - totalUsedBlocks, + FreeInodesCount: inodesPerGroup*groups - totalUsedInodes, + FirstDataBlock: 0, + LogBlockSize: 2, // 2^(10 + 2) + LogClusterSize: 2, + BlocksPerGroup: blocksPerGroup, + ClustersPerGroup: blocksPerGroup, + InodesPerGroup: inodesPerGroup, + Magic: format.SuperBlockMagic, + State: 1, // cleanly unmounted + Errors: 1, // continue on error? + CreatorOS: 0, // Linux + RevisionLevel: 1, // dynamic inode sizes + FirstInode: inodeFirst, + LpfInode: inodeLostAndFound, + InodeSize: inodeSize, + FeatureCompat: format.CompatSparseSuper2 | format.CompatExtAttr, + FeatureIncompat: format.IncompatFiletype | format.IncompatExtents | format.IncompatFlexBg, + FeatureRoCompat: format.RoCompatLargeFile | format.RoCompatHugeFile | format.RoCompatExtraIsize | format.RoCompatReadonly, + MinExtraIsize: extraIsize, + WantExtraIsize: extraIsize, + LogGroupsPerFlex: 31, + } + if w.supportInlineData { + sb.FeatureIncompat |= format.IncompatInlineData + } + binary.Write(b, binary.LittleEndian, sb) + w.seekBlock(0) + if _, err := w.write(blk[:]); err != nil { + return err + } + w.seekBlock(diskSize) + return w.err +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/ext4/internal/format/format.go b/components/engine/vendor/github.com/Microsoft/hcsshim/ext4/internal/format/format.go new file mode 100644 index 0000000000..9dc4c4e164 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/ext4/internal/format/format.go @@ -0,0 +1,411 @@ +package format + +type SuperBlock struct { + InodesCount uint32 + BlocksCountLow uint32 + RootBlocksCountLow uint32 + FreeBlocksCountLow uint32 + FreeInodesCount uint32 + FirstDataBlock uint32 + LogBlockSize uint32 + LogClusterSize uint32 + BlocksPerGroup uint32 + ClustersPerGroup uint32 + InodesPerGroup uint32 + Mtime uint32 + Wtime uint32 + MountCount uint16 + MaxMountCount uint16 + Magic uint16 + State uint16 + Errors uint16 + MinorRevisionLevel uint16 + LastCheck uint32 + CheckInterval uint32 + CreatorOS uint32 + RevisionLevel uint32 + DefaultReservedUid uint16 + DefaultReservedGid uint16 + FirstInode uint32 + InodeSize uint16 + BlockGroupNr uint16 + FeatureCompat CompatFeature + FeatureIncompat IncompatFeature + FeatureRoCompat RoCompatFeature + UUID [16]uint8 + VolumeName [16]byte + LastMounted [64]byte + AlgorithmUsageBitmap uint32 + PreallocBlocks uint8 + PreallocDirBlocks uint8 + ReservedGdtBlocks uint16 + JournalUUID [16]uint8 + JournalInum uint32 + JournalDev uint32 + LastOrphan uint32 + HashSeed [4]uint32 + DefHashVersion uint8 + JournalBackupType uint8 + DescSize uint16 + DefaultMountOpts uint32 + FirstMetaBg uint32 + MkfsTime uint32 + JournalBlocks [17]uint32 + BlocksCountHigh uint32 + RBlocksCountHigh uint32 + FreeBlocksCountHigh uint32 + MinExtraIsize uint16 + WantExtraIsize uint16 + Flags uint32 + RaidStride uint16 + MmpInterval uint16 + MmpBlock uint64 + RaidStripeWidth uint32 + LogGroupsPerFlex uint8 + ChecksumType uint8 + ReservedPad uint16 + KbytesWritten uint64 + SnapshotInum uint32 + SnapshotID uint32 + SnapshotRBlocksCount uint64 + SnapshotList uint32 + ErrorCount uint32 + FirstErrorTime uint32 + FirstErrorInode uint32 + FirstErrorBlock uint64 + FirstErrorFunc [32]uint8 + FirstErrorLine uint32 + LastErrorTime uint32 + LastErrorInode uint32 + LastErrorLine uint32 + LastErrorBlock uint64 + LastErrorFunc [32]uint8 + MountOpts [64]uint8 + UserQuotaInum uint32 + GroupQuotaInum uint32 + OverheadBlocks uint32 + BackupBgs [2]uint32 + EncryptAlgos [4]uint8 + EncryptPwSalt [16]uint8 + LpfInode uint32 + ProjectQuotaInum uint32 + ChecksumSeed uint32 + WtimeHigh uint8 + MtimeHigh uint8 + MkfsTimeHigh uint8 + LastcheckHigh uint8 + FirstErrorTimeHigh uint8 + LastErrorTimeHigh uint8 + Pad [2]uint8 + Reserved [96]uint32 + Checksum uint32 +} + +const SuperBlockMagic uint16 = 0xef53 + +type CompatFeature uint32 +type IncompatFeature uint32 +type RoCompatFeature uint32 + +const ( + CompatDirPrealloc CompatFeature = 0x1 + CompatImagicInodes CompatFeature = 0x2 + CompatHasJournal CompatFeature = 0x4 + CompatExtAttr CompatFeature = 0x8 + CompatResizeInode CompatFeature = 0x10 + CompatDirIndex CompatFeature = 0x20 + CompatLazyBg CompatFeature = 0x40 + CompatExcludeInode CompatFeature = 0x80 + CompatExcludeBitmap CompatFeature = 0x100 + CompatSparseSuper2 CompatFeature = 0x200 + + IncompatCompression IncompatFeature = 0x1 + IncompatFiletype IncompatFeature = 0x2 + IncompatRecover IncompatFeature = 0x4 + IncompatJournalDev IncompatFeature = 0x8 + IncompatMetaBg IncompatFeature = 0x10 + IncompatExtents IncompatFeature = 0x40 + Incompat_64Bit IncompatFeature = 0x80 + IncompatMmp IncompatFeature = 0x100 + IncompatFlexBg IncompatFeature = 0x200 + IncompatEaInode IncompatFeature = 0x400 + IncompatDirdata IncompatFeature = 0x1000 + IncompatCsumSeed IncompatFeature = 0x2000 + IncompatLargedir IncompatFeature = 0x4000 + IncompatInlineData IncompatFeature = 0x8000 + IncompatEncrypt IncompatFeature = 0x10000 + + RoCompatSparseSuper RoCompatFeature = 0x1 + RoCompatLargeFile RoCompatFeature = 0x2 + RoCompatBtreeDir RoCompatFeature = 0x4 + RoCompatHugeFile RoCompatFeature = 0x8 + RoCompatGdtCsum RoCompatFeature = 0x10 + RoCompatDirNlink RoCompatFeature = 0x20 + RoCompatExtraIsize RoCompatFeature = 0x40 + RoCompatHasSnapshot RoCompatFeature = 0x80 + RoCompatQuota RoCompatFeature = 0x100 + RoCompatBigalloc RoCompatFeature = 0x200 + RoCompatMetadataCsum RoCompatFeature = 0x400 + RoCompatReplica RoCompatFeature = 0x800 + RoCompatReadonly RoCompatFeature = 0x1000 + RoCompatProject RoCompatFeature = 0x2000 +) + +type BlockGroupFlag uint16 + +const ( + BlockGroupInodeUninit BlockGroupFlag = 0x1 + BlockGroupBlockUninit BlockGroupFlag = 0x2 + BlockGroupInodeZeroed BlockGroupFlag = 0x4 +) + +type GroupDescriptor struct { + BlockBitmapLow uint32 + InodeBitmapLow uint32 + InodeTableLow uint32 + FreeBlocksCountLow uint16 + FreeInodesCountLow uint16 + UsedDirsCountLow uint16 + Flags BlockGroupFlag + ExcludeBitmapLow uint32 + BlockBitmapCsumLow uint16 + InodeBitmapCsumLow uint16 + ItableUnusedLow uint16 + Checksum uint16 +} + +type GroupDescriptor64 struct { + GroupDescriptor + BlockBitmapHigh uint32 + InodeBitmapHigh uint32 + InodeTableHigh uint32 + FreeBlocksCountHigh uint16 + FreeInodesCountHigh uint16 + UsedDirsCountHigh uint16 + ItableUnusedHigh uint16 + ExcludeBitmapHigh uint32 + BlockBitmapCsumHigh uint16 + InodeBitmapCsumHigh uint16 + Reserved uint32 +} + +const ( + S_IXOTH = 0x1 + S_IWOTH = 0x2 + S_IROTH = 0x4 + S_IXGRP = 0x8 + S_IWGRP = 0x10 + S_IRGRP = 0x20 + S_IXUSR = 0x40 + S_IWUSR = 0x80 + S_IRUSR = 0x100 + S_ISVTX = 0x200 + S_ISGID = 0x400 + S_ISUID = 0x800 + S_IFIFO = 0x1000 + S_IFCHR = 0x2000 + S_IFDIR = 0x4000 + S_IFBLK = 0x6000 + S_IFREG = 0x8000 + S_IFLNK = 0xA000 + S_IFSOCK = 0xC000 + + TypeMask uint16 = 0xF000 +) + +type InodeNumber uint32 + +const ( + InodeRoot = 2 +) + +type Inode struct { + Mode uint16 + Uid uint16 + SizeLow uint32 + Atime uint32 + Ctime uint32 + Mtime uint32 + Dtime uint32 + Gid uint16 + LinksCount uint16 + BlocksLow uint32 + Flags InodeFlag + Version uint32 + Block [60]byte + Generation uint32 + XattrBlockLow uint32 + SizeHigh uint32 + ObsoleteFragmentAddr uint32 + BlocksHigh uint16 + XattrBlockHigh uint16 + UidHigh uint16 + GidHigh uint16 + ChecksumLow uint16 + Reserved uint16 + ExtraIsize uint16 + ChecksumHigh uint16 + CtimeExtra uint32 + MtimeExtra uint32 + AtimeExtra uint32 + Crtime uint32 + CrtimeExtra uint32 + VersionHigh uint32 + Projid uint32 +} + +type InodeFlag uint32 + +const ( + InodeFlagSecRm InodeFlag = 0x1 + InodeFlagUnRm InodeFlag = 0x2 + InodeFlagCompressed InodeFlag = 0x4 + InodeFlagSync InodeFlag = 0x8 + InodeFlagImmutable InodeFlag = 0x10 + InodeFlagAppend InodeFlag = 0x20 + InodeFlagNoDump InodeFlag = 0x40 + InodeFlagNoAtime InodeFlag = 0x80 + InodeFlagDirtyCompressed InodeFlag = 0x100 + InodeFlagCompressedClusters InodeFlag = 0x200 + InodeFlagNoCompress InodeFlag = 0x400 + InodeFlagEncrypted InodeFlag = 0x800 + InodeFlagHashedIndex InodeFlag = 0x1000 + InodeFlagMagic InodeFlag = 0x2000 + InodeFlagJournalData InodeFlag = 0x4000 + InodeFlagNoTail InodeFlag = 0x8000 + InodeFlagDirSync InodeFlag = 0x10000 + InodeFlagTopDir InodeFlag = 0x20000 + InodeFlagHugeFile InodeFlag = 0x40000 + InodeFlagExtents InodeFlag = 0x80000 + InodeFlagEaInode InodeFlag = 0x200000 + InodeFlagEOFBlocks InodeFlag = 0x400000 + InodeFlagSnapfile InodeFlag = 0x01000000 + InodeFlagSnapfileDeleted InodeFlag = 0x04000000 + InodeFlagSnapfileShrunk InodeFlag = 0x08000000 + InodeFlagInlineData InodeFlag = 0x10000000 + InodeFlagProjectIDInherit InodeFlag = 0x20000000 + InodeFlagReserved InodeFlag = 0x80000000 +) + +const ( + MaxLinks = 65000 +) + +type ExtentHeader struct { + Magic uint16 + Entries uint16 + Max uint16 + Depth uint16 + Generation uint32 +} + +const ExtentHeaderMagic uint16 = 0xf30a + +type ExtentIndexNode struct { + Block uint32 + LeafLow uint32 + LeafHigh uint16 + Unused uint16 +} + +type ExtentLeafNode struct { + Block uint32 + Length uint16 + StartHigh uint16 + StartLow uint32 +} + +type ExtentTail struct { + Checksum uint32 +} + +type DirectoryEntry struct { + Inode InodeNumber + RecordLength uint16 + NameLength uint8 + FileType FileType + //Name []byte +} + +type FileType uint8 + +const ( + FileTypeUnknown FileType = 0x0 + FileTypeRegular FileType = 0x1 + FileTypeDirectory FileType = 0x2 + FileTypeCharacter FileType = 0x3 + FileTypeBlock FileType = 0x4 + FileTypeFIFO FileType = 0x5 + FileTypeSocket FileType = 0x6 + FileTypeSymbolicLink FileType = 0x7 +) + +type DirectoryEntryTail struct { + ReservedZero1 uint32 + RecordLength uint16 + ReservedZero2 uint8 + FileType uint8 + Checksum uint32 +} + +type DirectoryTreeRoot struct { + Dot DirectoryEntry + DotName [4]byte + DotDot DirectoryEntry + DotDotName [4]byte + ReservedZero uint32 + HashVersion uint8 + InfoLength uint8 + IndirectLevels uint8 + UnusedFlags uint8 + Limit uint16 + Count uint16 + Block uint32 + //Entries []DirectoryTreeEntry +} + +type DirectoryTreeNode struct { + FakeInode uint32 + FakeRecordLength uint16 + NameLength uint8 + FileType uint8 + Limit uint16 + Count uint16 + Block uint32 + //Entries []DirectoryTreeEntry +} + +type DirectoryTreeEntry struct { + Hash uint32 + Block uint32 +} + +type DirectoryTreeTail struct { + Reserved uint32 + Checksum uint32 +} + +type XAttrInodeBodyHeader struct { + Magic uint32 +} + +type XAttrHeader struct { + Magic uint32 + ReferenceCount uint32 + Blocks uint32 + Hash uint32 + Checksum uint32 + Reserved [3]uint32 +} + +const XAttrHeaderMagic uint32 = 0xea020000 + +type XAttrEntry struct { + NameLength uint8 + NameIndex uint8 + ValueOffset uint16 + ValueInum uint32 + ValueSize uint32 + Hash uint32 + //Name []byte +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/ext4/tar2ext4/tar2ext4.go b/components/engine/vendor/github.com/Microsoft/hcsshim/ext4/tar2ext4/tar2ext4.go new file mode 100644 index 0000000000..ad09210469 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/ext4/tar2ext4/tar2ext4.go @@ -0,0 +1,174 @@ +package tar2ext4 + +import ( + "archive/tar" + "bufio" + "encoding/binary" + "io" + "path" + "strings" + + "github.com/Microsoft/hcsshim/ext4/internal/compactext4" +) + +type params struct { + convertWhiteout bool + appendVhdFooter bool + ext4opts []compactext4.Option +} + +// Option is the type for optional parameters to Convert. +type Option func(*params) + +// ConvertWhiteout instructs the converter to convert OCI-style whiteouts +// (beginning with .wh.) to overlay-style whiteouts. +func ConvertWhiteout(p *params) { + p.convertWhiteout = true +} + +// AppendVhdFooter instructs the converter to add a fixed VHD footer to the +// file. +func AppendVhdFooter(p *params) { + p.appendVhdFooter = true +} + +// InlineData instructs the converter to write small files into the inode +// structures directly. This creates smaller images but currently is not +// compatible with DAX. +func InlineData(p *params) { + p.ext4opts = append(p.ext4opts, compactext4.InlineData) +} + +// MaximumDiskSize instructs the writer to limit the disk size to the specified +// value. This also reserves enough metadata space for the specified disk size. +// If not provided, then 16GB is the default. +func MaximumDiskSize(size int64) Option { + return func(p *params) { + p.ext4opts = append(p.ext4opts, compactext4.MaximumDiskSize(size)) + } +} + +const ( + whiteoutPrefix = ".wh." + opaqueWhiteout = ".wh..wh..opq" +) + +// Convert writes a compact ext4 file system image that contains the files in the +// input tar stream. +func Convert(r io.Reader, w io.ReadWriteSeeker, options ...Option) error { + var p params + for _, opt := range options { + opt(&p) + } + t := tar.NewReader(bufio.NewReader(r)) + fs := compactext4.NewWriter(w, p.ext4opts...) + for { + hdr, err := t.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + if p.convertWhiteout { + dir, name := path.Split(hdr.Name) + if strings.HasPrefix(name, whiteoutPrefix) { + if name == opaqueWhiteout { + // Update the directory with the appropriate xattr. + f, err := fs.Stat(dir) + if err != nil { + return err + } + f.Xattrs["trusted.overlay.opaque"] = []byte("y") + err = fs.Create(dir, f) + if err != nil { + return err + } + } else { + // Create an overlay-style whiteout. + f := &compactext4.File{ + Mode: compactext4.S_IFCHR, + Devmajor: 0, + Devminor: 0, + } + err = fs.Create(path.Join(dir, name[len(whiteoutPrefix):]), f) + if err != nil { + return err + } + } + + continue + } + } + + if hdr.Typeflag == tar.TypeLink { + err = fs.Link(hdr.Linkname, hdr.Name) + if err != nil { + return err + } + } else { + f := &compactext4.File{ + Mode: uint16(hdr.Mode), + Atime: hdr.AccessTime, + Mtime: hdr.ModTime, + Ctime: hdr.ChangeTime, + Crtime: hdr.ModTime, + Size: hdr.Size, + Uid: uint32(hdr.Uid), + Gid: uint32(hdr.Gid), + Linkname: hdr.Linkname, + Devmajor: uint32(hdr.Devmajor), + Devminor: uint32(hdr.Devminor), + Xattrs: make(map[string][]byte), + } + for key, value := range hdr.PAXRecords { + const xattrPrefix = "SCHILY.xattr." + if strings.HasPrefix(key, xattrPrefix) { + f.Xattrs[key[len(xattrPrefix):]] = []byte(value) + } + } + + var typ uint16 + switch hdr.Typeflag { + case tar.TypeReg, tar.TypeRegA: + typ = compactext4.S_IFREG + case tar.TypeSymlink: + typ = compactext4.S_IFLNK + case tar.TypeChar: + typ = compactext4.S_IFCHR + case tar.TypeBlock: + typ = compactext4.S_IFBLK + case tar.TypeDir: + typ = compactext4.S_IFDIR + case tar.TypeFifo: + typ = compactext4.S_IFIFO + } + f.Mode &= ^compactext4.TypeMask + f.Mode |= typ + err = fs.Create(hdr.Name, f) + if err != nil { + return err + } + _, err = io.Copy(fs, t) + if err != nil { + return err + } + } + } + err := fs.Close() + if err != nil { + return err + } + if p.appendVhdFooter { + size, err := w.Seek(0, io.SeekEnd) + if err != nil { + return err + } + err = binary.Write(w, binary.BigEndian, makeFixedVHDFooter(size)) + if err != nil { + return err + } + } + return nil +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/ext4/tar2ext4/vhdfooter.go b/components/engine/vendor/github.com/Microsoft/hcsshim/ext4/tar2ext4/vhdfooter.go new file mode 100644 index 0000000000..c98740302f --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/ext4/tar2ext4/vhdfooter.go @@ -0,0 +1,76 @@ +package tar2ext4 + +import ( + "bytes" + "crypto/rand" + "encoding/binary" +) + +// Constants for the VHD footer +const ( + cookieMagic = "conectix" + featureMask = 0x2 + fileFormatVersionMagic = 0x00010000 + fixedDataOffset = -1 + creatorVersionMagic = 0x000a0000 + diskTypeFixed = 2 +) + +type vhdFooter struct { + Cookie [8]byte + Features uint32 + FileFormatVersion uint32 + DataOffset int64 + TimeStamp uint32 + CreatorApplication [4]byte + CreatorVersion uint32 + CreatorHostOS [4]byte + OriginalSize int64 + CurrentSize int64 + DiskGeometry uint32 + DiskType uint32 + Checksum uint32 + UniqueID [16]uint8 + SavedState uint8 + Reserved [427]uint8 +} + +func makeFixedVHDFooter(size int64) *vhdFooter { + footer := &vhdFooter{ + Features: featureMask, + FileFormatVersion: fileFormatVersionMagic, + DataOffset: fixedDataOffset, + CreatorVersion: creatorVersionMagic, + OriginalSize: size, + CurrentSize: size, + DiskType: diskTypeFixed, + UniqueID: generateUUID(), + } + copy(footer.Cookie[:], cookieMagic) + footer.Checksum = calculateCheckSum(footer) + return footer +} + +func calculateCheckSum(footer *vhdFooter) uint32 { + oldchk := footer.Checksum + footer.Checksum = 0 + + buf := &bytes.Buffer{} + binary.Write(buf, binary.BigEndian, footer) + + var chk uint32 + bufBytes := buf.Bytes() + for i := 0; i < len(bufBytes); i++ { + chk += uint32(bufBytes[i]) + } + footer.Checksum = oldchk + return uint32(^chk) +} + +func generateUUID() [16]byte { + res := [16]byte{} + if _, err := rand.Read(res[:]); err != nil { + panic(err) + } + return res +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/version.go b/components/engine/vendor/github.com/Microsoft/hcsshim/version.go deleted file mode 100644 index 9ebb257b3e..0000000000 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/version.go +++ /dev/null @@ -1,6 +0,0 @@ -package hcsshim - -// IsTP4 returns whether the currently running Windows build is at least TP4. -func IsTP4() bool { - return false -} From 02673c7372acd51422dc7cfa00ffb539a029844e Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Tue, 16 Oct 2018 14:31:19 +0900 Subject: [PATCH 06/25] bump up runc Changes: https://github.com/opencontainers/runc/compare/69663f0bd4b60df09991c08812a60108003fa340...a00bf0190895aa465a5fbed0268888e2c8ddfe85 Signed-off-by: Akihiro Suda (cherry picked from commit 275044bbc374b563a2039229660df58a75bdc9f3) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 0afe0309bd9580bc76496c9e0da75216795c1c01 Component: engine --- .../engine/hack/dockerfile/install/runc.installer | 2 +- components/engine/vendor.conf | 2 +- .../runc/libcontainer/configs/config.go | 13 ++++++++++--- .../runc/libcontainer/nsenter/nsexec.c | 12 ++++++------ 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/components/engine/hack/dockerfile/install/runc.installer b/components/engine/hack/dockerfile/install/runc.installer index ed483e0f40..f7c73b7dd2 100755 --- a/components/engine/hack/dockerfile/install/runc.installer +++ b/components/engine/hack/dockerfile/install/runc.installer @@ -1,7 +1,7 @@ #!/bin/sh # When updating RUNC_COMMIT, also update runc in vendor.conf accordingly -RUNC_COMMIT=69663f0bd4b60df09991c08812a60108003fa340 +RUNC_COMMIT=a00bf0190895aa465a5fbed0268888e2c8ddfe85 install_runc() { # Do not build with ambient capabilities support diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 73a11723a6..91e6118e0a 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -75,7 +75,7 @@ github.com/pborman/uuid v1.0 google.golang.org/grpc v1.12.0 # This does not need to match RUNC_COMMIT as it is used for helper packages but should be newer or equal -github.com/opencontainers/runc 00dc70017d222b178a002ed30e9321b12647af2d +github.com/opencontainers/runc a00bf0190895aa465a5fbed0268888e2c8ddfe85 github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d github.com/opencontainers/image-spec v1.0.1 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go index b1c4762fe2..f6d1f134db 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go @@ -186,12 +186,19 @@ type Config struct { // callers keyring in this case. NoNewKeyring bool `json:"no_new_keyring"` - // Rootless specifies whether the container is a rootless container. - Rootless bool `json:"rootless"` - // IntelRdt specifies settings for Intel RDT/CAT group that the container is placed into // to limit the resources (e.g., L3 cache) the container has available IntelRdt *IntelRdt `json:"intel_rdt,omitempty"` + + // RootlessEUID is set when the runc was launched with non-zero EUID. + // Note that RootlessEUID is set to false when launched with EUID=0 in userns. + // When RootlessEUID is set, runc creates a new userns for the container. + // (config.json needs to contain userns settings) + RootlessEUID bool `json:"rootless_euid,omitempty"` + + // RootlessCgroups is set when unlikely to have the full access to cgroups. + // When RootlessCgroups is set, cgroups errors are ignored. + RootlessCgroups bool `json:"rootless_cgroups,omitempty"` } type Hooks struct { diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c index cb22431473..d7cb0af030 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c @@ -82,7 +82,7 @@ struct nlconfig_t { uint8_t is_setgroup; /* Rootless container settings. */ - uint8_t is_rootless; + uint8_t is_rootless_euid; /* boolean */ char *uidmappath; size_t uidmappath_len; char *gidmappath; @@ -100,7 +100,7 @@ struct nlconfig_t { #define GIDMAP_ATTR 27284 #define SETGROUP_ATTR 27285 #define OOM_SCORE_ADJ_ATTR 27286 -#define ROOTLESS_ATTR 27287 +#define ROOTLESS_EUID_ATTR 27287 #define UIDMAPPATH_ATTR 27288 #define GIDMAPPATH_ATTR 27289 @@ -419,8 +419,8 @@ static void nl_parse(int fd, struct nlconfig_t *config) case CLONE_FLAGS_ATTR: config->cloneflags = readint32(current); break; - case ROOTLESS_ATTR: - config->is_rootless = readint8(current); + case ROOTLESS_EUID_ATTR: + config->is_rootless_euid = readint8(current); /* boolean */ break; case OOM_SCORE_ADJ_ATTR: config->oom_score_adj = current; @@ -687,7 +687,7 @@ void nsexec(void) * newuidmap/newgidmap shall be used. */ - if (config.is_rootless && !config.is_setgroup) + if (config.is_rootless_euid && !config.is_setgroup) update_setgroups(child, SETGROUPS_DENY); /* Set up mappings. */ @@ -953,7 +953,7 @@ void nsexec(void) if (setgid(0) < 0) bail("setgid failed"); - if (!config.is_rootless && config.is_setgroup) { + if (!config.is_rootless_euid && config.is_setgroup) { if (setgroups(0, NULL) < 0) bail("setgroups failed"); } From 90250502de6084f13a3a3fce620b08648333fb37 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 18 Oct 2018 21:37:23 +0200 Subject: [PATCH 07/25] update containerd client and dependencies to v1.2.0 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit dd7799afd40b74e538af01fc658c861d225a4b82) Signed-off-by: Sebastiaan van Stijn Upstream-commit: c27094289aadaad4ad4d78aefcc44e95278d3508 Component: engine --- .../libcontainerd/supervisor/remote_daemon.go | 6 +- components/engine/vendor.conf | 10 +- .../Microsoft/hcsshim/hnsendpoint.go | 1 + .../hcsshim/internal/guestrequest/types.go | 85 ++++ .../Microsoft/hcsshim/internal/hcs/hcs.go | 1 + .../Microsoft/hcsshim/internal/hcs/process.go | 34 ++ .../hcsshim/internal/hcs/zsyscall_windows.go | 33 +- .../hcsshim/internal/schema1/schema1.go | 23 +- .../hcsshim/internal/schema2/attachment.go | 31 ++ .../hcsshim/internal/schema2/battery.go | 13 + .../schema2/cache_query_stats_response.go | 19 + .../hcsshim/internal/schema2/chipset.go | 25 ++ .../hcsshim/internal/schema2/close_handle.go | 15 + .../hcsshim/internal/schema2/com_port.go | 18 + .../internal/schema2/compute_system.go | 27 ++ .../hcsshim/internal/schema2/configuration.go | 72 ++++ .../hcsshim/internal/schema2/console_size.go | 17 + .../hcsshim/internal/schema2/container.go | 35 ++ .../container_credential_guard_state.go | 25 ++ .../schema2/container_memory_information.go | 26 ++ .../hcsshim/internal/schema2/device.go | 16 + .../hcsshim/internal/schema2/devices.go | 43 +++ .../internal/schema2/enhanced_mode_video.go | 15 + .../internal/schema2/flexible_io_device.go | 19 + .../internal/schema2/guest_connection.go | 19 + .../internal/schema2/guest_connection_info.go | 21 + .../internal/schema2/guest_crash_reporting.go | 15 + .../hcsshim/internal/schema2/guest_os.go | 15 + .../hcsshim/internal/schema2/guest_state.go | 22 ++ .../hcsshim/internal/schema2/hosted_system.go | 17 + .../hcsshim/internal/schema2/hv_socket.go | 17 + .../hcsshim/internal/schema2/hv_socket_2.go | 16 + .../schema2/hv_socket_service_config.go | 22 ++ .../schema2/hv_socket_system_config.go | 22 ++ .../hcsshim/internal/schema2/keyboard.go | 13 + .../hcsshim/internal/schema2/layer.go | 22 ++ .../internal/schema2/mapped_directory.go | 21 + .../hcsshim/internal/schema2/mapped_pipe.go | 19 + .../hcsshim/internal/schema2/memory.go | 15 + .../hcsshim/internal/schema2/memory_2.go | 23 ++ .../schema2/memory_information_for_vm.go | 19 + .../hcsshim/internal/schema2/memory_stats.go | 20 + .../schema2/modify_setting_request.go | 20 + .../hcsshim/internal/schema2/mouse.go | 13 + .../internal/schema2/network_adapter.go | 17 + .../hcsshim/internal/schema2/networking.go | 24 ++ .../internal/schema2/pause_notification.go | 16 + .../hcsshim/internal/schema2/pause_options.go | 18 + .../hcsshim/internal/schema2/plan9.go | 15 + .../hcsshim/internal/schema2/plan9_share.go | 26 ++ .../internal/schema2/process_details.go | 34 ++ .../schema2/process_modify_request.go | 20 + .../internal/schema2/process_parameters.go | 47 +++ .../internal/schema2/process_status.go | 22 ++ .../hcsshim/internal/schema2/processor.go | 19 + .../hcsshim/internal/schema2/processor_2.go | 21 + .../internal/schema2/processor_stats.go | 20 + .../hcsshim/internal/schema2/properties.go | 47 +++ .../internal/schema2/property_query.go | 16 + .../schema2/rdp_connection_options.go | 17 + .../internal/schema2/registry_changes.go | 17 + .../hcsshim/internal/schema2/registry_key.go | 19 + .../internal/schema2/registry_value.go | 31 ++ .../hcsshim/internal/schema2/restore_state.go | 19 + .../hcsshim/internal/schema2/save_options.go | 19 + .../hcsshim/internal/schema2/scsi.go | 16 + .../schema2/shared_memory_configuration.go | 15 + .../internal/schema2/shared_memory_region.go | 23 ++ .../schema2/shared_memory_region_info.go | 17 + .../internal/schema2/silo_properties.go | 18 + .../hcsshim/internal/schema2/statistics.go | 30 ++ .../hcsshim/internal/schema2/storage.go | 21 + .../hcsshim/internal/schema2/storage_qo_s.go | 17 + .../hcsshim/internal/schema2/storage_stats.go | 22 ++ .../hcsshim/internal/schema2/topology.go | 17 + .../hcsshim/internal/schema2/uefi.go | 21 + .../internal/schema2/uefi_boot_entry.go | 23 ++ .../hcsshim/internal/schema2/version.go | 17 + .../hcsshim/internal/schema2/video_monitor.go | 19 + .../internal/schema2/virtual_machine.go | 29 ++ .../internal/schema2/virtual_node_info.go | 21 + .../schema2/virtual_p_mem_controller.go | 21 + .../internal/schema2/virtual_p_mem_device.go | 19 + .../hcsshim/internal/schema2/virtual_smb.go | 17 + .../internal/schema2/virtual_smb_share.go | 21 + .../schema2/virtual_smb_share_options.go | 63 +++ .../hcsshim/internal/schema2/vm_memory.go | 27 ++ .../schema2/windows_crash_reporting.go | 17 + .../github.com/Microsoft/hcsshim/layer.go | 2 + .../containerd/containerd/README.md | 15 +- .../containerd/archive/time_unix.go | 2 +- .../containerd/containerd/content/content.go | 7 +- .../containerd/content/local/store.go | 5 +- .../containerd/content/local/writer.go | 71 ++-- .../containerd/containerd/events.go | 3 + .../containerd/images/archive/importer.go | 4 +- .../containerd/containerd/metadata/content.go | 26 +- .../containerd/containerd/mount/mount_unix.go | 2 +- ...{mountinfo_freebsd.go => mountinfo_bsd.go} | 2 + .../containerd/mount/mountinfo_unsupported.go | 2 +- .../containerd/platforms/cpuinfo.go | 16 + .../containerd/remotes/docker/authorizer.go | 6 +- .../remotes/docker/schema1/converter.go | 22 +- .../services/server/{ => config}/config.go | 2 +- .../containerd/services/server/server.go | 362 ------------------ .../services/server/server_linux.go | 54 --- .../services/server/server_solaris.go | 23 -- .../services/server/server_unsupported.go | 25 -- .../services/server/server_windows.go | 27 -- .../containerd/containerd/vendor.conf | 22 +- .../github.com/containerd/cri/vendor.conf | 28 +- .../runc/libcontainer/configs/config.go | 4 +- .../runc/libcontainer/configs/intelrdt.go | 4 + .../runc/libcontainer/user/lookup_unix.go | 22 +- .../opencontainers/runc/vendor.conf | 2 +- .../runtime-spec/specs-go/config.go | 14 +- 116 files changed, 2142 insertions(+), 607 deletions(-) create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/attachment.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/battery.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/cache_query_stats_response.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/chipset.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/close_handle.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/com_port.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/compute_system.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/configuration.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/console_size.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/container.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_state.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_memory_information.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/devices.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/enhanced_mode_video.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/flexible_io_device.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection_info.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_crash_reporting.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_os.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_state.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hosted_system.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_2.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_service_config.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_system_config.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/keyboard.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/layer.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_directory.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_pipe.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_2.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_information_for_vm.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_stats.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/modify_setting_request.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/mouse.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/network_adapter.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/networking.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_notification.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_options.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9_share.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_details.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_modify_request.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_parameters.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_status.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_2.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_stats.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/properties.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/property_query.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/rdp_connection_options.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_changes.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_key.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_value.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/restore_state.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/save_options.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/scsi.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_configuration.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region_info.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/silo_properties.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/statistics.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_qo_s.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_stats.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/topology.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi_boot_entry.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/version.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/video_monitor.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_machine.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_node_info.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_controller.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_device.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share_options.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/vm_memory.go create mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/windows_crash_reporting.go rename components/engine/vendor/github.com/containerd/containerd/mount/{mountinfo_freebsd.go => mountinfo_bsd.go} (98%) rename components/engine/vendor/github.com/containerd/containerd/services/server/{ => config}/config.go (99%) delete mode 100644 components/engine/vendor/github.com/containerd/containerd/services/server/server.go delete mode 100644 components/engine/vendor/github.com/containerd/containerd/services/server/server_linux.go delete mode 100644 components/engine/vendor/github.com/containerd/containerd/services/server/server_solaris.go delete mode 100644 components/engine/vendor/github.com/containerd/containerd/services/server/server_unsupported.go delete mode 100644 components/engine/vendor/github.com/containerd/containerd/services/server/server_windows.go diff --git a/components/engine/libcontainerd/supervisor/remote_daemon.go b/components/engine/libcontainerd/supervisor/remote_daemon.go index 095300f753..439162de70 100644 --- a/components/engine/libcontainerd/supervisor/remote_daemon.go +++ b/components/engine/libcontainerd/supervisor/remote_daemon.go @@ -15,7 +15,7 @@ import ( "github.com/BurntSushi/toml" "github.com/containerd/containerd" - "github.com/containerd/containerd/services/server" + "github.com/containerd/containerd/services/server/config" "github.com/docker/docker/pkg/system" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -37,7 +37,7 @@ type pluginConfigs struct { type remote struct { sync.RWMutex - server.Config + config.Config daemonPid int logger *logrus.Entry @@ -65,7 +65,7 @@ func Start(ctx context.Context, rootDir, stateDir string, opts ...DaemonOpt) (Da r := &remote{ rootDir: rootDir, stateDir: stateDir, - Config: server.Config{ + Config: config.Config{ Root: filepath.Join(rootDir, "daemon"), State: filepath.Join(stateDir, "daemon"), }, diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 91e6118e0a..d10a349b02 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -1,6 +1,6 @@ # the following lines are in sorted order, FYI github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109 -github.com/Microsoft/hcsshim v0.7.9 +github.com/Microsoft/hcsshim v0.7.12 github.com/Microsoft/go-winio v0.4.11 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git @@ -75,8 +75,8 @@ github.com/pborman/uuid v1.0 google.golang.org/grpc v1.12.0 # This does not need to match RUNC_COMMIT as it is used for helper packages but should be newer or equal -github.com/opencontainers/runc a00bf0190895aa465a5fbed0268888e2c8ddfe85 -github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d +github.com/opencontainers/runc 58592df56734acf62e574865fe40b9e53e967910 +github.com/opencontainers/runtime-spec 5684b8af48c1ac3b1451fa499724e30e3c20a294 # v1.0.1-49-g5684b8a github.com/opencontainers/image-spec v1.0.1 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 @@ -114,12 +114,12 @@ github.com/googleapis/gax-go v2.0.0 google.golang.org/genproto 694d95ba50e67b2e363f3483057db5d4910c18f9 # containerd -github.com/containerd/containerd 0c5f8f63c3368856c320ae8a1c125e703b73b51d # v1.2.0-rc.1 +github.com/containerd/containerd c4446665cb9c30056f4998ed953e6d4ff22c7c39 # v1.2.0 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4 github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2 github.com/containerd/console c12b1e7919c14469339a5d38f2f8ed9b64a9de23 -github.com/containerd/cri 9f39e3289533fc228c5e5fcac0a6dbdd60c6047b # release/1.2 branch +github.com/containerd/cri f913714917d2456d7e65a0be84962b1ce8acb487 # release/1.2 branch github.com/containerd/go-runc 5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3 github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40 github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go b/components/engine/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go index f2eedbe3d3..eb013d2c42 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go @@ -6,6 +6,7 @@ import ( // HNSEndpoint represents a network endpoint in HNS type HNSEndpoint = hns.HNSEndpoint + // Namespace represents a Compartment. type Namespace = hns.Namespace diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go new file mode 100644 index 0000000000..9f926c6be7 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go @@ -0,0 +1,85 @@ +package guestrequest + +import "github.com/Microsoft/hcsshim/internal/schema2" + +// Arguably, many of these (at least CombinedLayers) should have been generated +// by swagger. +// +// This will also change package name due to an inbound breaking change. + +// This class is used by a modify request to add or remove a combined layers +// structure in the guest. For windows, the GCS applies a filter in ContainerRootPath +// using the specified layers as the parent content. Ignores property ScratchPath +// since the container path is already the scratch path. For linux, the GCS unions +// the specified layers and ScratchPath together, placing the resulting union +// filesystem at ContainerRootPath. +type CombinedLayers struct { + ContainerRootPath string `json:"ContainerRootPath,omitempty"` + Layers []hcsschema.Layer `json:"Layers,omitempty"` + ScratchPath string `json:"ScratchPath,omitempty"` +} + +// Defines the schema for hosted settings passed to GCS and/or OpenGCS + +// SCSI. Scratch space for remote file-system commands, or R/W layer for containers +type LCOWMappedVirtualDisk struct { + MountPath string `json:"MountPath,omitempty"` // /tmp/scratch for an LCOW utility VM being used as a service VM + Lun uint8 `json:"Lun,omitempty"` + Controller uint8 `json:"Controller,omitempty"` + ReadOnly bool `json:"ReadOnly,omitempty"` +} + +type WCOWMappedVirtualDisk struct { + ContainerPath string `json:"ContainerPath,omitempty"` + Lun int32 `json:"Lun,omitempty"` +} + +type LCOWMappedDirectory struct { + MountPath string `json:"MountPath,omitempty"` + Port int32 `json:"Port,omitempty"` + ShareName string `json:"ShareName,omitempty"` // If empty not using ANames (not currently supported) + ReadOnly bool `json:"ReadOnly,omitempty"` +} + +// Read-only layers over VPMem +type LCOWMappedVPMemDevice struct { + DeviceNumber uint32 `json:"DeviceNumber,omitempty"` + MountPath string `json:"MountPath,omitempty"` // /tmp/pN +} + +type ResourceType string + +const ( + // These are constants for v2 schema modify guest requests. + ResourceTypeMappedDirectory ResourceType = "MappedDirectory" + ResourceTypeMappedVirtualDisk ResourceType = "MappedVirtualDisk" + ResourceTypeNetwork ResourceType = "Network" + ResourceTypeNetworkNamespace ResourceType = "NetworkNamespace" + ResourceTypeCombinedLayers ResourceType = "CombinedLayers" + ResourceTypeVPMemDevice ResourceType = "VPMemDevice" +) + +// GuestRequest is for modify commands passed to the guest. +type GuestRequest struct { + RequestType string `json:"RequestType,omitempty"` + ResourceType ResourceType `json:"ResourceType,omitempty"` + Settings interface{} `json:"Settings,omitempty"` +} + +type NetworkModifyRequest struct { + AdapterId string `json:"AdapterId,omitempty"` + RequestType string `json:"RequestType,omitempty"` + Settings interface{} `json:"Settings,omitempty"` +} + +type RS4NetworkModifyRequest struct { + AdapterInstanceId string `json:"AdapterInstanceId,omitempty"` + RequestType string `json:"RequestType,omitempty"` + Settings interface{} `json:"Settings,omitempty"` +} + +// SignalProcessOptions is the options passed to either WCOW or LCOW +// to signal a given process. +type SignalProcessOptions struct { + Signal int `json:,omitempty` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/hcs/hcs.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/hcs/hcs.go index b8e30eba17..b0d49cbcf1 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/hcs/hcs.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/hcs/hcs.go @@ -27,6 +27,7 @@ import ( //sys hcsOpenProcess(computeSystem hcsSystem, pid uint32, process *hcsProcess, result **uint16) (hr error) = vmcompute.HcsOpenProcess? //sys hcsCloseProcess(process hcsProcess) (hr error) = vmcompute.HcsCloseProcess? //sys hcsTerminateProcess(process hcsProcess, result **uint16) (hr error) = vmcompute.HcsTerminateProcess? +//sys hcsSignalProcess(process hcsProcess, options string, result **uint16) (hr error) = vmcompute.HcsTerminateProcess? //sys hcsGetProcessInfo(process hcsProcess, processInformation *hcsProcessInformation, result **uint16) (hr error) = vmcompute.HcsGetProcessInfo? //sys hcsGetProcessProperties(process hcsProcess, processProperties **uint16, result **uint16) (hr error) = vmcompute.HcsGetProcessProperties? //sys hcsModifyProcess(process hcsProcess, settings string, result **uint16) (hr error) = vmcompute.HcsModifyProcess? diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go index 8294d66d7b..d356cdc4d6 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go @@ -8,6 +8,7 @@ import ( "syscall" "time" + "github.com/Microsoft/hcsshim/internal/guestrequest" "github.com/Microsoft/hcsshim/internal/interop" "github.com/sirupsen/logrus" ) @@ -71,6 +72,39 @@ func (process *Process) SystemID() string { return process.system.ID() } +// Signal signals the process with `options`. +func (process *Process) Signal(options guestrequest.SignalProcessOptions) error { + process.handleLock.RLock() + defer process.handleLock.RUnlock() + operation := "Signal" + title := "hcsshim::Process::" + operation + logrus.Debugf(title+" processid=%d", process.processID) + + if process.handle == 0 { + return makeProcessError(process, operation, ErrAlreadyClosed, nil) + } + + optionsb, err := json.Marshal(options) + if err != nil { + return err + } + + optionsStr := string(optionsb) + + var resultp *uint16 + completed := false + go syscallWatcher(fmt.Sprintf("SignalProcess %s: %d", process.SystemID(), process.Pid()), &completed) + err = hcsSignalProcess(process.handle, optionsStr, &resultp) + completed = true + events := processHcsResult(resultp) + if err != nil { + return makeProcessError(process, operation, err, events) + } + + logrus.Debugf(title+" succeeded processid=%d", process.processID) + return nil +} + // Kill signals the process to terminate but does not wait for it to finish terminating. func (process *Process) Kill() error { process.handleLock.RLock() diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/hcs/zsyscall_windows.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/hcs/zsyscall_windows.go index 48d5cd32b9..925c65e28d 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/hcs/zsyscall_windows.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/hcs/zsyscall_windows.go @@ -57,12 +57,13 @@ var ( procHcsOpenProcess = modvmcompute.NewProc("HcsOpenProcess") procHcsCloseProcess = modvmcompute.NewProc("HcsCloseProcess") procHcsTerminateProcess = modvmcompute.NewProc("HcsTerminateProcess") - procHcsGetProcessInfo = modvmcompute.NewProc("HcsGetProcessInfo") - procHcsGetProcessProperties = modvmcompute.NewProc("HcsGetProcessProperties") - procHcsModifyProcess = modvmcompute.NewProc("HcsModifyProcess") - procHcsGetServiceProperties = modvmcompute.NewProc("HcsGetServiceProperties") - procHcsRegisterProcessCallback = modvmcompute.NewProc("HcsRegisterProcessCallback") - procHcsUnregisterProcessCallback = modvmcompute.NewProc("HcsUnregisterProcessCallback") + + procHcsGetProcessInfo = modvmcompute.NewProc("HcsGetProcessInfo") + procHcsGetProcessProperties = modvmcompute.NewProc("HcsGetProcessProperties") + procHcsModifyProcess = modvmcompute.NewProc("HcsModifyProcess") + procHcsGetServiceProperties = modvmcompute.NewProc("HcsGetServiceProperties") + procHcsRegisterProcessCallback = modvmcompute.NewProc("HcsRegisterProcessCallback") + procHcsUnregisterProcessCallback = modvmcompute.NewProc("HcsUnregisterProcessCallback") ) func hcsEnumerateComputeSystems(query string, computeSystems **uint16, result **uint16) (hr error) { @@ -356,6 +357,26 @@ func hcsTerminateProcess(process hcsProcess, result **uint16) (hr error) { return } +func hcsSignalProcess(process hcsProcess, options string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(options) + if hr != nil { + return + } + return _hcsSignalProcess(process, _p0, result) +} + +func _hcsSignalProcess(process hcsProcess, options *uint16, result **uint16) (hr error) { + if hr = procHcsTerminateProcess.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsTerminateProcess.Addr(), 3, uintptr(process), uintptr(unsafe.Pointer(options)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = interop.Win32FromHresult(r0) + } + return +} + func hcsGetProcessInfo(process hcsProcess, processInformation *hcsProcessInformation, result **uint16) (hr error) { if hr = procHcsGetProcessInfo.Find(); hr != nil { return diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go index 6fa3bbc73d..995433ace6 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go @@ -3,6 +3,8 @@ package schema1 import ( "encoding/json" "time" + + "github.com/Microsoft/hcsshim/internal/schema2" ) // ProcessConfig is used as both the input of Container.CreateProcess @@ -115,9 +117,10 @@ type ComputeSystemQuery struct { type PropertyType string const ( - PropertyTypeStatistics PropertyType = "Statistics" - PropertyTypeProcessList = "ProcessList" - PropertyTypeMappedVirtualDisk = "MappedVirtualDisk" + PropertyTypeStatistics PropertyType = "Statistics" // V1 and V2 + PropertyTypeProcessList = "ProcessList" // V1 and V2 + PropertyTypeMappedVirtualDisk = "MappedVirtualDisk" // Not supported in V2 schema call + PropertyTypeGuestConnection = "GuestConnection" // V1 and V2. Nil return from HCS before RS5 ) type PropertyQuery struct { @@ -142,6 +145,7 @@ type ContainerProperties struct { Statistics Statistics `json:",omitempty"` ProcessList []ProcessListItem `json:",omitempty"` MappedVirtualDiskControllers map[int]MappedVirtualDiskController `json:",omitempty"` + GuestConnectionInfo GuestConnectionInfo `json:",omitempty"` } // MemoryStats holds the memory statistics for a container @@ -206,6 +210,19 @@ type MappedVirtualDiskController struct { MappedVirtualDisks map[int]MappedVirtualDisk `json:",omitempty"` } +// GuestDefinedCapabilities is part of the GuestConnectionInfo returned by a GuestConnection call on a utility VM +type GuestDefinedCapabilities struct { + NamespaceAddRequestSupported bool `json:",omitempty"` + SignalProcessSupported bool `json:",omitempty"` +} + +// GuestConnectionInfo is the structure of an iterm return by a GuestConnection call on a utility VM +type GuestConnectionInfo struct { + SupportedSchemaVersions []hcsschema.Version `json:",omitempty"` + ProtocolVersion uint32 `json:",omitempty"` + GuestDefinedCapabilities GuestDefinedCapabilities `json:",omitempty"` +} + // Type of Request Support in ModifySystem type RequestType string diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/attachment.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/attachment.go new file mode 100644 index 0000000000..09456cbc21 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/attachment.go @@ -0,0 +1,31 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Attachment struct { + + Type_ string `json:"Type,omitempty"` + + Path string `json:"Path,omitempty"` + + IgnoreFlushes bool `json:"IgnoreFlushes,omitempty"` + + CachingMode string `json:"CachingMode,omitempty"` + + NoWriteHardening bool `json:"NoWriteHardening,omitempty"` + + DisableExpansionOptimization bool `json:"DisableExpansionOptimization,omitempty"` + + IgnoreRelativeLocator bool `json:"IgnoreRelativeLocator,omitempty"` + + CaptureIoAttributionContext bool `json:"CaptureIoAttributionContext,omitempty"` + + ReadOnly bool `json:"ReadOnly,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/battery.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/battery.go new file mode 100644 index 0000000000..ecbbed4c23 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/battery.go @@ -0,0 +1,13 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Battery struct { +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/cache_query_stats_response.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/cache_query_stats_response.go new file mode 100644 index 0000000000..243779eab6 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/cache_query_stats_response.go @@ -0,0 +1,19 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type CacheQueryStatsResponse struct { + + L3OccupancyBytes int32 `json:"L3OccupancyBytes,omitempty"` + + L3TotalBwBytes int32 `json:"L3TotalBwBytes,omitempty"` + + L3LocalBwBytes int32 `json:"L3LocalBwBytes,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/chipset.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/chipset.go new file mode 100644 index 0000000000..3fb24e2505 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/chipset.go @@ -0,0 +1,25 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Chipset struct { + + Uefi *Uefi `json:"Uefi,omitempty"` + + IsNumLockDisabled bool `json:"IsNumLockDisabled,omitempty"` + + BaseBoardSerialNumber string `json:"BaseBoardSerialNumber,omitempty"` + + ChassisSerialNumber string `json:"ChassisSerialNumber,omitempty"` + + ChassisAssetTag string `json:"ChassisAssetTag,omitempty"` + + UseUtc bool `json:"UseUtc,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/close_handle.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/close_handle.go new file mode 100644 index 0000000000..88f01707a7 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/close_handle.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type CloseHandle struct { + + Handle string `json:"Handle,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/com_port.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/com_port.go new file mode 100644 index 0000000000..c665be3d5a --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/com_port.go @@ -0,0 +1,18 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// ComPort specifies the named pipe that will be used for the port, with empty string indicating a disconnected port. +type ComPort struct { + + NamedPipe string `json:"NamedPipe,omitempty"` + + OptimizeForDebugger bool `json:"OptimizeForDebugger,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/compute_system.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/compute_system.go new file mode 100644 index 0000000000..85785d2858 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/compute_system.go @@ -0,0 +1,27 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type ComputeSystem struct { + + Owner string `json:"Owner,omitempty"` + + SchemaVersion *Version `json:"SchemaVersion,omitempty"` + + HostingSystemId string `json:"HostingSystemId,omitempty"` + + HostedSystem *HostedSystem `json:"HostedSystem,omitempty"` + + Container *Container `json:"Container,omitempty"` + + VirtualMachine *VirtualMachine `json:"VirtualMachine,omitempty"` + + ShouldTerminateOnLastHandleClosed bool `json:"ShouldTerminateOnLastHandleClosed,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/configuration.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/configuration.go new file mode 100644 index 0000000000..1a47db7d95 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/configuration.go @@ -0,0 +1,72 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +import ( + "net/http" +) + +// contextKeys are used to identify the type of value in the context. +// Since these are string, it is possible to get a short description of the +// context key for logging and debugging using key.String(). + +type contextKey string + +func (c contextKey) String() string { + return "auth " + string(c) +} + +var ( + // ContextOAuth2 takes a oauth2.TokenSource as authentication for the request. + ContextOAuth2 = contextKey("token") + + // ContextBasicAuth takes BasicAuth as authentication for the request. + ContextBasicAuth = contextKey("basic") + + // ContextAccessToken takes a string oauth2 access token as authentication for the request. + ContextAccessToken = contextKey("accesstoken") + + // ContextAPIKey takes an APIKey as authentication for the request + ContextAPIKey = contextKey("apikey") +) + +// BasicAuth provides basic http authentication to a request passed via context using ContextBasicAuth +type BasicAuth struct { + UserName string `json:"userName,omitempty"` + Password string `json:"password,omitempty"` +} + +// APIKey provides API key based authentication to a request passed via context using ContextAPIKey +type APIKey struct { + Key string + Prefix string +} + +type Configuration struct { + BasePath string `json:"basePath,omitempty"` + Host string `json:"host,omitempty"` + Scheme string `json:"scheme,omitempty"` + DefaultHeader map[string]string `json:"defaultHeader,omitempty"` + UserAgent string `json:"userAgent,omitempty"` + HTTPClient *http.Client +} + +func NewConfiguration() *Configuration { + cfg := &Configuration{ + BasePath: "https://localhost", + DefaultHeader: make(map[string]string), + UserAgent: "Swagger-Codegen/2.1.0/go", + } + return cfg +} + +func (c *Configuration) AddDefaultHeader(key string, value string) { + c.DefaultHeader[key] = value +} \ No newline at end of file diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/console_size.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/console_size.go new file mode 100644 index 0000000000..adbe07fe55 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/console_size.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type ConsoleSize struct { + + Height int32 `json:"Height,omitempty"` + + Width int32 `json:"Width,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/container.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/container.go new file mode 100644 index 0000000000..17dce28bc7 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/container.go @@ -0,0 +1,35 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Container struct { + + GuestOs *GuestOs `json:"GuestOs,omitempty"` + + Storage *Storage `json:"Storage,omitempty"` + + MappedDirectories []MappedDirectory `json:"MappedDirectories,omitempty"` + + MappedPipes []MappedPipe `json:"MappedPipes,omitempty"` + + Memory *Memory `json:"Memory,omitempty"` + + Processor *Processor `json:"Processor,omitempty"` + + Networking *Networking `json:"Networking,omitempty"` + + HvSocket *HvSocket `json:"HvSocket,omitempty"` + + ContainerCredentialGuard *ContainerCredentialGuardState `json:"ContainerCredentialGuard,omitempty"` + + RegistryChanges *RegistryChanges `json:"RegistryChanges,omitempty"` + + AssignedDevices []Device `json:"AssignedDevices,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_state.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_state.go new file mode 100644 index 0000000000..0f8f644379 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_state.go @@ -0,0 +1,25 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type ContainerCredentialGuardState struct { + + // Authentication cookie for calls to a Container Credential Guard instance. + Cookie string `json:"Cookie,omitempty"` + + // Name of the RPC endpoint of the Container Credential Guard instance. + RpcEndpoint string `json:"RpcEndpoint,omitempty"` + + // Transport used for the configured Container Credential Guard instance. + Transport string `json:"Transport,omitempty"` + + // Credential spec used for the configured Container Credential Guard instance. + CredentialSpec string `json:"CredentialSpec,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_memory_information.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_memory_information.go new file mode 100644 index 0000000000..754797e213 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_memory_information.go @@ -0,0 +1,26 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// memory usage as viewed from within the container +type ContainerMemoryInformation struct { + + TotalPhysicalBytes int32 `json:"TotalPhysicalBytes,omitempty"` + + TotalUsage int32 `json:"TotalUsage,omitempty"` + + CommittedBytes int32 `json:"CommittedBytes,omitempty"` + + SharedCommittedBytes int32 `json:"SharedCommittedBytes,omitempty"` + + CommitLimitBytes int32 `json:"CommitLimitBytes,omitempty"` + + PeakCommitmentBytes int32 `json:"PeakCommitmentBytes,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go new file mode 100644 index 0000000000..ca319bbbce --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go @@ -0,0 +1,16 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Device struct { + + // The interface class guid of the device to assign to container. + InterfaceClassGuid string `json:"InterfaceClassGuid,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/devices.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/devices.go new file mode 100644 index 0000000000..b2191c571d --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/devices.go @@ -0,0 +1,43 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Devices struct { + + ComPorts map[string]ComPort `json:"ComPorts,omitempty"` + + Scsi map[string]Scsi `json:"Scsi,omitempty"` + + VirtualPMem *VirtualPMemController `json:"VirtualPMem,omitempty"` + + NetworkAdapters map[string]NetworkAdapter `json:"NetworkAdapters,omitempty"` + + VideoMonitor *VideoMonitor `json:"VideoMonitor,omitempty"` + + Keyboard *Keyboard `json:"Keyboard,omitempty"` + + Mouse *Mouse `json:"Mouse,omitempty"` + + HvSocket *HvSocket2 `json:"HvSocket,omitempty"` + + EnhancedModeVideo *EnhancedModeVideo `json:"EnhancedModeVideo,omitempty"` + + GuestCrashReporting *GuestCrashReporting `json:"GuestCrashReporting,omitempty"` + + VirtualSmb *VirtualSmb `json:"VirtualSmb,omitempty"` + + Plan9 *Plan9 `json:"Plan9,omitempty"` + + Battery *Battery `json:"Battery,omitempty"` + + FlexibleIov map[string]FlexibleIoDevice `json:"FlexibleIov,omitempty"` + + SharedMemory *SharedMemoryConfiguration `json:"SharedMemory,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/enhanced_mode_video.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/enhanced_mode_video.go new file mode 100644 index 0000000000..4fe592f711 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/enhanced_mode_video.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type EnhancedModeVideo struct { + + ConnectionOptions *RdpConnectionOptions `json:"ConnectionOptions,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/flexible_io_device.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/flexible_io_device.go new file mode 100644 index 0000000000..51011afe40 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/flexible_io_device.go @@ -0,0 +1,19 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type FlexibleIoDevice struct { + + EmulatorId string `json:"EmulatorId,omitempty"` + + HostingModel string `json:"HostingModel,omitempty"` + + Configuration []string `json:"Configuration,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection.go new file mode 100644 index 0000000000..7db29495b3 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection.go @@ -0,0 +1,19 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type GuestConnection struct { + + // Use Vsock rather than Hyper-V sockets to communicate with the guest service. + UseVsock bool `json:"UseVsock,omitempty"` + + // Don't disconnect the guest connection when pausing the virtual machine. + UseConnectedSuspend bool `json:"UseConnectedSuspend,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection_info.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection_info.go new file mode 100644 index 0000000000..8a369bab71 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection_info.go @@ -0,0 +1,21 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// Information about the guest. +type GuestConnectionInfo struct { + + // Each schema version x.y stands for the range of versions a.b where a==x and b<=y. This list comes from the SupportedSchemaVersions field in GcsCapabilities. + SupportedSchemaVersions []Version `json:"SupportedSchemaVersions,omitempty"` + + ProtocolVersion int32 `json:"ProtocolVersion,omitempty"` + + GuestDefinedCapabilities *interface{} `json:"GuestDefinedCapabilities,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_crash_reporting.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_crash_reporting.go new file mode 100644 index 0000000000..c5fa767352 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_crash_reporting.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type GuestCrashReporting struct { + + WindowsCrashSettings *WindowsCrashReporting `json:"WindowsCrashSettings,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_os.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_os.go new file mode 100644 index 0000000000..c708fc7c3f --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_os.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type GuestOs struct { + + HostName string `json:"HostName,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_state.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_state.go new file mode 100644 index 0000000000..ef1eec8865 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_state.go @@ -0,0 +1,22 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type GuestState struct { + + // The path to an existing file uses for persistent guest state storage. An empty string indicates the system should initialize new transient, in-memory guest state. + GuestStateFilePath string `json:"GuestStateFilePath,omitempty"` + + // The path to an existing file for persistent runtime state storage. An empty string indicates the system should initialize new transient, in-memory runtime state. + RuntimeStateFilePath string `json:"RuntimeStateFilePath,omitempty"` + + // If true, the guest state and runtime state files will be used as templates to populate transient, in-memory state instead of using the files as persistent backing store. + ForceTransientState bool `json:"ForceTransientState,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hosted_system.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hosted_system.go new file mode 100644 index 0000000000..0797584c51 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hosted_system.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type HostedSystem struct { + + SchemaVersion *Version `json:"SchemaVersion,omitempty"` + + Container *Container `json:"Container,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket.go new file mode 100644 index 0000000000..ef9ffb8dd9 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type HvSocket struct { + + Config *HvSocketSystemConfig `json:"Config,omitempty"` + + EnablePowerShellDirect bool `json:"EnablePowerShellDirect,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_2.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_2.go new file mode 100644 index 0000000000..a19ba15c15 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_2.go @@ -0,0 +1,16 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// HvSocket configuration for a VM +type HvSocket2 struct { + + HvSocketConfig *HvSocketSystemConfig `json:"HvSocketConfig,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_service_config.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_service_config.go new file mode 100644 index 0000000000..a848e91e69 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_service_config.go @@ -0,0 +1,22 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type HvSocketServiceConfig struct { + + // SDDL string that HvSocket will check before allowing a host process to bind to this specific service. If not specified, defaults to the system DefaultBindSecurityDescriptor, defined in HvSocketSystemWpConfig in V1. + BindSecurityDescriptor string `json:"BindSecurityDescriptor,omitempty"` + + // SDDL string that HvSocket will check before allowing a host process to connect to this specific service. If not specified, defaults to the system DefaultConnectSecurityDescriptor, defined in HvSocketSystemWpConfig in V1. + ConnectSecurityDescriptor string `json:"ConnectSecurityDescriptor,omitempty"` + + // If true, HvSocket will process wildcard binds for this service/system combination. Wildcard binds are secured in the registry at SOFTWARE/Microsoft/Windows NT/CurrentVersion/Virtualization/HvSocket/WildcardDescriptors + AllowWildcardBinds bool `json:"AllowWildcardBinds,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_system_config.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_system_config.go new file mode 100644 index 0000000000..69f4f9d39b --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_system_config.go @@ -0,0 +1,22 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// This is the HCS Schema version of the HvSocket configuration. The VMWP version is located in Config.Devices.IC in V1. +type HvSocketSystemConfig struct { + + // SDDL string that HvSocket will check before allowing a host process to bind to an unlisted service for this specific container/VM (not wildcard binds). + DefaultBindSecurityDescriptor string `json:"DefaultBindSecurityDescriptor,omitempty"` + + // SDDL string that HvSocket will check before allowing a host process to connect to an unlisted service in the VM/container. + DefaultConnectSecurityDescriptor string `json:"DefaultConnectSecurityDescriptor,omitempty"` + + ServiceTable map[string]HvSocketServiceConfig `json:"ServiceTable,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/keyboard.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/keyboard.go new file mode 100644 index 0000000000..3d3fa3b1c7 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/keyboard.go @@ -0,0 +1,13 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Keyboard struct { +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/layer.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/layer.go new file mode 100644 index 0000000000..b63b8ef12c --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/layer.go @@ -0,0 +1,22 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Layer struct { + + Id string `json:"Id,omitempty"` + + Path string `json:"Path,omitempty"` + + PathType string `json:"PathType,omitempty"` + + // Unspecified defaults to Enabled + Cache string `json:"Cache,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_directory.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_directory.go new file mode 100644 index 0000000000..a823a6d3b8 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_directory.go @@ -0,0 +1,21 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type MappedDirectory struct { + + HostPath string `json:"HostPath,omitempty"` + + HostPathType string `json:"HostPathType,omitempty"` + + ContainerPath string `json:"ContainerPath,omitempty"` + + ReadOnly bool `json:"ReadOnly,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_pipe.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_pipe.go new file mode 100644 index 0000000000..2d1d2604a9 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_pipe.go @@ -0,0 +1,19 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type MappedPipe struct { + + ContainerPipeName string `json:"ContainerPipeName,omitempty"` + + HostPath string `json:"HostPath,omitempty"` + + HostPathType string `json:"HostPathType,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory.go new file mode 100644 index 0000000000..e1d135a3a4 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Memory struct { + + SizeInMB int32 `json:"SizeInMB,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_2.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_2.go new file mode 100644 index 0000000000..fd766f41a3 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_2.go @@ -0,0 +1,23 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Memory2 struct { + + SizeInMB int32 `json:"SizeInMB,omitempty"` + + AllowOvercommit bool `json:"AllowOvercommit,omitempty"` + + EnableHotHint bool `json:"EnableHotHint,omitempty"` + + EnableColdHint bool `json:"EnableColdHint,omitempty"` + + EnableEpf bool `json:"EnableEpf,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_information_for_vm.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_information_for_vm.go new file mode 100644 index 0000000000..bdd87dffd8 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_information_for_vm.go @@ -0,0 +1,19 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type MemoryInformationForVm struct { + + VirtualNodeCount int32 `json:"VirtualNodeCount,omitempty"` + + VirtualMachineMemory *VmMemory `json:"VirtualMachineMemory,omitempty"` + + VirtualNodes []VirtualNodeInfo `json:"VirtualNodes,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_stats.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_stats.go new file mode 100644 index 0000000000..6214970f69 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_stats.go @@ -0,0 +1,20 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// Memory runtime statistics +type MemoryStats struct { + + MemoryUsageCommitBytes int32 `json:"MemoryUsageCommitBytes,omitempty"` + + MemoryUsageCommitPeakBytes int32 `json:"MemoryUsageCommitPeakBytes,omitempty"` + + MemoryUsagePrivateWorkingSetBytes int32 `json:"MemoryUsagePrivateWorkingSetBytes,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/modify_setting_request.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/modify_setting_request.go new file mode 100644 index 0000000000..d29455a3e4 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/modify_setting_request.go @@ -0,0 +1,20 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type ModifySettingRequest struct { + ResourcePath string `json:"ResourcePath,omitempty"` + + RequestType string `json:"RequestType,omitempty"` + + Settings interface{} `json:"Settings,omitempty"` // NOTE: Swagger generated as *interface{}. Locally updated + + GuestRequest interface{} `json:"GuestRequest,omitempty"` // NOTE: Swagger generated as *interface{}. Locally updated +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/mouse.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/mouse.go new file mode 100644 index 0000000000..ccf8b938f3 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/mouse.go @@ -0,0 +1,13 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Mouse struct { +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/network_adapter.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/network_adapter.go new file mode 100644 index 0000000000..c586f66c25 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/network_adapter.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type NetworkAdapter struct { + + EndpointId string `json:"EndpointId,omitempty"` + + MacAddress string `json:"MacAddress,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/networking.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/networking.go new file mode 100644 index 0000000000..12c47827c5 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/networking.go @@ -0,0 +1,24 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Networking struct { + + AllowUnqualifiedDnsQuery bool `json:"AllowUnqualifiedDnsQuery,omitempty"` + + DnsSearchList string `json:"DnsSearchList,omitempty"` + + NetworkSharedContainerName string `json:"NetworkSharedContainerName,omitempty"` + + // Guid in windows; string in linux + Namespace string `json:"Namespace,omitempty"` + + NetworkAdapters []string `json:"NetworkAdapters,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_notification.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_notification.go new file mode 100644 index 0000000000..1cd70d1790 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_notification.go @@ -0,0 +1,16 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// Notification data that is indicated to components running in the Virtual Machine. +type PauseNotification struct { + + Reason string `json:"Reason,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_options.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_options.go new file mode 100644 index 0000000000..780a5cae2c --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_options.go @@ -0,0 +1,18 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// Options for HcsPauseComputeSystem +type PauseOptions struct { + + SuspensionLevel string `json:"SuspensionLevel,omitempty"` + + HostedNotification *PauseNotification `json:"HostedNotification,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9.go new file mode 100644 index 0000000000..705c677e1f --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Plan9 struct { + + Shares []Plan9Share `json:"Shares,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9_share.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9_share.go new file mode 100644 index 0000000000..b2bc58b83c --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9_share.go @@ -0,0 +1,26 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Plan9Share struct { + + Name string `json:"Name,omitempty"` + + // The name by which the guest operation system can access this share, via the aname parameter in the Plan9 protocol. + AccessName string `json:"AccessName,omitempty"` + + Path string `json:"Path,omitempty"` + + Port int32 `json:"Port,omitempty"` + + ReadOnly bool `json:"ReadOnly,omitempty"` + + UseShareRootIdentity bool `json:"UseShareRootIdentity,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_details.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_details.go new file mode 100644 index 0000000000..63e0b7f8fe --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_details.go @@ -0,0 +1,34 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +import ( + "time" +) + +// Information about a process running in a container +type ProcessDetails struct { + + ProcessId int32 `json:"ProcessId,omitempty"` + + ImageName string `json:"ImageName,omitempty"` + + CreateTimestamp time.Time `json:"CreateTimestamp,omitempty"` + + UserTime100ns int32 `json:"UserTime100ns,omitempty"` + + KernelTime100ns int32 `json:"KernelTime100ns,omitempty"` + + MemoryCommitBytes int32 `json:"MemoryCommitBytes,omitempty"` + + MemoryWorkingSetPrivateBytes int32 `json:"MemoryWorkingSetPrivateBytes,omitempty"` + + MemoryWorkingSetSharedBytes int32 `json:"MemoryWorkingSetSharedBytes,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_modify_request.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_modify_request.go new file mode 100644 index 0000000000..29bc2e3d00 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_modify_request.go @@ -0,0 +1,20 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// Passed to HcsRpc_ModifyProcess +type ProcessModifyRequest struct { + + Operation string `json:"Operation,omitempty"` + + ConsoleSize *ConsoleSize `json:"ConsoleSize,omitempty"` + + CloseHandle *CloseHandle `json:"CloseHandle,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_parameters.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_parameters.go new file mode 100644 index 0000000000..470c55734e --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_parameters.go @@ -0,0 +1,47 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type ProcessParameters struct { + + ApplicationName string `json:"ApplicationName,omitempty"` + + CommandLine string `json:"CommandLine,omitempty"` + + // optional alternative to CommandLine, currently only supported by Linux GCS + CommandArgs []string `json:"CommandArgs,omitempty"` + + User string `json:"User,omitempty"` + + WorkingDirectory string `json:"WorkingDirectory,omitempty"` + + Environment map[string]string `json:"Environment,omitempty"` + + // if set, will run as low-privilege process + RestrictedToken bool `json:"RestrictedToken,omitempty"` + + // if set, ignore StdErrPipe + EmulateConsole bool `json:"EmulateConsole,omitempty"` + + CreateStdInPipe bool `json:"CreateStdInPipe,omitempty"` + + CreateStdOutPipe bool `json:"CreateStdOutPipe,omitempty"` + + CreateStdErrPipe bool `json:"CreateStdErrPipe,omitempty"` + + // height then width + ConsoleSize []int32 `json:"ConsoleSize,omitempty"` + + // if set, find an existing session for the user and create the process in it + UseExistingLogin bool `json:"UseExistingLogin,omitempty"` + + // if set, use the legacy console instead of conhost + UseLegacyConsole bool `json:"UseLegacyConsole,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_status.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_status.go new file mode 100644 index 0000000000..20793d1503 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_status.go @@ -0,0 +1,22 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// Status of a process running in a container +type ProcessStatus struct { + + ProcessId int32 `json:"ProcessId,omitempty"` + + Exited bool `json:"Exited,omitempty"` + + ExitCode int32 `json:"ExitCode,omitempty"` + + LastWaitResult int32 `json:"LastWaitResult,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor.go new file mode 100644 index 0000000000..7a60b0245a --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor.go @@ -0,0 +1,19 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Processor struct { + + Count int32 `json:"Count,omitempty"` + + Maximum int32 `json:"Maximum,omitempty"` + + Weight int32 `json:"Weight,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_2.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_2.go new file mode 100644 index 0000000000..40d3e7356d --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_2.go @@ -0,0 +1,21 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Processor2 struct { + + Count int32 `json:"Count,omitempty"` + + Limit int32 `json:"Limit,omitempty"` + + Weight int32 `json:"Weight,omitempty"` + + ExposeVirtualizationExtensions bool `json:"ExposeVirtualizationExtensions,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_stats.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_stats.go new file mode 100644 index 0000000000..9d3b77e572 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_stats.go @@ -0,0 +1,20 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// CPU runtime statistics +type ProcessorStats struct { + + TotalRuntime100ns int32 `json:"TotalRuntime100ns,omitempty"` + + RuntimeUser100ns int32 `json:"RuntimeUser100ns,omitempty"` + + RuntimeKernel100ns int32 `json:"RuntimeKernel100ns,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/properties.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/properties.go new file mode 100644 index 0000000000..6db2a48f66 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/properties.go @@ -0,0 +1,47 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Properties struct { + + Id string `json:"Id,omitempty"` + + SystemType string `json:"SystemType,omitempty"` + + RuntimeOsType string `json:"RuntimeOsType,omitempty"` + + Name string `json:"Name,omitempty"` + + Owner string `json:"Owner,omitempty"` + + RuntimeId string `json:"RuntimeId,omitempty"` + + RuntimeTemplateId string `json:"RuntimeTemplateId,omitempty"` + + State string `json:"State,omitempty"` + + Stopped bool `json:"Stopped,omitempty"` + + ExitType string `json:"ExitType,omitempty"` + + Memory *MemoryInformationForVm `json:"Memory,omitempty"` + + Statistics *Statistics `json:"Statistics,omitempty"` + + ProcessList []ProcessDetails `json:"ProcessList,omitempty"` + + TerminateOnLastHandleClosed bool `json:"TerminateOnLastHandleClosed,omitempty"` + + HostingSystemId string `json:"HostingSystemId,omitempty"` + + SharedMemoryRegionInfo []SharedMemoryRegionInfo `json:"SharedMemoryRegionInfo,omitempty"` + + GuestConnectionInfo *GuestConnectionInfo `json:"GuestConnectionInfo,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/property_query.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/property_query.go new file mode 100644 index 0000000000..22b92ffdfd --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/property_query.go @@ -0,0 +1,16 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// By default the basic properties will be returned. This query provides a way to request specific properties. +type PropertyQuery struct { + + PropertyTypes []string `json:"PropertyTypes,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/rdp_connection_options.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/rdp_connection_options.go new file mode 100644 index 0000000000..97e4531283 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/rdp_connection_options.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type RdpConnectionOptions struct { + + AccessSids []string `json:"AccessSids,omitempty"` + + NamedPipe string `json:"NamedPipe,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_changes.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_changes.go new file mode 100644 index 0000000000..fa574ccc80 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_changes.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type RegistryChanges struct { + + AddValues []RegistryValue `json:"AddValues,omitempty"` + + DeleteKeys []RegistryKey `json:"DeleteKeys,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_key.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_key.go new file mode 100644 index 0000000000..fab03bc60b --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_key.go @@ -0,0 +1,19 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type RegistryKey struct { + + Hive string `json:"Hive,omitempty"` + + Name string `json:"Name,omitempty"` + + Volatile bool `json:"Volatile,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_value.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_value.go new file mode 100644 index 0000000000..1589f48413 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_value.go @@ -0,0 +1,31 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type RegistryValue struct { + + Key *RegistryKey `json:"Key,omitempty"` + + Name string `json:"Name,omitempty"` + + Type_ string `json:"Type,omitempty"` + + // One and only one value type must be set. + StringValue string `json:"StringValue,omitempty"` + + BinaryValue string `json:"BinaryValue,omitempty"` + + DWordValue int32 `json:"DWordValue,omitempty"` + + QWordValue int32 `json:"QWordValue,omitempty"` + + // Only used if RegistryValueType is CustomType The data is in BinaryValue + CustomType int32 `json:"CustomType,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/restore_state.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/restore_state.go new file mode 100644 index 0000000000..778ff58735 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/restore_state.go @@ -0,0 +1,19 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type RestoreState struct { + + // The path to the save state file to restore the system from. + SaveStateFilePath string `json:"SaveStateFilePath,omitempty"` + + // The ID of the template system to clone this new system off of. An empty string indicates the system should not be cloned from a template. + TemplateSystemId string `json:"TemplateSystemId,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/save_options.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/save_options.go new file mode 100644 index 0000000000..e55fa1d98a --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/save_options.go @@ -0,0 +1,19 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type SaveOptions struct { + + // The type of save operation to be performed. + SaveType string `json:"SaveType,omitempty"` + + // The path to the file that will container the saved state. + SaveStateFilePath string `json:"SaveStateFilePath,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/scsi.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/scsi.go new file mode 100644 index 0000000000..bf253a470b --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/scsi.go @@ -0,0 +1,16 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Scsi struct { + + // Map of attachments, where the key is the integer LUN number on the controller. + Attachments map[string]Attachment `json:"Attachments,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_configuration.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_configuration.go new file mode 100644 index 0000000000..bd573f6cd4 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_configuration.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type SharedMemoryConfiguration struct { + + Regions []SharedMemoryRegion `json:"Regions,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region.go new file mode 100644 index 0000000000..a57b2cba73 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region.go @@ -0,0 +1,23 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type SharedMemoryRegion struct { + + SectionName string `json:"SectionName,omitempty"` + + StartOffset int32 `json:"StartOffset,omitempty"` + + Length int32 `json:"Length,omitempty"` + + AllowGuestWrite bool `json:"AllowGuestWrite,omitempty"` + + HiddenFromGuest bool `json:"HiddenFromGuest,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region_info.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region_info.go new file mode 100644 index 0000000000..d9a50cc7da --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region_info.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type SharedMemoryRegionInfo struct { + + SectionName string `json:"SectionName,omitempty"` + + GuestPhysicalAddress int32 `json:"GuestPhysicalAddress,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/silo_properties.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/silo_properties.go new file mode 100644 index 0000000000..599c06e8aa --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/silo_properties.go @@ -0,0 +1,18 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// Silo job information +type SiloProperties struct { + + Enabled bool `json:"Enabled,omitempty"` + + JobName string `json:"JobName,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/statistics.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/statistics.go new file mode 100644 index 0000000000..5cb3ed93b5 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/statistics.go @@ -0,0 +1,30 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +import ( + "time" +) + +// Runtime statistics for a container +type Statistics struct { + + Timestamp time.Time `json:"Timestamp,omitempty"` + + ContainerStartTime time.Time `json:"ContainerStartTime,omitempty"` + + Uptime100ns int32 `json:"Uptime100ns,omitempty"` + + Processor *ProcessorStats `json:"Processor,omitempty"` + + Memory *MemoryStats `json:"Memory,omitempty"` + + Storage *StorageStats `json:"Storage,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage.go new file mode 100644 index 0000000000..2627af9132 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage.go @@ -0,0 +1,21 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Storage struct { + + // List of layers that describe the parent hierarchy for a container's storage. These layers combined together, presented as a disposable and/or committable working storage, are used by the container to record all changes done to the parent layers. + Layers []Layer `json:"Layers,omitempty"` + + // Path that points to the scratch space of a container, where parent layers are combined together to present a new disposable and/or committable layer with the changes done during its runtime. + Path string `json:"Path,omitempty"` + + QoS *StorageQoS `json:"QoS,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_qo_s.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_qo_s.go new file mode 100644 index 0000000000..8c5255df1e --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_qo_s.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type StorageQoS struct { + + IopsMaximum int32 `json:"IopsMaximum,omitempty"` + + BandwidthMaximum int32 `json:"BandwidthMaximum,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_stats.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_stats.go new file mode 100644 index 0000000000..198ea57d75 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_stats.go @@ -0,0 +1,22 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// Storage runtime statistics +type StorageStats struct { + + ReadCountNormalized int32 `json:"ReadCountNormalized,omitempty"` + + ReadSizeBytes int32 `json:"ReadSizeBytes,omitempty"` + + WriteCountNormalized int32 `json:"WriteCountNormalized,omitempty"` + + WriteSizeBytes int32 `json:"WriteSizeBytes,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/topology.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/topology.go new file mode 100644 index 0000000000..af2e3c8234 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/topology.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Topology struct { + + Memory *Memory2 `json:"Memory,omitempty"` + + Processor *Processor2 `json:"Processor,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi.go new file mode 100644 index 0000000000..ba91178f96 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi.go @@ -0,0 +1,21 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Uefi struct { + + EnableDebugger bool `json:"EnableDebugger,omitempty"` + + SecureBootTemplateId string `json:"SecureBootTemplateId,omitempty"` + + BootThis *UefiBootEntry `json:"BootThis,omitempty"` + + Console string `json:"Console,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi_boot_entry.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi_boot_entry.go new file mode 100644 index 0000000000..6620fb2bcf --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi_boot_entry.go @@ -0,0 +1,23 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type UefiBootEntry struct { + + DeviceType string `json:"DeviceType,omitempty"` + + DevicePath string `json:"DevicePath,omitempty"` + + DiskNumber int32 `json:"DiskNumber,omitempty"` + + OptionalData string `json:"OptionalData,omitempty"` + + VmbFsRootPath string `json:"VmbFsRootPath,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/version.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/version.go new file mode 100644 index 0000000000..62c0e4d12a --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/version.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type Version struct { + + Major int32 `json:"Major,omitempty"` + + Minor int32 `json:"Minor,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/video_monitor.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/video_monitor.go new file mode 100644 index 0000000000..0958e56062 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/video_monitor.go @@ -0,0 +1,19 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type VideoMonitor struct { + + HorizontalResolution int32 `json:"HorizontalResolution,omitempty"` + + VerticalResolution int32 `json:"VerticalResolution,omitempty"` + + ConnectionOptions *RdpConnectionOptions `json:"ConnectionOptions,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_machine.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_machine.go new file mode 100644 index 0000000000..11f39eea7b --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_machine.go @@ -0,0 +1,29 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type VirtualMachine struct { + + Chipset *Chipset `json:"Chipset,omitempty"` + + ComputeTopology *Topology `json:"ComputeTopology,omitempty"` + + Devices *Devices `json:"Devices,omitempty"` + + GuestState *GuestState `json:"GuestState,omitempty"` + + RestoreState *RestoreState `json:"RestoreState,omitempty"` + + RegistryChanges *RegistryChanges `json:"RegistryChanges,omitempty"` + + StorageQoS *StorageQoS `json:"StorageQoS,omitempty"` + + GuestConnection *GuestConnection `json:"GuestConnection,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_node_info.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_node_info.go new file mode 100644 index 0000000000..48402d8ecb --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_node_info.go @@ -0,0 +1,21 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type VirtualNodeInfo struct { + + VirtualNodeIndex int32 `json:"VirtualNodeIndex,omitempty"` + + PhysicalNodeNumber int32 `json:"PhysicalNodeNumber,omitempty"` + + VirtualProcessorCount int32 `json:"VirtualProcessorCount,omitempty"` + + MemoryUsageInPages int32 `json:"MemoryUsageInPages,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_controller.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_controller.go new file mode 100644 index 0000000000..2ef1a47979 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_controller.go @@ -0,0 +1,21 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type VirtualPMemController struct { + + Devices map[string]VirtualPMemDevice `json:"Devices,omitempty"` + + MaximumCount int32 `json:"MaximumCount,omitempty"` + + MaximumSizeBytes int32 `json:"MaximumSizeBytes,omitempty"` + + Backing string `json:"Backing,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_device.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_device.go new file mode 100644 index 0000000000..47714444aa --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_device.go @@ -0,0 +1,19 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type VirtualPMemDevice struct { + + HostPath string `json:"HostPath,omitempty"` + + ReadOnly bool `json:"ReadOnly,omitempty"` + + ImageFormat string `json:"ImageFormat,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb.go new file mode 100644 index 0000000000..76131b3a71 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type VirtualSmb struct { + + Shares []VirtualSmbShare `json:"Shares,omitempty"` + + DirectFileMappingInMB int64 `json:"DirectFileMappingInMB,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share.go new file mode 100644 index 0000000000..b50098a423 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share.go @@ -0,0 +1,21 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type VirtualSmbShare struct { + + Name string `json:"Name,omitempty"` + + Path string `json:"Path,omitempty"` + + AllowedFiles []string `json:"AllowedFiles,omitempty"` + + Options *VirtualSmbShareOptions `json:"Options,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share_options.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share_options.go new file mode 100644 index 0000000000..c1894279dc --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share_options.go @@ -0,0 +1,63 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type VirtualSmbShareOptions struct { + + ReadOnly bool `json:"ReadOnly,omitempty"` + + // convert exclusive access to shared read access + ShareRead bool `json:"ShareRead,omitempty"` + + // all opens will use cached I/O + CacheIo bool `json:"CacheIo,omitempty"` + + // disable oplock support + NoOplocks bool `json:"NoOplocks,omitempty"` + + // Acquire the backup privilege when attempting to open + TakeBackupPrivilege bool `json:"TakeBackupPrivilege,omitempty"` + + // Use the identity of the share root when opening + UseShareRootIdentity bool `json:"UseShareRootIdentity,omitempty"` + + // disable Direct Mapping + NoDirectmap bool `json:"NoDirectmap,omitempty"` + + // disable Byterange locks + NoLocks bool `json:"NoLocks,omitempty"` + + // disable Directory CHange Notifications + NoDirnotify bool `json:"NoDirnotify,omitempty"` + + // share is use for VM shared memory + VmSharedMemory bool `json:"VmSharedMemory,omitempty"` + + // allow access only to the files specified in AllowedFiles + RestrictFileAccess bool `json:"RestrictFileAccess,omitempty"` + + // disable all oplocks except Level II + ForceLevelIIOplocks bool `json:"ForceLevelIIOplocks,omitempty"` + + // Allow the host to reparse this base layer + ReparseBaseLayer bool `json:"ReparseBaseLayer,omitempty"` + + // Enable pseudo-oplocks + PseudoOplocks bool `json:"PseudoOplocks,omitempty"` + + // All opens will use non-cached IO + NonCacheIo bool `json:"NonCacheIo,omitempty"` + + // Enable pseudo directory change notifications + PseudoDirnotify bool `json:"PseudoDirnotify,omitempty"` + + // Block directory enumeration, renames, and deletes. + SingleFileMapping bool `json:"SingleFileMapping,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/vm_memory.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/vm_memory.go new file mode 100644 index 0000000000..39f628667c --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/vm_memory.go @@ -0,0 +1,27 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type VmMemory struct { + + AvailableMemory int32 `json:"AvailableMemory,omitempty"` + + AvailableMemoryBuffer int32 `json:"AvailableMemoryBuffer,omitempty"` + + ReservedMemory int32 `json:"ReservedMemory,omitempty"` + + AssignedMemory int32 `json:"AssignedMemory,omitempty"` + + SlpActive bool `json:"SlpActive,omitempty"` + + BalancingEnabled bool `json:"BalancingEnabled,omitempty"` + + DmOperationInProgress bool `json:"DmOperationInProgress,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/windows_crash_reporting.go b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/windows_crash_reporting.go new file mode 100644 index 0000000000..cf632bbc83 --- /dev/null +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/internal/schema2/windows_crash_reporting.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.1 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type WindowsCrashReporting struct { + + DumpFileName string `json:"DumpFileName,omitempty"` + + MaxDumpSize int64 `json:"MaxDumpSize,omitempty"` +} diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/layer.go b/components/engine/vendor/github.com/Microsoft/hcsshim/layer.go index 8cdc247dcd..d143efc49a 100644 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/layer.go +++ b/components/engine/vendor/github.com/Microsoft/hcsshim/layer.go @@ -19,6 +19,7 @@ func ActivateLayer(info DriverInfo, id string) error { func CreateLayer(info DriverInfo, id, parent string) error { return wclayer.CreateLayer(layerPath(&info, id), parent) } + // New clients should use CreateScratchLayer instead. Kept in to preserve API compatibility. func CreateSandboxLayer(info DriverInfo, layerId, parentId string, parentLayerPaths []string) error { return wclayer.CreateScratchLayer(layerPath(&info, layerId), parentLayerPaths) @@ -32,6 +33,7 @@ func DeactivateLayer(info DriverInfo, id string) error { func DestroyLayer(info DriverInfo, id string) error { return wclayer.DestroyLayer(layerPath(&info, id)) } + // New clients should use ExpandScratchSize instead. Kept in to preserve API compatibility. func ExpandSandboxSize(info DriverInfo, layerId string, size uint64) error { return wclayer.ExpandScratchSize(layerPath(&info, layerId), size) diff --git a/components/engine/vendor/github.com/containerd/containerd/README.md b/components/engine/vendor/github.com/containerd/containerd/README.md index c4d9dffdbd..2055404b5b 100644 --- a/components/engine/vendor/github.com/containerd/containerd/README.md +++ b/components/engine/vendor/github.com/containerd/containerd/README.md @@ -1,4 +1,4 @@ -![banner](https://github.com/containerd/containerd.io/blob/master/static/img/containerd-dark.png?raw=true) +![containerd banner](https://raw.githubusercontent.com/cncf/artwork/master/containerd/horizontal/color/containerd-horizontal-color.png) [![GoDoc](https://godoc.org/github.com/containerd/containerd?status.svg)](https://godoc.org/github.com/containerd/containerd) [![Build Status](https://travis-ci.org/containerd/containerd.svg?branch=master)](https://travis-ci.org/containerd/containerd) @@ -236,3 +236,16 @@ The containerd codebase is released under the [Apache 2.0 license](LICENSE.code) The README.md file, and files in the "docs" folder are licensed under the Creative Commons Attribution 4.0 International License. You may obtain a copy of the license, titled CC-BY-4.0, at http://creativecommons.org/licenses/by/4.0/. + +## Project details + +**containerd** is the primary open source project within the broader containerd GitHub repository. +However, all projects within the repo have common maintainership, governance, and contributing +guidelines which are stored in a `project` repository commonly for all containerd projects. + +Please find all these core project documents, including the: + * [Project governance](https://github.com/containerd/project/blob/master/GOVERNANCE.md), + * [Maintainers](https://github.com/containerd/project/blob/master/MAINTAINERS), + * and [Contributing guidelines](https://github.com/containerd/project/blob/master/CONTRIBUTING.md) + +information in our [`containerd/project`](https://github.com/containerd/project) repository. diff --git a/components/engine/vendor/github.com/containerd/containerd/archive/time_unix.go b/components/engine/vendor/github.com/containerd/containerd/archive/time_unix.go index 4a69cb7d0e..53d655be46 100644 --- a/components/engine/vendor/github.com/containerd/containerd/archive/time_unix.go +++ b/components/engine/vendor/github.com/containerd/containerd/archive/time_unix.go @@ -1,4 +1,4 @@ -// +build linux freebsd solaris +// +build freebsd linux openbsd solaris /* Copyright The containerd Authors. diff --git a/components/engine/vendor/github.com/containerd/containerd/content/content.go b/components/engine/vendor/github.com/containerd/containerd/content/content.go index aabf4c8f31..d8141a68bc 100644 --- a/components/engine/vendor/github.com/containerd/containerd/content/content.go +++ b/components/engine/vendor/github.com/containerd/containerd/content/content.go @@ -110,8 +110,9 @@ type IngestManager interface { // Writer handles the write of content into a content store type Writer interface { - // Close is expected to be called after Commit() when commission is needed. - // Closing a writer without commit allows resuming or aborting. + // Close closes the writer, if the writer has not been + // committed this allows resuming or aborting. + // Calling Close on a closed writer will not error. io.WriteCloser // Digest may return empty digest or panics until committed. @@ -119,6 +120,8 @@ type Writer interface { // Commit commits the blob (but no roll-back is guaranteed on an error). // size and expected can be zero-value when unknown. + // Commit always closes the writer, even on error. + // ErrAlreadyExists aborts the writer. Commit(ctx context.Context, size int64, expected digest.Digest, opts ...Opt) error // Status returns the current state of write diff --git a/components/engine/vendor/github.com/containerd/containerd/content/local/store.go b/components/engine/vendor/github.com/containerd/containerd/content/local/store.go index 7fa9bb736a..996724057f 100644 --- a/components/engine/vendor/github.com/containerd/containerd/content/local/store.go +++ b/components/engine/vendor/github.com/containerd/containerd/content/local/store.go @@ -524,12 +524,11 @@ func (s *store) writer(ctx context.Context, ref string, total int64, expected di if err != nil { return nil, err } - defer fp.Close() p := bufPool.Get().(*[]byte) - defer bufPool.Put(p) - offset, err = io.CopyBuffer(digester.Hash(), fp, *p) + bufPool.Put(p) + fp.Close() if err != nil { return nil, err } diff --git a/components/engine/vendor/github.com/containerd/containerd/content/local/writer.go b/components/engine/vendor/github.com/containerd/containerd/content/local/writer.go index 10df4a4c54..223b145444 100644 --- a/components/engine/vendor/github.com/containerd/containerd/content/local/writer.go +++ b/components/engine/vendor/github.com/containerd/containerd/content/local/writer.go @@ -26,6 +26,7 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/log" "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) @@ -80,43 +81,36 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, } } - if w.fp == nil { + // Ensure even on error the writer is fully closed + defer unlock(w.ref) + fp := w.fp + w.fp = nil + + if fp == nil { return errors.Wrap(errdefs.ErrFailedPrecondition, "cannot commit on closed writer") } - if err := w.fp.Sync(); err != nil { + if err := fp.Sync(); err != nil { + fp.Close() return errors.Wrap(err, "sync failed") } - fi, err := w.fp.Stat() + fi, err := fp.Stat() + closeErr := fp.Close() if err != nil { return errors.Wrap(err, "stat on ingest file failed") } - - // change to readonly, more important for read, but provides _some_ - // protection from this point on. We use the existing perms with a mask - // only allowing reads honoring the umask on creation. - // - // This removes write and exec, only allowing read per the creation umask. - // - // NOTE: Windows does not support this operation - if runtime.GOOS != "windows" { - if err := w.fp.Chmod((fi.Mode() & os.ModePerm) &^ 0333); err != nil { - return errors.Wrap(err, "failed to change ingest file permissions") - } + if closeErr != nil { + return errors.Wrap(err, "failed to close ingest file") } if size > 0 && size != fi.Size() { - return errors.Errorf("unexpected commit size %d, expected %d", fi.Size(), size) - } - - if err := w.fp.Close(); err != nil { - return errors.Wrap(err, "failed closing ingest") + return errors.Wrapf(errdefs.ErrFailedPrecondition, "unexpected commit size %d, expected %d", fi.Size(), size) } dgst := w.digester.Digest() if expected != "" && expected != dgst { - return errors.Errorf("unexpected commit digest %s, expected %s", dgst, expected) + return errors.Wrapf(errdefs.ErrFailedPrecondition, "unexpected commit digest %s, expected %s", dgst, expected) } var ( @@ -129,27 +123,48 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, return err } - // clean up!! - defer os.RemoveAll(w.path) - if _, err := os.Stat(target); err == nil { // collision with the target file! + if err := os.RemoveAll(w.path); err != nil { + log.G(ctx).WithField("ref", w.ref).WithField("path", w.path).Errorf("failed to remove ingest directory") + } return errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", dgst) } + if err := os.Rename(ingest, target); err != nil { return err } + + // Ingest has now been made available in the content store, attempt to complete + // setting metadata but errors should only be logged and not returned since + // the content store cannot be cleanly rolled back. + commitTime := time.Now() if err := os.Chtimes(target, commitTime, commitTime); err != nil { - return err + log.G(ctx).WithField("digest", dgst).Errorf("failed to change file time to commit time") } - w.fp = nil - unlock(w.ref) + // clean up!! + if err := os.RemoveAll(w.path); err != nil { + log.G(ctx).WithField("ref", w.ref).WithField("path", w.path).Errorf("failed to remove ingest directory") + } if w.s.ls != nil && base.Labels != nil { if err := w.s.ls.Set(dgst, base.Labels); err != nil { - return err + log.G(ctx).WithField("digest", dgst).Errorf("failed to set labels") + } + } + + // change to readonly, more important for read, but provides _some_ + // protection from this point on. We use the existing perms with a mask + // only allowing reads honoring the umask on creation. + // + // This removes write and exec, only allowing read per the creation umask. + // + // NOTE: Windows does not support this operation + if runtime.GOOS != "windows" { + if err := os.Chmod(target, (fi.Mode()&os.ModePerm)&^0333); err != nil { + log.G(ctx).WithField("ref", w.ref).Errorf("failed to make readonly") } } diff --git a/components/engine/vendor/github.com/containerd/containerd/events.go b/components/engine/vendor/github.com/containerd/containerd/events.go index 92e9cd5104..3577b7c3a9 100644 --- a/components/engine/vendor/github.com/containerd/containerd/events.go +++ b/components/engine/vendor/github.com/containerd/containerd/events.go @@ -110,6 +110,9 @@ func (e *eventRemote) Subscribe(ctx context.Context, filters ...string) (ch <-ch Event: ev.Event, }: case <-ctx.Done(): + if cerr := ctx.Err(); cerr != context.Canceled { + errq <- cerr + } return } } diff --git a/components/engine/vendor/github.com/containerd/containerd/images/archive/importer.go b/components/engine/vendor/github.com/containerd/containerd/images/archive/importer.go index c1522df461..da83275c3a 100644 --- a/components/engine/vendor/github.com/containerd/containerd/images/archive/importer.go +++ b/components/engine/vendor/github.com/containerd/containerd/images/archive/importer.go @@ -36,7 +36,7 @@ import ( "github.com/pkg/errors" ) -// ImportIndex imports an index from a tar achive image bundle +// ImportIndex imports an index from a tar archive image bundle // - implements Docker v1.1, v1.2 and OCI v1. // - prefers OCI v1 when provided // - creates OCI index for Docker formats @@ -164,7 +164,7 @@ func ImportIndex(ctx context.Context, store content.Store, reader io.Reader) (oc if len(platforms) > 0 { // Only one platform can be resolved from non-index manifest, // The platform can only come from the config included above, - // if the config has no platform it can be safely ommitted. + // if the config has no platform it can be safely omitted. desc.Platform = &platforms[0] } diff --git a/components/engine/vendor/github.com/containerd/containerd/metadata/content.go b/components/engine/vendor/github.com/containerd/containerd/metadata/content.go index f51b0aadde..8ee0f2e201 100644 --- a/components/engine/vendor/github.com/containerd/containerd/metadata/content.go +++ b/components/engine/vendor/github.com/containerd/containerd/metadata/content.go @@ -556,12 +556,6 @@ func (nw *namespacedWriter) Commit(ctx context.Context, size int64, expected dig var innerErr error if err := update(ctx, nw.db, func(tx *bolt.Tx) error { - bkt := getIngestsBucket(tx, nw.namespace) - if bkt != nil { - if err := bkt.DeleteBucket([]byte(nw.ref)); err != nil && err != bolt.ErrBucketNotFound { - return err - } - } dgst, err := nw.commit(ctx, tx, size, expected, opts...) if err != nil { if !errdefs.IsAlreadyExists(err) { @@ -569,6 +563,12 @@ func (nw *namespacedWriter) Commit(ctx context.Context, size int64, expected dig } innerErr = err } + bkt := getIngestsBucket(tx, nw.namespace) + if bkt != nil { + if err := bkt.DeleteBucket([]byte(nw.ref)); err != nil && err != bolt.ErrBucketNotFound { + return err + } + } if err := removeIngestLease(ctx, tx, nw.ref); err != nil { return err } @@ -584,30 +584,38 @@ func (nw *namespacedWriter) commit(ctx context.Context, tx *bolt.Tx, size int64, var base content.Info for _, opt := range opts { if err := opt(&base); err != nil { + if nw.w != nil { + nw.w.Close() + } return "", err } } if err := validateInfo(&base); err != nil { + if nw.w != nil { + nw.w.Close() + } return "", err } var actual digest.Digest if nw.w == nil { if size != 0 && size != nw.desc.Size { - return "", errors.Errorf("%q failed size validation: %v != %v", nw.ref, nw.desc.Size, size) + return "", errors.Wrapf(errdefs.ErrFailedPrecondition, "%q failed size validation: %v != %v", nw.ref, nw.desc.Size, size) } if expected != "" && expected != nw.desc.Digest { - return "", errors.Errorf("%q unexpected digest", nw.ref) + return "", errors.Wrapf(errdefs.ErrFailedPrecondition, "%q unexpected digest", nw.ref) } size = nw.desc.Size actual = nw.desc.Digest } else { status, err := nw.w.Status() if err != nil { + nw.w.Close() return "", err } if size != 0 && size != status.Offset { - return "", errors.Errorf("%q failed size validation: %v != %v", nw.ref, status.Offset, size) + nw.w.Close() + return "", errors.Wrapf(errdefs.ErrFailedPrecondition, "%q failed size validation: %v != %v", nw.ref, status.Offset, size) } size = status.Offset actual = nw.w.Digest() diff --git a/components/engine/vendor/github.com/containerd/containerd/mount/mount_unix.go b/components/engine/vendor/github.com/containerd/containerd/mount/mount_unix.go index 6741293f89..95da9428e6 100644 --- a/components/engine/vendor/github.com/containerd/containerd/mount/mount_unix.go +++ b/components/engine/vendor/github.com/containerd/containerd/mount/mount_unix.go @@ -1,4 +1,4 @@ -// +build darwin freebsd +// +build darwin freebsd openbsd /* Copyright The containerd Authors. diff --git a/components/engine/vendor/github.com/containerd/containerd/mount/mountinfo_freebsd.go b/components/engine/vendor/github.com/containerd/containerd/mount/mountinfo_bsd.go similarity index 98% rename from components/engine/vendor/github.com/containerd/containerd/mount/mountinfo_freebsd.go rename to components/engine/vendor/github.com/containerd/containerd/mount/mountinfo_bsd.go index bbe79767e3..8f8dbf95a4 100644 --- a/components/engine/vendor/github.com/containerd/containerd/mount/mountinfo_freebsd.go +++ b/components/engine/vendor/github.com/containerd/containerd/mount/mountinfo_bsd.go @@ -1,3 +1,5 @@ +// +build freebsd openbsd + /* Copyright The containerd Authors. diff --git a/components/engine/vendor/github.com/containerd/containerd/mount/mountinfo_unsupported.go b/components/engine/vendor/github.com/containerd/containerd/mount/mountinfo_unsupported.go index eba602f1a6..ae998db6b5 100644 --- a/components/engine/vendor/github.com/containerd/containerd/mount/mountinfo_unsupported.go +++ b/components/engine/vendor/github.com/containerd/containerd/mount/mountinfo_unsupported.go @@ -1,4 +1,4 @@ -// +build !linux,!freebsd,!solaris freebsd,!cgo solaris,!cgo +// +build !linux,!freebsd,!solaris,!openbsd freebsd,!cgo solaris,!cgo openbsd,!cgo /* Copyright The containerd Authors. diff --git a/components/engine/vendor/github.com/containerd/containerd/platforms/cpuinfo.go b/components/engine/vendor/github.com/containerd/containerd/platforms/cpuinfo.go index a5c5ab42b9..bf6476b641 100644 --- a/components/engine/vendor/github.com/containerd/containerd/platforms/cpuinfo.go +++ b/components/engine/vendor/github.com/containerd/containerd/platforms/cpuinfo.go @@ -74,6 +74,22 @@ func getCPUInfo(pattern string) (info string, err error) { } func getCPUVariant() string { + if runtime.GOOS == "windows" { + // Windows only supports v7 for ARM32 and v8 for ARM64 and so we can use + // runtime.GOARCH to determine the variants + var variant string + switch runtime.GOARCH { + case "arm64": + variant = "v8" + case "arm": + variant = "v7" + default: + variant = "unknown" + } + + return variant + } + variant, err := getCPUInfo("Cpu architecture") if err != nil { log.L.WithError(err).Error("failure getting variant") diff --git a/components/engine/vendor/github.com/containerd/containerd/remotes/docker/authorizer.go b/components/engine/vendor/github.com/containerd/containerd/remotes/docker/authorizer.go index bb691f183f..2d88c9f173 100644 --- a/components/engine/vendor/github.com/containerd/containerd/remotes/docker/authorizer.go +++ b/components/engine/vendor/github.com/containerd/containerd/remotes/docker/authorizer.go @@ -80,11 +80,7 @@ func (a *dockerAuthorizer) AddResponses(ctx context.Context, responses []*http.R // TODO(dmcg): Store challenge, not token // Move token fetching to authorize - if err := a.setTokenAuth(ctx, host, c.parameters); err != nil { - return err - } - - return nil + return a.setTokenAuth(ctx, host, c.parameters) } else if c.scheme == basicAuth { // TODO: Resolve credentials on authorize username, secret, err := a.credentials(host) diff --git a/components/engine/vendor/github.com/containerd/containerd/remotes/docker/schema1/converter.go b/components/engine/vendor/github.com/containerd/containerd/remotes/docker/schema1/converter.go index 45ac1933fd..766c24a26d 100644 --- a/components/engine/vendor/github.com/containerd/containerd/remotes/docker/schema1/converter.go +++ b/components/engine/vendor/github.com/containerd/containerd/remotes/docker/schema1/converter.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "io/ioutil" + "strconv" "strings" "sync" "time" @@ -42,7 +43,10 @@ import ( "github.com/pkg/errors" ) -const manifestSizeLimit = 8e6 // 8MB +const ( + manifestSizeLimit = 8e6 // 8MB + labelDockerSchema1EmptyLayer = "containerd.io/docker.schema1.empty-layer" +) type blobState struct { diffID digest.Digest @@ -353,10 +357,11 @@ func (c *Converter) fetchBlob(ctx context.Context, desc ocispec.Descriptor) erro Digest: desc.Digest, Labels: map[string]string{ "containerd.io/uncompressed": state.diffID.String(), + labelDockerSchema1EmptyLayer: strconv.FormatBool(state.empty), }, } - if _, err := c.contentStore.Update(ctx, cinfo, "labels.containerd.io/uncompressed"); err != nil { + if _, err := c.contentStore.Update(ctx, cinfo, "labels.containerd.io/uncompressed", fmt.Sprintf("labels.%s", labelDockerSchema1EmptyLayer)); err != nil { return errors.Wrap(err, "failed to update uncompressed label") } @@ -380,7 +385,18 @@ func (c *Converter) reuseLabelBlobState(ctx context.Context, desc ocispec.Descri return false, nil } - bState := blobState{empty: false} + emptyVal, ok := cinfo.Labels[labelDockerSchema1EmptyLayer] + if !ok { + return false, nil + } + + isEmpty, err := strconv.ParseBool(emptyVal) + if err != nil { + log.G(ctx).WithField("id", desc.Digest).Warnf("failed to parse bool from label %s: %v", labelDockerSchema1EmptyLayer, isEmpty) + return false, nil + } + + bState := blobState{empty: isEmpty} if bState.diffID, err = digest.Parse(diffID); err != nil { log.G(ctx).WithField("id", desc.Digest).Warnf("failed to parse digest from label containerd.io/uncompressed: %v", diffID) diff --git a/components/engine/vendor/github.com/containerd/containerd/services/server/config.go b/components/engine/vendor/github.com/containerd/containerd/services/server/config/config.go similarity index 99% rename from components/engine/vendor/github.com/containerd/containerd/services/server/config.go rename to components/engine/vendor/github.com/containerd/containerd/services/server/config/config.go index a91d5fffd9..2124d86710 100644 --- a/components/engine/vendor/github.com/containerd/containerd/services/server/config.go +++ b/components/engine/vendor/github.com/containerd/containerd/services/server/config/config.go @@ -14,7 +14,7 @@ limitations under the License. */ -package server +package config import ( "github.com/BurntSushi/toml" diff --git a/components/engine/vendor/github.com/containerd/containerd/services/server/server.go b/components/engine/vendor/github.com/containerd/containerd/services/server/server.go deleted file mode 100644 index ed10766f3d..0000000000 --- a/components/engine/vendor/github.com/containerd/containerd/services/server/server.go +++ /dev/null @@ -1,362 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package server - -import ( - "context" - "expvar" - "io" - "net" - "net/http" - "net/http/pprof" - "os" - "path/filepath" - "strings" - "sync" - "time" - - csapi "github.com/containerd/containerd/api/services/content/v1" - ssapi "github.com/containerd/containerd/api/services/snapshots/v1" - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/content/local" - csproxy "github.com/containerd/containerd/content/proxy" - "github.com/containerd/containerd/defaults" - "github.com/containerd/containerd/events/exchange" - "github.com/containerd/containerd/log" - "github.com/containerd/containerd/metadata" - "github.com/containerd/containerd/pkg/dialer" - "github.com/containerd/containerd/plugin" - "github.com/containerd/containerd/snapshots" - ssproxy "github.com/containerd/containerd/snapshots/proxy" - metrics "github.com/docker/go-metrics" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" - "google.golang.org/grpc" -) - -// New creates and initializes a new containerd server -func New(ctx context.Context, config *Config) (*Server, error) { - switch { - case config.Root == "": - return nil, errors.New("root must be specified") - case config.State == "": - return nil, errors.New("state must be specified") - case config.Root == config.State: - return nil, errors.New("root and state must be different paths") - } - - if err := os.MkdirAll(config.Root, 0711); err != nil { - return nil, err - } - if err := os.MkdirAll(config.State, 0711); err != nil { - return nil, err - } - if err := apply(ctx, config); err != nil { - return nil, err - } - plugins, err := LoadPlugins(ctx, config) - if err != nil { - return nil, err - } - - serverOpts := []grpc.ServerOption{ - grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor), - grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), - } - if config.GRPC.MaxRecvMsgSize > 0 { - serverOpts = append(serverOpts, grpc.MaxRecvMsgSize(config.GRPC.MaxRecvMsgSize)) - } - if config.GRPC.MaxSendMsgSize > 0 { - serverOpts = append(serverOpts, grpc.MaxSendMsgSize(config.GRPC.MaxSendMsgSize)) - } - rpc := grpc.NewServer(serverOpts...) - var ( - services []plugin.Service - s = &Server{ - rpc: rpc, - events: exchange.NewExchange(), - config: config, - } - initialized = plugin.NewPluginSet() - ) - for _, p := range plugins { - id := p.URI() - log.G(ctx).WithField("type", p.Type).Infof("loading plugin %q...", id) - - initContext := plugin.NewContext( - ctx, - p, - initialized, - config.Root, - config.State, - ) - initContext.Events = s.events - initContext.Address = config.GRPC.Address - - // load the plugin specific configuration if it is provided - if p.Config != nil { - pluginConfig, err := config.Decode(p.ID, p.Config) - if err != nil { - return nil, err - } - initContext.Config = pluginConfig - } - result := p.Init(initContext) - if err := initialized.Add(result); err != nil { - return nil, errors.Wrapf(err, "could not add plugin result to plugin set") - } - - instance, err := result.Instance() - if err != nil { - if plugin.IsSkipPlugin(err) { - log.G(ctx).WithField("type", p.Type).Infof("skip loading plugin %q...", id) - } else { - log.G(ctx).WithError(err).Warnf("failed to load plugin %s", id) - } - continue - } - // check for grpc services that should be registered with the server - if service, ok := instance.(plugin.Service); ok { - services = append(services, service) - } - s.plugins = append(s.plugins, result) - } - // register services after all plugins have been initialized - for _, service := range services { - if err := service.Register(rpc); err != nil { - return nil, err - } - } - return s, nil -} - -// Server is the containerd main daemon -type Server struct { - rpc *grpc.Server - events *exchange.Exchange - config *Config - plugins []*plugin.Plugin -} - -// ServeGRPC provides the containerd grpc APIs on the provided listener -func (s *Server) ServeGRPC(l net.Listener) error { - if s.config.Metrics.GRPCHistogram { - // enable grpc time histograms to measure rpc latencies - grpc_prometheus.EnableHandlingTimeHistogram() - } - // before we start serving the grpc API register the grpc_prometheus metrics - // handler. This needs to be the last service registered so that it can collect - // metrics for every other service - grpc_prometheus.Register(s.rpc) - return trapClosedConnErr(s.rpc.Serve(l)) -} - -// ServeMetrics provides a prometheus endpoint for exposing metrics -func (s *Server) ServeMetrics(l net.Listener) error { - m := http.NewServeMux() - m.Handle("/v1/metrics", metrics.Handler()) - return trapClosedConnErr(http.Serve(l, m)) -} - -// ServeDebug provides a debug endpoint -func (s *Server) ServeDebug(l net.Listener) error { - // don't use the default http server mux to make sure nothing gets registered - // that we don't want to expose via containerd - m := http.NewServeMux() - m.Handle("/debug/vars", expvar.Handler()) - m.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index)) - m.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) - m.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) - m.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) - m.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) - return trapClosedConnErr(http.Serve(l, m)) -} - -// Stop the containerd server canceling any open connections -func (s *Server) Stop() { - s.rpc.Stop() - for i := len(s.plugins) - 1; i >= 0; i-- { - p := s.plugins[i] - instance, err := p.Instance() - if err != nil { - log.L.WithError(err).WithField("id", p.Registration.ID). - Errorf("could not get plugin instance") - continue - } - closer, ok := instance.(io.Closer) - if !ok { - continue - } - if err := closer.Close(); err != nil { - log.L.WithError(err).WithField("id", p.Registration.ID). - Errorf("failed to close plugin") - } - } -} - -// LoadPlugins loads all plugins into containerd and generates an ordered graph -// of all plugins. -func LoadPlugins(ctx context.Context, config *Config) ([]*plugin.Registration, error) { - // load all plugins into containerd - if err := plugin.Load(filepath.Join(config.Root, "plugins")); err != nil { - return nil, err - } - // load additional plugins that don't automatically register themselves - plugin.Register(&plugin.Registration{ - Type: plugin.ContentPlugin, - ID: "content", - InitFn: func(ic *plugin.InitContext) (interface{}, error) { - ic.Meta.Exports["root"] = ic.Root - return local.NewStore(ic.Root) - }, - }) - plugin.Register(&plugin.Registration{ - Type: plugin.MetadataPlugin, - ID: "bolt", - Requires: []plugin.Type{ - plugin.ContentPlugin, - plugin.SnapshotPlugin, - }, - InitFn: func(ic *plugin.InitContext) (interface{}, error) { - if err := os.MkdirAll(ic.Root, 0711); err != nil { - return nil, err - } - cs, err := ic.Get(plugin.ContentPlugin) - if err != nil { - return nil, err - } - - snapshottersRaw, err := ic.GetByType(plugin.SnapshotPlugin) - if err != nil { - return nil, err - } - - snapshotters := make(map[string]snapshots.Snapshotter) - for name, sn := range snapshottersRaw { - sn, err := sn.Instance() - if err != nil { - log.G(ic.Context).WithError(err). - Warnf("could not use snapshotter %v in metadata plugin", name) - continue - } - snapshotters[name] = sn.(snapshots.Snapshotter) - } - - path := filepath.Join(ic.Root, "meta.db") - ic.Meta.Exports["path"] = path - - db, err := bolt.Open(path, 0644, nil) - if err != nil { - return nil, err - } - mdb := metadata.NewDB(db, cs.(content.Store), snapshotters) - if err := mdb.Init(ic.Context); err != nil { - return nil, err - } - return mdb, nil - }, - }) - - clients := &proxyClients{} - for name, pp := range config.ProxyPlugins { - var ( - t plugin.Type - f func(*grpc.ClientConn) interface{} - - address = pp.Address - ) - - switch pp.Type { - case string(plugin.SnapshotPlugin), "snapshot": - t = plugin.SnapshotPlugin - ssname := name - f = func(conn *grpc.ClientConn) interface{} { - return ssproxy.NewSnapshotter(ssapi.NewSnapshotsClient(conn), ssname) - } - - case string(plugin.ContentPlugin), "content": - t = plugin.ContentPlugin - f = func(conn *grpc.ClientConn) interface{} { - return csproxy.NewContentStore(csapi.NewContentClient(conn)) - } - default: - log.G(ctx).WithField("type", pp.Type).Warn("unknown proxy plugin type") - } - - plugin.Register(&plugin.Registration{ - Type: t, - ID: name, - InitFn: func(ic *plugin.InitContext) (interface{}, error) { - ic.Meta.Exports["address"] = address - conn, err := clients.getClient(address) - if err != nil { - return nil, err - } - return f(conn), nil - }, - }) - - } - - // return the ordered graph for plugins - return plugin.Graph(config.DisabledPlugins), nil -} - -type proxyClients struct { - m sync.Mutex - clients map[string]*grpc.ClientConn -} - -func (pc *proxyClients) getClient(address string) (*grpc.ClientConn, error) { - pc.m.Lock() - defer pc.m.Unlock() - if pc.clients == nil { - pc.clients = map[string]*grpc.ClientConn{} - } else if c, ok := pc.clients[address]; ok { - return c, nil - } - - gopts := []grpc.DialOption{ - grpc.WithInsecure(), - grpc.WithBackoffMaxDelay(3 * time.Second), - grpc.WithDialer(dialer.Dialer), - - // TODO(stevvooe): We may need to allow configuration of this on the client. - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)), - grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)), - } - - conn, err := grpc.Dial(dialer.DialAddress(address), gopts...) - if err != nil { - return nil, errors.Wrapf(err, "failed to dial %q", address) - } - - pc.clients[address] = conn - - return conn, nil -} - -func trapClosedConnErr(err error) error { - if err == nil { - return nil - } - if strings.Contains(err.Error(), "use of closed network connection") { - return nil - } - return err -} diff --git a/components/engine/vendor/github.com/containerd/containerd/services/server/server_linux.go b/components/engine/vendor/github.com/containerd/containerd/services/server/server_linux.go deleted file mode 100644 index c45ccd3d51..0000000000 --- a/components/engine/vendor/github.com/containerd/containerd/services/server/server_linux.go +++ /dev/null @@ -1,54 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package server - -import ( - "context" - "os" - - "github.com/containerd/cgroups" - "github.com/containerd/containerd/log" - "github.com/containerd/containerd/sys" - specs "github.com/opencontainers/runtime-spec/specs-go" -) - -// apply sets config settings on the server process -func apply(ctx context.Context, config *Config) error { - if config.OOMScore != 0 { - log.G(ctx).Debugf("changing OOM score to %d", config.OOMScore) - if err := sys.SetOOMScore(os.Getpid(), config.OOMScore); err != nil { - log.G(ctx).WithError(err).Errorf("failed to change OOM score to %d", config.OOMScore) - } - } - if config.Cgroup.Path != "" { - cg, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(config.Cgroup.Path)) - if err != nil { - if err != cgroups.ErrCgroupDeleted { - return err - } - if cg, err = cgroups.New(cgroups.V1, cgroups.StaticPath(config.Cgroup.Path), &specs.LinuxResources{}); err != nil { - return err - } - } - if err := cg.Add(cgroups.Process{ - Pid: os.Getpid(), - }); err != nil { - return err - } - } - return nil -} diff --git a/components/engine/vendor/github.com/containerd/containerd/services/server/server_solaris.go b/components/engine/vendor/github.com/containerd/containerd/services/server/server_solaris.go deleted file mode 100644 index 0dbbb9feac..0000000000 --- a/components/engine/vendor/github.com/containerd/containerd/services/server/server_solaris.go +++ /dev/null @@ -1,23 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package server - -import "context" - -func apply(_ context.Context, _ *Config) error { - return nil -} diff --git a/components/engine/vendor/github.com/containerd/containerd/services/server/server_unsupported.go b/components/engine/vendor/github.com/containerd/containerd/services/server/server_unsupported.go deleted file mode 100644 index c6211dbb2a..0000000000 --- a/components/engine/vendor/github.com/containerd/containerd/services/server/server_unsupported.go +++ /dev/null @@ -1,25 +0,0 @@ -// +build !linux,!windows,!solaris - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package server - -import "context" - -func apply(_ context.Context, _ *Config) error { - return nil -} diff --git a/components/engine/vendor/github.com/containerd/containerd/services/server/server_windows.go b/components/engine/vendor/github.com/containerd/containerd/services/server/server_windows.go deleted file mode 100644 index ac0b8481c2..0000000000 --- a/components/engine/vendor/github.com/containerd/containerd/services/server/server_windows.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build windows - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package server - -import ( - "context" -) - -func apply(_ context.Context, _ *Config) error { - return nil -} diff --git a/components/engine/vendor/github.com/containerd/containerd/vendor.conf b/components/engine/vendor/github.com/containerd/containerd/vendor.conf index c4cd8f1f42..dbc3eecd96 100644 --- a/components/engine/vendor/github.com/containerd/containerd/vendor.conf +++ b/components/engine/vendor/github.com/containerd/containerd/vendor.conf @@ -20,7 +20,7 @@ github.com/gogo/protobuf v1.0.0 github.com/gogo/googleapis 08a7655d27152912db7aaf4f983275eaf8d128ef github.com/golang/protobuf v1.1.0 github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d -github.com/opencontainers/runc 00dc70017d222b178a002ed30e9321b12647af2d +github.com/opencontainers/runc 58592df56734acf62e574865fe40b9e53e967910 github.com/sirupsen/logrus v1.0.0 github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c golang.org/x/net b3756b4b77d7b13260a0a2ec658753cf48922eac @@ -32,8 +32,8 @@ github.com/opencontainers/image-spec v1.0.1 golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895 github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0 -github.com/Microsoft/go-winio v0.4.10 -github.com/Microsoft/hcsshim v0.7.6 +github.com/Microsoft/go-winio v0.4.11 +github.com/Microsoft/hcsshim v0.7.12 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a @@ -43,8 +43,8 @@ github.com/google/go-cmp v0.1.0 go.etcd.io/bbolt v1.3.1-etcd.8 # cri dependencies -github.com/containerd/cri 9f39e3289533fc228c5e5fcac0a6dbdd60c6047b # release/1.2 branch -github.com/containerd/go-cni 6d7b509a054a3cb1c35ed1865d4fde2f0cb547cd +github.com/containerd/cri f913714917d2456d7e65a0be84962b1ce8acb487 # release/1.2 branch +github.com/containerd/go-cni 40bcf8ec8acd7372be1d77031d585d5d8e561c90 github.com/blang/semver v3.1.0 github.com/containernetworking/cni v0.6.0 github.com/containernetworking/plugins v0.7.0 @@ -73,12 +73,12 @@ golang.org/x/oauth2 a6bd8cefa1811bd24b86f8902872e4e8225f74c4 golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631 gopkg.in/inf.v0 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 gopkg.in/yaml.v2 v2.2.1 -k8s.io/api 012f271b5d41baad56190c5f1ae19bff16df0fd8 -k8s.io/apimachinery 6429050ef506887d121f3e7306e894f8900d8a63 -k8s.io/apiserver e9312c15296b6c2c923ebd5031ff5d1d5fd022d7 -k8s.io/client-go 37c3c02ec96533daec0dbda1f39a6b1d68505c79 -k8s.io/kubernetes v1.12.0-beta.1 -k8s.io/utils 982821ea41da7e7c15f3d3738921eb2e7e241ccd +k8s.io/api kubernetes-1.12.0 +k8s.io/apimachinery kubernetes-1.12.0 +k8s.io/apiserver kubernetes-1.12.0 +k8s.io/client-go kubernetes-1.12.0 +k8s.io/kubernetes v1.12.0 +k8s.io/utils cd34563cd63c2bd7c6fe88a73c4dcf34ed8a67cb # zfs dependencies github.com/containerd/zfs 9a0b8b8b5982014b729cd34eb7cd7a11062aa6ec diff --git a/components/engine/vendor/github.com/containerd/cri/vendor.conf b/components/engine/vendor/github.com/containerd/cri/vendor.conf index a9ffca873a..c81758046f 100644 --- a/components/engine/vendor/github.com/containerd/cri/vendor.conf +++ b/components/engine/vendor/github.com/containerd/cri/vendor.conf @@ -1,15 +1,14 @@ github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 github.com/blang/semver v3.1.0 -github.com/boltdb/bolt v1.3.1 github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895 github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2 github.com/containerd/console c12b1e7919c14469339a5d38f2f8ed9b64a9de23 -github.com/containerd/containerd 1950f791d9225ffe061c77e74e292bcb3c428a04 -github.com/containerd/continuity f44b615e492bdfb371aae2f76ec694d9da1db537 +github.com/containerd/containerd 15f19d7a67fa322e6de0ef4c6a1bf9da0f056554 +github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c -github.com/containerd/go-cni 6d7b509a054a3cb1c35ed1865d4fde2f0cb547cd +github.com/containerd/go-cni 40bcf8ec8acd7372be1d77031d585d5d8e561c90 github.com/containerd/go-runc 5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3 -github.com/containerd/ttrpc 94dde388801693c54f88a6596f713b51a8b30b2d +github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40 github.com/containernetworking/cni v0.6.0 github.com/containernetworking/plugins v0.7.0 @@ -35,13 +34,13 @@ github.com/hashicorp/go-multierror ed905158d87462226a13fe39ddf685ea65f1c11f github.com/json-iterator/go 1.1.5 github.com/matttproud/golang_protobuf_extensions v1.0.0 github.com/Microsoft/go-winio v0.4.10 -github.com/Microsoft/hcsshim 44c060121b68e8bdc40b411beba551f3b4ee9e55 +github.com/Microsoft/hcsshim v0.7.6 github.com/modern-go/concurrent 1.0.3 github.com/modern-go/reflect2 1.0.1 github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 github.com/opencontainers/image-spec v1.0.1 -github.com/opencontainers/runc 20aff4f0488c6d4b8df4d85b4f63f1f704c11abd -github.com/opencontainers/runtime-spec d810dbc60d8c5aeeb3d054bd1132fab2121968ce +github.com/opencontainers/runc 00dc70017d222b178a002ed30e9321b12647af2d +github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 github.com/opencontainers/runtime-tools v0.6.0 github.com/opencontainers/selinux b6fa367ed7f534f9ba25391cc2d467085dbb445a github.com/pkg/errors v0.8.0 @@ -59,6 +58,7 @@ github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6 github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b github.com/xeipuuv/gojsonschema 1d523034197ff1f222f6429836dd36a2457a1874 +go.etcd.io/bbolt v1.3.1-etcd.8 golang.org/x/crypto 49796115aa4b964c318aad4f3084fdb41e9aa067 golang.org/x/net b3756b4b77d7b13260a0a2ec658753cf48922eac golang.org/x/oauth2 a6bd8cefa1811bd24b86f8902872e4e8225f74c4 @@ -70,9 +70,9 @@ google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 google.golang.org/grpc v1.12.0 gopkg.in/inf.v0 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 gopkg.in/yaml.v2 53feefa2559fb8dfa8d81baad31be332c97d6c77 -k8s.io/api 012f271b5d41baad56190c5f1ae19bff16df0fd8 -k8s.io/apimachinery 6429050ef506887d121f3e7306e894f8900d8a63 -k8s.io/apiserver e9312c15296b6c2c923ebd5031ff5d1d5fd022d7 -k8s.io/client-go 37c3c02ec96533daec0dbda1f39a6b1d68505c79 -k8s.io/kubernetes v1.12.0-beta.1 -k8s.io/utils 982821ea41da7e7c15f3d3738921eb2e7e241ccd +k8s.io/api kubernetes-1.12.0 +k8s.io/apimachinery kubernetes-1.12.0 +k8s.io/apiserver kubernetes-1.12.0 +k8s.io/client-go kubernetes-1.12.0 +k8s.io/kubernetes v1.12.0 +k8s.io/utils cd34563cd63c2bd7c6fe88a73c4dcf34ed8a67cb diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go index f6d1f134db..b065f7f28e 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go @@ -186,8 +186,8 @@ type Config struct { // callers keyring in this case. NoNewKeyring bool `json:"no_new_keyring"` - // IntelRdt specifies settings for Intel RDT/CAT group that the container is placed into - // to limit the resources (e.g., L3 cache) the container has available + // IntelRdt specifies settings for Intel RDT group that the container is placed into + // to limit the resources (e.g., L3 cache, memory bandwidth) the container has available IntelRdt *IntelRdt `json:"intel_rdt,omitempty"` // RootlessEUID is set when the runc was launched with non-zero EUID. diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/intelrdt.go b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/intelrdt.go index 36bd5f96a1..6f47aac077 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/intelrdt.go +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/intelrdt.go @@ -4,4 +4,8 @@ type IntelRdt struct { // The schema for L3 cache id and capacity bitmask (CBM) // Format: "L3:=;=;..." L3CacheSchema string `json:"l3_cache_schema,omitempty"` + + // The schema of memory bandwidth percentage per L3 cache id + // Format: "MB:=bandwidth0;=bandwidth1;..." + MemBwSchema string `json:"memBwSchema,omitempty"` } diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/user/lookup_unix.go b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/user/lookup_unix.go index c1e634c949..92b5ae8de0 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/user/lookup_unix.go +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/user/lookup_unix.go @@ -5,6 +5,7 @@ package user import ( "io" "os" + "strconv" "golang.org/x/sys/unix" ) @@ -115,22 +116,23 @@ func CurrentGroup() (Group, error) { return LookupGid(unix.Getgid()) } -func CurrentUserSubUIDs() ([]SubID, error) { +func currentUserSubIDs(fileName string) ([]SubID, error) { u, err := CurrentUser() if err != nil { return nil, err } - return ParseSubIDFileFilter("/etc/subuid", - func(entry SubID) bool { return entry.Name == u.Name }) + filter := func(entry SubID) bool { + return entry.Name == u.Name || entry.Name == strconv.Itoa(u.Uid) + } + return ParseSubIDFileFilter(fileName, filter) } -func CurrentGroupSubGIDs() ([]SubID, error) { - g, err := CurrentGroup() - if err != nil { - return nil, err - } - return ParseSubIDFileFilter("/etc/subgid", - func(entry SubID) bool { return entry.Name == g.Name }) +func CurrentUserSubUIDs() ([]SubID, error) { + return currentUserSubIDs("/etc/subuid") +} + +func CurrentUserSubGIDs() ([]SubID, error) { + return currentUserSubIDs("/etc/subgid") } func CurrentProcessUIDMap() ([]IDMap, error) { diff --git a/components/engine/vendor/github.com/opencontainers/runc/vendor.conf b/components/engine/vendor/github.com/opencontainers/runc/vendor.conf index e2b519e673..fadbe07071 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/vendor.conf +++ b/components/engine/vendor/github.com/opencontainers/runc/vendor.conf @@ -1,7 +1,7 @@ # OCI runtime-spec. When updating this, make sure you use a version tag rather # than a commit ID so it's much more obvious what version of the spec we are # using. -github.com/opencontainers/runtime-spec v1.0.0 +github.com/opencontainers/runtime-spec 5684b8af48c1ac3b1451fa499724e30e3c20a294 # Core libcontainer functionality. github.com/mrunalp/fileutils ed869b029674c0e9ce4c0dfa781405c2d9946d08 github.com/opencontainers/selinux v1.0.0-rc1 diff --git a/components/engine/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go b/components/engine/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go index f32698cab2..9984bbce30 100644 --- a/components/engine/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go +++ b/components/engine/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go @@ -160,8 +160,8 @@ type Linux struct { ReadonlyPaths []string `json:"readonlyPaths,omitempty"` // MountLabel specifies the selinux context for the mounts in the container. MountLabel string `json:"mountLabel,omitempty"` - // IntelRdt contains Intel Resource Director Technology (RDT) information - // for handling resource constraints (e.g., L3 cache) for the container + // IntelRdt contains Intel Resource Director Technology (RDT) information for + // handling resource constraints (e.g., L3 cache, memory bandwidth) for the container IntelRdt *LinuxIntelRdt `json:"intelRdt,omitempty"` } @@ -623,10 +623,16 @@ type LinuxSyscall struct { Args []LinuxSeccompArg `json:"args,omitempty"` } -// LinuxIntelRdt has container runtime resource constraints -// for Intel RDT/CAT which introduced in Linux 4.10 kernel +// LinuxIntelRdt has container runtime resource constraints for Intel RDT +// CAT and MBA features which introduced in Linux 4.10 and 4.12 kernel type LinuxIntelRdt struct { + // The identity for RDT Class of Service + ClosID string `json:"closID,omitempty"` // The schema for L3 cache id and capacity bitmask (CBM) // Format: "L3:=;=;..." L3CacheSchema string `json:"l3CacheSchema,omitempty"` + + // The schema of memory bandwidth percentage per L3 cache id + // Format: "MB:=bandwidth0;=bandwidth1;..." + MemBwSchema string `json:"memBwSchema,omitempty"` } From e2c5b23d4959ab373e530425892a2e2d40955ffa Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 6 Nov 2018 12:33:50 +0100 Subject: [PATCH 08/25] [18.09 backport] update libnetwork to fix iptables compatibility on debian Fixes a compatibility issue on recent debian versions, where iptables now uses nft by default. Signed-off-by: Sebastiaan van Stijn Upstream-commit: e1783a72d1b84bc3e32470c468d14445e5fba8db Component: engine --- .../engine/hack/dockerfile/install/proxy.installer | 2 +- components/engine/vendor.conf | 2 +- .../github.com/docker/libnetwork/iptables/iptables.go | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/components/engine/hack/dockerfile/install/proxy.installer b/components/engine/hack/dockerfile/install/proxy.installer index f1d3f93507..1027edf823 100755 --- a/components/engine/hack/dockerfile/install/proxy.installer +++ b/components/engine/hack/dockerfile/install/proxy.installer @@ -3,7 +3,7 @@ # LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When # updating the binary version, consider updating github.com/docker/libnetwork # in vendor.conf accordingly -LIBNETWORK_COMMIT=6da50d1978302f04c3e2089e29112ea24812f05b +LIBNETWORK_COMMIT=8bc51fd276b549b106dbc7bf8ab18b3a01d66e64 # bump_18.09 branch install_proxy() { case "$1" in diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index b822620bd3..4bee6cbbe0 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -37,7 +37,7 @@ github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b #get libnetwork packages # When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly -github.com/docker/libnetwork 6da50d1978302f04c3e2089e29112ea24812f05b +github.com/docker/libnetwork 8bc51fd276b549b106dbc7bf8ab18b3a01d66e64 # bump_18.09 branch github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec diff --git a/components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go b/components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go index 4b8d8832e9..6a108e93d0 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go +++ b/components/engine/vendor/github.com/docker/libnetwork/iptables/iptables.go @@ -87,11 +87,16 @@ func initFirewalld() { } func detectIptables() { - path, err := exec.LookPath("iptables") + path, err := exec.LookPath("iptables-legacy") // debian has iptables-legacy and iptables-nft now if err != nil { - return + path, err = exec.LookPath("iptables") + if err != nil { + return + } } + iptablesPath = path + supportsXlock = exec.Command(iptablesPath, "--wait", "-L", "-n").Run() == nil mj, mn, mc, err := GetVersion() if err != nil { From 3cd1faba5a8a16792b901f116c2a2815b235d565 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 5 Nov 2018 14:50:33 +0100 Subject: [PATCH 09/25] API: properly handle invalid JSON to return a 400 status The API did not treat invalid JSON payloads as a 400 error, as a result returning a 500 error; Before this change, an invalid JSON body would return a 500 error; ```bash curl -v \ --unix-socket /var/run/docker.sock \ -X POST \ "http://localhost/v1.30/networks/create" \ -H "Content-Type: application/json" \ -d '{invalid json' ``` ``` > POST /v1.30/networks/create HTTP/1.1 > Host: localhost > User-Agent: curl/7.52.1 > Accept: */* > Content-Type: application/json > Content-Length: 13 > * upload completely sent off: 13 out of 13 bytes < HTTP/1.1 500 Internal Server Error < Api-Version: 1.40 < Content-Type: application/json < Docker-Experimental: false < Ostype: linux < Server: Docker/dev (linux) < Date: Mon, 05 Nov 2018 11:55:20 GMT < Content-Length: 79 < {"message":"invalid character 'i' looking for beginning of object key string"} ``` Empty request: ```bash curl -v \ --unix-socket /var/run/docker.sock \ -X POST \ "http://localhost/v1.30/networks/create" \ -H "Content-Type: application/json" ``` ``` > POST /v1.30/networks/create HTTP/1.1 > Host: localhost > User-Agent: curl/7.54.0 > Accept: */* > Content-Type: application/json > < HTTP/1.1 500 Internal Server Error < Api-Version: 1.38 < Content-Length: 18 < Content-Type: application/json < Date: Mon, 05 Nov 2018 12:00:18 GMT < Docker-Experimental: true < Ostype: linux < Server: Docker/18.06.1-ce (linux) < {"message":"EOF"} ``` After this change, a 400 is returned; ```bash curl -v \ --unix-socket /var/run/docker.sock \ -X POST \ "http://localhost/v1.30/networks/create" \ -H "Content-Type: application/json" \ -d '{invalid json' ``` ``` > POST /v1.30/networks/create HTTP/1.1 > Host: localhost > User-Agent: curl/7.52.1 > Accept: */* > Content-Type: application/json > Content-Length: 13 > * upload completely sent off: 13 out of 13 bytes < HTTP/1.1 400 Bad Request < Api-Version: 1.40 < Content-Type: application/json < Docker-Experimental: false < Ostype: linux < Server: Docker/dev (linux) < Date: Mon, 05 Nov 2018 11:57:15 GMT < Content-Length: 79 < {"message":"invalid character 'i' looking for beginning of object key string"} ``` Empty request: ```bash curl -v \ --unix-socket /var/run/docker.sock \ -X POST \ "http://localhost/v1.30/networks/create" \ -H "Content-Type: application/json" ``` ``` > POST /v1.30/networks/create HTTP/1.1 > Host: localhost > User-Agent: curl/7.52.1 > Accept: */* > Content-Type: application/json > < HTTP/1.1 400 Bad Request < Api-Version: 1.40 < Content-Type: application/json < Docker-Experimental: false < Ostype: linux < Server: Docker/dev (linux) < Date: Mon, 05 Nov 2018 11:59:22 GMT < Content-Length: 49 < {"message":"got EOF while reading request body"} ``` Signed-off-by: Sebastiaan van Stijn (cherry picked from commit c7b488fbc82604ec23b862ec1edc5a0be9c7793d) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 9e06a421234f0bba8392b9a8908a94ff74f0c254 Component: engine --- .../api/server/router/container/copy.go | 7 ++- .../api/server/router/container/exec.go | 11 +++- .../server/router/network/network_routes.go | 16 ++++-- .../api/server/router/plugin/plugin_routes.go | 7 ++- .../api/server/router/swarm/cluster_routes.go | 52 +++++++++++++++---- .../api/server/router/volume/volume_routes.go | 2 +- .../integration/container/container_test.go | 42 +++++++++++++++ .../integration/network/network_test.go | 34 ++++++++++++ .../integration/plugin/common/main_test.go | 27 ++++++++++ .../integration/plugin/common/plugin_test.go | 38 ++++++++++++++ .../engine/integration/volume/volume_test.go | 29 +++++++++++ 11 files changed, 248 insertions(+), 17 deletions(-) create mode 100644 components/engine/integration/container/container_test.go create mode 100644 components/engine/integration/plugin/common/main_test.go create mode 100644 components/engine/integration/plugin/common/plugin_test.go diff --git a/components/engine/api/server/router/container/copy.go b/components/engine/api/server/router/container/copy.go index 837836d001..47de35a455 100644 --- a/components/engine/api/server/router/container/copy.go +++ b/components/engine/api/server/router/container/copy.go @@ -6,12 +6,14 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "io" "net/http" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/versions" + "github.com/docker/docker/errdefs" gddohttputil "github.com/golang/gddo/httputil" ) @@ -37,7 +39,10 @@ func (s *containerRouter) postContainersCopy(ctx context.Context, w http.Respons cfg := types.CopyConfig{} if err := json.NewDecoder(r.Body).Decode(&cfg); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } if cfg.Resource == "" { diff --git a/components/engine/api/server/router/container/exec.go b/components/engine/api/server/router/container/exec.go index 25125edb5f..cf92be1d2b 100644 --- a/components/engine/api/server/router/container/exec.go +++ b/components/engine/api/server/router/container/exec.go @@ -3,6 +3,7 @@ package container // import "github.com/docker/docker/api/server/router/containe import ( "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -44,7 +45,10 @@ func (s *containerRouter) postContainerExecCreate(ctx context.Context, w http.Re execConfig := &types.ExecConfig{} if err := json.NewDecoder(r.Body).Decode(execConfig); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } if len(execConfig.Cmd) == 0 { @@ -84,7 +88,10 @@ func (s *containerRouter) postContainerExecStart(ctx context.Context, w http.Res execStartCheck := &types.ExecStartCheck{} if err := json.NewDecoder(r.Body).Decode(execStartCheck); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } if exists, err := s.backend.ExecExists(execName); !exists { diff --git a/components/engine/api/server/router/network/network_routes.go b/components/engine/api/server/router/network/network_routes.go index 8b9382cb45..aa8280c8ac 100644 --- a/components/engine/api/server/router/network/network_routes.go +++ b/components/engine/api/server/router/network/network_routes.go @@ -3,6 +3,7 @@ package network // import "github.com/docker/docker/api/server/router/network" import ( "context" "encoding/json" + "io" "net/http" "strconv" "strings" @@ -215,7 +216,10 @@ func (n *networkRouter) postNetworkCreate(ctx context.Context, w http.ResponseWr } if err := json.NewDecoder(r.Body).Decode(&create); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } if nws, err := n.cluster.GetNetworksByName(create.Name); err == nil && len(nws) > 0 { @@ -261,7 +265,10 @@ func (n *networkRouter) postNetworkConnect(ctx context.Context, w http.ResponseW } if err := json.NewDecoder(r.Body).Decode(&connect); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } // Unlike other operations, we does not check ambiguity of the name/ID here. @@ -282,7 +289,10 @@ func (n *networkRouter) postNetworkDisconnect(ctx context.Context, w http.Respon } if err := json.NewDecoder(r.Body).Decode(&disconnect); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } return n.backend.DisconnectContainerFromNetwork(disconnect.Container, vars["id"], disconnect.Force) diff --git a/components/engine/api/server/router/plugin/plugin_routes.go b/components/engine/api/server/router/plugin/plugin_routes.go index 4e816391d1..9508ca8c9d 100644 --- a/components/engine/api/server/router/plugin/plugin_routes.go +++ b/components/engine/api/server/router/plugin/plugin_routes.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "encoding/json" + "io" "net/http" "strconv" "strings" @@ -12,6 +13,7 @@ import ( "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/streamformatter" "github.com/pkg/errors" @@ -276,7 +278,10 @@ func (pr *pluginRouter) pushPlugin(ctx context.Context, w http.ResponseWriter, r func (pr *pluginRouter) setPlugin(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { var args []string if err := json.NewDecoder(r.Body).Decode(&args); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } if err := pr.backend.Set(vars["name"], args); err != nil { return err diff --git a/components/engine/api/server/router/swarm/cluster_routes.go b/components/engine/api/server/router/swarm/cluster_routes.go index a702488602..60785519a8 100644 --- a/components/engine/api/server/router/swarm/cluster_routes.go +++ b/components/engine/api/server/router/swarm/cluster_routes.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "strconv" @@ -21,7 +22,10 @@ import ( func (sr *swarmRouter) initCluster(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { var req types.InitRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } nodeID, err := sr.backend.Init(req) if err != nil { @@ -34,7 +38,10 @@ func (sr *swarmRouter) initCluster(ctx context.Context, w http.ResponseWriter, r func (sr *swarmRouter) joinCluster(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { var req types.JoinRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } return sr.backend.Join(req) } @@ -61,7 +68,10 @@ func (sr *swarmRouter) inspectCluster(ctx context.Context, w http.ResponseWriter func (sr *swarmRouter) updateCluster(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { var swarm types.Spec if err := json.NewDecoder(r.Body).Decode(&swarm); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } rawVersion := r.URL.Query().Get("version") @@ -112,7 +122,10 @@ func (sr *swarmRouter) updateCluster(ctx context.Context, w http.ResponseWriter, func (sr *swarmRouter) unlockCluster(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { var req types.UnlockRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } if err := sr.backend.UnlockSwarm(req); err != nil { @@ -175,7 +188,10 @@ func (sr *swarmRouter) getService(ctx context.Context, w http.ResponseWriter, r func (sr *swarmRouter) createService(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { var service types.ServiceSpec if err := json.NewDecoder(r.Body).Decode(&service); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } // Get returns "" if the header does not exist @@ -198,7 +214,10 @@ func (sr *swarmRouter) createService(ctx context.Context, w http.ResponseWriter, func (sr *swarmRouter) updateService(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { var service types.ServiceSpec if err := json.NewDecoder(r.Body).Decode(&service); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } rawVersion := r.URL.Query().Get("version") @@ -291,7 +310,10 @@ func (sr *swarmRouter) getNode(ctx context.Context, w http.ResponseWriter, r *ht func (sr *swarmRouter) updateNode(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { var node types.NodeSpec if err := json.NewDecoder(r.Body).Decode(&node); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } rawVersion := r.URL.Query().Get("version") @@ -370,7 +392,10 @@ func (sr *swarmRouter) getSecrets(ctx context.Context, w http.ResponseWriter, r func (sr *swarmRouter) createSecret(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { var secret types.SecretSpec if err := json.NewDecoder(r.Body).Decode(&secret); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } version := httputils.VersionFromContext(ctx) if secret.Templating != nil && versions.LessThan(version, "1.37") { @@ -408,6 +433,9 @@ func (sr *swarmRouter) getSecret(ctx context.Context, w http.ResponseWriter, r * func (sr *swarmRouter) updateSecret(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { var secret types.SecretSpec if err := json.NewDecoder(r.Body).Decode(&secret); err != nil { + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } return errdefs.InvalidParameter(err) } @@ -441,7 +469,10 @@ func (sr *swarmRouter) getConfigs(ctx context.Context, w http.ResponseWriter, r func (sr *swarmRouter) createConfig(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { var config types.ConfigSpec if err := json.NewDecoder(r.Body).Decode(&config); err != nil { - return err + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } + return errdefs.InvalidParameter(err) } version := httputils.VersionFromContext(ctx) @@ -480,6 +511,9 @@ func (sr *swarmRouter) getConfig(ctx context.Context, w http.ResponseWriter, r * func (sr *swarmRouter) updateConfig(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { var config types.ConfigSpec if err := json.NewDecoder(r.Body).Decode(&config); err != nil { + if err == io.EOF { + return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) + } return errdefs.InvalidParameter(err) } diff --git a/components/engine/api/server/router/volume/volume_routes.go b/components/engine/api/server/router/volume/volume_routes.go index e892d1a524..4fa4cb53aa 100644 --- a/components/engine/api/server/router/volume/volume_routes.go +++ b/components/engine/api/server/router/volume/volume_routes.go @@ -56,7 +56,7 @@ func (v *volumeRouter) postVolumesCreate(ctx context.Context, w http.ResponseWri if err == io.EOF { return errdefs.InvalidParameter(errors.New("got EOF while reading request body")) } - return err + return errdefs.InvalidParameter(err) } volume, err := v.backend.Create(ctx, req.Name, req.Driver, opts.WithCreateOptions(req.DriverOpts), opts.WithCreateLabels(req.Labels)) diff --git a/components/engine/integration/container/container_test.go b/components/engine/integration/container/container_test.go new file mode 100644 index 0000000000..a4ae31e948 --- /dev/null +++ b/components/engine/integration/container/container_test.go @@ -0,0 +1,42 @@ +package container // import "github.com/docker/docker/integration/container" + +import ( + "net/http" + "testing" + + "github.com/docker/docker/internal/test/request" + "gotest.tools/assert" + is "gotest.tools/assert/cmp" +) + +func TestContainerInvalidJSON(t *testing.T) { + defer setupTest(t)() + + endpoints := []string{ + "/containers/foobar/copy", + "/containers/foobar/exec", + "/exec/foobar/start", + } + + for _, ep := range endpoints { + t.Run(ep, func(t *testing.T) { + t.Parallel() + + res, body, err := request.Post(ep, request.RawString("{invalid json"), request.JSON) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + buf, err := request.ReadBody(body) + assert.NilError(t, err) + assert.Check(t, is.Contains(string(buf), "invalid character 'i' looking for beginning of object key string")) + + res, body, err = request.Post(ep, request.JSON) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + buf, err = request.ReadBody(body) + assert.NilError(t, err) + assert.Check(t, is.Contains(string(buf), "got EOF while reading request body")) + }) + } +} diff --git a/components/engine/integration/network/network_test.go b/components/engine/integration/network/network_test.go index 3829dd728e..5f62550f05 100644 --- a/components/engine/integration/network/network_test.go +++ b/components/engine/integration/network/network_test.go @@ -3,6 +3,7 @@ package network // import "github.com/docker/docker/integration/network" import ( "bytes" "context" + "net/http" "os/exec" "strings" "testing" @@ -10,6 +11,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/internal/test/daemon" + "github.com/docker/docker/internal/test/request" "gotest.tools/assert" is "gotest.tools/assert/cmp" "gotest.tools/skip" @@ -56,3 +58,35 @@ func TestRunContainerWithBridgeNone(t *testing.T) { assert.NilError(t, err) assert.Check(t, is.Equal(stdout.String(), result.Combined()), "The network namspace of container should be the same with host when --net=host and bridge network is disabled") } + +func TestNetworkInvalidJSON(t *testing.T) { + defer setupTest(t)() + + endpoints := []string{ + "/networks/create", + "/networks/bridge/connect", + "/networks/bridge/disconnect", + } + + for _, ep := range endpoints { + t.Run(ep, func(t *testing.T) { + t.Parallel() + + res, body, err := request.Post(ep, request.RawString("{invalid json"), request.JSON) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + buf, err := request.ReadBody(body) + assert.NilError(t, err) + assert.Check(t, is.Contains(string(buf), "invalid character 'i' looking for beginning of object key string")) + + res, body, err = request.Post(ep, request.JSON) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + buf, err = request.ReadBody(body) + assert.NilError(t, err) + assert.Check(t, is.Contains(string(buf), "got EOF while reading request body")) + }) + } +} diff --git a/components/engine/integration/plugin/common/main_test.go b/components/engine/integration/plugin/common/main_test.go new file mode 100644 index 0000000000..47400129b6 --- /dev/null +++ b/components/engine/integration/plugin/common/main_test.go @@ -0,0 +1,27 @@ +package common // import "github.com/docker/docker/integration/plugin/common" + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/internal/test/environment" +) + +var testEnv *environment.Execution + +func TestMain(m *testing.M) { + var err error + testEnv, err = environment.New() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + testEnv.Print() + os.Exit(m.Run()) +} + +func setupTest(t *testing.T) func() { + environment.ProtectAll(t, testEnv) + return func() { testEnv.Clean(t) } +} diff --git a/components/engine/integration/plugin/common/plugin_test.go b/components/engine/integration/plugin/common/plugin_test.go new file mode 100644 index 0000000000..5b6bfef765 --- /dev/null +++ b/components/engine/integration/plugin/common/plugin_test.go @@ -0,0 +1,38 @@ +package common // import "github.com/docker/docker/integration/plugin/common" + +import ( + "net/http" + "testing" + + "github.com/docker/docker/internal/test/request" + "gotest.tools/assert" + is "gotest.tools/assert/cmp" +) + +func TestPluginInvalidJSON(t *testing.T) { + defer setupTest(t)() + + endpoints := []string{"/plugins/foobar/set"} + + for _, ep := range endpoints { + t.Run(ep, func(t *testing.T) { + t.Parallel() + + res, body, err := request.Post(ep, request.RawString("{invalid json"), request.JSON) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + buf, err := request.ReadBody(body) + assert.NilError(t, err) + assert.Check(t, is.Contains(string(buf), "invalid character 'i' looking for beginning of object key string")) + + res, body, err = request.Post(ep, request.JSON) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + buf, err = request.ReadBody(body) + assert.NilError(t, err) + assert.Check(t, is.Contains(string(buf), "got EOF while reading request body")) + }) + } +} diff --git a/components/engine/integration/volume/volume_test.go b/components/engine/integration/volume/volume_test.go index f6ffb5eb50..f29e3669d1 100644 --- a/components/engine/integration/volume/volume_test.go +++ b/components/engine/integration/volume/volume_test.go @@ -2,6 +2,7 @@ package volume import ( "context" + "net/http" "path/filepath" "strings" "testing" @@ -95,6 +96,34 @@ func TestVolumesInspect(t *testing.T) { assert.Check(t, createdAt.Truncate(time.Minute).Equal(now.Truncate(time.Minute)), "CreatedAt (%s) not equal to creation time (%s)", createdAt, now) } +func TestVolumesInvalidJSON(t *testing.T) { + defer setupTest(t)() + + endpoints := []string{"/volumes/create"} + + for _, ep := range endpoints { + t.Run(ep, func(t *testing.T) { + t.Parallel() + + res, body, err := request.Post(ep, request.RawString("{invalid json"), request.JSON) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + buf, err := request.ReadBody(body) + assert.NilError(t, err) + assert.Check(t, is.Contains(string(buf), "invalid character 'i' looking for beginning of object key string")) + + res, body, err = request.Post(ep, request.JSON) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + buf, err = request.ReadBody(body) + assert.NilError(t, err) + assert.Check(t, is.Contains(string(buf), "got EOF while reading request body")) + }) + } +} + func getPrefixAndSlashFromDaemonPlatform() (prefix, slash string) { if testEnv.OSType == "windows" { return "c:", `\` From 6ad9059d367403b060a649000dc01cd33fda3bb9 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Fri, 5 Oct 2018 16:30:41 -0700 Subject: [PATCH 10/25] awslogs: account for UTF-8 normalization in limits The CloudWatch Logs API defines its limits in terms of bytes, but its inputs in terms of UTF-8 encoded strings. Byte-sequences which are not valid UTF-8 encodings are normalized to the Unicode replacement character U+FFFD, which is a 3-byte sequence in UTF-8. This replacement can cause the input to grow, exceeding the API limit and causing failed API calls. This commit adds logic for counting the effective byte length after normalization and splitting input without splitting valid UTF-8 byte-sequences into two invalid byte-sequences. Fixes https://github.com/moby/moby/issues/37747 Signed-off-by: Samuel Karp (cherry picked from commit 1e8ef386279e2e28aff199047e798fad660efbdd) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 757650e8dcca87f95ba083a80639769a0b6ca1cc Component: engine --- .../daemon/logger/awslogs/cloudwatchlogs.go | 64 ++++-- .../logger/awslogs/cloudwatchlogs_test.go | 182 ++++++++++++++++++ 2 files changed, 233 insertions(+), 13 deletions(-) diff --git a/components/engine/daemon/logger/awslogs/cloudwatchlogs.go b/components/engine/daemon/logger/awslogs/cloudwatchlogs.go index 0f989af379..44e35e52b2 100644 --- a/components/engine/daemon/logger/awslogs/cloudwatchlogs.go +++ b/components/engine/daemon/logger/awslogs/cloudwatchlogs.go @@ -11,6 +11,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -46,6 +47,10 @@ const ( maximumLogEventsPerPut = 10000 // See: http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_limits.html + // Because the events are interpreted as UTF-8 encoded Unicode, invalid UTF-8 byte sequences are replaced with the + // Unicode replacement character (U+FFFD), which is a 3-byte sequence in UTF-8. To compensate for that and to avoid + // splitting valid UTF-8 characters into invalid byte sequences, we calculate the length of each event assuming that + // this replacement happens. maximumBytesPerEvent = 262144 - perEventBytes resourceAlreadyExistsCode = "ResourceAlreadyExistsException" @@ -495,15 +500,16 @@ func (l *logStream) collectBatch(created chan bool) { } line := msg.Line if l.multilinePattern != nil { - if l.multilinePattern.Match(line) || len(eventBuffer)+len(line) > maximumBytesPerEvent { + lineEffectiveLen := effectiveLen(string(line)) + if l.multilinePattern.Match(line) || effectiveLen(string(eventBuffer))+lineEffectiveLen > maximumBytesPerEvent { // This is a new log event or we will exceed max bytes per event // so flush the current eventBuffer to events and reset timestamp l.processEvent(batch, eventBuffer, eventBufferTimestamp) eventBufferTimestamp = msg.Timestamp.UnixNano() / int64(time.Millisecond) eventBuffer = eventBuffer[:0] } - // Append new line if event is less than max event size - if len(line) < maximumBytesPerEvent { + // Append newline if event is less than max event size + if lineEffectiveLen < maximumBytesPerEvent { line = append(line, "\n"...) } eventBuffer = append(eventBuffer, line...) @@ -524,16 +530,17 @@ func (l *logStream) collectBatch(created chan bool) { // batch (defined in maximumBytesPerPut). Log messages are split by the maximum // bytes per event (defined in maximumBytesPerEvent). There is a fixed per-event // byte overhead (defined in perEventBytes) which is accounted for in split- and -// batch-calculations. -func (l *logStream) processEvent(batch *eventBatch, events []byte, timestamp int64) { - for len(events) > 0 { +// batch-calculations. Because the events are interpreted as UTF-8 encoded +// Unicode, invalid UTF-8 byte sequences are replaced with the Unicode +// replacement character (U+FFFD), which is a 3-byte sequence in UTF-8. To +// compensate for that and to avoid splitting valid UTF-8 characters into +// invalid byte sequences, we calculate the length of each event assuming that +// this replacement happens. +func (l *logStream) processEvent(batch *eventBatch, bytes []byte, timestamp int64) { + for len(bytes) > 0 { // Split line length so it does not exceed the maximum - lineBytes := len(events) - if lineBytes > maximumBytesPerEvent { - lineBytes = maximumBytesPerEvent - } - line := events[:lineBytes] - + splitOffset, lineBytes := findValidSplit(string(bytes), maximumBytesPerEvent) + line := bytes[:splitOffset] event := wrappedEvent{ inputLogEvent: &cloudwatchlogs.InputLogEvent{ Message: aws.String(string(line)), @@ -544,7 +551,7 @@ func (l *logStream) processEvent(batch *eventBatch, events []byte, timestamp int added := batch.add(event, lineBytes) if added { - events = events[lineBytes:] + bytes = bytes[splitOffset:] } else { l.publishBatch(batch) batch.reset() @@ -552,6 +559,37 @@ func (l *logStream) processEvent(batch *eventBatch, events []byte, timestamp int } } +// effectiveLen counts the effective number of bytes in the string, after +// UTF-8 normalization. UTF-8 normalization includes replacing bytes that do +// not constitute valid UTF-8 encoded Unicode codepoints with the Unicode +// replacement codepoint U+FFFD (a 3-byte UTF-8 sequence, represented in Go as +// utf8.RuneError) +func effectiveLen(line string) int { + effectiveBytes := 0 + for _, rune := range line { + effectiveBytes += utf8.RuneLen(rune) + } + return effectiveBytes +} + +// findValidSplit finds the byte offset to split a string without breaking valid +// Unicode codepoints given a maximum number of total bytes. findValidSplit +// returns the byte offset for splitting a string or []byte, as well as the +// effective number of bytes if the string were normalized to replace invalid +// UTF-8 encoded bytes with the Unicode replacement character (a 3-byte UTF-8 +// sequence, represented in Go as utf8.RuneError) +func findValidSplit(line string, maxBytes int) (splitOffset, effectiveBytes int) { + for offset, rune := range line { + splitOffset = offset + if effectiveBytes+utf8.RuneLen(rune) > maxBytes { + return splitOffset, effectiveBytes + } + effectiveBytes += utf8.RuneLen(rune) + } + splitOffset = len(line) + return +} + // publishBatch calls PutLogEvents for a given set of InputLogEvents, // accounting for sequencing requirements (each request must reference the // sequence token returned by the previous request). diff --git a/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go b/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go index 172be51a30..fdae99c76d 100644 --- a/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go +++ b/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go @@ -938,6 +938,62 @@ func TestCollectBatchClose(t *testing.T) { } } +func TestEffectiveLen(t *testing.T) { + tests := []struct { + str string + effectiveBytes int + }{ + {"Hello", 5}, + {string([]byte{1, 2, 3, 4}), 4}, + {"🙃", 4}, + {string([]byte{0xFF, 0xFF, 0xFF, 0xFF}), 12}, + {"He\xff\xffo", 9}, + {"", 0}, + } + for i, tc := range tests { + t.Run(fmt.Sprintf("%d/%s", i, tc.str), func(t *testing.T) { + assert.Equal(t, tc.effectiveBytes, effectiveLen(tc.str)) + }) + } +} + +func TestFindValidSplit(t *testing.T) { + tests := []struct { + str string + maxEffectiveBytes int + splitOffset int + effectiveBytes int + }{ + {"", 10, 0, 0}, + {"Hello", 6, 5, 5}, + {"Hello", 2, 2, 2}, + {"Hello", 0, 0, 0}, + {"🙃", 3, 0, 0}, + {"🙃", 4, 4, 4}, + {string([]byte{'a', 0xFF}), 2, 1, 1}, + {string([]byte{'a', 0xFF}), 4, 2, 4}, + } + for i, tc := range tests { + t.Run(fmt.Sprintf("%d/%s", i, tc.str), func(t *testing.T) { + splitOffset, effectiveBytes := findValidSplit(tc.str, tc.maxEffectiveBytes) + assert.Equal(t, tc.splitOffset, splitOffset, "splitOffset") + assert.Equal(t, tc.effectiveBytes, effectiveBytes, "effectiveBytes") + t.Log(tc.str[:tc.splitOffset]) + t.Log(tc.str[tc.splitOffset:]) + }) + } +} + +func TestProcessEventEmoji(t *testing.T) { + stream := &logStream{} + batch := &eventBatch{} + bytes := []byte(strings.Repeat("🙃", maximumBytesPerEvent/4+1)) + stream.processEvent(batch, bytes, 0) + assert.Equal(t, 2, len(batch.batch), "should be two events in the batch") + assert.Equal(t, strings.Repeat("🙃", maximumBytesPerEvent/4), aws.StringValue(batch.batch[0].inputLogEvent.Message)) + assert.Equal(t, "🙃", aws.StringValue(batch.batch[1].inputLogEvent.Message)) +} + func TestCollectBatchLineSplit(t *testing.T) { mockClient := newMockClient() stream := &logStream{ @@ -987,6 +1043,55 @@ func TestCollectBatchLineSplit(t *testing.T) { } } +func TestCollectBatchLineSplitWithBinary(t *testing.T) { + mockClient := newMockClient() + stream := &logStream{ + client: mockClient, + logGroupName: groupName, + logStreamName: streamName, + sequenceToken: aws.String(sequenceToken), + messages: make(chan *logger.Message), + } + mockClient.putLogEventsResult <- &putLogEventsResult{ + successResult: &cloudwatchlogs.PutLogEventsOutput{ + NextSequenceToken: aws.String(nextSequenceToken), + }, + } + var ticks = make(chan time.Time) + newTicker = func(_ time.Duration) *time.Ticker { + return &time.Ticker{ + C: ticks, + } + } + + d := make(chan bool) + close(d) + go stream.collectBatch(d) + + longline := strings.Repeat("\xFF", maximumBytesPerEvent/3) // 0xFF is counted as the 3-byte utf8.RuneError + stream.Log(&logger.Message{ + Line: []byte(longline + "\xFD"), + Timestamp: time.Time{}, + }) + + // no ticks + stream.Close() + + argument := <-mockClient.putLogEventsArgument + if argument == nil { + t.Fatal("Expected non-nil PutLogEventsInput") + } + if len(argument.LogEvents) != 2 { + t.Errorf("Expected LogEvents to contain 2 elements, but contains %d", len(argument.LogEvents)) + } + if *argument.LogEvents[0].Message != longline { + t.Errorf("Expected message to be %s but was %s", longline, *argument.LogEvents[0].Message) + } + if *argument.LogEvents[1].Message != "\xFD" { + t.Errorf("Expected message to be %s but was %s", "\xFD", *argument.LogEvents[1].Message) + } +} + func TestCollectBatchMaxEvents(t *testing.T) { mockClient := newMockClientBuffered(1) stream := &logStream{ @@ -1125,6 +1230,83 @@ func TestCollectBatchMaxTotalBytes(t *testing.T) { } } +func TestCollectBatchMaxTotalBytesWithBinary(t *testing.T) { + expectedPuts := 2 + mockClient := newMockClientBuffered(expectedPuts) + stream := &logStream{ + client: mockClient, + logGroupName: groupName, + logStreamName: streamName, + sequenceToken: aws.String(sequenceToken), + messages: make(chan *logger.Message), + } + for i := 0; i < expectedPuts; i++ { + mockClient.putLogEventsResult <- &putLogEventsResult{ + successResult: &cloudwatchlogs.PutLogEventsOutput{ + NextSequenceToken: aws.String(nextSequenceToken), + }, + } + } + + var ticks = make(chan time.Time) + newTicker = func(_ time.Duration) *time.Ticker { + return &time.Ticker{ + C: ticks, + } + } + + d := make(chan bool) + close(d) + go stream.collectBatch(d) + + // maxline is the maximum line that could be submitted after + // accounting for its overhead. + maxline := strings.Repeat("\xFF", (maximumBytesPerPut-perEventBytes)/3) // 0xFF is counted as the 3-byte utf8.RuneError + // This will be split and batched up to the `maximumBytesPerPut' + // (+/- `maximumBytesPerEvent'). This /should/ be aligned, but + // should also tolerate an offset within that range. + stream.Log(&logger.Message{ + Line: []byte(maxline), + Timestamp: time.Time{}, + }) + stream.Log(&logger.Message{ + Line: []byte("B"), + Timestamp: time.Time{}, + }) + + // no ticks, guarantee batch by size (and chan close) + stream.Close() + + argument := <-mockClient.putLogEventsArgument + if argument == nil { + t.Fatal("Expected non-nil PutLogEventsInput") + } + + // Should total to the maximum allowed bytes. + eventBytes := 0 + for _, event := range argument.LogEvents { + eventBytes += effectiveLen(*event.Message) + } + eventsOverhead := len(argument.LogEvents) * perEventBytes + payloadTotal := eventBytes + eventsOverhead + // lowestMaxBatch allows the payload to be offset if the messages + // don't lend themselves to align with the maximum event size. + lowestMaxBatch := maximumBytesPerPut - maximumBytesPerEvent + + if payloadTotal > maximumBytesPerPut { + t.Errorf("Expected <= %d bytes but was %d", maximumBytesPerPut, payloadTotal) + } + if payloadTotal < lowestMaxBatch { + t.Errorf("Batch to be no less than %d but was %d", lowestMaxBatch, payloadTotal) + } + + argument = <-mockClient.putLogEventsArgument + message := *argument.LogEvents[len(argument.LogEvents)-1].Message + if message[len(message)-1:] != "B" { + t.Errorf("Expected message to be %s but was %s", "B", message[len(message)-1:]) + } +} + func TestCollectBatchWithDuplicateTimestamps(t *testing.T) { mockClient := newMockClient() stream := &logStream{ From 825b77191a9bee67294e36c6a13794e0fcf3d912 Mon Sep 17 00:00:00 2001 From: John Howard Date: Thu, 23 Aug 2018 15:56:27 -0700 Subject: [PATCH 11/25] Windows: DetachVhd attempt in cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: John Howard This is a fix for a few related scenarios where it's impossible to remove layers or containers until the host is rebooted. Generally (or at least easiest to repro) through a forced daemon kill while a container is running. Possibly slightly worse than that, as following a host reboot, the scratch layer would possibly be leaked and left on disk under the dataroot\windowsfilter directory after the container is removed. One such example of a failure: 1. run a long running container with the --rm flag docker run --rm -d --name test microsoft/windowsservercore powershell sleep 30 2. Force kill the daemon not allowing it to cleanup. Simulates a crash or a host power-cycle. 3. (re-)Start daemon 4. docker ps -a PS C:\control> docker ps -a CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 7aff773d782b malloc "powershell start-sl…" 11 seconds ago Removal In Progress malloc 5. Try to remove PS C:\control> docker rm 7aff Error response from daemon: container 7aff773d782bbf35d95095369ffcb170b7b8f0e6f8f65d5aff42abf61234855d: driver "windowsfilter" failed to remove root filesystem: rename C:\control\windowsfilter\7aff773d782bbf35d95095369ffcb170b7b8f0e6f8f65d5aff42abf61234855d C:\control\windowsfilter\7aff773d782bbf35d95095369ffcb170b7b8f0e6f8f65d5aff42abf61234855d-removing: Access is denied. PS C:\control> Step 5 fails. (cherry picked from commit efdad5374465a2a889d7572834a2dcca147af4fb) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 02fe71843e0e45ddc986a6c8182370b042349a27 Component: engine --- .../engine/daemon/graphdriver/windows/windows.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/components/engine/daemon/graphdriver/windows/windows.go b/components/engine/daemon/graphdriver/windows/windows.go index 13bfac2d2e..52b0c6d845 100644 --- a/components/engine/daemon/graphdriver/windows/windows.go +++ b/components/engine/daemon/graphdriver/windows/windows.go @@ -6,7 +6,6 @@ import ( "bufio" "bytes" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -23,6 +22,7 @@ import ( "github.com/Microsoft/go-winio" "github.com/Microsoft/go-winio/archive/tar" "github.com/Microsoft/go-winio/backuptar" + "github.com/Microsoft/go-winio/vhd" "github.com/Microsoft/hcsshim" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" @@ -33,6 +33,7 @@ import ( "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/pkg/system" units "github.com/docker/go-units" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) @@ -331,7 +332,18 @@ func (d *Driver) Remove(id string) error { tmpID := fmt.Sprintf("%s-removing", rID) tmpLayerPath := filepath.Join(d.info.HomeDir, tmpID) if err := os.Rename(layerPath, tmpLayerPath); err != nil && !os.IsNotExist(err) { - return err + if !os.IsPermission(err) { + return err + } + // If permission denied, it's possible that the scratch is still mounted, an + // artifact after a hard daemon crash for example. Worth a shot to try detaching it + // before retrying the rename. + if detachErr := vhd.DetachVhd(filepath.Join(layerPath, "sandbox.vhdx")); detachErr != nil { + return errors.Wrapf(err, "failed to detach VHD: %s", detachErr) + } + if renameErr := os.Rename(layerPath, tmpLayerPath); renameErr != nil && !os.IsNotExist(renameErr) { + return errors.Wrapf(err, "second rename attempt following detach failed: %s", renameErr) + } } if err := hcsshim.DestroyLayer(d.info, tmpID); err != nil { logrus.Errorf("Failed to DestroyLayer %s: %s", id, err) From c539968eac7349e363299f891c86c64d3f9b52de Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 11 Oct 2018 01:37:39 +0900 Subject: [PATCH 12/25] client: use io.LimitedReader for reading HTTP error client.checkResponseErr() was hanging and consuming infinite memory when the serverResp.Body io.Reader returns infinite stream. This commit prohibits reading more than 1MiB. Signed-off-by: Akihiro Suda (cherry picked from commit 1db4be0c32267b7a8a27998089ce96440bed492e) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 49556e047059d81b64f6cd12f4c602c9a9c471c7 Component: engine --- components/engine/client/request.go | 10 +++++++++- components/engine/client/request_test.go | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/components/engine/client/request.go b/components/engine/client/request.go index a19d62aa52..9b8639a1e5 100644 --- a/components/engine/client/request.go +++ b/components/engine/client/request.go @@ -195,10 +195,18 @@ func (cli *Client) checkResponseErr(serverResp serverResponse) error { return nil } - body, err := ioutil.ReadAll(serverResp.body) + bodyMax := 1 * 1024 * 1024 // 1 MiB + bodyR := &io.LimitedReader{ + R: serverResp.body, + N: int64(bodyMax), + } + body, err := ioutil.ReadAll(bodyR) if err != nil { return err } + if bodyR.N == 0 { + return fmt.Errorf("request returned %s with a message (> %d bytes) for API route and version %s, check if the server supports the requested API version", http.StatusText(serverResp.statusCode), bodyMax, serverResp.reqURL) + } if len(body) == 0 { return fmt.Errorf("request returned %s for API route and version %s, check if the server supports the requested API version", http.StatusText(serverResp.statusCode), serverResp.reqURL) } diff --git a/components/engine/client/request_test.go b/components/engine/client/request_test.go index fda4d88aa1..2847a9a4a3 100644 --- a/components/engine/client/request_test.go +++ b/components/engine/client/request_test.go @@ -5,12 +5,14 @@ import ( "context" "fmt" "io/ioutil" + "math/rand" "net/http" "strings" "testing" "github.com/docker/docker/api/types" "gotest.tools/assert" + is "gotest.tools/assert/cmp" ) // TestSetHostHeader should set fake host for local communications, set real host @@ -87,3 +89,18 @@ func TestPlainTextError(t *testing.T) { t.Fatalf("expected a Server Error, got %v", err) } } + +func TestInfiniteError(t *testing.T) { + infinitR := rand.New(rand.NewSource(42)) + client := &Client{ + client: newMockClient(func(req *http.Request) (*http.Response, error) { + resp := &http.Response{StatusCode: http.StatusInternalServerError} + resp.Header = http.Header{} + resp.Body = ioutil.NopCloser(infinitR) + return resp, nil + }), + } + + _, err := client.Ping(context.Background()) + assert.Check(t, is.ErrorContains(err, "request returned Internal Server Error")) +} From fd839bc74951ca3987b1bef35521adfac8145699 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Thu, 1 Nov 2018 16:07:06 -0700 Subject: [PATCH 13/25] Bump runc Changes: https://github.com/opencontainers/runc/compare/a00bf0190895...9f1e94488e5e4 Signed-off-by: Kir Kolyshkin (cherry picked from commit 335736fb0167a31a00cf2967c181a565c61334be) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 5b8cee93b5b6a2449d9af225e17d85c612f64ed2 Component: engine --- components/engine/hack/dockerfile/install/runc.installer | 2 +- components/engine/vendor.conf | 2 +- .../engine/vendor/github.com/opencontainers/runc/README.md | 4 ++++ .../github.com/opencontainers/runc/libcontainer/README.md | 7 ++++--- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/components/engine/hack/dockerfile/install/runc.installer b/components/engine/hack/dockerfile/install/runc.installer index f7c73b7dd2..f10375a799 100755 --- a/components/engine/hack/dockerfile/install/runc.installer +++ b/components/engine/hack/dockerfile/install/runc.installer @@ -1,7 +1,7 @@ #!/bin/sh # When updating RUNC_COMMIT, also update runc in vendor.conf accordingly -RUNC_COMMIT=a00bf0190895aa465a5fbed0268888e2c8ddfe85 +RUNC_COMMIT=9f1e94488e5e478e084fef997f022565b64b01d9 install_runc() { # Do not build with ambient capabilities support diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index d10a349b02..0ec740f5dc 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -75,7 +75,7 @@ github.com/pborman/uuid v1.0 google.golang.org/grpc v1.12.0 # This does not need to match RUNC_COMMIT as it is used for helper packages but should be newer or equal -github.com/opencontainers/runc 58592df56734acf62e574865fe40b9e53e967910 +github.com/opencontainers/runc 9f1e94488e5e478e084fef997f022565b64b01d9 github.com/opencontainers/runtime-spec 5684b8af48c1ac3b1451fa499724e30e3c20a294 # v1.0.1-49-g5684b8a github.com/opencontainers/image-spec v1.0.1 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 diff --git a/components/engine/vendor/github.com/opencontainers/runc/README.md b/components/engine/vendor/github.com/opencontainers/runc/README.md index 83379d9623..671571d1ef 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/README.md +++ b/components/engine/vendor/github.com/opencontainers/runc/README.md @@ -263,3 +263,7 @@ PIDFile=/run/mycontainerid.pid [Install] WantedBy=multi-user.target ``` + +## License + +The code and docs are released under the [Apache 2.0 license](LICENSE). diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/README.md b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/README.md index 42f3efe563..f2a2f0c6c4 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/README.md +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/README.md @@ -323,6 +323,7 @@ generated when building libcontainer with docker. ## Copyright and license -Code and documentation copyright 2014 Docker, inc. Code released under the Apache 2.0 license. -Docs released under Creative commons. - +Code and documentation copyright 2014 Docker, inc. +The code and documentation are released under the [Apache 2.0 license](../LICENSE). +The documentation is also released under Creative Commons Attribution 4.0 International License. +You may obtain a copy of the license, titled CC-BY-4.0, at http://creativecommons.org/licenses/by/4.0/. From a4d9985c381c672630a84076a7be8166d1b38b92 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Tue, 6 Nov 2018 11:15:36 -0800 Subject: [PATCH 14/25] runc.installer: add nokmem build tag for rhel7 kernel In case we're running on RHEL7 kernel, which has non-working and broken kernel memory controller, add 'nokmem' build tag so that runc never enables kmem accounting. For more info, see the following runc commit: https://github.com/opencontainers/runc/commit/6a2c1559684 This behavior can be overriden by having `RUNC_NOKMEM` environment variable set (e.g. to empty value to disable setting nokmem). Signed-off-by: Kir Kolyshkin (cherry picked from commit 8972aa9350d52e4a7e58242447b7a9d2f0c27f37) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 8486ea11ae800a1e6d634b741dfb007ba29f6003 Component: engine --- .../engine/hack/dockerfile/install/runc.installer | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/components/engine/hack/dockerfile/install/runc.installer b/components/engine/hack/dockerfile/install/runc.installer index f10375a799..18723dc605 100755 --- a/components/engine/hack/dockerfile/install/runc.installer +++ b/components/engine/hack/dockerfile/install/runc.installer @@ -4,10 +4,15 @@ RUNC_COMMIT=9f1e94488e5e478e084fef997f022565b64b01d9 install_runc() { - # Do not build with ambient capabilities support - RUNC_BUILDTAGS="${RUNC_BUILDTAGS:-"seccomp apparmor selinux"}" + # If using RHEL7 kernels (3.10.0 el7), disable kmem accounting/limiting + if uname -r | grep -q '^3\.10\.0.*\.el7\.'; then + : ${RUNC_NOKMEM='nokmem'} + fi - echo "Install runc version $RUNC_COMMIT" + # Do not build with ambient capabilities support + RUNC_BUILDTAGS="${RUNC_BUILDTAGS:-"seccomp apparmor selinux $RUNC_NOKMEM"}" + + echo "Install runc version $RUNC_COMMIT (build tags: $RUNC_BUILDTAGS)" git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" cd "$GOPATH/src/github.com/opencontainers/runc" git checkout -q "$RUNC_COMMIT" From 87524aec577d362f9391dea0825b6ec7242fd19f Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 18 Nov 2018 01:52:28 +0100 Subject: [PATCH 15/25] Add missing default address pool fields to swagger Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 2e8c913dbdfbba2591b6531ad2428c59eb261e0b) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 56cc26f927e6a1de51731f88baf5b0af3a5688bc Component: engine --- components/engine/api/swagger.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index f58a64f29e..2b10ae9998 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -2459,6 +2459,22 @@ definitions: description: "Whether there is currently a root CA rotation in progress for the swarm" type: "boolean" example: false + DefaultAddrPool: + description: | + Default Address Pool specifies default subnet pools for global scope networks. + type: "array" + items: + type: "string" + format: "CIDR" + example: ["10.10.0.0/16", "20.20.0.0/16"] + SubnetSize: + description: | + SubnetSize specifies the subnet size of the networks created from the default subnet pool + type: "integer" + format: "uint32" + maximum: 29 + default: 24 + example: 24 JoinTokens: description: | From bcbad46784e22effe2989c30606a444e0b42c5ba Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 14 Nov 2018 14:35:21 +0100 Subject: [PATCH 16/25] Add CONFIG_IP_VS_PROTO_TCP, CONFIG_IP_VS_PROTO_UDP, IP_NF_TARGET_REDIRECT to check-config.sh On kernels without this options set, publishing ports for swarm services does not work, making the published port not accessible ("connection refused") Thanks to Wenbo Wang for reporting, and Tianon for finding this. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 44e1c6ce81ff62c76ba630caf16a73922950abc3) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 25bec4665b6f011e52f7b2765ba1579c7430481d Component: engine --- components/engine/contrib/check-config.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/contrib/check-config.sh b/components/engine/contrib/check-config.sh index 88eb8aa753..fcbd874ff2 100755 --- a/components/engine/contrib/check-config.sh +++ b/components/engine/contrib/check-config.sh @@ -265,8 +265,11 @@ flags=( CGROUP_HUGETLB NET_CLS_CGROUP $netprio CFS_BANDWIDTH FAIR_GROUP_SCHED RT_GROUP_SCHED + IP_NF_TARGET_REDIRECT IP_VS IP_VS_NFCT + IP_VS_PROTO_TCP + IP_VS_PROTO_UDP IP_VS_RR ) check_flags "${flags[@]}" From 4056fb2fc05972e03dc88be09af5d0d8d1208a54 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 27 Sep 2018 17:48:24 +0200 Subject: [PATCH 17/25] Update containerd to v1.1.4 Fixes a potential content store bug, backported from 1.2 - v1.1.3 release notes: https://github.com/containerd/containerd/releases/tag/v1.1.3 - v1.1.4 release notes: https://github.com/containerd/containerd/releases/tag/v1.1.4 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit b3c3c7a5a310ba340f1a86a4a708de34b9602e74) Signed-off-by: Sebastiaan van Stijn Upstream-commit: a4decd0c4cd6033907fe85576a3d7dc8990aa758 Component: engine --- components/engine/hack/dockerfile/install/containerd.installer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/hack/dockerfile/install/containerd.installer b/components/engine/hack/dockerfile/install/containerd.installer index 4e5680d1ec..811348fce2 100755 --- a/components/engine/hack/dockerfile/install/containerd.installer +++ b/components/engine/hack/dockerfile/install/containerd.installer @@ -4,7 +4,7 @@ # containerd is also pinned in vendor.conf. When updating the binary # version you may also need to update the vendor version to pick up bug # fixes or new APIs. -CONTAINERD_COMMIT=468a545b9edcd5932818eb9de8e72413e616e86e # v1.1.2 +CONTAINERD_COMMIT=9f2e07b1fc1342d1c48fe4d7bbb94cb6d1bf278b # v1.1.4 install_containerd() { echo "Install containerd version $CONTAINERD_COMMIT" From d1211604a72cb5773ea6690383c16c88705c4ebb Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 16 Nov 2018 14:31:57 +0100 Subject: [PATCH 18/25] Add a note about updating runc / runc vendoring Containerd should be "leading" when specifying which version of runc to use. From the RUNC.MD document in the containerd repository (https://github.com/containerd/containerd/blob/b1e202c32724e82779544365528a1a082 b335553/RUNC.md); > We depend on a specific runc version when dealing with advanced features. You > should have a specific runc build for development. The current supported runc > commit is described in vendor.conf. Please refer to the line that starts with > github.com/opencontainers/runc. This patch adds a note to vendor.conf and runc.installer to describe the order in which runc should be updated. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit da3810d235bc0bd0197243d0128f258394554704) Signed-off-by: Sebastiaan van Stijn Upstream-commit: c9c87d76d651d57d72e52c575a2c9600170b5212 Component: engine --- components/engine/hack/dockerfile/install/runc.installer | 3 +++ components/engine/vendor.conf | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/components/engine/hack/dockerfile/install/runc.installer b/components/engine/hack/dockerfile/install/runc.installer index 18723dc605..32ef5f3dcb 100755 --- a/components/engine/hack/dockerfile/install/runc.installer +++ b/components/engine/hack/dockerfile/install/runc.installer @@ -1,6 +1,9 @@ #!/bin/sh # When updating RUNC_COMMIT, also update runc in vendor.conf accordingly +# The version of runc should match the version that is used by the containerd +# version that is used. If you need to update runc, open a pull request in +# the containerd project first, and update both after that is merged. RUNC_COMMIT=9f1e94488e5e478e084fef997f022565b64b01d9 install_runc() { diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 0ec740f5dc..587e193694 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -74,7 +74,11 @@ github.com/pborman/uuid v1.0 google.golang.org/grpc v1.12.0 -# This does not need to match RUNC_COMMIT as it is used for helper packages but should be newer or equal +# The version of runc should match the version that is used by the containerd +# version that is used. If you need to update runc, open a pull request in +# the containerd project first, and update both after that is merged. +# This commit does not need to match RUNC_COMMIT as it is used for helper +# packages but should be newer or equal. github.com/opencontainers/runc 9f1e94488e5e478e084fef997f022565b64b01d9 github.com/opencontainers/runtime-spec 5684b8af48c1ac3b1451fa499724e30e3c20a294 # v1.0.1-49-g5684b8a github.com/opencontainers/image-spec v1.0.1 From d9478a175ea6351683bc5c9f5f3d1d3318e20c64 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 28 Sep 2018 11:19:57 +0200 Subject: [PATCH 19/25] Update containerd to v1.2.0 release notes: https://github.com/containerd/containerd/releases/tag/v1.2.0 - New V2 Runtime with a stable gRPC interface for managing containers through external shims. - Updated CRI Plugin, validated against Kubernetes v1.11 and v1.12, but it is also compatible with Kubernetes v1.10. - Support for Kubernetes Runtime Class, introduced in Kubernetes 1.12 - A new proxy plugin configuration has been added to allow external snapshotters be connected to containerd using gRPC.- - A new Install method on the containerd client allows users to publish host level binaries using standard container build tooling and container distribution tooling to download containerd related binaries on their systems. - Add support for cleaning up leases and content ingests to garbage collections. - Improved multi-arch image support using more precise matching and ranking - Added a runtime `options` field for shim v2 runtime. Use the `options` field to config runtime specific options, e.g. `NoPivotRoot` and `SystemdCgroup` for runtime type `io.containerd.runc.v1`. - Some Minor API additions - Add `ListStream` method to containers API. This allows listing a larger number of containers without hitting message size limts. - Add `Sync` flag to `Delete` in leases API. Setting this option will ensure a garbage collection completes before the removal call is returned. This can be used to guarantee unreferenced objects are removed from disk after a lease. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 8674930c84140c990451adb148165422d008b661) Signed-off-by: Sebastiaan van Stijn Upstream-commit: e137337fe6083da91fd6d83d699cff3a857f636e Component: engine --- components/engine/hack/dockerfile/install/containerd.installer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/hack/dockerfile/install/containerd.installer b/components/engine/hack/dockerfile/install/containerd.installer index 811348fce2..0dd6dd0a4c 100755 --- a/components/engine/hack/dockerfile/install/containerd.installer +++ b/components/engine/hack/dockerfile/install/containerd.installer @@ -4,7 +4,7 @@ # containerd is also pinned in vendor.conf. When updating the binary # version you may also need to update the vendor version to pick up bug # fixes or new APIs. -CONTAINERD_COMMIT=9f2e07b1fc1342d1c48fe4d7bbb94cb6d1bf278b # v1.1.4 +CONTAINERD_COMMIT=c4446665cb9c30056f4998ed953e6d4ff22c7c39 # v1.2.0 install_containerd() { echo "Install containerd version $CONTAINERD_COMMIT" From 1372540bda0a9a9a690671bb0ec5383e4d256f67 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 18 Oct 2018 21:38:15 +0200 Subject: [PATCH 20/25] Update runc to 58592df56734acf62e574865fe40b9e53e967910 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit fc0038a3edd518b7a4de7a8168d4888b08130c28) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 262abed3d2e84756e16c73c7c241aa62918c51c8 Component: engine --- components/engine/hack/dockerfile/install/runc.installer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/hack/dockerfile/install/runc.installer b/components/engine/hack/dockerfile/install/runc.installer index 32ef5f3dcb..efb430a6e1 100755 --- a/components/engine/hack/dockerfile/install/runc.installer +++ b/components/engine/hack/dockerfile/install/runc.installer @@ -4,7 +4,7 @@ # The version of runc should match the version that is used by the containerd # version that is used. If you need to update runc, open a pull request in # the containerd project first, and update both after that is merged. -RUNC_COMMIT=9f1e94488e5e478e084fef997f022565b64b01d9 +RUNC_COMMIT=10d38b660a77168360df3522881e2dc2be5056bd install_runc() { # If using RHEL7 kernels (3.10.0 el7), disable kmem accounting/limiting From 309dc64b35304b5b5e52e1f7cdff848b1b62b99f Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 30 Oct 2018 10:32:58 +0100 Subject: [PATCH 21/25] Bump containerd binary to fix shim hang Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 7af4c904b31887e28776cd514e0b8cf8c0261310) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 64a05e3d162b7234f8a7aa32d10434db4c5e6364 Component: engine --- components/engine/hack/dockerfile/install/containerd.installer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/hack/dockerfile/install/containerd.installer b/components/engine/hack/dockerfile/install/containerd.installer index 0dd6dd0a4c..bb3e95202e 100755 --- a/components/engine/hack/dockerfile/install/containerd.installer +++ b/components/engine/hack/dockerfile/install/containerd.installer @@ -4,7 +4,7 @@ # containerd is also pinned in vendor.conf. When updating the binary # version you may also need to update the vendor version to pick up bug # fixes or new APIs. -CONTAINERD_COMMIT=c4446665cb9c30056f4998ed953e6d4ff22c7c39 # v1.2.0 +CONTAINERD_COMMIT=8afcade1a6041b8301b6f6d8b88bae909993b989 # v1.2.0-13-g8afcade1 install_containerd() { echo "Install containerd version $CONTAINERD_COMMIT" From 9a2dacb99c8ef391a1f19805dc120b11866ade05 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 9 Nov 2018 13:11:04 -0500 Subject: [PATCH 22/25] wip: bump containerd and runc version Signed-off-by: Michael Crosby (cherry picked from commit d13528c6350a4b359a7b8dc9a8a7bcba12a3c1a8) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 7d6ec38402f4e2a5e1c83a981a88bf1a5f202858 Component: engine --- .../dockerfile/install/containerd.installer | 2 +- components/engine/vendor.conf | 4 +- .../containerd/containerd/README.md | 8 +- .../containerd/remotes/docker/resolver.go | 2 +- .../containerd/runtime/proc/proc.go | 5 - .../containerd/runtime/v1/linux/proc/exec.go | 37 +++- .../runtime/v1/linux/proc/exec_state.go | 59 ++---- .../containerd/runtime/v1/linux/proc/init.go | 124 +++++++++--- .../runtime/v1/linux/proc/init_state.go | 180 +++--------------- .../containerd/runtime/v1/shim/reaper.go | 3 +- .../containerd/runtime/v1/shim/service.go | 156 ++++++++------- .../containerd/containerd/vendor.conf | 10 +- .../github.com/opencontainers/runc/README.md | 1 + .../runc/libcontainer/README.md | 1 + .../runc/libcontainer/cgroups/utils.go | 8 +- .../libcontainer/configs/namespaces_linux.go | 16 +- .../configs/namespaces_syscall.go | 13 +- .../runc/libcontainer/nsenter/nsexec.c | 63 +++--- 18 files changed, 342 insertions(+), 350 deletions(-) diff --git a/components/engine/hack/dockerfile/install/containerd.installer b/components/engine/hack/dockerfile/install/containerd.installer index bb3e95202e..8981ab37ed 100755 --- a/components/engine/hack/dockerfile/install/containerd.installer +++ b/components/engine/hack/dockerfile/install/containerd.installer @@ -4,7 +4,7 @@ # containerd is also pinned in vendor.conf. When updating the binary # version you may also need to update the vendor version to pick up bug # fixes or new APIs. -CONTAINERD_COMMIT=8afcade1a6041b8301b6f6d8b88bae909993b989 # v1.2.0-13-g8afcade1 +CONTAINERD_COMMIT=aa537a67b3a9026c423a63dbaf71e4cc0776b2ad # v1.2.0-13-g8afcade1 install_containerd() { echo "Install containerd version $CONTAINERD_COMMIT" diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 587e193694..6421f98ebb 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -79,7 +79,7 @@ google.golang.org/grpc v1.12.0 # the containerd project first, and update both after that is merged. # This commit does not need to match RUNC_COMMIT as it is used for helper # packages but should be newer or equal. -github.com/opencontainers/runc 9f1e94488e5e478e084fef997f022565b64b01d9 +github.com/opencontainers/runc 10d38b660a77168360df3522881e2dc2be5056bd github.com/opencontainers/runtime-spec 5684b8af48c1ac3b1451fa499724e30e3c20a294 # v1.0.1-49-g5684b8a github.com/opencontainers/image-spec v1.0.1 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 @@ -118,7 +118,7 @@ github.com/googleapis/gax-go v2.0.0 google.golang.org/genproto 694d95ba50e67b2e363f3483057db5d4910c18f9 # containerd -github.com/containerd/containerd c4446665cb9c30056f4998ed953e6d4ff22c7c39 # v1.2.0 +github.com/containerd/containerd aa537a67b3a9026c423a63dbaf71e4cc0776b2ad # v1.2.0 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4 github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2 diff --git a/components/engine/vendor/github.com/containerd/containerd/README.md b/components/engine/vendor/github.com/containerd/containerd/README.md index 2055404b5b..4a90fc8a0e 100644 --- a/components/engine/vendor/github.com/containerd/containerd/README.md +++ b/components/engine/vendor/github.com/containerd/containerd/README.md @@ -224,7 +224,8 @@ This will be the best place to discuss design and implementation. For sync communication we have a community slack with a #containerd channel that everyone is welcome to join and chat about development. -**Slack:** https://join.slack.com/t/dockercommunity/shared_invite/enQtNDM4NjAwNDMyOTUwLWZlMDZmYWRjZjk4Zjc5ZGQ5NWZkOWI1Yjk2NGE3ZWVlYjYxM2VhYjczOWIyZDFhZTE3NTUwZWQzMjhmNGYyZTg +**Slack:** Catch us in the #containerd and #containerd-dev channels on dockercommunity.slack.com. +[Click here for an invite to docker community slack.](https://join.slack.com/t/dockercommunity/shared_invite/enQtNDY4MDc1Mzc0MzIwLTgxZDBlMmM4ZGEyNDc1N2FkMzlhODJkYmE1YTVkYjM1MDE3ZjAwZjBkOGFlOTJkZjRmZGYzNjYyY2M3ZTUxYzQ) ### Reporting security issues @@ -249,3 +250,8 @@ Please find all these core project documents, including the: * and [Contributing guidelines](https://github.com/containerd/project/blob/master/CONTRIBUTING.md) information in our [`containerd/project`](https://github.com/containerd/project) repository. + +## Adoption + +Interested to see who is using containerd? Are you using containerd in a project? +Please add yourself via pull request to our [ADOPTERS.md](./ADOPTERS.md) file. diff --git a/components/engine/vendor/github.com/containerd/containerd/remotes/docker/resolver.go b/components/engine/vendor/github.com/containerd/containerd/remotes/docker/resolver.go index 5cccdecba0..b89bd98c73 100644 --- a/components/engine/vendor/github.com/containerd/containerd/remotes/docker/resolver.go +++ b/components/engine/vendor/github.com/containerd/containerd/remotes/docker/resolver.go @@ -75,7 +75,7 @@ type ResolverOptions struct { // Credentials provides username and secret given a host. // If username is empty but a secret is given, that secret - // is interpretted as a long lived token. + // is interpreted as a long lived token. // Deprecated: use Authorizer Credentials func(string) (string, string, error) diff --git a/components/engine/vendor/github.com/containerd/containerd/runtime/proc/proc.go b/components/engine/vendor/github.com/containerd/containerd/runtime/proc/proc.go index 02bc9bda81..91ca59bb15 100644 --- a/components/engine/vendor/github.com/containerd/containerd/runtime/proc/proc.go +++ b/components/engine/vendor/github.com/containerd/containerd/runtime/proc/proc.go @@ -40,7 +40,6 @@ func (s Stdio) IsNull() bool { // Process on a system type Process interface { - State // ID returns the id for the process ID() string // Pid returns the pid for the process @@ -57,10 +56,6 @@ type Process interface { Status(context.Context) (string, error) // Wait blocks until the process has exited Wait() -} - -// State of a process -type State interface { // Resize resizes the process console Resize(ws console.WinSize) error // Start execution of the process diff --git a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/exec.go b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/exec.go index 96c425dd96..4b932455b0 100644 --- a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/exec.go +++ b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/exec.go @@ -41,7 +41,7 @@ import ( type execProcess struct { wg sync.WaitGroup - proc.State + execState execState mu sync.Mutex id string @@ -86,6 +86,13 @@ func (e *execProcess) ExitedAt() time.Time { return e.exited } +func (e *execProcess) SetExited(status int) { + e.mu.Lock() + defer e.mu.Unlock() + + e.execState.SetExited(status) +} + func (e *execProcess) setExited(status int) { e.status = status e.exited = time.Now() @@ -93,6 +100,13 @@ func (e *execProcess) setExited(status int) { close(e.waitBlock) } +func (e *execProcess) Delete(ctx context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + + return e.execState.Delete(ctx) +} + func (e *execProcess) delete(ctx context.Context) error { e.wg.Wait() if e.io != nil { @@ -107,6 +121,13 @@ func (e *execProcess) delete(ctx context.Context) error { return nil } +func (e *execProcess) Resize(ws console.WinSize) error { + e.mu.Lock() + defer e.mu.Unlock() + + return e.execState.Resize(ws) +} + func (e *execProcess) resize(ws console.WinSize) error { if e.console == nil { return nil @@ -114,6 +135,13 @@ func (e *execProcess) resize(ws console.WinSize) error { return e.console.Resize(ws) } +func (e *execProcess) Kill(ctx context.Context, sig uint32, _ bool) error { + e.mu.Lock() + defer e.mu.Unlock() + + return e.execState.Kill(ctx, sig, false) +} + func (e *execProcess) kill(ctx context.Context, sig uint32, _ bool) error { pid := e.pid if pid != 0 { @@ -132,6 +160,13 @@ func (e *execProcess) Stdio() proc.Stdio { return e.stdio } +func (e *execProcess) Start(ctx context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + + return e.execState.Start(ctx) +} + func (e *execProcess) start(ctx context.Context) (err error) { var ( socket *runc.Socket diff --git a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/exec_state.go b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/exec_state.go index ac54675528..12489501ba 100644 --- a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/exec_state.go +++ b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/exec_state.go @@ -25,6 +25,14 @@ import ( "github.com/pkg/errors" ) +type execState interface { + Resize(console.WinSize) error + Start(context.Context) error + Delete(context.Context) error + Kill(context.Context, uint32, bool) error + SetExited(int) +} + type execCreatedState struct { p *execProcess } @@ -32,11 +40,11 @@ type execCreatedState struct { func (s *execCreatedState) transition(name string) error { switch name { case "running": - s.p.State = &execRunningState{p: s.p} + s.p.execState = &execRunningState{p: s.p} case "stopped": - s.p.State = &execStoppedState{p: s.p} + s.p.execState = &execStoppedState{p: s.p} case "deleted": - s.p.State = &deletedState{} + s.p.execState = &deletedState{} default: return errors.Errorf("invalid state transition %q to %q", stateName(s), name) } @@ -44,15 +52,10 @@ func (s *execCreatedState) transition(name string) error { } func (s *execCreatedState) Resize(ws console.WinSize) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.resize(ws) } func (s *execCreatedState) Start(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() if err := s.p.start(ctx); err != nil { return err } @@ -63,22 +66,15 @@ func (s *execCreatedState) Delete(ctx context.Context) error { if err := s.p.delete(ctx); err != nil { return err } - s.p.mu.Lock() - defer s.p.mu.Unlock() + return s.transition("deleted") } func (s *execCreatedState) Kill(ctx context.Context, sig uint32, all bool) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.kill(ctx, sig, all) } func (s *execCreatedState) SetExited(status int) { - s.p.mu.Lock() - defer s.p.mu.Unlock() - s.p.setExited(status) if err := s.transition("stopped"); err != nil { @@ -93,7 +89,7 @@ type execRunningState struct { func (s *execRunningState) transition(name string) error { switch name { case "stopped": - s.p.State = &execStoppedState{p: s.p} + s.p.execState = &execStoppedState{p: s.p} default: return errors.Errorf("invalid state transition %q to %q", stateName(s), name) } @@ -101,37 +97,22 @@ func (s *execRunningState) transition(name string) error { } func (s *execRunningState) Resize(ws console.WinSize) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.resize(ws) } func (s *execRunningState) Start(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot start a running process") } func (s *execRunningState) Delete(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot delete a running process") } func (s *execRunningState) Kill(ctx context.Context, sig uint32, all bool) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.kill(ctx, sig, all) } func (s *execRunningState) SetExited(status int) { - s.p.mu.Lock() - defer s.p.mu.Unlock() - s.p.setExited(status) if err := s.transition("stopped"); err != nil { @@ -146,7 +127,7 @@ type execStoppedState struct { func (s *execStoppedState) transition(name string) error { switch name { case "deleted": - s.p.State = &deletedState{} + s.p.execState = &deletedState{} default: return errors.Errorf("invalid state transition %q to %q", stateName(s), name) } @@ -154,16 +135,10 @@ func (s *execStoppedState) transition(name string) error { } func (s *execStoppedState) Resize(ws console.WinSize) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot resize a stopped container") } func (s *execStoppedState) Start(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot start a stopped process") } @@ -171,15 +146,11 @@ func (s *execStoppedState) Delete(ctx context.Context) error { if err := s.p.delete(ctx); err != nil { return err } - s.p.mu.Lock() - defer s.p.mu.Unlock() + return s.transition("deleted") } func (s *execStoppedState) Kill(ctx context.Context, sig uint32, all bool) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.kill(ctx, sig, all) } diff --git a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/init.go b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/init.go index 5bf5f83443..3ce7a43418 100644 --- a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/init.go +++ b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/init.go @@ -46,8 +46,8 @@ const InitPidFile = "init.pid" // Init represents an initial process for a container type Init struct { - wg sync.WaitGroup - initState + wg sync.WaitGroup + initState initState // mu is used to ensure that `Start()` and `Exited()` calls return in // the right order when invoked in separate go routines. @@ -212,6 +212,7 @@ func (p *Init) Pid() int { func (p *Init) ExitStatus() int { p.mu.Lock() defer p.mu.Unlock() + return p.status } @@ -219,6 +220,7 @@ func (p *Init) ExitStatus() int { func (p *Init) ExitedAt() time.Time { p.mu.Lock() defer p.mu.Unlock() + return p.exited } @@ -226,6 +228,7 @@ func (p *Init) ExitedAt() time.Time { func (p *Init) Status(ctx context.Context) (string, error) { p.mu.Lock() defer p.mu.Unlock() + c, err := p.runtime.State(ctx, p.id) if err != nil { if strings.Contains(err.Error(), "does not exist") { @@ -236,11 +239,27 @@ func (p *Init) Status(ctx context.Context) (string, error) { return c.Status, nil } -func (p *Init) start(context context.Context) error { - err := p.runtime.Start(context, p.id) +// Start the init process +func (p *Init) Start(ctx context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + + return p.initState.Start(ctx) +} + +func (p *Init) start(ctx context.Context) error { + err := p.runtime.Start(ctx, p.id) return p.runtimeError(err, "OCI runtime start failed") } +// SetExited of the init process with the next status +func (p *Init) SetExited(status int) { + p.mu.Lock() + defer p.mu.Unlock() + + p.initState.SetExited(status) +} + func (p *Init) setExited(status int) { p.exited = time.Now() p.status = status @@ -248,9 +267,17 @@ func (p *Init) setExited(status int) { close(p.waitBlock) } -func (p *Init) delete(context context.Context) error { +// Delete the init process +func (p *Init) Delete(ctx context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + + return p.initState.Delete(ctx) +} + +func (p *Init) delete(ctx context.Context) error { p.wg.Wait() - err := p.runtime.Delete(context, p.id, nil) + err := p.runtime.Delete(ctx, p.id, nil) // ignore errors if a runtime has already deleted the process // but we still hold metadata and pipes // @@ -270,7 +297,7 @@ func (p *Init) delete(context context.Context) error { p.io.Close() } if err2 := mount.UnmountAll(p.Rootfs, 0); err2 != nil { - log.G(context).WithError(err2).Warn("failed to cleanup rootfs mount") + log.G(ctx).WithError(err2).Warn("failed to cleanup rootfs mount") if err == nil { err = errors.Wrap(err2, "failed rootfs umount") } @@ -278,6 +305,17 @@ func (p *Init) delete(context context.Context) error { return err } +// Resize the init processes console +func (p *Init) Resize(ws console.WinSize) error { + p.mu.Lock() + defer p.mu.Unlock() + + if p.console == nil { + return nil + } + return p.console.Resize(ws) +} + func (p *Init) resize(ws console.WinSize) error { if p.console == nil { return nil @@ -285,26 +323,40 @@ func (p *Init) resize(ws console.WinSize) error { return p.console.Resize(ws) } -func (p *Init) pause(context context.Context) error { - err := p.runtime.Pause(context, p.id) - return p.runtimeError(err, "OCI runtime pause failed") +// Pause the init process and all its child processes +func (p *Init) Pause(ctx context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + + return p.initState.Pause(ctx) } -func (p *Init) resume(context context.Context) error { - err := p.runtime.Resume(context, p.id) - return p.runtimeError(err, "OCI runtime resume failed") +// Resume the init process and all its child processes +func (p *Init) Resume(ctx context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + + return p.initState.Resume(ctx) } -func (p *Init) kill(context context.Context, signal uint32, all bool) error { - err := p.runtime.Kill(context, p.id, int(signal), &runc.KillOpts{ +// Kill the init process +func (p *Init) Kill(ctx context.Context, signal uint32, all bool) error { + p.mu.Lock() + defer p.mu.Unlock() + + return p.initState.Kill(ctx, signal, all) +} + +func (p *Init) kill(ctx context.Context, signal uint32, all bool) error { + err := p.runtime.Kill(ctx, p.id, int(signal), &runc.KillOpts{ All: all, }) return checkKillError(err) } // KillAll processes belonging to the init process -func (p *Init) KillAll(context context.Context) error { - err := p.runtime.Kill(context, p.id, int(syscall.SIGKILL), &runc.KillOpts{ +func (p *Init) KillAll(ctx context.Context) error { + err := p.runtime.Kill(ctx, p.id, int(syscall.SIGKILL), &runc.KillOpts{ All: true, }) return p.runtimeError(err, "OCI runtime killall failed") @@ -320,8 +372,16 @@ func (p *Init) Runtime() *runc.Runc { return p.runtime } +// Exec returns a new child process +func (p *Init) Exec(ctx context.Context, path string, r *ExecConfig) (proc.Process, error) { + p.mu.Lock() + defer p.mu.Unlock() + + return p.initState.Exec(ctx, path, r) +} + // exec returns a new exec'd process -func (p *Init) exec(context context.Context, path string, r *ExecConfig) (proc.Process, error) { +func (p *Init) exec(ctx context.Context, path string, r *ExecConfig) (proc.Process, error) { // process exec request var spec specs.Process if err := json.Unmarshal(r.Spec.Value, &spec); err != nil { @@ -342,18 +402,26 @@ func (p *Init) exec(context context.Context, path string, r *ExecConfig) (proc.P }, waitBlock: make(chan struct{}), } - e.State = &execCreatedState{p: e} + e.execState = &execCreatedState{p: e} return e, nil } -func (p *Init) checkpoint(context context.Context, r *CheckpointConfig) error { +// Checkpoint the init process +func (p *Init) Checkpoint(ctx context.Context, r *CheckpointConfig) error { + p.mu.Lock() + defer p.mu.Unlock() + + return p.initState.Checkpoint(ctx, r) +} + +func (p *Init) checkpoint(ctx context.Context, r *CheckpointConfig) error { var actions []runc.CheckpointAction if !r.Exit { actions = append(actions, runc.LeaveRunning) } work := filepath.Join(p.WorkDir, "criu-work") defer os.RemoveAll(work) - if err := p.runtime.Checkpoint(context, p.id, &runc.CheckpointOpts{ + if err := p.runtime.Checkpoint(ctx, p.id, &runc.CheckpointOpts{ WorkDir: work, ImagePath: r.Path, AllowOpenTCP: r.AllowOpenTCP, @@ -364,19 +432,27 @@ func (p *Init) checkpoint(context context.Context, r *CheckpointConfig) error { }, actions...); err != nil { dumpLog := filepath.Join(p.Bundle, "criu-dump.log") if cerr := copyFile(dumpLog, filepath.Join(work, "dump.log")); cerr != nil { - log.G(context).Error(err) + log.G(ctx).Error(err) } return fmt.Errorf("%s path= %s", criuError(err), dumpLog) } return nil } -func (p *Init) update(context context.Context, r *google_protobuf.Any) error { +// Update the processes resource configuration +func (p *Init) Update(ctx context.Context, r *google_protobuf.Any) error { + p.mu.Lock() + defer p.mu.Unlock() + + return p.initState.Update(ctx, r) +} + +func (p *Init) update(ctx context.Context, r *google_protobuf.Any) error { var resources specs.LinuxResources if err := json.Unmarshal(r.Value, &resources); err != nil { return err } - return p.runtime.Update(context, p.id, &resources) + return p.runtime.Update(ctx, p.id, &resources) } // Stdio of the process diff --git a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/init_state.go b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/init_state.go index 6a6b448d3a..06a5c7f3d7 100644 --- a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/init_state.go +++ b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/init_state.go @@ -30,16 +30,20 @@ import ( runc "github.com/containerd/go-runc" google_protobuf "github.com/gogo/protobuf/types" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) type initState interface { - proc.State - + Resize(console.WinSize) error + Start(context.Context) error + Delete(context.Context) error Pause(context.Context) error Resume(context.Context) error Update(context.Context, *google_protobuf.Any) error Checkpoint(context.Context, *CheckpointConfig) error Exec(context.Context, string, *ExecConfig) (proc.Process, error) + Kill(context.Context, uint32, bool) error + SetExited(int) } type createdState struct { @@ -61,43 +65,26 @@ func (s *createdState) transition(name string) error { } func (s *createdState) Pause(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot pause task in created state") } func (s *createdState) Resume(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot resume task in created state") } -func (s *createdState) Update(context context.Context, r *google_protobuf.Any) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - - return s.p.update(context, r) +func (s *createdState) Update(ctx context.Context, r *google_protobuf.Any) error { + return s.p.update(ctx, r) } -func (s *createdState) Checkpoint(context context.Context, r *CheckpointConfig) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - +func (s *createdState) Checkpoint(ctx context.Context, r *CheckpointConfig) error { return errors.Errorf("cannot checkpoint a task in created state") } func (s *createdState) Resize(ws console.WinSize) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.resize(ws) } func (s *createdState) Start(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() if err := s.p.start(ctx); err != nil { return err } @@ -105,8 +92,6 @@ func (s *createdState) Start(ctx context.Context) error { } func (s *createdState) Delete(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() if err := s.p.delete(ctx); err != nil { return err } @@ -114,16 +99,10 @@ func (s *createdState) Delete(ctx context.Context) error { } func (s *createdState) Kill(ctx context.Context, sig uint32, all bool) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.kill(ctx, sig, all) } func (s *createdState) SetExited(status int) { - s.p.mu.Lock() - defer s.p.mu.Unlock() - s.p.setExited(status) if err := s.transition("stopped"); err != nil { @@ -132,8 +111,6 @@ func (s *createdState) SetExited(status int) { } func (s *createdState) Exec(ctx context.Context, path string, r *ExecConfig) (proc.Process, error) { - s.p.mu.Lock() - defer s.p.mu.Unlock() return s.p.exec(ctx, path, r) } @@ -157,43 +134,26 @@ func (s *createdCheckpointState) transition(name string) error { } func (s *createdCheckpointState) Pause(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot pause task in created state") } func (s *createdCheckpointState) Resume(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot resume task in created state") } -func (s *createdCheckpointState) Update(context context.Context, r *google_protobuf.Any) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - - return s.p.update(context, r) +func (s *createdCheckpointState) Update(ctx context.Context, r *google_protobuf.Any) error { + return s.p.update(ctx, r) } -func (s *createdCheckpointState) Checkpoint(context context.Context, r *CheckpointConfig) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - +func (s *createdCheckpointState) Checkpoint(ctx context.Context, r *CheckpointConfig) error { return errors.Errorf("cannot checkpoint a task in created state") } func (s *createdCheckpointState) Resize(ws console.WinSize) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.resize(ws) } func (s *createdCheckpointState) Start(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() p := s.p sio := p.stdio @@ -247,8 +207,6 @@ func (s *createdCheckpointState) Start(ctx context.Context) error { } func (s *createdCheckpointState) Delete(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() if err := s.p.delete(ctx); err != nil { return err } @@ -256,16 +214,10 @@ func (s *createdCheckpointState) Delete(ctx context.Context) error { } func (s *createdCheckpointState) Kill(ctx context.Context, sig uint32, all bool) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.kill(ctx, sig, all) } func (s *createdCheckpointState) SetExited(status int) { - s.p.mu.Lock() - defer s.p.mu.Unlock() - s.p.setExited(status) if err := s.transition("stopped"); err != nil { @@ -274,9 +226,6 @@ func (s *createdCheckpointState) SetExited(status int) { } func (s *createdCheckpointState) Exec(ctx context.Context, path string, r *ExecConfig) (proc.Process, error) { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return nil, errors.Errorf("cannot exec in a created state") } @@ -297,67 +246,42 @@ func (s *runningState) transition(name string) error { } func (s *runningState) Pause(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - if err := s.p.pause(ctx); err != nil { - return err + if err := s.p.runtime.Pause(ctx, s.p.id); err != nil { + return s.p.runtimeError(err, "OCI runtime pause failed") } + return s.transition("paused") } func (s *runningState) Resume(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot resume a running process") } -func (s *runningState) Update(context context.Context, r *google_protobuf.Any) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - - return s.p.update(context, r) +func (s *runningState) Update(ctx context.Context, r *google_protobuf.Any) error { + return s.p.update(ctx, r) } func (s *runningState) Checkpoint(ctx context.Context, r *CheckpointConfig) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.checkpoint(ctx, r) } func (s *runningState) Resize(ws console.WinSize) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.resize(ws) } func (s *runningState) Start(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot start a running process") } func (s *runningState) Delete(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot delete a running process") } func (s *runningState) Kill(ctx context.Context, sig uint32, all bool) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.kill(ctx, sig, all) } func (s *runningState) SetExited(status int) { - s.p.mu.Lock() - defer s.p.mu.Unlock() - s.p.setExited(status) if err := s.transition("stopped"); err != nil { @@ -366,8 +290,6 @@ func (s *runningState) SetExited(status int) { } func (s *runningState) Exec(ctx context.Context, path string, r *ExecConfig) (proc.Process, error) { - s.p.mu.Lock() - defer s.p.mu.Unlock() return s.p.exec(ctx, path, r) } @@ -388,79 +310,54 @@ func (s *pausedState) transition(name string) error { } func (s *pausedState) Pause(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot pause a paused container") } func (s *pausedState) Resume(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - - if err := s.p.resume(ctx); err != nil { - return err + if err := s.p.runtime.Resume(ctx, s.p.id); err != nil { + return s.p.runtimeError(err, "OCI runtime resume failed") } + return s.transition("running") } -func (s *pausedState) Update(context context.Context, r *google_protobuf.Any) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - - return s.p.update(context, r) +func (s *pausedState) Update(ctx context.Context, r *google_protobuf.Any) error { + return s.p.update(ctx, r) } func (s *pausedState) Checkpoint(ctx context.Context, r *CheckpointConfig) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.checkpoint(ctx, r) } func (s *pausedState) Resize(ws console.WinSize) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.resize(ws) } func (s *pausedState) Start(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot start a paused process") } func (s *pausedState) Delete(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot delete a paused process") } func (s *pausedState) Kill(ctx context.Context, sig uint32, all bool) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return s.p.kill(ctx, sig, all) } func (s *pausedState) SetExited(status int) { - s.p.mu.Lock() - defer s.p.mu.Unlock() - s.p.setExited(status) + if err := s.p.runtime.Resume(context.Background(), s.p.id); err != nil { + logrus.WithError(err).Error("resuming exited container from paused state") + } + if err := s.transition("stopped"); err != nil { panic(err) } } func (s *pausedState) Exec(ctx context.Context, path string, r *ExecConfig) (proc.Process, error) { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return nil, errors.Errorf("cannot exec in a paused state") } @@ -479,50 +376,30 @@ func (s *stoppedState) transition(name string) error { } func (s *stoppedState) Pause(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot pause a stopped container") } func (s *stoppedState) Resume(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot resume a stopped container") } -func (s *stoppedState) Update(context context.Context, r *google_protobuf.Any) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - +func (s *stoppedState) Update(ctx context.Context, r *google_protobuf.Any) error { return errors.Errorf("cannot update a stopped container") } func (s *stoppedState) Checkpoint(ctx context.Context, r *CheckpointConfig) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot checkpoint a stopped container") } func (s *stoppedState) Resize(ws console.WinSize) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot resize a stopped container") } func (s *stoppedState) Start(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return errors.Errorf("cannot start a stopped process") } func (s *stoppedState) Delete(ctx context.Context) error { - s.p.mu.Lock() - defer s.p.mu.Unlock() if err := s.p.delete(ctx); err != nil { return err } @@ -538,8 +415,5 @@ func (s *stoppedState) SetExited(status int) { } func (s *stoppedState) Exec(ctx context.Context, path string, r *ExecConfig) (proc.Process, error) { - s.p.mu.Lock() - defer s.p.mu.Unlock() - return nil, errors.Errorf("cannot exec in a stopped state") } diff --git a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/shim/reaper.go b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/shim/reaper.go index 2937f1a9ef..45a88db12b 100644 --- a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/shim/reaper.go +++ b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/shim/reaper.go @@ -31,7 +31,7 @@ import ( // ErrNoSuchProcess is returned when the process no longer exists var ErrNoSuchProcess = errors.New("no such process") -const bufferSize = 32 +const bufferSize = 2048 // Reap should be called when the process receives an SIGCHLD. Reap will reap // all exited processes and close their wait channels @@ -47,7 +47,6 @@ func Reap() error { Status: e.Status, } } - } Default.Unlock() return err diff --git a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/shim/service.go b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/shim/service.go index d76d5803d6..df6d8b64e2 100644 --- a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/shim/service.go +++ b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/shim/service.go @@ -114,9 +114,6 @@ type Service struct { // Create a new initial process and container with the underlying OCI runtime func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *shimapi.CreateTaskResponse, err error) { - s.mu.Lock() - defer s.mu.Unlock() - var mounts []proc.Mount for _, m := range r.Rootfs { mounts = append(mounts, proc.Mount{ @@ -158,6 +155,10 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ * return nil, errors.Wrapf(err, "failed to mount rootfs component %v", m) } } + + s.mu.Lock() + defer s.mu.Unlock() + process, err := newInit( ctx, s.config.Path, @@ -187,11 +188,9 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ * // Start a process func (s *Service) Start(ctx context.Context, r *shimapi.StartRequest) (*shimapi.StartResponse, error) { - s.mu.Lock() - defer s.mu.Unlock() - p := s.processes[r.ID] - if p == nil { - return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process %s", r.ID) + p, err := s.getExecProcess(r.ID) + if err != nil { + return nil, err } if err := p.Start(ctx); err != nil { return nil, err @@ -204,16 +203,16 @@ func (s *Service) Start(ctx context.Context, r *shimapi.StartRequest) (*shimapi. // Delete the initial process and container func (s *Service) Delete(ctx context.Context, r *ptypes.Empty) (*shimapi.DeleteResponse, error) { - s.mu.Lock() - defer s.mu.Unlock() - p := s.processes[s.id] - if p == nil { - return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") + p, err := s.getInitProcess() + if err != nil { + return nil, err } if err := p.Delete(ctx); err != nil { return nil, err } + s.mu.Lock() delete(s.processes, s.id) + s.mu.Unlock() s.platform.Close() return &shimapi.DeleteResponse{ ExitStatus: uint32(p.ExitStatus()), @@ -227,11 +226,9 @@ func (s *Service) DeleteProcess(ctx context.Context, r *shimapi.DeleteProcessReq if r.ID == s.id { return nil, status.Errorf(codes.InvalidArgument, "cannot delete init process with DeleteProcess") } - s.mu.Lock() - p := s.processes[r.ID] - s.mu.Unlock() - if p == nil { - return nil, errors.Wrapf(errdefs.ErrNotFound, "process %s", r.ID) + p, err := s.getExecProcess(r.ID) + if err != nil { + return nil, err } if err := p.Delete(ctx); err != nil { return nil, err @@ -249,13 +246,14 @@ func (s *Service) DeleteProcess(ctx context.Context, r *shimapi.DeleteProcessReq // Exec an additional process inside the container func (s *Service) Exec(ctx context.Context, r *shimapi.ExecProcessRequest) (*ptypes.Empty, error) { s.mu.Lock() - defer s.mu.Unlock() if p := s.processes[r.ID]; p != nil { + s.mu.Unlock() return nil, errdefs.ToGRPCf(errdefs.ErrAlreadyExists, "id %s", r.ID) } p := s.processes[s.id] + s.mu.Unlock() if p == nil { return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") } @@ -271,14 +269,14 @@ func (s *Service) Exec(ctx context.Context, r *shimapi.ExecProcessRequest) (*pty if err != nil { return nil, errdefs.ToGRPC(err) } + s.mu.Lock() s.processes[r.ID] = process + s.mu.Unlock() return empty, nil } // ResizePty of a process func (s *Service) ResizePty(ctx context.Context, r *shimapi.ResizePtyRequest) (*ptypes.Empty, error) { - s.mu.Lock() - defer s.mu.Unlock() if r.ID == "" { return nil, errdefs.ToGRPCf(errdefs.ErrInvalidArgument, "id not provided") } @@ -286,7 +284,9 @@ func (s *Service) ResizePty(ctx context.Context, r *shimapi.ResizePtyRequest) (* Width: uint16(r.Width), Height: uint16(r.Height), } + s.mu.Lock() p := s.processes[r.ID] + s.mu.Unlock() if p == nil { return nil, errors.Errorf("process does not exist %s", r.ID) } @@ -298,11 +298,9 @@ func (s *Service) ResizePty(ctx context.Context, r *shimapi.ResizePtyRequest) (* // State returns runtime state information for a process func (s *Service) State(ctx context.Context, r *shimapi.StateRequest) (*shimapi.StateResponse, error) { - s.mu.Lock() - defer s.mu.Unlock() - p := s.processes[r.ID] - if p == nil { - return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process id %s", r.ID) + p, err := s.getExecProcess(r.ID) + if err != nil { + return nil, err } st, err := p.Status(ctx) if err != nil { @@ -338,11 +336,9 @@ func (s *Service) State(ctx context.Context, r *shimapi.StateRequest) (*shimapi. // Pause the container func (s *Service) Pause(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, error) { - s.mu.Lock() - defer s.mu.Unlock() - p := s.processes[s.id] - if p == nil { - return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") + p, err := s.getInitProcess() + if err != nil { + return nil, err } if err := p.(*proc.Init).Pause(ctx); err != nil { return nil, err @@ -352,11 +348,9 @@ func (s *Service) Pause(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, er // Resume the container func (s *Service) Resume(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, error) { - s.mu.Lock() - defer s.mu.Unlock() - p := s.processes[s.id] - if p == nil { - return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") + p, err := s.getInitProcess() + if err != nil { + return nil, err } if err := p.(*proc.Init).Resume(ctx); err != nil { return nil, err @@ -366,12 +360,10 @@ func (s *Service) Resume(ctx context.Context, r *ptypes.Empty) (*ptypes.Empty, e // Kill a process with the provided signal func (s *Service) Kill(ctx context.Context, r *shimapi.KillRequest) (*ptypes.Empty, error) { - s.mu.Lock() - defer s.mu.Unlock() if r.ID == "" { - p := s.processes[s.id] - if p == nil { - return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") + p, err := s.getInitProcess() + if err != nil { + return nil, err } if err := p.Kill(ctx, r.Signal, r.All); err != nil { return nil, errdefs.ToGRPC(err) @@ -379,9 +371,9 @@ func (s *Service) Kill(ctx context.Context, r *shimapi.KillRequest) (*ptypes.Emp return empty, nil } - p := s.processes[r.ID] - if p == nil { - return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process id %s not found", r.ID) + p, err := s.getExecProcess(r.ID) + if err != nil { + return nil, err } if err := p.Kill(ctx, r.Signal, r.All); err != nil { return nil, errdefs.ToGRPC(err) @@ -422,11 +414,9 @@ func (s *Service) ListPids(ctx context.Context, r *shimapi.ListPidsRequest) (*sh // CloseIO of a process func (s *Service) CloseIO(ctx context.Context, r *shimapi.CloseIORequest) (*ptypes.Empty, error) { - s.mu.Lock() - defer s.mu.Unlock() - p := s.processes[r.ID] - if p == nil { - return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process does not exist %s", r.ID) + p, err := s.getExecProcess(r.ID) + if err != nil { + return nil, err } if stdin := p.Stdin(); stdin != nil { if err := stdin.Close(); err != nil { @@ -438,11 +428,9 @@ func (s *Service) CloseIO(ctx context.Context, r *shimapi.CloseIORequest) (*ptyp // Checkpoint the container func (s *Service) Checkpoint(ctx context.Context, r *shimapi.CheckpointTaskRequest) (*ptypes.Empty, error) { - s.mu.Lock() - defer s.mu.Unlock() - p := s.processes[s.id] - if p == nil { - return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") + p, err := s.getInitProcess() + if err != nil { + return nil, err } var options runctypes.CheckpointOptions if r.Options != nil { @@ -475,11 +463,9 @@ func (s *Service) ShimInfo(ctx context.Context, r *ptypes.Empty) (*shimapi.ShimI // Update a running container func (s *Service) Update(ctx context.Context, r *shimapi.UpdateTaskRequest) (*ptypes.Empty, error) { - s.mu.Lock() - defer s.mu.Unlock() - p := s.processes[s.id] - if p == nil { - return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") + p, err := s.getInitProcess() + if err != nil { + return nil, err } if err := p.(*proc.Init).Update(ctx, r.Resources); err != nil { return nil, errdefs.ToGRPC(err) @@ -489,11 +475,9 @@ func (s *Service) Update(ctx context.Context, r *shimapi.UpdateTaskRequest) (*pt // Wait for a process to exit func (s *Service) Wait(ctx context.Context, r *shimapi.WaitRequest) (*shimapi.WaitResponse, error) { - s.mu.Lock() - p := s.processes[r.ID] - s.mu.Unlock() - if p == nil { - return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") + p, err := s.getExecProcess(r.ID) + if err != nil { + return nil, err } p.Wait() @@ -509,16 +493,24 @@ func (s *Service) processExits() { } } -func (s *Service) checkProcesses(e runc.Exit) { +func (s *Service) allProcesses() []rproc.Process { s.mu.Lock() defer s.mu.Unlock() + res := make([]rproc.Process, 0, len(s.processes)) + for _, p := range s.processes { + res = append(res, p) + } + return res +} + +func (s *Service) checkProcesses(e runc.Exit) { shouldKillAll, err := shouldKillAllOnExit(s.bundle) if err != nil { log.G(s.context).WithError(err).Error("failed to check shouldKillAll") } - for _, p := range s.processes { + for _, p := range s.allProcesses() { if p.Pid() == e.Pid { if shouldKillAll { @@ -563,11 +555,9 @@ func shouldKillAllOnExit(bundlePath string) (bool, error) { } func (s *Service) getContainerPids(ctx context.Context, id string) ([]uint32, error) { - s.mu.Lock() - defer s.mu.Unlock() - p := s.processes[s.id] - if p == nil { - return nil, errors.Wrapf(errdefs.ErrFailedPrecondition, "container must be created") + p, err := s.getInitProcess() + if err != nil { + return nil, err } ps, err := p.(*proc.Init).Runtime().Ps(ctx, id) @@ -589,6 +579,30 @@ func (s *Service) forward(publisher events.Publisher) { } } +// getInitProcess returns initial process +func (s *Service) getInitProcess() (rproc.Process, error) { + s.mu.Lock() + defer s.mu.Unlock() + + p := s.processes[s.id] + if p == nil { + return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created") + } + return p, nil +} + +// getExecProcess returns exec process +func (s *Service) getExecProcess(id string) (rproc.Process, error) { + s.mu.Lock() + defer s.mu.Unlock() + + p := s.processes[id] + if p == nil { + return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process %s does not exist", id) + } + return p, nil +} + func getTopic(ctx context.Context, e interface{}) string { switch e.(type) { case *eventstypes.TaskCreate: diff --git a/components/engine/vendor/github.com/containerd/containerd/vendor.conf b/components/engine/vendor/github.com/containerd/containerd/vendor.conf index dbc3eecd96..efba4b6bff 100644 --- a/components/engine/vendor/github.com/containerd/containerd/vendor.conf +++ b/components/engine/vendor/github.com/containerd/containerd/vendor.conf @@ -20,8 +20,8 @@ github.com/gogo/protobuf v1.0.0 github.com/gogo/googleapis 08a7655d27152912db7aaf4f983275eaf8d128ef github.com/golang/protobuf v1.1.0 github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d -github.com/opencontainers/runc 58592df56734acf62e574865fe40b9e53e967910 -github.com/sirupsen/logrus v1.0.0 +github.com/opencontainers/runc 10d38b660a77168360df3522881e2dc2be5056bd +github.com/sirupsen/logrus v1.0.3 github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c golang.org/x/net b3756b4b77d7b13260a0a2ec658753cf48922eac google.golang.org/grpc v1.12.0 @@ -33,7 +33,7 @@ golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895 github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0 github.com/Microsoft/go-winio v0.4.11 -github.com/Microsoft/hcsshim v0.7.12 +github.com/Microsoft/hcsshim v0.8.1 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a @@ -81,9 +81,9 @@ k8s.io/kubernetes v1.12.0 k8s.io/utils cd34563cd63c2bd7c6fe88a73c4dcf34ed8a67cb # zfs dependencies -github.com/containerd/zfs 9a0b8b8b5982014b729cd34eb7cd7a11062aa6ec +github.com/containerd/zfs 9f6ef3b1fe5144bd91fe5855b4eba81bc0d17d03 github.com/mistifyio/go-zfs 166add352731e515512690329794ee593f1aaff2 github.com/pborman/uuid c65b2f87fee37d1c7854c9164a450713c28d50cd # aufs dependencies -github.com/containerd/aufs ffa39970e26ad01d81f540b21e65f9c1841a5f92 +github.com/containerd/aufs da3cf16bfbe68ba8f114f1536a05c01528a25434 diff --git a/components/engine/vendor/github.com/opencontainers/runc/README.md b/components/engine/vendor/github.com/opencontainers/runc/README.md index 671571d1ef..e755fb7bcd 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/README.md +++ b/components/engine/vendor/github.com/opencontainers/runc/README.md @@ -68,6 +68,7 @@ make BUILDTAGS='seccomp apparmor' | selinux | selinux process and mount labeling | | | apparmor | apparmor profile support | | | ambient | ambient capability support | kernel 4.3 | +| nokmem | disable kernel memory account | | ### Running the test suite diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/README.md b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/README.md index f2a2f0c6c4..1d7fa04c08 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/README.md +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/README.md @@ -148,6 +148,7 @@ config := &configs.Config{ {Type: configs.NEWPID}, {Type: configs.NEWUSER}, {Type: configs.NEWNET}, + {Type: configs.NEWCGROUP}, }), Cgroups: &configs.Cgroup{ Name: "test-container", diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go index 7c61ff13fc..cd32a67353 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go @@ -17,7 +17,7 @@ import ( ) const ( - cgroupNamePrefix = "name=" + CgroupNamePrefix = "name=" CgroupProcesses = "cgroup.procs" ) @@ -156,8 +156,8 @@ func getCgroupMountsHelper(ss map[string]bool, mi io.Reader, all bool) ([]Mount, continue } ss[opt] = true - if strings.HasPrefix(opt, cgroupNamePrefix) { - opt = opt[len(cgroupNamePrefix):] + if strings.HasPrefix(opt, CgroupNamePrefix) { + opt = opt[len(CgroupNamePrefix):] } m.Subsystems = append(m.Subsystems, opt) numFound++ @@ -343,7 +343,7 @@ func getControllerPath(subsystem string, cgroups map[string]string) (string, err return p, nil } - if p, ok := cgroups[cgroupNamePrefix+subsystem]; ok { + if p, ok := cgroups[CgroupNamePrefix+subsystem]; ok { return p, nil } diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_linux.go b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_linux.go index 5fc171a57b..1bbaef9bd9 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_linux.go +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_linux.go @@ -7,12 +7,13 @@ import ( ) const ( - NEWNET NamespaceType = "NEWNET" - NEWPID NamespaceType = "NEWPID" - NEWNS NamespaceType = "NEWNS" - NEWUTS NamespaceType = "NEWUTS" - NEWIPC NamespaceType = "NEWIPC" - NEWUSER NamespaceType = "NEWUSER" + NEWNET NamespaceType = "NEWNET" + NEWPID NamespaceType = "NEWPID" + NEWNS NamespaceType = "NEWNS" + NEWUTS NamespaceType = "NEWUTS" + NEWIPC NamespaceType = "NEWIPC" + NEWUSER NamespaceType = "NEWUSER" + NEWCGROUP NamespaceType = "NEWCGROUP" ) var ( @@ -35,6 +36,8 @@ func NsName(ns NamespaceType) string { return "user" case NEWUTS: return "uts" + case NEWCGROUP: + return "cgroup" } return "" } @@ -68,6 +71,7 @@ func NamespaceTypes() []NamespaceType { NEWNET, NEWPID, NEWNS, + NEWCGROUP, } } diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_syscall.go b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_syscall.go index 4ce6813d23..2dc7adfc96 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_syscall.go +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_syscall.go @@ -9,12 +9,13 @@ func (n *Namespace) Syscall() int { } var namespaceInfo = map[NamespaceType]int{ - NEWNET: unix.CLONE_NEWNET, - NEWNS: unix.CLONE_NEWNS, - NEWUSER: unix.CLONE_NEWUSER, - NEWIPC: unix.CLONE_NEWIPC, - NEWUTS: unix.CLONE_NEWUTS, - NEWPID: unix.CLONE_NEWPID, + NEWNET: unix.CLONE_NEWNET, + NEWNS: unix.CLONE_NEWNS, + NEWUSER: unix.CLONE_NEWUSER, + NEWIPC: unix.CLONE_NEWIPC, + NEWUTS: unix.CLONE_NEWUTS, + NEWPID: unix.CLONE_NEWPID, + NEWCGROUP: unix.CLONE_NEWCGROUP, } // CloneFlags parses the container's Namespaces options to set the correct diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c index d7cb0af030..28269dfc02 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c @@ -42,6 +42,12 @@ enum sync_t { SYNC_ERR = 0xFF, /* Fatal error, no turning back. The error code follows. */ }; +/* + * Synchronisation value for cgroup namespace setup. + * The same constant is defined in process_linux.go as "createCgroupns". + */ +#define CREATECGROUPNS 0x80 + /* longjmp() arguments. */ #define JUMP_PARENT 0x00 #define JUMP_CHILD 0xA0 @@ -640,7 +646,6 @@ void nsexec(void) case JUMP_PARENT:{ int len; pid_t child, first_child = -1; - char buf[JSON_MAX]; bool ready = false; /* For debugging. */ @@ -716,6 +721,18 @@ void nsexec(void) kill(child, SIGKILL); bail("failed to sync with child: write(SYNC_RECVPID_ACK)"); } + + /* Send the init_func pid back to our parent. + * + * Send the init_func pid and the pid of the first child back to our parent. + * We need to send both back because we can't reap the first child we created (CLONE_PARENT). + * It becomes the responsibility of our parent to reap the first child. + */ + len = dprintf(pipenum, "{\"pid\": %d, \"pid_first\": %d}\n", child, first_child); + if (len < 0) { + kill(child, SIGKILL); + bail("unable to generate JSON for child pid"); + } } break; case SYNC_CHILD_READY: @@ -759,23 +776,6 @@ void nsexec(void) bail("unexpected sync value: %u", s); } } - - /* - * Send the init_func pid and the pid of the first child back to our parent. - * - * We need to send both back because we can't reap the first child we created (CLONE_PARENT). - * It becomes the responsibility of our parent to reap the first child. - */ - len = snprintf(buf, JSON_MAX, "{\"pid\": %d, \"pid_first\": %d}\n", child, first_child); - if (len < 0) { - kill(child, SIGKILL); - bail("unable to generate JSON for child pid"); - } - if (write(pipenum, buf, len) != len) { - kill(child, SIGKILL); - bail("unable to send child pid to bootstrapper"); - } - exit(0); } @@ -862,14 +862,17 @@ void nsexec(void) if (setresuid(0, 0, 0) < 0) bail("failed to become root in user namespace"); } - /* - * Unshare all of the namespaces. Note that we don't merge this - * with clone() because there were some old kernel versions where - * clone(CLONE_PARENT | CLONE_NEWPID) was broken, so we'll just do - * it the long way. + * Unshare all of the namespaces. Now, it should be noted that this + * ordering might break in the future (especially with rootless + * containers). But for now, it's not possible to split this into + * CLONE_NEWUSER + [the rest] because of some RHEL SELinux issues. + * + * Note that we don't merge this with clone() because there were + * some old kernel versions where clone(CLONE_PARENT | CLONE_NEWPID) + * was broken, so we'll just do it the long way anyway. */ - if (unshare(config.cloneflags) < 0) + if (unshare(config.cloneflags & ~CLONE_NEWCGROUP) < 0) bail("failed to unshare namespaces"); /* @@ -958,6 +961,18 @@ void nsexec(void) bail("setgroups failed"); } + /* ... wait until our topmost parent has finished cgroup setup in p.manager.Apply() ... */ + if (config.cloneflags & CLONE_NEWCGROUP) { + uint8_t value; + if (read(pipenum, &value, sizeof(value)) != sizeof(value)) + bail("read synchronisation value failed"); + if (value == CREATECGROUPNS) { + if (unshare(CLONE_NEWCGROUP) < 0) + bail("failed to unshare cgroup namespace"); + } else + bail("received unknown synchronisation value"); + } + s = SYNC_CHILD_READY; if (write(syncfd, &s, sizeof(s)) != sizeof(s)) bail("failed to sync with patent: write(SYNC_CHILD_READY)"); From 9baf0f2b61e511610df6ffdbf49e87647b3eb812 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 21 Nov 2018 00:05:16 +0100 Subject: [PATCH 23/25] Update containerd to v1.2.1-rc.0 The previous update used a commit from master. Now that all the fixes are backported to the containerd 1.2 release branch, we can switch back to that branch. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 2fb5de68a9bd05b1dbf3ae3f7ae82bcd5e64dc5c) Signed-off-by: Sebastiaan van Stijn Upstream-commit: db7f375d6a2aaf6d79f5c690e2f302c640bdde04 Component: engine --- .../engine/hack/dockerfile/install/containerd.installer | 2 +- components/engine/vendor.conf | 2 +- .../vendor/github.com/containerd/containerd/README.md | 8 +------- .../containerd/containerd/remotes/docker/resolver.go | 2 +- .../containerd/containerd/runtime/v1/linux/proc/init.go | 3 +++ .../containerd/runtime/v1/shim/service_linux.go | 8 +++++--- .../vendor/github.com/containerd/containerd/vendor.conf | 8 ++++---- 7 files changed, 16 insertions(+), 17 deletions(-) diff --git a/components/engine/hack/dockerfile/install/containerd.installer b/components/engine/hack/dockerfile/install/containerd.installer index 8981ab37ed..b62aac0793 100755 --- a/components/engine/hack/dockerfile/install/containerd.installer +++ b/components/engine/hack/dockerfile/install/containerd.installer @@ -4,7 +4,7 @@ # containerd is also pinned in vendor.conf. When updating the binary # version you may also need to update the vendor version to pick up bug # fixes or new APIs. -CONTAINERD_COMMIT=aa537a67b3a9026c423a63dbaf71e4cc0776b2ad # v1.2.0-13-g8afcade1 +CONTAINERD_COMMIT=de1f167ab96338a9f5c2b17347abf84bdf1dd411 # v1.2.1-rc.0 install_containerd() { echo "Install containerd version $CONTAINERD_COMMIT" diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 6421f98ebb..0e07a4c6a5 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -118,7 +118,7 @@ github.com/googleapis/gax-go v2.0.0 google.golang.org/genproto 694d95ba50e67b2e363f3483057db5d4910c18f9 # containerd -github.com/containerd/containerd aa537a67b3a9026c423a63dbaf71e4cc0776b2ad # v1.2.0 +github.com/containerd/containerd de1f167ab96338a9f5c2b17347abf84bdf1dd411 # v1.2.1-rc.0 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4 github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2 diff --git a/components/engine/vendor/github.com/containerd/containerd/README.md b/components/engine/vendor/github.com/containerd/containerd/README.md index 4a90fc8a0e..2055404b5b 100644 --- a/components/engine/vendor/github.com/containerd/containerd/README.md +++ b/components/engine/vendor/github.com/containerd/containerd/README.md @@ -224,8 +224,7 @@ This will be the best place to discuss design and implementation. For sync communication we have a community slack with a #containerd channel that everyone is welcome to join and chat about development. -**Slack:** Catch us in the #containerd and #containerd-dev channels on dockercommunity.slack.com. -[Click here for an invite to docker community slack.](https://join.slack.com/t/dockercommunity/shared_invite/enQtNDY4MDc1Mzc0MzIwLTgxZDBlMmM4ZGEyNDc1N2FkMzlhODJkYmE1YTVkYjM1MDE3ZjAwZjBkOGFlOTJkZjRmZGYzNjYyY2M3ZTUxYzQ) +**Slack:** https://join.slack.com/t/dockercommunity/shared_invite/enQtNDM4NjAwNDMyOTUwLWZlMDZmYWRjZjk4Zjc5ZGQ5NWZkOWI1Yjk2NGE3ZWVlYjYxM2VhYjczOWIyZDFhZTE3NTUwZWQzMjhmNGYyZTg ### Reporting security issues @@ -250,8 +249,3 @@ Please find all these core project documents, including the: * and [Contributing guidelines](https://github.com/containerd/project/blob/master/CONTRIBUTING.md) information in our [`containerd/project`](https://github.com/containerd/project) repository. - -## Adoption - -Interested to see who is using containerd? Are you using containerd in a project? -Please add yourself via pull request to our [ADOPTERS.md](./ADOPTERS.md) file. diff --git a/components/engine/vendor/github.com/containerd/containerd/remotes/docker/resolver.go b/components/engine/vendor/github.com/containerd/containerd/remotes/docker/resolver.go index b89bd98c73..5cccdecba0 100644 --- a/components/engine/vendor/github.com/containerd/containerd/remotes/docker/resolver.go +++ b/components/engine/vendor/github.com/containerd/containerd/remotes/docker/resolver.go @@ -75,7 +75,7 @@ type ResolverOptions struct { // Credentials provides username and secret given a host. // If username is empty but a secret is given, that secret - // is interpreted as a long lived token. + // is interpretted as a long lived token. // Deprecated: use Authorizer Credentials func(string) (string, string, error) diff --git a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/init.go b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/init.go index 3ce7a43418..fa23b5e883 100644 --- a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/init.go +++ b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/linux/proc/init.go @@ -356,6 +356,9 @@ func (p *Init) kill(ctx context.Context, signal uint32, all bool) error { // KillAll processes belonging to the init process func (p *Init) KillAll(ctx context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + err := p.runtime.Kill(ctx, p.id, int(syscall.SIGKILL), &runc.KillOpts{ All: true, }) diff --git a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/shim/service_linux.go b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/shim/service_linux.go index 18ae6503b4..307e20dabd 100644 --- a/components/engine/vendor/github.com/containerd/containerd/runtime/v1/shim/service_linux.go +++ b/components/engine/vendor/github.com/containerd/containerd/runtime/v1/shim/service_linux.go @@ -49,9 +49,11 @@ func (p *linuxPlatform) CopyConsole(ctx context.Context, console console.Console cwg.Add(1) go func() { cwg.Done() - p := bufPool.Get().(*[]byte) - defer bufPool.Put(p) - io.CopyBuffer(epollConsole, in, *p) + bp := bufPool.Get().(*[]byte) + defer bufPool.Put(bp) + io.CopyBuffer(epollConsole, in, *bp) + // we need to shutdown epollConsole when pipe broken + epollConsole.Shutdown(p.epoller.CloseConsole) }() } diff --git a/components/engine/vendor/github.com/containerd/containerd/vendor.conf b/components/engine/vendor/github.com/containerd/containerd/vendor.conf index efba4b6bff..650710a9b8 100644 --- a/components/engine/vendor/github.com/containerd/containerd/vendor.conf +++ b/components/engine/vendor/github.com/containerd/containerd/vendor.conf @@ -21,7 +21,7 @@ github.com/gogo/googleapis 08a7655d27152912db7aaf4f983275eaf8d128ef github.com/golang/protobuf v1.1.0 github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d github.com/opencontainers/runc 10d38b660a77168360df3522881e2dc2be5056bd -github.com/sirupsen/logrus v1.0.3 +github.com/sirupsen/logrus v1.0.0 github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c golang.org/x/net b3756b4b77d7b13260a0a2ec658753cf48922eac google.golang.org/grpc v1.12.0 @@ -43,7 +43,7 @@ github.com/google/go-cmp v0.1.0 go.etcd.io/bbolt v1.3.1-etcd.8 # cri dependencies -github.com/containerd/cri f913714917d2456d7e65a0be84962b1ce8acb487 # release/1.2 branch +github.com/containerd/cri 2bb57d27203d82fc79c496aea724aec593b2705a # release/1.2 branch github.com/containerd/go-cni 40bcf8ec8acd7372be1d77031d585d5d8e561c90 github.com/blang/semver v3.1.0 github.com/containernetworking/cni v0.6.0 @@ -81,9 +81,9 @@ k8s.io/kubernetes v1.12.0 k8s.io/utils cd34563cd63c2bd7c6fe88a73c4dcf34ed8a67cb # zfs dependencies -github.com/containerd/zfs 9f6ef3b1fe5144bd91fe5855b4eba81bc0d17d03 +github.com/containerd/zfs 9a0b8b8b5982014b729cd34eb7cd7a11062aa6ec github.com/mistifyio/go-zfs 166add352731e515512690329794ee593f1aaff2 github.com/pborman/uuid c65b2f87fee37d1c7854c9164a450713c28d50cd # aufs dependencies -github.com/containerd/aufs da3cf16bfbe68ba8f114f1536a05c01528a25434 +github.com/containerd/aufs ffa39970e26ad01d81f540b21e65f9c1841a5f92 From 66a497c4f1c30b394ce889656c7a4eba8e10bf5d Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Sun, 8 Apr 2018 20:21:30 +1000 Subject: [PATCH 24/25] apparmor: allow receiving of signals from 'docker kill' In newer kernels, AppArmor will reject attempts to send signals to a container because the signal originated from outside of that AppArmor profile. Correct this by allowing all unconfined signals to be received. Signed-off-by: Goldwyn Rodrigues Signed-off-by: Aleksa Sarai (cherry picked from commit 4822fb1e2423d88cdf0ad5d039b8fd3274b05401) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 67c602c3fe3d5d817fb7210e50d7ed1688b28801 Component: engine --- .../engine/profiles/apparmor/apparmor.go | 21 +++++++++++++++++++ .../engine/profiles/apparmor/template.go | 6 ++++++ 2 files changed, 27 insertions(+) diff --git a/components/engine/profiles/apparmor/apparmor.go b/components/engine/profiles/apparmor/apparmor.go index b021668c8e..2f58ee852c 100644 --- a/components/engine/profiles/apparmor/apparmor.go +++ b/components/engine/profiles/apparmor/apparmor.go @@ -23,6 +23,8 @@ var ( type profileData struct { // Name is profile name. Name string + // DaemonProfile is the profile name of our daemon. + DaemonProfile string // Imports defines the apparmor functions to import, before defining the profile. Imports []string // InnerImports defines the apparmor functions to import in the profile. @@ -70,6 +72,25 @@ func InstallDefault(name string) error { Name: name, } + // Figure out the daemon profile. + currentProfile, err := ioutil.ReadFile("/proc/self/attr/current") + if err != nil { + // If we couldn't get the daemon profile, assume we are running + // unconfined which is generally the default. + currentProfile = nil + } + daemonProfile := string(currentProfile) + // Normally profiles are suffixed by " (enforcing)" or similar. AppArmor + // profiles cannot contain spaces so this doesn't restrict daemon profile + // names. + if parts := strings.SplitN(daemonProfile, " ", 2); len(parts) >= 1 { + daemonProfile = parts[0] + } + if daemonProfile == "" { + daemonProfile = "unconfined" + } + p.DaemonProfile = daemonProfile + // Install to a temporary directory. f, err := ioutil.TempFile("", name) if err != nil { diff --git a/components/engine/profiles/apparmor/template.go b/components/engine/profiles/apparmor/template.go index c00a3f70e9..400b3bd50a 100644 --- a/components/engine/profiles/apparmor/template.go +++ b/components/engine/profiles/apparmor/template.go @@ -17,6 +17,12 @@ profile {{.Name}} flags=(attach_disconnected,mediate_deleted) { capability, file, umount, +{{if ge .Version 208096}} +{{/* Allow 'docker kill' to actually send signals to container processes. */}} + signal (receive) peer={{.DaemonProfile}}, +{{/* Allow container processes to send signals amongst themselves. */}} + signal (send,receive) peer={{.Name}}, +{{end}} deny @{PROC}/* w, # deny write for all files directly in /proc (not in a subdir) # deny write to files not in /proc//** or /proc/sys/** From 78dccb265dd7578319a3f9f014065604ea3c77af Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 13 Nov 2018 14:04:06 +0100 Subject: [PATCH 25/25] Ignore default address-pools on API < 1.39 These options were added in API 1.39, so should be ignored when using an older version of the API. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 7632ccbc66c82179547b96524672bd5082ae7481) Signed-off-by: Sebastiaan van Stijn Upstream-commit: 4cc45d91eb44cbaddcfd75335c3f72ede035c440 Component: engine --- components/engine/api/server/router/swarm/cluster_routes.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/engine/api/server/router/swarm/cluster_routes.go b/components/engine/api/server/router/swarm/cluster_routes.go index a702488602..5d460741f1 100644 --- a/components/engine/api/server/router/swarm/cluster_routes.go +++ b/components/engine/api/server/router/swarm/cluster_routes.go @@ -23,6 +23,12 @@ func (sr *swarmRouter) initCluster(ctx context.Context, w http.ResponseWriter, r if err := json.NewDecoder(r.Body).Decode(&req); err != nil { return err } + version := httputils.VersionFromContext(ctx) + // DefaultAddrPool and SubnetSize were added in API 1.39. Ignore on older API versions. + if versions.LessThan(version, "1.39") { + req.DefaultAddrPool = nil + req.SubnetSize = 0 + } nodeID, err := sr.backend.Init(req) if err != nil { logrus.Errorf("Error initializing swarm: %v", err)