[engine] Graceful upgrade of containerd and runc state files upon live-restore

Vendors new dependency github.com/crosbymichael/upgrade

Signed-off-by: Tibor Vass <tibor@docker.com>
(cherry picked from commit 358c36e930)
Signed-off-by: Victor Vieux <victorvieux@gmail.com>

components/engine/libcontainerd/remote_unix.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	containerd "github.com/containerd/containerd/api/grpc/types"
+	"github.com/crosbymichael/upgrade/v17_06_1"
 	"github.com/docker/docker/pkg/locker"
 	"github.com/docker/docker/pkg/system"
 	"github.com/golang/protobuf/ptypes"
@@ -39,7 +40,13 @@ const (
 	containerdPidFilename        = "docker-containerd.pid"
 	containerdSockFilename       = "docker-containerd.sock"
 	containerdStateDir           = "containerd"
+	containerdInitDir            = "init"
 	eventTimestampFilename       = "event.ts"
+	processFilename              = "process.json"
+
+	// TODO: Use user's --root parameter for runc, if possible
+	runcStateDir      = "/run/runc"
+	runcStateFilename = "state.json"
 )
 
 type remote struct {
@@ -89,6 +96,7 @@ func New(stateDir string, options ...RemoteOption) (_ Remote, err error) {
 	}
 
 	if r.startDaemon {
+		r.makeUpgradeProof()
 		if err := r.runContainerdDaemon(); err != nil {
 			return nil, err
 		}
@@ -128,6 +136,37 @@ func New(stateDir string, options ...RemoteOption) (_ Remote, err error) {
 	return r, nil
 }
 
+func (r *remote) makeUpgradeProof() {
+	dir := filepath.Join(r.stateDir, containerdStateDir)
+	f, err := os.Open(dir)
+	if err != nil {
+		logrus.Warnf("libcontainerd: makeUpgradeProof could not open %s", dir)
+		return
+	}
+	fis, err := f.Readdir(0)
+	if err != nil {
+		logrus.Warnf("libcontainerd: makeUpgradeProof could not read directory entries in %s", dir)
+		f.Close()
+		return
+	}
+	containerIds := make([]string, 0, len(fis))
+	for _, fi := range fis {
+		if fi.IsDir() {
+			containerIds = append(containerIds, fi.Name())
+		}
+	}
+	f.Close()
+	for _, id := range containerIds {
+		if err := v17_06_1.Upgrade(
+			filepath.Join(runcStateDir, id, runcStateFilename),
+			filepath.Join(r.stateDir, id, configFilename),
+			filepath.Join(dir, id, containerdInitDir, processFilename),
+		); err != nil {
+			logrus.Warnf("libcontainerd: could not upgrade state files during live restore for container %s: %v", id, err)
+		}
+	}
+}
+
 func (r *remote) UpdateOptions(options ...RemoteOption) error {
 	for _, option := range options {
 		if err := option.Apply(r); err != nil {

components/engine/vendor.conf

@@ -3,6 +3,7 @@ github.com/Azure/go-ansiterm 388960b655244e76e24c75f48631564eaefade62
 github.com/Microsoft/hcsshim v0.5.25
 github.com/Microsoft/go-winio v0.4.2
 github.com/Sirupsen/logrus v0.11.0
+github.com/crosbymichael/upgrade 3ee9eb41518034a2dfe45d8273297f309a9d94da
 github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76
 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a
 github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git

components/engine/vendor/github.com/crosbymichael/upgrade/v17_06_1/README.md

@@ -0,0 +1,3 @@
+# How to generate
+
+go generate ./template.go

components/engine/vendor/github.com/crosbymichael/upgrade/v17_06_1/process_state_gen.go

@@ -0,0 +1,29 @@
+// DO NOT EDIT
+// This file has been auto-generated with go generate.
+
+package v17_06_1
+
+import specs "github.com/opencontainers/runtime-spec/specs-go" // a45ba0989fc26c695fe166a49c45bb8b7618ab36 https://github.com/docker/runtime-spec
+
+type ProcessState struct {
+	Terminal        bool                `json:"terminal,omitempty"`
+	ConsoleSize     specs.Box           `json:"consoleSize,omitempty"`
+	User            specs.User          `json:"user"`
+	Args            []string            `json:"args"`
+	Env             []string            `json:"env,omitempty"`
+	Cwd             string              `json:"cwd"`
+	Capabilities    linuxCapabilities   `json:"capabilities,omitempty" platform:"linux"`
+	Rlimits         []specs.LinuxRlimit `json:"rlimits,omitempty" platform:"linux"`
+	NoNewPrivileges bool                `json:"noNewPrivileges,omitempty" platform:"linux"`
+	ApparmorProfile string              `json:"apparmorProfile,omitempty" platform:"linux"`
+	SelinuxLabel    string              `json:"selinuxLabel,omitempty" platform:"linux"`
+	Exec            bool                `json:"exec"`
+	Stdin           string              `json:"containerdStdin"`
+	Stdout          string              `json:"containerdStdout"`
+	Stderr          string              `json:"containerdStderr"`
+	RuntimeArgs     []string            `json:"runtimeArgs"`
+	NoPivotRoot     bool                `json:"noPivotRoot"`
+	Checkpoint      string              `json:"checkpoint"`
+	RootUID         int                 `json:"rootUID"`
+	RootGID         int                 `json:"rootGID"`
+}

components/engine/vendor/github.com/crosbymichael/upgrade/v17_06_1/spec_gen.go

@@ -0,0 +1,66 @@
+// DO NOT EDIT
+// This file has been auto-generated with go generate.
+
+package v17_06_1
+
+import specs "github.com/opencontainers/runtime-spec/specs-go" // a45ba0989fc26c695fe166a49c45bb8b7618ab36 https://github.com/docker/runtime-spec
+
+type Spec struct {
+	Version  string         `json:"ociVersion"`
+	Platform specs.Platform `json:"platform"`
+	Process  struct {
+		Terminal        bool                `json:"terminal,omitempty"`
+		ConsoleSize     specs.Box           `json:"consoleSize,omitempty"`
+		User            specs.User          `json:"user"`
+		Args            []string            `json:"args"`
+		Env             []string            `json:"env,omitempty"`
+		Cwd             string              `json:"cwd"`
+		Capabilities    linuxCapabilities   `json:"capabilities,omitempty" platform:"linux"`
+		Rlimits         []specs.LinuxRlimit `json:"rlimits,omitempty" platform:"linux"`
+		NoNewPrivileges bool                `json:"noNewPrivileges,omitempty" platform:"linux"`
+		ApparmorProfile string              `json:"apparmorProfile,omitempty" platform:"linux"`
+		SelinuxLabel    string              `json:"selinuxLabel,omitempty" platform:"linux"`
+	} `json:"process"`
+	Root        specs.Root        `json:"root"`
+	Hostname    string            `json:"hostname,omitempty"`
+	Mounts      []specs.Mount     `json:"mounts,omitempty"`
+	Hooks       *specs.Hooks      `json:"hooks,omitempty"`
+	Annotations map[string]string `json:"annotations,omitempty"`
+	Linux       *struct {
+		UIDMappings []specs.LinuxIDMapping `json:"uidMappings,omitempty"`
+		GIDMappings []specs.LinuxIDMapping `json:"gidMappings,omitempty"`
+		Sysctl      map[string]string      `json:"sysctl,omitempty"`
+		Resources   *struct {
+			Devices          []specs.LinuxDeviceCgroup `json:"devices,omitempty"`
+			DisableOOMKiller *bool                     `json:"disableOOMKiller,omitempty"`
+			OOMScoreAdj      *int                      `json:"oomScoreAdj,omitempty"`
+			Memory           *struct {
+				Limit       *int64           `json:"limit,omitempty"`
+				Reservation *int64           `json:"reservation,omitempty"`
+				Swap        *int64           `json:"swap,omitempty"`
+				Kernel      *int64           `json:"kernel,omitempty"`
+				KernelTCP   *int64           `json:"kernelTCP,omitempty"`
+				Swappiness  memorySwappiness `json:"swappiness,omitempty"`
+			} `json:"memory,omitempty"`
+			CPU            *specs.LinuxCPU            `json:"cpu,omitempty"`
+			Pids           *specs.LinuxPids           `json:"pids,omitempty"`
+			BlockIO        *specs.LinuxBlockIO        `json:"blockIO,omitempty"`
+			HugepageLimits []specs.LinuxHugepageLimit `json:"hugepageLimits,omitempty"`
+			Network        *specs.LinuxNetwork        `json:"network,omitempty"`
+		} `json:"resources,omitempty"`
+		CgroupsPath string                 `json:"cgroupsPath,omitempty"`
+		Namespaces  []specs.LinuxNamespace `json:"namespaces,omitempty"`
+		Devices     []specs.LinuxDevice    `json:"devices,omitempty"`
+		Seccomp     *struct {
+			DefaultAction specs.LinuxSeccompAction `json:"defaultAction"`
+			Architectures []specs.Arch             `json:"architectures,omitempty"`
+			Syscalls      linuxSyscalls            `json:"syscalls"`
+		} `json:"seccomp,omitempty"`
+		RootfsPropagation string   `json:"rootfsPropagation,omitempty"`
+		MaskedPaths       []string `json:"maskedPaths,omitempty"`
+		ReadonlyPaths     []string `json:"readonlyPaths,omitempty"`
+		MountLabel        string   `json:"mountLabel,omitempty"`
+	} `json:"linux,omitempty" platform:"linux"`
+	Solaris *specs.Solaris `json:"solaris,omitempty" platform:"solaris"`
+	Windows *specs.Windows `json:"windows,omitempty" platform:"windows"`
+}

components/engine/vendor/github.com/crosbymichael/upgrade/v17_06_1/state_gen.go

@@ -0,0 +1,89 @@
+// DO NOT EDIT
+// This file has been auto-generated with go generate.
+
+package v17_06_1
+
+import (
+	"time"
+
+	"github.com/opencontainers/runc/libcontainer/configs" // 810190ceaa507aa2727d7ae6f4790c76ec150bd2 https://github.com/docker/runc
+)
+
+type State struct {
+	ID                   string    `json:"id"`
+	InitProcessPid       int       `json:"init_process_pid"`
+	InitProcessStartTime string    `json:"init_process_start"`
+	Created              time.Time `json:"created"`
+	Config               struct {
+		NoPivotRoot       bool               `json:"no_pivot_root"`
+		ParentDeathSignal int                `json:"parent_death_signal"`
+		Rootfs            string             `json:"rootfs"`
+		Readonlyfs        bool               `json:"readonlyfs"`
+		RootPropagation   int                `json:"rootPropagation"`
+		Mounts            []*configs.Mount   `json:"mounts"`
+		Devices           []*configs.Device  `json:"devices"`
+		MountLabel        string             `json:"mount_label"`
+		Hostname          string             `json:"hostname"`
+		Namespaces        configs.Namespaces `json:"namespaces"`
+		Capabilities      linuxCapabilities  `json:"capabilities"`
+		Networks          []*configs.Network `json:"networks"`
+		Routes            []*configs.Route   `json:"routes"`
+		Cgroups           *struct {
+			Name                         string `json:"name,omitempty"`
+			Parent                       string `json:"parent,omitempty"`
+			Path                         string `json:"path"`
+			ScopePrefix                  string `json:"scope_prefix"`
+			Paths                        map[string]string
+			AllowAllDevices              *bool                     `json:"allow_all_devices,omitempty"`
+			AllowedDevices               []*configs.Device         `json:"allowed_devices,omitempty"`
+			DeniedDevices                []*configs.Device         `json:"denied_devices,omitempty"`
+			Devices                      []*configs.Device         `json:"devices"`
+			Memory                       int64                     `json:"memory"`
+			MemoryReservation            int64                     `json:"memory_reservation"`
+			MemorySwap                   int64                     `json:"memory_swap"`
+			KernelMemory                 int64                     `json:"kernel_memory"`
+			KernelMemoryTCP              int64                     `json:"kernel_memory_tcp"`
+			CpuShares                    uint64                    `json:"cpu_shares"`
+			CpuQuota                     int64                     `json:"cpu_quota"`
+			CpuPeriod                    uint64                    `json:"cpu_period"`
+			CpuRtRuntime                 int64                     `json:"cpu_rt_quota"`
+			CpuRtPeriod                  uint64                    `json:"cpu_rt_period"`
+			CpusetCpus                   string                    `json:"cpuset_cpus"`
+			CpusetMems                   string                    `json:"cpuset_mems"`
+			PidsLimit                    int64                     `json:"pids_limit"`
+			BlkioWeight                  uint16                    `json:"blkio_weight"`
+			BlkioLeafWeight              uint16                    `json:"blkio_leaf_weight"`
+			BlkioWeightDevice            []*configs.WeightDevice   `json:"blkio_weight_device"`
+			BlkioThrottleReadBpsDevice   []*configs.ThrottleDevice `json:"blkio_throttle_read_bps_device"`
+			BlkioThrottleWriteBpsDevice  []*configs.ThrottleDevice `json:"blkio_throttle_write_bps_device"`
+			BlkioThrottleReadIOPSDevice  []*configs.ThrottleDevice `json:"blkio_throttle_read_iops_device"`
+			BlkioThrottleWriteIOPSDevice []*configs.ThrottleDevice `json:"blkio_throttle_write_iops_device"`
+			Freezer                      configs.FreezerState      `json:"freezer"`
+			HugetlbLimit                 []*configs.HugepageLimit  `json:"hugetlb_limit"`
+			OomKillDisable               bool                      `json:"oom_kill_disable"`
+			MemorySwappiness             memorySwappiness          `json:"memory_swappiness"`
+			NetPrioIfpriomap             []*configs.IfPrioMap      `json:"net_prio_ifpriomap"`
+			NetClsClassid                uint32                    `json:"net_cls_classid_u"`
+		} `json:"cgroups"`
+		AppArmorProfile string            `json:"apparmor_profile,omitempty"`
+		ProcessLabel    string            `json:"process_label,omitempty"`
+		Rlimits         []configs.Rlimit  `json:"rlimits,omitempty"`
+		OomScoreAdj     int               `json:"oom_score_adj"`
+		UidMappings     []configs.IDMap   `json:"uid_mappings"`
+		GidMappings     []configs.IDMap   `json:"gid_mappings"`
+		MaskPaths       []string          `json:"mask_paths"`
+		ReadonlyPaths   []string          `json:"readonly_paths"`
+		Sysctl          map[string]string `json:"sysctl"`
+		Seccomp         *configs.Seccomp  `json:"seccomp"`
+		NoNewPrivileges bool              `json:"no_new_privileges,omitempty"`
+		Hooks           *configs.Hooks
+		Version         string   `json:"version"`
+		Labels          []string `json:"labels"`
+		NoNewKeyring    bool     `json:"no_new_keyring"`
+		Rootless        bool     `json:"rootless"`
+	} `json:"config"`
+	Rootless            bool                             `json:"rootless"`
+	CgroupPaths         map[string]string                `json:"cgroup_paths"`
+	NamespacePaths      map[configs.NamespaceType]string `json:"namespace_paths"`
+	ExternalDescriptors []string                         `json:"external_descriptors,omitempty"`
+}

components/engine/vendor/github.com/crosbymichael/upgrade/v17_06_1/template.go

@@ -0,0 +1,20 @@
+//+build ignore
+
+package v17_06_1
+
+import (
+	"github.com/containerd/containerd/runtime"
+	"github.com/opencontainers/runc/libcontainer"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+//go:generate -command rewrite go run ../gen/rewrite-structs.go --
+
+//go:generate rewrite spec_gen.go .Process.Capabilities->linuxCapabilities .Linux.Resources.Memory.Swappiness->memorySwappiness .Linux.Seccomp.Syscalls->linuxSyscalls
+type Spec specs.Spec
+
+//go:generate rewrite process_state_gen.go .Capabilities->linuxCapabilities
+type ProcessState runtime.ProcessState
+
+//go:generate rewrite state_gen.go .Config.Capabilities->linuxCapabilities .Config.Cgroups.MemorySwappiness->memorySwappiness
+type State libcontainer.State

components/engine/vendor/github.com/crosbymichael/upgrade/v17_06_1/unmarshal.go

@@ -0,0 +1,119 @@
+package v17_06_1
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+type linuxSyscalls []linuxSyscall
+
+type linuxSyscall struct {
+	specs.LinuxSyscall
+}
+
+func (ls *linuxSyscall) UnmarshalJSON(b []byte) error {
+	var t struct {
+		specs.LinuxSyscall
+		Name *string `json:"name,omitempty"`
+	}
+	if err := json.Unmarshal(b, &t); err != nil {
+		return err
+	}
+	ls.LinuxSyscall = t.LinuxSyscall
+	if t.Name != nil {
+		if ls.LinuxSyscall.Names != nil {
+			return fmt.Errorf("found incompatible 'name' and 'names' fields")
+		}
+		ls.LinuxSyscall.Names = []string{*t.Name}
+		t.Name = nil
+	}
+	return nil
+}
+
+// TODO: figure out how to omitempty when pointer is nil
+type memorySwappiness struct {
+	V *uint64 `json:",omitempty"`
+}
+
+func (m memorySwappiness) String() string {
+	if m.V == nil {
+		return "<nil>"
+	}
+	return fmt.Sprintf("%d", *m.V)
+}
+
+var null = []byte("null")
+
+func (m *memorySwappiness) MarshalJSON() ([]byte, error) {
+	if m.V == nil {
+		return null, nil
+	}
+	return []byte(fmt.Sprintf("%d", *m.V)), nil
+}
+
+func (m *memorySwappiness) UnmarshalJSON(b []byte) error {
+	if bytes.Compare(b, null) == 0 {
+		return nil
+	}
+
+	var n uint64
+	var i int64
+	err := json.Unmarshal(b, &i)
+	switch err.(type) {
+	case nil:
+		n = uint64(i)
+	case *json.UnmarshalTypeError:
+		// The only valid reason for accepting a uint64 that does not fit into an int64
+		// is for erroneous -1 values being converted to uint64.
+		// Nevertheless, try unmarshaling it and error out if it's not a number at all.
+		if err := json.Unmarshal(b, &n); err != nil {
+			return err
+		}
+	default:
+		return err
+	}
+	if n >= 0 && n <= 100 {
+		m.V = &n
+	} else {
+		m.V = nil
+	}
+	return nil
+}
+
+type linuxCapabilities struct {
+	V *specs.LinuxCapabilities
+}
+
+func (l *linuxCapabilities) MarshalJSON() ([]byte, error) {
+	return json.Marshal(l.V)
+}
+
+func (l *linuxCapabilities) UnmarshalJSON(b []byte) error {
+	if bytes.Compare(b, null) == 0 {
+		return nil
+	}
+	var s specs.LinuxCapabilities
+	err := json.Unmarshal(b, &s)
+	switch err.(type) {
+	case nil:
+		l.V = &s
+	case *json.UnmarshalTypeError:
+		var caps []string
+		err = json.Unmarshal(b, &caps)
+		if err != nil {
+			return err
+		}
+		// TODO: copy caps or not copy caps?
+		l.V = &specs.LinuxCapabilities{
+			Bounding:    caps,
+			Effective:   caps,
+			Inheritable: caps,
+			Permitted:   caps,
+			Ambient:     nil,
+		}
+	}
+	return err
+}

components/engine/vendor/github.com/crosbymichael/upgrade/v17_06_1/upgrade.go

@@ -0,0 +1,63 @@
+package v17_06_1
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/docker/docker/pkg/ioutils"
+)
+
+type file struct {
+	name string
+	x    interface{}
+	buf  bytes.Buffer
+	w    io.WriteCloser
+}
+
+func Upgrade(runcState, containerdConfig, containerdProcess string) error {
+	files := []*file{
+		&file{name: runcState, x: new(State)},
+		&file{name: containerdConfig, x: new(Spec)},
+		&file{name: containerdProcess, x: new(ProcessState)},
+	}
+	for _, f := range files {
+		fd, err := os.Open(f.name)
+		if err != nil {
+			return err
+		}
+		defer fd.Close()
+		// error out if any of the files have issues being decoded
+		// before overwriting them, to prevent being in a mixed state.
+		if err := json.NewDecoder(fd).Decode(f.x); err != nil {
+			return err
+		}
+		// error out if any of the files have issues being encoded
+		// before overwriting them, to prevent being in a mixed state.
+		if err := json.NewEncoder(&f.buf).Encode(f.x); err != nil {
+			return err
+		}
+		fi, err := fd.Stat()
+		if err != nil {
+			return err
+		}
+		f.w, err = ioutils.NewAtomicFileWriter(f.name, fi.Mode())
+		if err != nil {
+			return err
+		}
+		defer f.w.Close()
+	}
+	var errs []string
+	for _, f := range files {
+		if _, err := f.w.Write(f.buf.Bytes()); err != nil {
+			errs = append(errs, fmt.Sprintf("error writing to %s: %v", f.name, err))
+		}
+	}
+	if errs != nil {
+		return fmt.Errorf(strings.Join(errs, ", "))
+	}
+	return nil
+}

components/engine/vendor/github.com/crosbymichael/upgrade/v17_06_1/vendor.conf

@@ -0,0 +1,23 @@
+# runtime-spec
+github.com/opencontainers/runtime-spec a45ba0989fc26c695fe166a49c45bb8b7618ab36 https://github.com/docker/runtime-spec
+
+# runc
+github.com/opencontainers/runc 810190ceaa507aa2727d7ae6f4790c76ec150bd2 https://github.com/docker/runc
+github.com/mrunalp/fileutils ed869b029674c0e9ce4c0dfa781405c2d9946d08
+github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0
+github.com/syndtr/gocapability e7cb7fa329f456b3855136a2642b197bad7366ba
+github.com/golang/protobuf f7137ae6b19afbfd61a94b746fda3b3fe
+github.com/docker/go-units v0.2.0
+github.com/vishvananda/netlink 1e2e08e8a2dcdacaae3f14ac44c5c
+github.com/docker/docker 0f5c9d301b9b1cca66b3ea0f9dec3b5317d3686d
+github.com/opencontainers/selinux v1.0.0-rc1
+github.com/coreos/go-systemd v14
+github.com/coreos/pkg v3
+github.com/godbus/dbus v3
+
+# containerd
+github.com/containerd/containerd 6e23458c129b551d5c9871e5174f6b1b7f6d1170 https://github.com/docker/containerd
+golang.org/x/net 991d3e32f76f19ee6d9caadb3a22eae8d23315f7 https://github.com/golang/net.git
+golang.org/x/sys d4feaf1a7e61e1d9e79e6c4e76c6349e9 https://github.com/golang/sys.git
+github.com/Sirupsen/logrus v0.11.2
+

components/engine/vendor/github.com/crosbymichael/upgrade/vendor.conf

@@ -0,0 +1 @@
+v17_06_1/vendor.conf