Add --cpus flag to control cpu resources

This fix tries to address the proposal raised in 27921 and add `--cpus` flag for `docker run/create`. Basically, `--cpus` will allow user to specify a number (possibly partial) about how many CPUs the container will use. For example, on a 2-CPU system `--cpus 1.5` means the container will take 75% (1.5/2) of the CPU share. This fix adds a `NanoCPUs` field to `HostConfig` since swarmkit alreay have a concept of NanoCPUs for tasks. The `--cpus` flag will translate the number into reused `NanoCPUs` to be consistent. This fix adds integration tests to cover the changes. Related docs (`docker run` and Remote APIs) have been updated. This fix fixes 27921. Signed-off-by: Yong Tang <yong.tang.github@outlook.com> Upstream-commit: 846baf1fd3efcbfbf9d3eb99e436ca9a59d3e185 Component: engine
2016-11-01 10:12:29 -07:00
parent 71fb2940a4
commit 133eec9cfa
16 changed files with 122 additions and 32 deletions
@@ -234,6 +234,7 @@ type Resources struct {
 	// Applicable to all platforms
 	CPUShares int64 `json:"CpuShares"` // CPU shares (relative weight vs. other containers)
 	Memory    int64 // Memory limit (in bytes)
+	NanoCPUs  int64 `json:"NanoCpus"` // CPU quota in units of 10<sup>-9</sup> CPUs.

 	// Applicable to UNIX platforms
 	CgroupParent         string // Parent cgroup.
@@ -2,7 +2,6 @@ package service

 import (
 	"fmt"
-	"math/big"
 	"strconv"
 	"strings"
 	"time"
@@ -40,33 +39,6 @@ func (m *memBytes) Value() int64 {
 	return int64(*m)
 }

-type nanoCPUs int64
-
-func (c *nanoCPUs) String() string {
-	return big.NewRat(c.Value(), 1e9).FloatString(3)
-}
-
-func (c *nanoCPUs) Set(value string) error {
-	cpu, ok := new(big.Rat).SetString(value)
-	if !ok {
-		return fmt.Errorf("Failed to parse %v as a rational number", value)
-	}
-	nano := cpu.Mul(cpu, big.NewRat(1e9, 1))
-	if !nano.IsInt() {
-		return fmt.Errorf("value is too precise")
-	}
-	*c = nanoCPUs(nano.Num().Int64())
-	return nil
-}
-
-func (c *nanoCPUs) Type() string {
-	return "NanoCPUs"
-}
-
-func (c *nanoCPUs) Value() int64 {
-	return int64(*c)
-}
-
 // PositiveDurationOpt is an option type for time.Duration that uses a pointer.
 // It bahave similarly to DurationOpt but only allows positive duration values.
 type PositiveDurationOpt struct {
@@ -156,9 +128,9 @@ type updateOptions struct {
 }

 type resourceOptions struct {
-	limitCPU      nanoCPUs
+	limitCPU      opts.NanoCPUs
 	limitMemBytes memBytes
-	resCPU        nanoCPUs
+	resCPU        opts.NanoCPUs
 	resMemBytes   memBytes
 }

@@ -6,6 +6,7 @@ import (
 	"time"

 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/opts"
 	"github.com/docker/docker/pkg/testutil/assert"
 )

@@ -21,12 +22,12 @@ func TestMemBytesSetAndValue(t *testing.T) {
 }

 func TestNanoCPUsString(t *testing.T) {
-	var cpus nanoCPUs = 6100000000
+	var cpus opts.NanoCPUs = 6100000000
 	assert.Equal(t, cpus.String(), "6.100")
 }

 func TestNanoCPUsSetAndValue(t *testing.T) {
-	var cpus nanoCPUs
+	var cpus opts.NanoCPUs
 	assert.NilError(t, cpus.Set("0.35"))
 	assert.Equal(t, cpus.Value(), int64(350000000))
 }
@@ -15,6 +15,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"

 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/api/types"
@@ -110,6 +111,16 @@ func getCPUResources(config containertypes.Resources) *specs.CPU {
 		cpu.Mems = &cpuset
 	}

+	if config.NanoCPUs > 0 {
+		// Use the default setting of 100ms, as is specified in:
+		// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
+		//    	cpu.cfs_period_us=100ms
+		period := uint64(100 * time.Millisecond / time.Microsecond)
+		quota := uint64(config.NanoCPUs) * period / 1e9
+		cpu.Period = &period
+		cpu.Quota = &quota
+	}
+
 	if config.CPUPeriod != 0 {
 		period := uint64(config.CPUPeriod)
 		cpu.Period = &period
@@ -341,6 +352,19 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi
 	}

 	// cpu subsystem checks and adjustments
+	if resources.NanoCPUs > 0 && resources.CPUPeriod > 0 {
+		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Period cannot both be set")
+	}
+	if resources.NanoCPUs > 0 && resources.CPUQuota > 0 {
+		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Quota cannot both be set")
+	}
+	if resources.NanoCPUs > 0 && (!sysInfo.CPUCfsPeriod || !sysInfo.CPUCfsQuota) {
+		return warnings, fmt.Errorf("NanoCPUs can not be set, as your kernel does not support CPU cfs period/quota or the cgroup is not mounted")
+	}
+	if resources.NanoCPUs < 0 || resources.NanoCPUs > int64(sysinfo.NumCPU())*1e9 {
+		return warnings, fmt.Errorf("Range of Nano CPUs is from 1 to %d", int64(sysinfo.NumCPU())*1e9)
+	}
+
 	if resources.CPUShares > 0 && !sysInfo.CPUShares {
 		warnings = append(warnings, "Your kernel does not support CPU shares or the cgroup is not mounted. Shares discarded.")
 		logrus.Warn("Your kernel does not support CPU shares or the cgroup is not mounted. Shares discarded.")
@@ -103,6 +103,17 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi
 		return warnings, fmt.Errorf("Conflicting options: CPU Shares and CPU Percent cannot both be set")
 	}

+	if resources.NanoCPUs > 0 && resources.CPUPercent > 0 {
+		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Percent cannot both be set")
+	}
+
+	if resources.NanoCPUs > 0 && resources.CPUShares > 0 {
+		return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Shares cannot both be set")
+	}
+	if resources.NanoCPUs < 0 || resources.NanoCPUs > int64(sysinfo.NumCPU())*1e9 {
+		return warnings, fmt.Errorf("Range of Nano CPUs is from 1 to %d", int64(sysinfo.NumCPU())*1e9)
+	}
+
 	// TODO Windows: Add more validation of resource settings not supported on Windows

 	if resources.BlkioWeight > 0 {
@@ -6,6 +6,7 @@ import (
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/oci"
+	"github.com/docker/docker/pkg/sysinfo"
 	"github.com/opencontainers/runtime-spec/specs-go"
 )

@@ -82,6 +83,9 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
 	// @darrenstahlmsft implement these resources
 	cpuShares := uint16(c.HostConfig.CPUShares)
 	cpuPercent := uint8(c.HostConfig.CPUPercent)
+	if c.HostConfig.NanoCPUs > 0 {
+		cpuPercent = uint8(c.HostConfig.NanoCPUs * 100 / int64(sysinfo.NumCPU()) / 1e9)
+	}
 	memoryLimit := uint64(c.HostConfig.Memory)
 	s.Windows.Resources = &specs.WindowsResources{
 		CPU: &specs.WindowsCPUResources{
@@ -164,6 +164,7 @@ This section lists each version from latest to oldest.  Each listing includes a
 * The `hostConfig` option now accepts the fields `CpuRealtimePeriod` and `CpuRtRuntime` to allocate cpu runtime to rt tasks when `CONFIG_RT_GROUP_SCHED` is enabled in the kernel.
 * The `SecurityOptions` field within the `GET /info` response now includes `userns` if user namespaces are enabled in the daemon.
 * `GET /nodes` and `GET /node/(id or name)` now return `Addr` as part of a node's `Status`, which is the address that that node connects to the manager from.
+* The `HostConfig` field now includes `NanoCPUs` that represents CPU quota in units of 10<sup>-9</sup> CPUs.

 ### v1.24 API changes

@@ -302,6 +302,7 @@ Create a container
             "MemorySwap": 0,
             "MemoryReservation": 0,
             "KernelMemory": 0,
+             "NanoCPUs": 500000,
             "CpuPercent": 80,
             "CpuShares": 512,
             "CpuPeriod": 100000,
@@ -425,6 +426,7 @@ Create a container
          You must use this with `memory` and make the swap value larger than `memory`.
    -   **MemoryReservation** - Memory soft limit in bytes.
    -   **KernelMemory** - Kernel memory limit in bytes.
+    -   **NanoCPUs** - CPU quota in units of 10<sup>-9</sup> CPUs.
    -   **CpuPercent** - An integer value containing the usable percentage of the available CPUs. (Windows daemon only)
    -   **CpuShares** - An integer value containing the container's CPU Shares
          (ie. the relative weight vs other containers).
@@ -35,6 +35,7 @@ Options:
      --cpu-period int              Limit CPU CFS (Completely Fair Scheduler) period
      --cpu-quota int               Limit CPU CFS (Completely Fair Scheduler) quota
  -c, --cpu-shares int              CPU shares (relative weight)
+      --cpus NanoCPUs               Number of CPUs (default 0.000)
      --cpu-rt-period int           Limit the CPU real-time period in microseconds
      --cpu-rt-runtime int          Limit the CPU real-time runtime in microseconds
      --cpuset-cpus string          CPUs in which to allow execution (0-3, 0,1)
@@ -33,6 +33,7 @@ Options:
      --cpu-period int              Limit CPU CFS (Completely Fair Scheduler) period
      --cpu-quota int               Limit CPU CFS (Completely Fair Scheduler) quota
  -c, --cpu-shares int              CPU shares (relative weight)
+      --cpus NanoCPUs               Number of CPUs (default 0.000)
      --cpu-rt-period int           Limit the CPU real-time period in microseconds
      --cpu-rt-runtime int          Limit the CPU real-time runtime in microseconds
      --cpuset-cpus string          CPUs in which to allow execution (0-3, 0,1)
@@ -686,6 +686,7 @@ container:
 | `--memory-reservation=""`  | Memory soft limit (format: `<number>[<unit>]`). Number is a positive integer. Unit can be one of `b`, `k`, `m`, or `g`.                         |
 | `--kernel-memory=""`       | Kernel memory limit (format: `<number>[<unit>]`). Number is a positive integer. Unit can be one of `b`, `k`, `m`, or `g`. Minimum is 4M.        |
 | `-c`, `--cpu-shares=0`     | CPU shares (relative weight)                                                                                                                    |
+| `--cpus=0.000`             | Number of CPUs. Number is a fractional number. 0.000 means no limit.                                                                            |
 | `--cpu-period=0`           | Limit the CPU CFS (Completely Fair Scheduler) period                                                                                            |
 | `--cpuset-cpus=""`         | CPUs in which to allow execution (0-3, 0,1)                                                                                                     |
 | `--cpuset-mems=""`         | Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems.                                                     |
@@ -970,6 +971,13 @@ Examples:

 If there is 1 CPU, this means the container can get 50% CPU worth of run-time every 50ms.

+In addition to use `--cpu-period` and `--cpu-quota` for setting CPU period constraints,
+it is possible to specify `--cpus` with a float number to achieve the same purpose.
+For example, if there is 1 CPU, then `--cpus=0.5` will achieve the same result as
+setting `--cpu-period=50000` and `--cpu-quota=25000` (50% CPU).
+
+The default value for `--cpus` is `0.000`, which means there is no limit.
+
 For more information, see the [CFS documentation on bandwidth limiting](https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt).

 ### Cpuset constraint
@@ -1409,3 +1409,23 @@ func (s *DockerDaemonSuite) TestRunWithDaemonDefaultSeccompProfile(c *check.C) {
 	c.Assert(err, check.NotNil)
 	c.Assert(out, checker.Contains, "Operation not permitted")
 }
+
+func (s *DockerSuite) TestRunWithNanoCPUs(c *check.C) {
+	testRequires(c, cpuCfsQuota, cpuCfsPeriod)
+
+	file1 := "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"
+	file2 := "/sys/fs/cgroup/cpu/cpu.cfs_period_us"
+	out, _ := dockerCmd(c, "run", "--cpus", "0.5", "--name", "test", "busybox", "sh", "-c", fmt.Sprintf("cat %s && cat %s", file1, file2))
+	c.Assert(strings.TrimSpace(out), checker.Equals, "50000\n100000")
+
+	out = inspectField(c, "test", "HostConfig.NanoCpus")
+	c.Assert(out, checker.Equals, "5e+08", check.Commentf("setting the Nano CPUs failed"))
+	out = inspectField(c, "test", "HostConfig.CpuQuota")
+	c.Assert(out, checker.Equals, "0", check.Commentf("CPU CFS quota should be 0"))
+	out = inspectField(c, "test", "HostConfig.CpuPeriod")
+	c.Assert(out, checker.Equals, "0", check.Commentf("CPU CFS period should be 0"))
+
+	out, _, err := dockerCmdWithError("run", "--cpus", "0.5", "--cpu-quota", "50000", "--cpu-period", "100000", "busybox", "sh")
+	c.Assert(err, check.NotNil)
+	c.Assert(out, checker.Contains, "Conflicting options: Nano CPUs and CPU Period cannot both be set")
+}
@@ -19,6 +19,7 @@ docker-create - Create a new container
 [**--cpu-quota**[=*0*]]
 [**--cpu-rt-period**[=*0*]]
 [**--cpu-rt-runtime**[=*0*]]
+[**--cpus**[=*0.0*]]
 [**--cpuset-cpus**[=*CPUSET-CPUS*]]
 [**--cpuset-mems**[=*CPUSET-MEMS*]]
 [**--device**[=*[]*]]
@@ -154,6 +155,9 @@ two memory nodes.

   The sum of all runtimes across containers cannot exceed the amount allotted to the parent cgroup.

+**--cpus**=0.0
+   Number of CPUs. The default is *0.0*.
+
 **--device**=[]
   Add a host device to the container (e.g. --device=/dev/sdc:/dev/xvdc:rwm)

@@ -19,6 +19,7 @@ docker-run - Run a command in a new container
 [**--cpu-quota**[=*0*]]
 [**--cpu-rt-period**[=*0*]]
 [**--cpu-rt-runtime**[=*0*]]
+[**--cpus**[=*0.0*]]
 [**--cpuset-cpus**[=*CPUSET-CPUS*]]
 [**--cpuset-mems**[=*CPUSET-MEMS*]]
 [**-d**|**--detach**]
@@ -208,6 +209,9 @@ to the quota you specify.

   The sum of all runtimes across containers cannot exceed the amount allotted to the parent cgroup.

+**--cpus**=0.0
+   Number of CPUs. The default is *0.0* which means no limit.
+
 **-d**, **--detach**=*true*|*false*
   Detached mode: run the container in the background and print the new container ID. The default is *false*.

@@ -2,6 +2,7 @@ package opts

 import (
 	"fmt"
+	"math/big"
 	"net"
 	"regexp"
 	"strings"
@@ -319,3 +320,35 @@ func (o *FilterOpt) Type() string {
 func (o *FilterOpt) Value() filters.Args {
 	return o.filter
 }
+
+// NanoCPUs is a type for fixed point fractional number.
+type NanoCPUs int64
+
+// String returns the string format of the number
+func (c *NanoCPUs) String() string {
+	return big.NewRat(c.Value(), 1e9).FloatString(3)
+}
+
+// Set sets the value of the NanoCPU by passing a string
+func (c *NanoCPUs) Set(value string) error {
+	cpu, ok := new(big.Rat).SetString(value)
+	if !ok {
+		return fmt.Errorf("Failed to parse %v as a rational number", value)
+	}
+	nano := cpu.Mul(cpu, big.NewRat(1e9, 1))
+	if !nano.IsInt() {
+		return fmt.Errorf("value is too precise")
+	}
+	*c = NanoCPUs(nano.Num().Int64())
+	return nil
+}
+
+// Type returns the type
+func (c *NanoCPUs) Type() string {
+	return "NanoCPUs"
+}
+
+// Value returns the value in int64
+func (c *NanoCPUs) Value() int64 {
+	return int64(*c)
+}
@@ -79,6 +79,7 @@ type ContainerOptions struct {
 	cpuRealtimePeriod  int64
 	cpuRealtimeRuntime int64
 	cpuQuota           int64
+	cpus               opts.NanoCPUs
 	cpusetCpus         string
 	cpusetMems         string
 	blkioWeight        uint16
@@ -232,6 +233,7 @@ func AddFlags(flags *pflag.FlagSet) *ContainerOptions {
 	flags.Int64Var(&copts.cpuRealtimePeriod, "cpu-rt-period", 0, "Limit CPU real-time period in microseconds")
 	flags.Int64Var(&copts.cpuRealtimeRuntime, "cpu-rt-runtime", 0, "Limit CPU real-time runtime in microseconds")
 	flags.Int64VarP(&copts.cpuShares, "cpu-shares", "c", 0, "CPU shares (relative weight)")
+	flags.Var(&copts.cpus, "cpus", "Number of CPUs")
 	flags.Var(&copts.deviceReadBps, "device-read-bps", "Limit read rate (bytes per second) from a device")
 	flags.Var(&copts.deviceReadIOps, "device-read-iops", "Limit read rate (IO per second) from a device")
 	flags.Var(&copts.deviceWriteBps, "device-write-bps", "Limit write rate (bytes per second) to a device")
@@ -526,6 +528,7 @@ func Parse(flags *pflag.FlagSet, copts *ContainerOptions) (*container.Config, *c
 		MemorySwappiness:     &copts.swappiness,
 		KernelMemory:         kernelMemory,
 		OomKillDisable:       &copts.oomKillDisable,
+		NanoCPUs:             copts.cpus.Value(),
 		CPUPercent:           copts.cpuPercent,
 		CPUShares:            copts.cpuShares,
 		CPUPeriod:            copts.cpuPeriod,