diff --git a/components/engine/api/client/build.go b/components/engine/api/client/build.go index dc54c22ffa..b4ee93b2ce 100644 --- a/components/engine/api/client/build.go +++ b/components/engine/api/client/build.go @@ -56,6 +56,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { flMemorySwap := cmd.String([]string{"-memory-swap"}, "", "Total memory (memory + swap), '-1' to disable swap") flCPUShares := cmd.Int64([]string{"c", "-cpu-shares"}, 0, "CPU shares (relative weight)") flCPUSetCpus := cmd.String([]string{"-cpuset-cpus"}, "", "CPUs in which to allow execution (0-3, 0,1)") + flCPUSetMems := cmd.String([]string{"-cpuset-mems"}, "", "MEMs in which to allow execution (0-3, 0,1)") cmd.Require(flag.Exact, 1) cmd.ParseFlags(args, true) @@ -278,6 +279,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { } v.Set("cpusetcpus", *flCPUSetCpus) + v.Set("cpusetmems", *flCPUSetMems) v.Set("cpushares", strconv.FormatInt(*flCPUShares, 10)) v.Set("memory", strconv.FormatInt(memory, 10)) v.Set("memswap", strconv.FormatInt(memorySwap, 10)) diff --git a/components/engine/api/server/server.go b/components/engine/api/server/server.go index 7ebeb4b6aa..3e8d9bae0f 100644 --- a/components/engine/api/server/server.go +++ b/components/engine/api/server/server.go @@ -1213,6 +1213,7 @@ func postBuild(eng *engine.Engine, version version.Version, w http.ResponseWrite job.Setenv("memswap", r.FormValue("memswap")) job.Setenv("memory", r.FormValue("memory")) job.Setenv("cpusetcpus", r.FormValue("cpusetcpus")) + job.Setenv("cpusetmems", r.FormValue("cpusetmems")) job.Setenv("cpushares", r.FormValue("cpushares")) // Job cancellation. Note: not all job types support this. diff --git a/components/engine/builder/evaluator.go b/components/engine/builder/evaluator.go index 6237f26630..c159e51bf9 100644 --- a/components/engine/builder/evaluator.go +++ b/components/engine/builder/evaluator.go @@ -124,6 +124,7 @@ type Builder struct { // Set resource restrictions for build containers cpuSetCpus string + cpuSetMems string cpuShares int64 memory int64 memorySwap int64 diff --git a/components/engine/builder/internals.go b/components/engine/builder/internals.go index 728ccde8ae..caa94ef2be 100644 --- a/components/engine/builder/internals.go +++ b/components/engine/builder/internals.go @@ -541,6 +541,7 @@ func (b *Builder) create() (*daemon.Container, error) { hostConfig := &runconfig.HostConfig{ CpuShares: b.cpuShares, CpusetCpus: b.cpuSetCpus, + CpusetMems: b.cpuSetMems, Memory: b.memory, MemorySwap: b.memorySwap, } diff --git a/components/engine/builder/job.go b/components/engine/builder/job.go index b0ce8ddc09..f9ff610396 100644 --- a/components/engine/builder/job.go +++ b/components/engine/builder/job.go @@ -63,6 +63,7 @@ func (b *BuilderJob) CmdBuild(job *engine.Job) error { memorySwap = job.GetenvInt64("memswap") cpuShares = job.GetenvInt64("cpushares") cpuSetCpus = job.Getenv("cpusetcpus") + cpuSetMems = job.Getenv("cpusetmems") authConfig = ®istry.AuthConfig{} configFile = ®istry.ConfigFile{} tag string @@ -153,6 +154,7 @@ func (b *BuilderJob) CmdBuild(job *engine.Job) error { dockerfileName: dockerfileName, cpuShares: cpuShares, cpuSetCpus: cpuSetCpus, + cpuSetMems: cpuSetMems, memory: memory, memorySwap: memorySwap, cancelled: job.WaitCancelled(), diff --git a/components/engine/daemon/container.go b/components/engine/daemon/container.go index f89db5cfcf..97d1afbb01 100644 --- a/components/engine/daemon/container.go +++ b/components/engine/daemon/container.go @@ -355,6 +355,7 @@ func populateCommand(c *Container, env []string) error { MemorySwap: c.hostConfig.MemorySwap, CpuShares: c.hostConfig.CpuShares, CpusetCpus: c.hostConfig.CpusetCpus, + CpusetMems: c.hostConfig.CpusetMems, Rlimits: rlimits, } diff --git a/components/engine/daemon/execdriver/driver.go b/components/engine/daemon/execdriver/driver.go index 637f7d779e..fc3b5caba4 100644 --- a/components/engine/daemon/execdriver/driver.go +++ b/components/engine/daemon/execdriver/driver.go @@ -110,6 +110,7 @@ type Resources struct { MemorySwap int64 `json:"memory_swap"` CpuShares int64 `json:"cpu_shares"` CpusetCpus string `json:"cpuset_cpus"` + CpusetMems string `json:"cpuset_mems"` Rlimits []*ulimit.Rlimit `json:"rlimits"` } @@ -204,6 +205,7 @@ func SetupCgroups(container *configs.Config, c *Command) error { container.Cgroups.MemoryReservation = c.Resources.Memory container.Cgroups.MemorySwap = c.Resources.MemorySwap container.Cgroups.CpusetCpus = c.Resources.CpusetCpus + container.Cgroups.CpusetMems = c.Resources.CpusetMems } return nil diff --git a/components/engine/daemon/execdriver/lxc/lxc_template.go b/components/engine/daemon/execdriver/lxc/lxc_template.go index 6c182ab394..ece924d38f 100644 --- a/components/engine/daemon/execdriver/lxc/lxc_template.go +++ b/components/engine/daemon/execdriver/lxc/lxc_template.go @@ -110,6 +110,9 @@ lxc.cgroup.cpu.shares = {{.Resources.CpuShares}} {{if .Resources.CpusetCpus}} lxc.cgroup.cpuset.cpus = {{.Resources.CpusetCpus}} {{end}} +{{if .Resources.CpusetMems}} +lxc.cgroup.cpuset.mems = {{.Resources.CpusetMems}} +{{end}} {{end}} {{if .LxcConfig}} diff --git a/components/engine/docs/man/docker-create.1.md b/components/engine/docs/man/docker-create.1.md index 1a0da1b8f4..6ce7fe6bdc 100644 --- a/components/engine/docs/man/docker-create.1.md +++ b/components/engine/docs/man/docker-create.1.md @@ -13,6 +13,7 @@ docker-create - Create a new container [**--cap-drop**[=*[]*]] [**--cidfile**[=*CIDFILE*]] [**--cpuset-cpus**[=*CPUSET-CPUS*]] +[**--cpuset-mems**[=*CPUSET-MEMS*]] [**--device**[=*[]*]] [**--dns-search**[=*[]*]] [**--dns**[=*[]*]] @@ -74,6 +75,13 @@ IMAGE [COMMAND] [ARG...] **--cpuset-cpus**="" CPUs in which to allow execution (0-3, 0,1) +**--cpuset-mems**="" + Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems. + + If you have four memory nodes on your system (0-3), use `--cpuset-mems=0,1` +then processes in your Docker container will only use memory from the first +two memory nodes. + **--device**=[] Add a host device to the container (e.g. --device=/dev/sdc:/dev/xvdc:rwm) diff --git a/components/engine/docs/man/docker-run.1.md b/components/engine/docs/man/docker-run.1.md index 8a856ca608..42eeeb349d 100644 --- a/components/engine/docs/man/docker-run.1.md +++ b/components/engine/docs/man/docker-run.1.md @@ -13,6 +13,7 @@ docker-run - Run a command in a new container [**--cap-drop**[=*[]*]] [**--cidfile**[=*CIDFILE*]] [**--cpuset-cpus**[=*CPUSET-CPUS*]] +[**--cpuset-mems**[=*CPUSET-MEMS*]] [**-d**|**--detach**[=*false*]] [**--device**[=*[]*]] [**--dns-search**[=*[]*]] @@ -134,6 +135,13 @@ division of CPU shares: **--cpuset-cpus**="" CPUs in which to allow execution (0-3, 0,1) +**--cpuset-mems**="" + Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems. + + If you have four memory nodes on your system (0-3), use `--cpuset-mems=0,1` +then processes in your Docker container will only use memory from the first +two memory nodes. + **-d**, **--detach**=*true*|*false* Detached mode: run the container in the background and print the new container ID. The default is *false*. diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.19.md b/components/engine/docs/sources/reference/api/docker_remote_api_v1.19.md index 524f77037f..18c52ef421 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.19.md +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.19.md @@ -147,6 +147,7 @@ Create a container "MemorySwap": 0, "CpuShares": 512, "CpusetCpus": "0,1", + "CpusetMems": "0,1", "PortBindings": { "22/tcp": [{ "HostPort": "11022" }] }, "PublishAllPorts": false, "Privileged": false, @@ -191,6 +192,7 @@ Json Parameters: (ie. the relative weight vs other containers). - **Cpuset** - The same as CpusetCpus, but deprecated, please don't use. - **CpusetCpus** - String value containing the cgroups CpusetCpus to use. +- **CpusetMems** - Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems. - **AttachStdin** - Boolean value, attaches to stdin. - **AttachStdout** - Boolean value, attaches to stdout. - **AttachStderr** - Boolean value, attaches to stderr. @@ -340,6 +342,7 @@ Return low-level information on the container `id` "CapDrop": null, "ContainerIDFile": "", "CpusetCpus": "", + "CpusetMems": "", "CpuShares": 0, "Devices": [], "Dns": null, diff --git a/components/engine/docs/sources/reference/commandline/cli.md b/components/engine/docs/sources/reference/commandline/cli.md index 7375261230..607f670762 100644 --- a/components/engine/docs/sources/reference/commandline/cli.md +++ b/components/engine/docs/sources/reference/commandline/cli.md @@ -597,6 +597,7 @@ is returned by the `docker attach` command to its caller too: --memory-swap="" Total memory (memory + swap), `-1` to disable swap -c, --cpu-shares CPU Shares (relative weight) --cpuset-cpus="" CPUs in which to allow execution, e.g. `0-3`, `0,1` + --cpuset-mems="" MEMs in which to allow execution, e.g. `0-3`, `0,1` Builds Docker images from a Dockerfile and a "context". A build's context is the files located in the specified `PATH` or `URL`. The build process can @@ -892,6 +893,7 @@ Creates a new container. --cgroup-parent="" Optional parent cgroup for the container --cidfile="" Write the container ID to the file --cpuset-cpus="" CPUs in which to allow execution (0-3, 0,1) + --cpuset-mems="" Memory nodes (MEMs) in which to allow execution (0-3, 0,1) --device=[] Add a host device to the container --dns=[] Set custom DNS servers --dns-search=[] Set custom DNS search domains @@ -1844,6 +1846,7 @@ To remove an image using its digest: --cap-drop=[] Drop Linux capabilities --cidfile="" Write the container ID to the file --cpuset-cpus="" CPUs in which to allow execution (0-3, 0,1) + --cpuset-mems="" Memory nodes (MEMs) in which to allow execution (0-3, 0,1) -d, --detach=false Run container in background and print container ID --device=[] Add a host device to the container --dns=[] Set custom DNS servers diff --git a/components/engine/docs/sources/reference/run.md b/components/engine/docs/sources/reference/run.md index d41b686c4b..b5784cba78 100644 --- a/components/engine/docs/sources/reference/run.md +++ b/components/engine/docs/sources/reference/run.md @@ -474,6 +474,7 @@ container: -memory-swap="": Total memory limit (memory + swap, format: , where unit = b, k, m or g) -c, --cpu-shares=0: CPU shares (relative weight) --cpuset-cpus="": CPUs in which to allow execution (0-3, 0,1) + --cpuset-mems="": Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems. ### Memory constraints @@ -599,6 +600,21 @@ This means processes in container can be executed on cpu 1 and cpu 3. This means processes in container can be executed on cpu 0, cpu 1 and cpu 2. +We can set mems in which to allow execution for containers. Only effective +on NUMA systems. + +Examples: + + $ docker run -ti --cpuset-mems="1,3" ubuntu:14.04 /bin/bash + +This example restricts the processes in the container to only use memory from +memory nodes 1 and 3. + + $ docker run -ti --cpuset-mems="0-2" ubuntu:14.04 /bin/bash + +This example restricts the processes in the container to only use memory from +memory nodes 0, 1 and 2. + ## Runtime privilege, Linux capabilities, and LXC configuration --cap-add: Add Linux capabilities diff --git a/components/engine/integration-cli/docker_cli_build_test.go b/components/engine/integration-cli/docker_cli_build_test.go index 40e038fc4b..1ef5088e24 100644 --- a/components/engine/integration-cli/docker_cli_build_test.go +++ b/components/engine/integration-cli/docker_cli_build_test.go @@ -5555,7 +5555,7 @@ func TestBuildResourceConstraintsAreUsed(t *testing.T) { t.Fatal(err) } - cmd := exec.Command(dockerBinary, "build", "--no-cache", "--rm=false", "--memory=64m", "--memory-swap=-1", "--cpuset-cpus=0", "--cpu-shares=100", "-t", name, ".") + cmd := exec.Command(dockerBinary, "build", "--no-cache", "--rm=false", "--memory=64m", "--memory-swap=-1", "--cpuset-cpus=0", "--cpuset-mems=0", "--cpu-shares=100", "-t", name, ".") cmd.Dir = ctx.Dir out, _, err := runCommandWithOutput(cmd) @@ -5573,6 +5573,7 @@ func TestBuildResourceConstraintsAreUsed(t *testing.T) { Memory float64 // Use float64 here since the json decoder sees it that way MemorySwap int CpusetCpus string + CpusetMems string CpuShares int } @@ -5586,9 +5587,9 @@ func TestBuildResourceConstraintsAreUsed(t *testing.T) { t.Fatal(err, cfg) } mem := int64(c1.Memory) - if mem != 67108864 || c1.MemorySwap != -1 || c1.CpusetCpus != "0" || c1.CpuShares != 100 { - t.Fatalf("resource constraints not set properly:\nMemory: %d, MemSwap: %d, CpusetCpus: %s, CpuShares: %d", - mem, c1.MemorySwap, c1.CpusetCpus, c1.CpuShares) + if mem != 67108864 || c1.MemorySwap != -1 || c1.CpusetCpus != "0" || c1.CpusetMems != "0" || c1.CpuShares != 100 { + t.Fatalf("resource constraints not set properly:\nMemory: %d, MemSwap: %d, CpusetCpus: %s, CpusetMems: %s, CpuShares: %d", + mem, c1.MemorySwap, c1.CpusetCpus, c1.CpusetMems, c1.CpuShares) } // Make sure constraints aren't saved to image @@ -5605,9 +5606,9 @@ func TestBuildResourceConstraintsAreUsed(t *testing.T) { t.Fatal(err, cfg) } mem = int64(c2.Memory) - if mem == 67108864 || c2.MemorySwap == -1 || c2.CpusetCpus == "0" || c2.CpuShares == 100 { - t.Fatalf("resource constraints leaked from build:\nMemory: %d, MemSwap: %d, CpusetCpus: %s, CpuShares: %d", - mem, c2.MemorySwap, c2.CpusetCpus, c2.CpuShares) + if mem == 67108864 || c2.MemorySwap == -1 || c2.CpusetCpus == "0" || c2.CpusetMems == "0" || c2.CpuShares == 100 { + t.Fatalf("resource constraints leaked from build:\nMemory: %d, MemSwap: %d, CpusetCpus: %s, CpusetMems: %s, CpuShares: %d", + mem, c2.MemorySwap, c2.CpusetCpus, c2.CpusetMems, c2.CpuShares) } logDone("build - resource constraints applied") diff --git a/components/engine/integration-cli/docker_cli_run_test.go b/components/engine/integration-cli/docker_cli_run_test.go index 7c931f8fac..e990f086ae 100644 --- a/components/engine/integration-cli/docker_cli_run_test.go +++ b/components/engine/integration-cli/docker_cli_run_test.go @@ -1367,7 +1367,7 @@ func TestRunWithCpuset(t *testing.T) { cmd := exec.Command(dockerBinary, "run", "--cpuset", "0", "busybox", "true") if code, err := runCommand(cmd); err != nil || code != 0 { - t.Fatalf("container should run successfuly with cpuset of 0: %s", err) + t.Fatalf("container should run successfully with cpuset of 0: %s", err) } logDone("run - cpuset 0") @@ -1378,12 +1378,23 @@ func TestRunWithCpusetCpus(t *testing.T) { cmd := exec.Command(dockerBinary, "run", "--cpuset-cpus", "0", "busybox", "true") if code, err := runCommand(cmd); err != nil || code != 0 { - t.Fatalf("container should run successfuly with cpuset-cpus of 0: %s", err) + t.Fatalf("container should run successfully with cpuset-cpus of 0: %s", err) } logDone("run - cpuset-cpus 0") } +func TestRunWithCpusetMems(t *testing.T) { + defer deleteAllContainers() + + cmd := exec.Command(dockerBinary, "run", "--cpuset-mems", "0", "busybox", "true") + if code, err := runCommand(cmd); err != nil || code != 0 { + t.Fatalf("container should run successfully with cpuset-mems of 0: %s", err) + } + + logDone("run - cpuset-mems 0") +} + func TestRunDeviceNumbers(t *testing.T) { defer deleteAllContainers() diff --git a/components/engine/runconfig/hostconfig.go b/components/engine/runconfig/hostconfig.go index 9d4eb26414..273634e63f 100644 --- a/components/engine/runconfig/hostconfig.go +++ b/components/engine/runconfig/hostconfig.go @@ -116,6 +116,7 @@ type HostConfig struct { MemorySwap int64 // Total memory usage (memory + swap); set `-1` to disable swap CpuShares int64 // CPU shares (relative weight vs. other containers) CpusetCpus string // CpusetCpus 0-2, 0,1 + CpusetMems string // CpusetMems 0-2, 0,1 Privileged bool PortBindings nat.PortMap Links []string diff --git a/components/engine/runconfig/parse.go b/components/engine/runconfig/parse.go index d302330c82..d9b21ca273 100644 --- a/components/engine/runconfig/parse.go +++ b/components/engine/runconfig/parse.go @@ -64,6 +64,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*Config, *HostConfig, *flag.FlagSe flWorkingDir = cmd.String([]string{"w", "-workdir"}, "", "Working directory inside the container") flCpuShares = cmd.Int64([]string{"c", "-cpu-shares"}, 0, "CPU shares (relative weight)") flCpusetCpus = cmd.String([]string{"#-cpuset", "-cpuset-cpus"}, "", "CPUs in which to allow execution (0-3, 0,1)") + flCpusetMems = cmd.String([]string{"-cpuset-mems"}, "", "MEMs in which to allow execution (0-3, 0,1)") flNetMode = cmd.String([]string{"-net"}, "bridge", "Set the Network mode for the container") flMacAddress = cmd.String([]string{"-mac-address"}, "", "Container MAC address (e.g. 92:d0:c6:0a:29:33)") flIpcMode = cmd.String([]string{"-ipc"}, "", "IPC namespace to use") @@ -313,6 +314,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*Config, *HostConfig, *flag.FlagSe MemorySwap: MemorySwap, CpuShares: *flCpuShares, CpusetCpus: *flCpusetCpus, + CpusetMems: *flCpusetMems, Privileged: *flPrivileged, PortBindings: portBindings, Links: flLinks.GetAll(),