Merge component 'engine' from git@github.com:moby/moby master

components/engine/api/swagger.yaml

@@ -722,7 +722,15 @@ definitions:
             description: "Gives the container full access to the host."
           PublishAllPorts:
             type: "boolean"
-            description: "Allocates a random host port for all of a container's exposed ports."
+            description: |
+              Allocates an ephemeral host port for all of a container's
+              exposed ports.
+
+              Ports are de-allocated when the container stops and allocated when the container starts.
+              The allocated port might be changed when restarting the container.
+
+              The port is selected from the ephemeral port range that depends on the kernel.
+              For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`.
           ReadonlyRootfs:
             type: "boolean"
             description: "Mount the container's root filesystem as read only."

components/engine/daemon/graphdriver/quota/projectquota.go

@@ -47,6 +47,8 @@ struct fsxattr {
 #ifndef Q_XGETPQUOTA
 #define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA)
 #endif
+
+const int Q_XGETQSTAT_PRJQUOTA = QCMD(Q_XGETQSTAT, PRJQUOTA);
 */
 import "C"
 import (
@@ -56,10 +58,15 @@ import (
 	"path/filepath"
 	"unsafe"
 
+	"errors"
+
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 )
 
+// ErrQuotaNotSupported indicates if were found the FS does not have projects quotas available
+var ErrQuotaNotSupported = errors.New("Filesystem does not support or has not enabled quotas")
+
 // Quota limit params - currently we only control blocks hard limit
 type Quota struct {
 	Size uint64
@@ -96,6 +103,24 @@ type Control struct {
 // project ids.
 //
 func NewControl(basePath string) (*Control, error) {
+	//
+	// create backing filesystem device node
+	//
+	backingFsBlockDev, err := makeBackingFsDev(basePath)
+	if err != nil {
+		return nil, err
+	}
+
+	// check if we can call quotactl with project quotas
+	// as a mechanism to determine (early) if we have support
+	hasQuotaSupport, err := hasQuotaSupport(backingFsBlockDev)
+	if err != nil {
+		return nil, err
+	}
+	if !hasQuotaSupport {
+		return nil, ErrQuotaNotSupported
+	}
+
 	//
 	// Get project id of parent dir as minimal id to be used by driver
 	//
@@ -105,14 +130,6 @@ func NewControl(basePath string) (*Control, error) {
 	}
 	minProjectID++
 
-	//
-	// create backing filesystem device node
-	//
-	backingFsBlockDev, err := makeBackingFsDev(basePath)
-	if err != nil {
-		return nil, err
-	}
-
 	//
 	// Test if filesystem supports project quotas by trying to set
 	// a quota on the first available project id
@@ -335,3 +352,23 @@ func makeBackingFsDev(home string) (string, error) {
 
 	return backingFsBlockDev, nil
 }
+
+func hasQuotaSupport(backingFsBlockDev string) (bool, error) {
+	var cs = C.CString(backingFsBlockDev)
+	defer free(cs)
+	var qstat C.fs_quota_stat_t
+
+	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, uintptr(C.Q_XGETQSTAT_PRJQUOTA), uintptr(unsafe.Pointer(cs)), 0, uintptr(unsafe.Pointer(&qstat)), 0, 0)
+	if errno == 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ENFD > 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ACCT > 0 {
+		return true, nil
+	}
+
+	switch errno {
+	// These are the known fatal errors, consider all other errors (ENOTTY, etc.. not supporting quota)
+	case unix.EFAULT, unix.ENOENT, unix.ENOTBLK, unix.EPERM:
+	default:
+		return false, nil
+	}
+
+	return false, errno
+}

components/engine/daemon/graphdriver/quota/projectquota_test.go

@@ -0,0 +1,161 @@
+// +build linux
+
+package quota
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// 10MB
+const testQuotaSize = 10 * 1024 * 1024
+const imageSize = 64 * 1024 * 1024
+
+func TestBlockDev(t *testing.T) {
+	mkfs, err := exec.LookPath("mkfs.xfs")
+	if err != nil {
+		t.Fatal("mkfs.xfs not installed")
+	}
+
+	// create a sparse image
+	imageFile, err := ioutil.TempFile("", "xfs-image")
+	if err != nil {
+		t.Fatal(err)
+	}
+	imageFileName := imageFile.Name()
+	defer os.Remove(imageFileName)
+	if _, err = imageFile.Seek(imageSize-1, 0); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = imageFile.Write([]byte{0}); err != nil {
+		t.Fatal(err)
+	}
+	if err = imageFile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// The reason for disabling these options is sometimes people run with a newer userspace
+	// than kernelspace
+	out, err := exec.Command(mkfs, "-m", "crc=0,finobt=0", imageFileName).CombinedOutput()
+	if len(out) > 0 {
+		t.Log(string(out))
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	runTest(t, "testBlockDevQuotaDisabled", wrapMountTest(imageFileName, false, testBlockDevQuotaDisabled))
+	runTest(t, "testBlockDevQuotaEnabled", wrapMountTest(imageFileName, true, testBlockDevQuotaEnabled))
+	runTest(t, "testSmallerThanQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testSmallerThanQuota)))
+	runTest(t, "testBiggerThanQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testBiggerThanQuota)))
+	runTest(t, "testRetrieveQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testRetrieveQuota)))
+}
+
+func runTest(t *testing.T, testName string, testFunc func(*testing.T)) {
+	if success := t.Run(testName, testFunc); !success {
+		out, _ := exec.Command("dmesg").CombinedOutput()
+		t.Log(string(out))
+	}
+}
+
+func wrapMountTest(imageFileName string, enableQuota bool, testFunc func(t *testing.T, mountPoint, backingFsDev string)) func(*testing.T) {
+	return func(t *testing.T) {
+		mountOptions := "loop"
+
+		if enableQuota {
+			mountOptions = mountOptions + ",prjquota"
+		}
+
+		// create a mountPoint
+		mountPoint, err := ioutil.TempDir("", "xfs-mountPoint")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.RemoveAll(mountPoint)
+
+		out, err := exec.Command("mount", "-o", mountOptions, imageFileName, mountPoint).CombinedOutput()
+		if len(out) > 0 {
+			t.Log(string(out))
+		}
+		if err != nil {
+			t.Fatal("mount failed")
+		}
+
+		defer func() {
+			if err := unix.Unmount(mountPoint, 0); err != nil {
+				t.Fatal(err)
+			}
+		}()
+
+		backingFsDev, err := makeBackingFsDev(mountPoint)
+		require.NoError(t, err)
+
+		testFunc(t, mountPoint, backingFsDev)
+	}
+}
+
+func testBlockDevQuotaDisabled(t *testing.T, mountPoint, backingFsDev string) {
+	hasSupport, err := hasQuotaSupport(backingFsDev)
+	require.NoError(t, err)
+	assert.False(t, hasSupport)
+}
+
+func testBlockDevQuotaEnabled(t *testing.T, mountPoint, backingFsDev string) {
+	hasSupport, err := hasQuotaSupport(backingFsDev)
+	require.NoError(t, err)
+	assert.True(t, hasSupport)
+}
+
+func wrapQuotaTest(testFunc func(t *testing.T, ctrl *Control, mountPoint, testDir, testSubDir string)) func(t *testing.T, mountPoint, backingFsDev string) {
+	return func(t *testing.T, mountPoint, backingFsDev string) {
+		testDir, err := ioutil.TempDir(mountPoint, "per-test")
+		require.NoError(t, err)
+		defer os.RemoveAll(testDir)
+
+		ctrl, err := NewControl(testDir)
+		require.NoError(t, err)
+
+		testSubDir, err := ioutil.TempDir(testDir, "quota-test")
+		require.NoError(t, err)
+		testFunc(t, ctrl, mountPoint, testDir, testSubDir)
+	}
+
+}
+
+func testSmallerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) {
+	require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize}))
+	smallerThanQuotaFile := filepath.Join(testSubDir, "smaller-than-quota")
+	require.NoError(t, ioutil.WriteFile(smallerThanQuotaFile, make([]byte, testQuotaSize/2), 0644))
+	require.NoError(t, os.Remove(smallerThanQuotaFile))
+}
+
+func testBiggerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) {
+	// Make sure the quota is being enforced
+	// TODO: When we implement this under EXT4, we need to shed CAP_SYS_RESOURCE, otherwise
+	// we're able to violate quota without issue
+	require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize}))
+
+	biggerThanQuotaFile := filepath.Join(testSubDir, "bigger-than-quota")
+	err := ioutil.WriteFile(biggerThanQuotaFile, make([]byte, testQuotaSize+1), 0644)
+	require.Error(t, err)
+	if err == io.ErrShortWrite {
+		require.NoError(t, os.Remove(biggerThanQuotaFile))
+	}
+}
+
+func testRetrieveQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) {
+	// Validate that we can retrieve quota
+	require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize}))
+
+	var q Quota
+	require.NoError(t, ctrl.GetQuota(testSubDir, &q))
+	assert.EqualValues(t, testQuotaSize, q.Size)
+}

components/engine/daemon/logger/awslogs/cloudwatchlogs.go

@@ -275,6 +275,10 @@ func (l *logStream) Name() string {
 	return name
 }
 
+func (l *logStream) BufSize() int {
+	return maximumBytesPerEvent
+}
+
 // Log submits messages for logging by an instance of the awslogs logging driver
 func (l *logStream) Log(msg *logger.Message) error {
 	l.lock.RLock()

components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go

@@ -1052,6 +1052,11 @@ func TestCreateTagSuccess(t *testing.T) {
 	}
 }
 
+func TestIsSizedLogger(t *testing.T) {
+	awslogs := &logStream{}
+	assert.Implements(t, (*logger.SizedLogger)(nil), awslogs, "awslogs should implement SizedLogger")
+}
+
 func BenchmarkUnwrapEvents(b *testing.B) {
 	events := make([]wrappedEvent, maximumLogEventsPerPut)
 	for i := 0; i < maximumLogEventsPerPut; i++ {

components/engine/daemon/logger/copier.go

@@ -10,8 +10,13 @@ import (
 )
 
 const (
-	bufSize  = 16 * 1024
+	// readSize is the maximum bytes read during a single read
+	// operation.
 	readSize = 2 * 1024
+
+	// defaultBufSize provides a reasonable default for loggers that do
+	// not have an external limit to impose on log line size.
+	defaultBufSize = 16 * 1024
 )
 
 // Copier can copy logs from specified sources to Logger and attach Timestamp.
@@ -44,7 +49,13 @@ func (c *Copier) Run() {
 
 func (c *Copier) copySrc(name string, src io.Reader) {
 	defer c.copyJobs.Done()
+
+	bufSize := defaultBufSize
+	if sizedLogger, ok := c.dst.(SizedLogger); ok {
+		bufSize = sizedLogger.BufSize()
+	}
 	buf := make([]byte, bufSize)
+
 	n := 0
 	eof := false
 

components/engine/daemon/logger/copier_test.go

@@ -31,6 +31,25 @@ func (l *TestLoggerJSON) Close() error { return nil }
 
 func (l *TestLoggerJSON) Name() string { return "json" }
 
+type TestSizedLoggerJSON struct {
+	*json.Encoder
+	mu sync.Mutex
+}
+
+func (l *TestSizedLoggerJSON) Log(m *Message) error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.Encode(m)
+}
+
+func (*TestSizedLoggerJSON) Close() error { return nil }
+
+func (*TestSizedLoggerJSON) Name() string { return "sized-json" }
+
+func (*TestSizedLoggerJSON) BufSize() int {
+	return 32 * 1024
+}
+
 func TestCopier(t *testing.T) {
 	stdoutLine := "Line that thinks that it is log line from docker stdout"
 	stderrLine := "Line that thinks that it is log line from docker stderr"
@@ -104,10 +123,9 @@ func TestCopier(t *testing.T) {
 
 // TestCopierLongLines tests long lines without line breaks
 func TestCopierLongLines(t *testing.T) {
-	// Long lines (should be split at "bufSize")
-	const bufSize = 16 * 1024
-	stdoutLongLine := strings.Repeat("a", bufSize)
-	stderrLongLine := strings.Repeat("b", bufSize)
+	// Long lines (should be split at "defaultBufSize")
+	stdoutLongLine := strings.Repeat("a", defaultBufSize)
+	stderrLongLine := strings.Repeat("b", defaultBufSize)
 	stdoutTrailingLine := "stdout trailing line"
 	stderrTrailingLine := "stderr trailing line"
 
@@ -205,6 +223,41 @@ func TestCopierSlow(t *testing.T) {
 	}
 }
 
+func TestCopierWithSized(t *testing.T) {
+	var jsonBuf bytes.Buffer
+	expectedMsgs := 2
+	sizedLogger := &TestSizedLoggerJSON{Encoder: json.NewEncoder(&jsonBuf)}
+	logbuf := bytes.NewBufferString(strings.Repeat(".", sizedLogger.BufSize()*expectedMsgs))
+	c := NewCopier(map[string]io.Reader{"stdout": logbuf}, sizedLogger)
+
+	c.Run()
+	// Wait for Copier to finish writing to the buffered logger.
+	c.Wait()
+	c.Close()
+
+	recvdMsgs := 0
+	dec := json.NewDecoder(&jsonBuf)
+	for {
+		var msg Message
+		if err := dec.Decode(&msg); err != nil {
+			if err == io.EOF {
+				break
+			}
+			t.Fatal(err)
+		}
+		if msg.Source != "stdout" {
+			t.Fatalf("Wrong Source: %q, should be %q", msg.Source, "stdout")
+		}
+		if len(msg.Line) != sizedLogger.BufSize() {
+			t.Fatalf("Line was not of expected max length %d, was %d", sizedLogger.BufSize(), len(msg.Line))
+		}
+		recvdMsgs++
+	}
+	if recvdMsgs != expectedMsgs {
+		t.Fatalf("expected to receive %d messages, actually received %d", expectedMsgs, recvdMsgs)
+	}
+}
+
 type BenchmarkLoggerDummy struct {
 }
 

components/engine/daemon/logger/logger.go

@@ -78,6 +78,13 @@ type Logger interface {
 	Close() error
 }
 
+// SizedLogger is the interface for logging drivers that can control
+// the size of buffer used for their messages.
+type SizedLogger interface {
+	Logger
+	BufSize() int
+}
+
 // ReadConfig is the configuration passed into ReadLogs.
 type ReadConfig struct {
 	Since  time.Time

components/engine/docs/api/v1.18.md

@@ -264,8 +264,14 @@ Create a container
           should map to. A JSON object in the form
           `{ <port>/<protocol>: [{ "HostPort": "<port>" }] }`
           Take note that `port` is specified as a string and not an integer value.
-    -   **PublishAllPorts** - Allocates a random host port for all of a container's
+    -   **PublishAllPorts** - Allocates an ephemeral host port for all of a container's
           exposed ports. Specified as a boolean value.
+
+          Ports are de-allocated when the container stops and allocated when the container starts.
+          The allocated port might be changed when restarting the container.
+
+          The port is selected from the ephemeral port range that depends on the kernel.
+          For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`.
     -   **Privileged** - Gives the container full access to the host. Specified as
           a boolean value.
     -   **ReadonlyRootfs** - Mount the container's root filesystem as read only.

components/engine/docs/api/v1.19.md

@@ -274,8 +274,14 @@ Create a container
           should map to. A JSON object in the form
           `{ <port>/<protocol>: [{ "HostPort": "<port>" }] }`
           Take note that `port` is specified as a string and not an integer value.
-    -   **PublishAllPorts** - Allocates a random host port for all of a container's
+    -   **PublishAllPorts** - Allocates an ephemeral host port for all of a container's
           exposed ports. Specified as a boolean value.
+
+          Ports are de-allocated when the container stops and allocated when the container starts.
+          The allocated port might be changed when restarting the container.
+
+          The port is selected from the ephemeral port range that depends on the kernel.
+          For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`.
     -   **Privileged** - Gives the container full access to the host. Specified as
           a boolean value.
     -   **ReadonlyRootfs** - Mount the container's root filesystem as read only.

components/engine/docs/api/v1.20.md

@@ -277,8 +277,14 @@ Create a container
           should map to. A JSON object in the form
           `{ <port>/<protocol>: [{ "HostPort": "<port>" }] }`
           Take note that `port` is specified as a string and not an integer value.
-    -   **PublishAllPorts** - Allocates a random host port for all of a container's
+    -   **PublishAllPorts** - Allocates an ephemeral host port for all of a container's
           exposed ports. Specified as a boolean value.
+
+          Ports are de-allocated when the container stops and allocated when the container starts.
+          The allocated port might be changed when restarting the container.
+
+          The port is selected from the ephemeral port range that depends on the kernel.
+          For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`.
     -   **Privileged** - Gives the container full access to the host. Specified as
           a boolean value.
     -   **ReadonlyRootfs** - Mount the container's root filesystem as read only.

components/engine/docs/api/v1.21.md

@@ -294,8 +294,14 @@ Create a container
           should map to. A JSON object in the form
           `{ <port>/<protocol>: [{ "HostPort": "<port>" }] }`
           Take note that `port` is specified as a string and not an integer value.
-    -   **PublishAllPorts** - Allocates a random host port for all of a container's
+    -   **PublishAllPorts** - Allocates an ephemeral host port for all of a container's
           exposed ports. Specified as a boolean value.
+
+          Ports are de-allocated when the container stops and allocated when the container starts.
+          The allocated port might be changed when restarting the container.
+
+          The port is selected from the ephemeral port range that depends on the kernel.
+          For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`.
     -   **Privileged** - Gives the container full access to the host. Specified as
           a boolean value.
     -   **ReadonlyRootfs** - Mount the container's root filesystem as read only.

components/engine/docs/api/v1.22.md

@@ -408,8 +408,14 @@ Create a container
           should map to. A JSON object in the form
           `{ <port>/<protocol>: [{ "HostPort": "<port>" }] }`
           Take note that `port` is specified as a string and not an integer value.
-    -   **PublishAllPorts** - Allocates a random host port for all of a container's
+    -   **PublishAllPorts** - Allocates an ephemeral host port for all of a container's
           exposed ports. Specified as a boolean value.
+
+          Ports are de-allocated when the container stops and allocated when the container starts.
+          The allocated port might be changed when restarting the container.
+
+          The port is selected from the ephemeral port range that depends on the kernel.
+          For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`.
     -   **Privileged** - Gives the container full access to the host. Specified as
           a boolean value.
     -   **ReadonlyRootfs** - Mount the container's root filesystem as read only.

components/engine/docs/api/v1.23.md

@@ -432,8 +432,14 @@ Create a container
           should map to. A JSON object in the form
           `{ <port>/<protocol>: [{ "HostPort": "<port>" }] }`
           Take note that `port` is specified as a string and not an integer value.
-    -   **PublishAllPorts** - Allocates a random host port for all of a container's
+    -   **PublishAllPorts** - Allocates an ephemeral host port for all of a container's
           exposed ports. Specified as a boolean value.
+
+          Ports are de-allocated when the container stops and allocated when the container starts.
+          The allocated port might be changed when restarting the container.
+
+          The port is selected from the ephemeral port range that depends on the kernel.
+          For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`.
     -   **Privileged** - Gives the container full access to the host. Specified as
           a boolean value.
     -   **ReadonlyRootfs** - Mount the container's root filesystem as read only.

components/engine/docs/api/v1.24.md

@@ -469,8 +469,14 @@ Create a container
           should map to. A JSON object in the form
           `{ <port>/<protocol>: [{ "HostPort": "<port>" }] }`
           Take note that `port` is specified as a string and not an integer value.
-    -   **PublishAllPorts** - Allocates a random host port for all of a container's
+    -   **PublishAllPorts** - Allocates an ephemeral host port for all of a container's
           exposed ports. Specified as a boolean value.
+
+          Ports are de-allocated when the container stops and allocated when the container starts.
+          The allocated port might be changed when restarting the container.
+
+          The port is selected from the ephemeral port range that depends on the kernel.
+          For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`.
     -   **Privileged** - Gives the container full access to the host. Specified as
           a boolean value.
     -   **ReadonlyRootfs** - Mount the container's root filesystem as read only.