diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index b0c0575fc0..80319ae7f3 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -722,7 +722,15 @@ definitions: description: "Gives the container full access to the host." PublishAllPorts: type: "boolean" - description: "Allocates a random host port for all of a container's exposed ports." + description: | + Allocates an ephemeral host port for all of a container's + exposed ports. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. ReadonlyRootfs: type: "boolean" description: "Mount the container's root filesystem as read only." diff --git a/components/engine/daemon/graphdriver/quota/projectquota.go b/components/engine/daemon/graphdriver/quota/projectquota.go index 0e70515434..84e391aa89 100644 --- a/components/engine/daemon/graphdriver/quota/projectquota.go +++ b/components/engine/daemon/graphdriver/quota/projectquota.go @@ -47,6 +47,8 @@ struct fsxattr { #ifndef Q_XGETPQUOTA #define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA) #endif + +const int Q_XGETQSTAT_PRJQUOTA = QCMD(Q_XGETQSTAT, PRJQUOTA); */ import "C" import ( @@ -56,10 +58,15 @@ import ( "path/filepath" "unsafe" + "errors" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) +// ErrQuotaNotSupported indicates if were found the FS does not have projects quotas available +var ErrQuotaNotSupported = errors.New("Filesystem does not support or has not enabled quotas") + // Quota limit params - currently we only control blocks hard limit type Quota struct { Size uint64 @@ -96,6 +103,24 @@ type Control struct { // project ids. // func NewControl(basePath string) (*Control, error) { + // + // create backing filesystem device node + // + backingFsBlockDev, err := makeBackingFsDev(basePath) + if err != nil { + return nil, err + } + + // check if we can call quotactl with project quotas + // as a mechanism to determine (early) if we have support + hasQuotaSupport, err := hasQuotaSupport(backingFsBlockDev) + if err != nil { + return nil, err + } + if !hasQuotaSupport { + return nil, ErrQuotaNotSupported + } + // // Get project id of parent dir as minimal id to be used by driver // @@ -105,14 +130,6 @@ func NewControl(basePath string) (*Control, error) { } minProjectID++ - // - // create backing filesystem device node - // - backingFsBlockDev, err := makeBackingFsDev(basePath) - if err != nil { - return nil, err - } - // // Test if filesystem supports project quotas by trying to set // a quota on the first available project id @@ -335,3 +352,23 @@ func makeBackingFsDev(home string) (string, error) { return backingFsBlockDev, nil } + +func hasQuotaSupport(backingFsBlockDev string) (bool, error) { + var cs = C.CString(backingFsBlockDev) + defer free(cs) + var qstat C.fs_quota_stat_t + + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, uintptr(C.Q_XGETQSTAT_PRJQUOTA), uintptr(unsafe.Pointer(cs)), 0, uintptr(unsafe.Pointer(&qstat)), 0, 0) + if errno == 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ENFD > 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ACCT > 0 { + return true, nil + } + + switch errno { + // These are the known fatal errors, consider all other errors (ENOTTY, etc.. not supporting quota) + case unix.EFAULT, unix.ENOENT, unix.ENOTBLK, unix.EPERM: + default: + return false, nil + } + + return false, errno +} diff --git a/components/engine/daemon/graphdriver/quota/projectquota_test.go b/components/engine/daemon/graphdriver/quota/projectquota_test.go new file mode 100644 index 0000000000..2b47a58db7 --- /dev/null +++ b/components/engine/daemon/graphdriver/quota/projectquota_test.go @@ -0,0 +1,161 @@ +// +build linux + +package quota + +import ( + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// 10MB +const testQuotaSize = 10 * 1024 * 1024 +const imageSize = 64 * 1024 * 1024 + +func TestBlockDev(t *testing.T) { + mkfs, err := exec.LookPath("mkfs.xfs") + if err != nil { + t.Fatal("mkfs.xfs not installed") + } + + // create a sparse image + imageFile, err := ioutil.TempFile("", "xfs-image") + if err != nil { + t.Fatal(err) + } + imageFileName := imageFile.Name() + defer os.Remove(imageFileName) + if _, err = imageFile.Seek(imageSize-1, 0); err != nil { + t.Fatal(err) + } + if _, err = imageFile.Write([]byte{0}); err != nil { + t.Fatal(err) + } + if err = imageFile.Close(); err != nil { + t.Fatal(err) + } + + // The reason for disabling these options is sometimes people run with a newer userspace + // than kernelspace + out, err := exec.Command(mkfs, "-m", "crc=0,finobt=0", imageFileName).CombinedOutput() + if len(out) > 0 { + t.Log(string(out)) + } + if err != nil { + t.Fatal(err) + } + + runTest(t, "testBlockDevQuotaDisabled", wrapMountTest(imageFileName, false, testBlockDevQuotaDisabled)) + runTest(t, "testBlockDevQuotaEnabled", wrapMountTest(imageFileName, true, testBlockDevQuotaEnabled)) + runTest(t, "testSmallerThanQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testSmallerThanQuota))) + runTest(t, "testBiggerThanQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testBiggerThanQuota))) + runTest(t, "testRetrieveQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testRetrieveQuota))) +} + +func runTest(t *testing.T, testName string, testFunc func(*testing.T)) { + if success := t.Run(testName, testFunc); !success { + out, _ := exec.Command("dmesg").CombinedOutput() + t.Log(string(out)) + } +} + +func wrapMountTest(imageFileName string, enableQuota bool, testFunc func(t *testing.T, mountPoint, backingFsDev string)) func(*testing.T) { + return func(t *testing.T) { + mountOptions := "loop" + + if enableQuota { + mountOptions = mountOptions + ",prjquota" + } + + // create a mountPoint + mountPoint, err := ioutil.TempDir("", "xfs-mountPoint") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(mountPoint) + + out, err := exec.Command("mount", "-o", mountOptions, imageFileName, mountPoint).CombinedOutput() + if len(out) > 0 { + t.Log(string(out)) + } + if err != nil { + t.Fatal("mount failed") + } + + defer func() { + if err := unix.Unmount(mountPoint, 0); err != nil { + t.Fatal(err) + } + }() + + backingFsDev, err := makeBackingFsDev(mountPoint) + require.NoError(t, err) + + testFunc(t, mountPoint, backingFsDev) + } +} + +func testBlockDevQuotaDisabled(t *testing.T, mountPoint, backingFsDev string) { + hasSupport, err := hasQuotaSupport(backingFsDev) + require.NoError(t, err) + assert.False(t, hasSupport) +} + +func testBlockDevQuotaEnabled(t *testing.T, mountPoint, backingFsDev string) { + hasSupport, err := hasQuotaSupport(backingFsDev) + require.NoError(t, err) + assert.True(t, hasSupport) +} + +func wrapQuotaTest(testFunc func(t *testing.T, ctrl *Control, mountPoint, testDir, testSubDir string)) func(t *testing.T, mountPoint, backingFsDev string) { + return func(t *testing.T, mountPoint, backingFsDev string) { + testDir, err := ioutil.TempDir(mountPoint, "per-test") + require.NoError(t, err) + defer os.RemoveAll(testDir) + + ctrl, err := NewControl(testDir) + require.NoError(t, err) + + testSubDir, err := ioutil.TempDir(testDir, "quota-test") + require.NoError(t, err) + testFunc(t, ctrl, mountPoint, testDir, testSubDir) + } + +} + +func testSmallerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) { + require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize})) + smallerThanQuotaFile := filepath.Join(testSubDir, "smaller-than-quota") + require.NoError(t, ioutil.WriteFile(smallerThanQuotaFile, make([]byte, testQuotaSize/2), 0644)) + require.NoError(t, os.Remove(smallerThanQuotaFile)) +} + +func testBiggerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) { + // Make sure the quota is being enforced + // TODO: When we implement this under EXT4, we need to shed CAP_SYS_RESOURCE, otherwise + // we're able to violate quota without issue + require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize})) + + biggerThanQuotaFile := filepath.Join(testSubDir, "bigger-than-quota") + err := ioutil.WriteFile(biggerThanQuotaFile, make([]byte, testQuotaSize+1), 0644) + require.Error(t, err) + if err == io.ErrShortWrite { + require.NoError(t, os.Remove(biggerThanQuotaFile)) + } +} + +func testRetrieveQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) { + // Validate that we can retrieve quota + require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize})) + + var q Quota + require.NoError(t, ctrl.GetQuota(testSubDir, &q)) + assert.EqualValues(t, testQuotaSize, q.Size) +} diff --git a/components/engine/daemon/logger/awslogs/cloudwatchlogs.go b/components/engine/daemon/logger/awslogs/cloudwatchlogs.go index 97882538b4..4ea942071d 100644 --- a/components/engine/daemon/logger/awslogs/cloudwatchlogs.go +++ b/components/engine/daemon/logger/awslogs/cloudwatchlogs.go @@ -275,6 +275,10 @@ func (l *logStream) Name() string { return name } +func (l *logStream) BufSize() int { + return maximumBytesPerEvent +} + // Log submits messages for logging by an instance of the awslogs logging driver func (l *logStream) Log(msg *logger.Message) error { l.lock.RLock() diff --git a/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go b/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go index 7d482d8196..7ebc5dede2 100644 --- a/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go +++ b/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go @@ -1052,6 +1052,11 @@ func TestCreateTagSuccess(t *testing.T) { } } +func TestIsSizedLogger(t *testing.T) { + awslogs := &logStream{} + assert.Implements(t, (*logger.SizedLogger)(nil), awslogs, "awslogs should implement SizedLogger") +} + func BenchmarkUnwrapEvents(b *testing.B) { events := make([]wrappedEvent, maximumLogEventsPerPut) for i := 0; i < maximumLogEventsPerPut; i++ { diff --git a/components/engine/daemon/logger/copier.go b/components/engine/daemon/logger/copier.go index c773fc6a29..a1d4f06f8b 100644 --- a/components/engine/daemon/logger/copier.go +++ b/components/engine/daemon/logger/copier.go @@ -10,8 +10,13 @@ import ( ) const ( - bufSize = 16 * 1024 + // readSize is the maximum bytes read during a single read + // operation. readSize = 2 * 1024 + + // defaultBufSize provides a reasonable default for loggers that do + // not have an external limit to impose on log line size. + defaultBufSize = 16 * 1024 ) // Copier can copy logs from specified sources to Logger and attach Timestamp. @@ -44,7 +49,13 @@ func (c *Copier) Run() { func (c *Copier) copySrc(name string, src io.Reader) { defer c.copyJobs.Done() + + bufSize := defaultBufSize + if sizedLogger, ok := c.dst.(SizedLogger); ok { + bufSize = sizedLogger.BufSize() + } buf := make([]byte, bufSize) + n := 0 eof := false diff --git a/components/engine/daemon/logger/copier_test.go b/components/engine/daemon/logger/copier_test.go index 4210022dcd..a911a703e9 100644 --- a/components/engine/daemon/logger/copier_test.go +++ b/components/engine/daemon/logger/copier_test.go @@ -31,6 +31,25 @@ func (l *TestLoggerJSON) Close() error { return nil } func (l *TestLoggerJSON) Name() string { return "json" } +type TestSizedLoggerJSON struct { + *json.Encoder + mu sync.Mutex +} + +func (l *TestSizedLoggerJSON) Log(m *Message) error { + l.mu.Lock() + defer l.mu.Unlock() + return l.Encode(m) +} + +func (*TestSizedLoggerJSON) Close() error { return nil } + +func (*TestSizedLoggerJSON) Name() string { return "sized-json" } + +func (*TestSizedLoggerJSON) BufSize() int { + return 32 * 1024 +} + func TestCopier(t *testing.T) { stdoutLine := "Line that thinks that it is log line from docker stdout" stderrLine := "Line that thinks that it is log line from docker stderr" @@ -104,10 +123,9 @@ func TestCopier(t *testing.T) { // TestCopierLongLines tests long lines without line breaks func TestCopierLongLines(t *testing.T) { - // Long lines (should be split at "bufSize") - const bufSize = 16 * 1024 - stdoutLongLine := strings.Repeat("a", bufSize) - stderrLongLine := strings.Repeat("b", bufSize) + // Long lines (should be split at "defaultBufSize") + stdoutLongLine := strings.Repeat("a", defaultBufSize) + stderrLongLine := strings.Repeat("b", defaultBufSize) stdoutTrailingLine := "stdout trailing line" stderrTrailingLine := "stderr trailing line" @@ -205,6 +223,41 @@ func TestCopierSlow(t *testing.T) { } } +func TestCopierWithSized(t *testing.T) { + var jsonBuf bytes.Buffer + expectedMsgs := 2 + sizedLogger := &TestSizedLoggerJSON{Encoder: json.NewEncoder(&jsonBuf)} + logbuf := bytes.NewBufferString(strings.Repeat(".", sizedLogger.BufSize()*expectedMsgs)) + c := NewCopier(map[string]io.Reader{"stdout": logbuf}, sizedLogger) + + c.Run() + // Wait for Copier to finish writing to the buffered logger. + c.Wait() + c.Close() + + recvdMsgs := 0 + dec := json.NewDecoder(&jsonBuf) + for { + var msg Message + if err := dec.Decode(&msg); err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + if msg.Source != "stdout" { + t.Fatalf("Wrong Source: %q, should be %q", msg.Source, "stdout") + } + if len(msg.Line) != sizedLogger.BufSize() { + t.Fatalf("Line was not of expected max length %d, was %d", sizedLogger.BufSize(), len(msg.Line)) + } + recvdMsgs++ + } + if recvdMsgs != expectedMsgs { + t.Fatalf("expected to receive %d messages, actually received %d", expectedMsgs, recvdMsgs) + } +} + type BenchmarkLoggerDummy struct { } diff --git a/components/engine/daemon/logger/logger.go b/components/engine/daemon/logger/logger.go index ee91b79c98..1108597c62 100644 --- a/components/engine/daemon/logger/logger.go +++ b/components/engine/daemon/logger/logger.go @@ -78,6 +78,13 @@ type Logger interface { Close() error } +// SizedLogger is the interface for logging drivers that can control +// the size of buffer used for their messages. +type SizedLogger interface { + Logger + BufSize() int +} + // ReadConfig is the configuration passed into ReadLogs. type ReadConfig struct { Since time.Time diff --git a/components/engine/docs/api/v1.18.md b/components/engine/docs/api/v1.18.md index 7ae32ccc93..327701427a 100644 --- a/components/engine/docs/api/v1.18.md +++ b/components/engine/docs/api/v1.18.md @@ -264,8 +264,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. diff --git a/components/engine/docs/api/v1.19.md b/components/engine/docs/api/v1.19.md index 3ecd312c34..448fe832ef 100644 --- a/components/engine/docs/api/v1.19.md +++ b/components/engine/docs/api/v1.19.md @@ -274,8 +274,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. diff --git a/components/engine/docs/api/v1.20.md b/components/engine/docs/api/v1.20.md index 1f94bdaddb..b971bab4be 100644 --- a/components/engine/docs/api/v1.20.md +++ b/components/engine/docs/api/v1.20.md @@ -277,8 +277,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. diff --git a/components/engine/docs/api/v1.21.md b/components/engine/docs/api/v1.21.md index a3b79ed8ba..2971d25c45 100644 --- a/components/engine/docs/api/v1.21.md +++ b/components/engine/docs/api/v1.21.md @@ -294,8 +294,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. diff --git a/components/engine/docs/api/v1.22.md b/components/engine/docs/api/v1.22.md index a6027bb760..a51fbcb7de 100644 --- a/components/engine/docs/api/v1.22.md +++ b/components/engine/docs/api/v1.22.md @@ -408,8 +408,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. diff --git a/components/engine/docs/api/v1.23.md b/components/engine/docs/api/v1.23.md index 19f9e5a961..677dcab789 100644 --- a/components/engine/docs/api/v1.23.md +++ b/components/engine/docs/api/v1.23.md @@ -432,8 +432,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. diff --git a/components/engine/docs/api/v1.24.md b/components/engine/docs/api/v1.24.md index 6ab5dc2483..a32325e2d9 100644 --- a/components/engine/docs/api/v1.24.md +++ b/components/engine/docs/api/v1.24.md @@ -469,8 +469,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only.