From 6bfaa3dd46fab9c416a89af55777408ddb782837 Mon Sep 17 00:00:00 2001 From: Boaz Shuster Date: Thu, 5 Oct 2017 23:57:11 +0300 Subject: [PATCH 1/3] Elaborate more about port allocation in docs Describe more how host port allocation is done when container is stopped/started in "PublishAllPorts". Signed-off-by: Boaz Shuster Upstream-commit: dfd706e1a59c3830c87b8240c466489ae6cc9d15 Component: engine --- components/engine/api/swagger.yaml | 10 +++++++++- components/engine/docs/api/v1.18.md | 8 +++++++- components/engine/docs/api/v1.19.md | 8 +++++++- components/engine/docs/api/v1.20.md | 8 +++++++- components/engine/docs/api/v1.21.md | 8 +++++++- components/engine/docs/api/v1.22.md | 8 +++++++- components/engine/docs/api/v1.23.md | 8 +++++++- components/engine/docs/api/v1.24.md | 8 +++++++- 8 files changed, 58 insertions(+), 8 deletions(-) diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index 1e300e7b40..3e2ead8425 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -720,7 +720,15 @@ definitions: description: "Gives the container full access to the host." PublishAllPorts: type: "boolean" - description: "Allocates a random host port for all of a container's exposed ports." + description: | + Allocates an ephemeral host port for all of a container's + exposed ports. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. ReadonlyRootfs: type: "boolean" description: "Mount the container's root filesystem as read only." diff --git a/components/engine/docs/api/v1.18.md b/components/engine/docs/api/v1.18.md index 973dca9f92..d3d97be08a 100644 --- a/components/engine/docs/api/v1.18.md +++ b/components/engine/docs/api/v1.18.md @@ -256,8 +256,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. diff --git a/components/engine/docs/api/v1.19.md b/components/engine/docs/api/v1.19.md index 3bcf7668f9..f82a5124fc 100644 --- a/components/engine/docs/api/v1.19.md +++ b/components/engine/docs/api/v1.19.md @@ -268,8 +268,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. diff --git a/components/engine/docs/api/v1.20.md b/components/engine/docs/api/v1.20.md index e5b8f32fce..91551815d5 100644 --- a/components/engine/docs/api/v1.20.md +++ b/components/engine/docs/api/v1.20.md @@ -269,8 +269,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. diff --git a/components/engine/docs/api/v1.21.md b/components/engine/docs/api/v1.21.md index f1863b8c11..57290e628f 100644 --- a/components/engine/docs/api/v1.21.md +++ b/components/engine/docs/api/v1.21.md @@ -288,8 +288,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. diff --git a/components/engine/docs/api/v1.22.md b/components/engine/docs/api/v1.22.md index 473ffd3c91..3253a68dba 100644 --- a/components/engine/docs/api/v1.22.md +++ b/components/engine/docs/api/v1.22.md @@ -400,8 +400,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. diff --git a/components/engine/docs/api/v1.23.md b/components/engine/docs/api/v1.23.md index c040d2460a..8a955ce93b 100644 --- a/components/engine/docs/api/v1.23.md +++ b/components/engine/docs/api/v1.23.md @@ -426,8 +426,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. diff --git a/components/engine/docs/api/v1.24.md b/components/engine/docs/api/v1.24.md index 29fed46a22..13d592b41c 100644 --- a/components/engine/docs/api/v1.24.md +++ b/components/engine/docs/api/v1.24.md @@ -461,8 +461,14 @@ Create a container should map to. A JSON object in the form `{ /: [{ "HostPort": "" }] }` Take note that `port` is specified as a string and not an integer value. - - **PublishAllPorts** - Allocates a random host port for all of a container's + - **PublishAllPorts** - Allocates an ephemeral host port for all of a container's exposed ports. Specified as a boolean value. + + Ports are de-allocated when the container stops and allocated when the container starts. + The allocated port might be changed when restarting the container. + + The port is selected from the ephemeral port range that depends on the kernel. + For example, on Linux the range is defined by `/proc/sys/net/ipv4/ip_local_port_range`. - **Privileged** - Gives the container full access to the host. Specified as a boolean value. - **ReadonlyRootfs** - Mount the container's root filesystem as read only. From 320e3a6526649228c47c094ca7aab369e6470458 Mon Sep 17 00:00:00 2001 From: Sargun Dhillon Date: Sat, 2 Sep 2017 21:25:36 -0700 Subject: [PATCH 2/3] Add tests to project quotas and detection mechanism This adds a mechanism (read-only) to check for project quota support in a standard way. This mechanism is leveraged by the tests, which test for the following: 1. Can we get a quota controller? 2. Can we set the quota for a particular directory? 3. Is the quota being over-enforced? 4. Is the quota being under-enforced? 5. Can we retrieve the quota? Signed-off-by: Sargun Dhillon Upstream-commit: 6966dc0aa9134c518babcbf1f02684cae5374843 Component: engine --- .../daemon/graphdriver/quota/projectquota.go | 53 +++++- .../graphdriver/quota/projectquota_test.go | 161 ++++++++++++++++++ 2 files changed, 206 insertions(+), 8 deletions(-) create mode 100644 components/engine/daemon/graphdriver/quota/projectquota_test.go diff --git a/components/engine/daemon/graphdriver/quota/projectquota.go b/components/engine/daemon/graphdriver/quota/projectquota.go index 0e70515434..84e391aa89 100644 --- a/components/engine/daemon/graphdriver/quota/projectquota.go +++ b/components/engine/daemon/graphdriver/quota/projectquota.go @@ -47,6 +47,8 @@ struct fsxattr { #ifndef Q_XGETPQUOTA #define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA) #endif + +const int Q_XGETQSTAT_PRJQUOTA = QCMD(Q_XGETQSTAT, PRJQUOTA); */ import "C" import ( @@ -56,10 +58,15 @@ import ( "path/filepath" "unsafe" + "errors" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) +// ErrQuotaNotSupported indicates if were found the FS does not have projects quotas available +var ErrQuotaNotSupported = errors.New("Filesystem does not support or has not enabled quotas") + // Quota limit params - currently we only control blocks hard limit type Quota struct { Size uint64 @@ -96,6 +103,24 @@ type Control struct { // project ids. // func NewControl(basePath string) (*Control, error) { + // + // create backing filesystem device node + // + backingFsBlockDev, err := makeBackingFsDev(basePath) + if err != nil { + return nil, err + } + + // check if we can call quotactl with project quotas + // as a mechanism to determine (early) if we have support + hasQuotaSupport, err := hasQuotaSupport(backingFsBlockDev) + if err != nil { + return nil, err + } + if !hasQuotaSupport { + return nil, ErrQuotaNotSupported + } + // // Get project id of parent dir as minimal id to be used by driver // @@ -105,14 +130,6 @@ func NewControl(basePath string) (*Control, error) { } minProjectID++ - // - // create backing filesystem device node - // - backingFsBlockDev, err := makeBackingFsDev(basePath) - if err != nil { - return nil, err - } - // // Test if filesystem supports project quotas by trying to set // a quota on the first available project id @@ -335,3 +352,23 @@ func makeBackingFsDev(home string) (string, error) { return backingFsBlockDev, nil } + +func hasQuotaSupport(backingFsBlockDev string) (bool, error) { + var cs = C.CString(backingFsBlockDev) + defer free(cs) + var qstat C.fs_quota_stat_t + + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, uintptr(C.Q_XGETQSTAT_PRJQUOTA), uintptr(unsafe.Pointer(cs)), 0, uintptr(unsafe.Pointer(&qstat)), 0, 0) + if errno == 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ENFD > 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ACCT > 0 { + return true, nil + } + + switch errno { + // These are the known fatal errors, consider all other errors (ENOTTY, etc.. not supporting quota) + case unix.EFAULT, unix.ENOENT, unix.ENOTBLK, unix.EPERM: + default: + return false, nil + } + + return false, errno +} diff --git a/components/engine/daemon/graphdriver/quota/projectquota_test.go b/components/engine/daemon/graphdriver/quota/projectquota_test.go new file mode 100644 index 0000000000..2b47a58db7 --- /dev/null +++ b/components/engine/daemon/graphdriver/quota/projectquota_test.go @@ -0,0 +1,161 @@ +// +build linux + +package quota + +import ( + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// 10MB +const testQuotaSize = 10 * 1024 * 1024 +const imageSize = 64 * 1024 * 1024 + +func TestBlockDev(t *testing.T) { + mkfs, err := exec.LookPath("mkfs.xfs") + if err != nil { + t.Fatal("mkfs.xfs not installed") + } + + // create a sparse image + imageFile, err := ioutil.TempFile("", "xfs-image") + if err != nil { + t.Fatal(err) + } + imageFileName := imageFile.Name() + defer os.Remove(imageFileName) + if _, err = imageFile.Seek(imageSize-1, 0); err != nil { + t.Fatal(err) + } + if _, err = imageFile.Write([]byte{0}); err != nil { + t.Fatal(err) + } + if err = imageFile.Close(); err != nil { + t.Fatal(err) + } + + // The reason for disabling these options is sometimes people run with a newer userspace + // than kernelspace + out, err := exec.Command(mkfs, "-m", "crc=0,finobt=0", imageFileName).CombinedOutput() + if len(out) > 0 { + t.Log(string(out)) + } + if err != nil { + t.Fatal(err) + } + + runTest(t, "testBlockDevQuotaDisabled", wrapMountTest(imageFileName, false, testBlockDevQuotaDisabled)) + runTest(t, "testBlockDevQuotaEnabled", wrapMountTest(imageFileName, true, testBlockDevQuotaEnabled)) + runTest(t, "testSmallerThanQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testSmallerThanQuota))) + runTest(t, "testBiggerThanQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testBiggerThanQuota))) + runTest(t, "testRetrieveQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testRetrieveQuota))) +} + +func runTest(t *testing.T, testName string, testFunc func(*testing.T)) { + if success := t.Run(testName, testFunc); !success { + out, _ := exec.Command("dmesg").CombinedOutput() + t.Log(string(out)) + } +} + +func wrapMountTest(imageFileName string, enableQuota bool, testFunc func(t *testing.T, mountPoint, backingFsDev string)) func(*testing.T) { + return func(t *testing.T) { + mountOptions := "loop" + + if enableQuota { + mountOptions = mountOptions + ",prjquota" + } + + // create a mountPoint + mountPoint, err := ioutil.TempDir("", "xfs-mountPoint") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(mountPoint) + + out, err := exec.Command("mount", "-o", mountOptions, imageFileName, mountPoint).CombinedOutput() + if len(out) > 0 { + t.Log(string(out)) + } + if err != nil { + t.Fatal("mount failed") + } + + defer func() { + if err := unix.Unmount(mountPoint, 0); err != nil { + t.Fatal(err) + } + }() + + backingFsDev, err := makeBackingFsDev(mountPoint) + require.NoError(t, err) + + testFunc(t, mountPoint, backingFsDev) + } +} + +func testBlockDevQuotaDisabled(t *testing.T, mountPoint, backingFsDev string) { + hasSupport, err := hasQuotaSupport(backingFsDev) + require.NoError(t, err) + assert.False(t, hasSupport) +} + +func testBlockDevQuotaEnabled(t *testing.T, mountPoint, backingFsDev string) { + hasSupport, err := hasQuotaSupport(backingFsDev) + require.NoError(t, err) + assert.True(t, hasSupport) +} + +func wrapQuotaTest(testFunc func(t *testing.T, ctrl *Control, mountPoint, testDir, testSubDir string)) func(t *testing.T, mountPoint, backingFsDev string) { + return func(t *testing.T, mountPoint, backingFsDev string) { + testDir, err := ioutil.TempDir(mountPoint, "per-test") + require.NoError(t, err) + defer os.RemoveAll(testDir) + + ctrl, err := NewControl(testDir) + require.NoError(t, err) + + testSubDir, err := ioutil.TempDir(testDir, "quota-test") + require.NoError(t, err) + testFunc(t, ctrl, mountPoint, testDir, testSubDir) + } + +} + +func testSmallerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) { + require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize})) + smallerThanQuotaFile := filepath.Join(testSubDir, "smaller-than-quota") + require.NoError(t, ioutil.WriteFile(smallerThanQuotaFile, make([]byte, testQuotaSize/2), 0644)) + require.NoError(t, os.Remove(smallerThanQuotaFile)) +} + +func testBiggerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) { + // Make sure the quota is being enforced + // TODO: When we implement this under EXT4, we need to shed CAP_SYS_RESOURCE, otherwise + // we're able to violate quota without issue + require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize})) + + biggerThanQuotaFile := filepath.Join(testSubDir, "bigger-than-quota") + err := ioutil.WriteFile(biggerThanQuotaFile, make([]byte, testQuotaSize+1), 0644) + require.Error(t, err) + if err == io.ErrShortWrite { + require.NoError(t, os.Remove(biggerThanQuotaFile)) + } +} + +func testRetrieveQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) { + // Validate that we can retrieve quota + require.NoError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize})) + + var q Quota + require.NoError(t, ctrl.GetQuota(testSubDir, &q)) + assert.EqualValues(t, testQuotaSize, q.Size) +} From bd6132334207f0d85d8a801995f13aea19ae95fa Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Tue, 22 Aug 2017 10:52:52 -0700 Subject: [PATCH 3/3] logger: copy to log driver's bufsize Log drivers may have an internal buffer size that can be accommodated by the copier as it is more effective to buffer and send fewer though larger messages that the log driver can consume. This eliminates the need for Partial handling for drivers that do not support the concept (ie: awslogs, which can only have events up to service limits). Signed-off-by: Jacob Vallejo Upstream-commit: e1ada0b885b31de0bb0e79b4d99ae4d48b65f721 Component: engine --- .../daemon/logger/awslogs/cloudwatchlogs.go | 4 ++ .../logger/awslogs/cloudwatchlogs_test.go | 5 ++ components/engine/daemon/logger/copier.go | 13 +++- .../engine/daemon/logger/copier_test.go | 61 +++++++++++++++++-- components/engine/daemon/logger/logger.go | 7 +++ 5 files changed, 85 insertions(+), 5 deletions(-) diff --git a/components/engine/daemon/logger/awslogs/cloudwatchlogs.go b/components/engine/daemon/logger/awslogs/cloudwatchlogs.go index 3a7f2f631d..de168cfd73 100644 --- a/components/engine/daemon/logger/awslogs/cloudwatchlogs.go +++ b/components/engine/daemon/logger/awslogs/cloudwatchlogs.go @@ -245,6 +245,10 @@ func (l *logStream) Name() string { return name } +func (l *logStream) BufSize() int { + return maximumBytesPerEvent +} + // Log submits messages for logging by an instance of the awslogs logging driver func (l *logStream) Log(msg *logger.Message) error { l.lock.RLock() diff --git a/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go b/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go index 989eb6f52c..d9bffc576f 100644 --- a/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go +++ b/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go @@ -1049,6 +1049,11 @@ func TestCreateTagSuccess(t *testing.T) { } } +func TestIsSizedLogger(t *testing.T) { + awslogs := &logStream{} + assert.Implements(t, (*logger.SizedLogger)(nil), awslogs, "awslogs should implement SizedLogger") +} + func BenchmarkUnwrapEvents(b *testing.B) { events := make([]wrappedEvent, maximumLogEventsPerPut) for i := 0; i < maximumLogEventsPerPut; i++ { diff --git a/components/engine/daemon/logger/copier.go b/components/engine/daemon/logger/copier.go index c773fc6a29..a1d4f06f8b 100644 --- a/components/engine/daemon/logger/copier.go +++ b/components/engine/daemon/logger/copier.go @@ -10,8 +10,13 @@ import ( ) const ( - bufSize = 16 * 1024 + // readSize is the maximum bytes read during a single read + // operation. readSize = 2 * 1024 + + // defaultBufSize provides a reasonable default for loggers that do + // not have an external limit to impose on log line size. + defaultBufSize = 16 * 1024 ) // Copier can copy logs from specified sources to Logger and attach Timestamp. @@ -44,7 +49,13 @@ func (c *Copier) Run() { func (c *Copier) copySrc(name string, src io.Reader) { defer c.copyJobs.Done() + + bufSize := defaultBufSize + if sizedLogger, ok := c.dst.(SizedLogger); ok { + bufSize = sizedLogger.BufSize() + } buf := make([]byte, bufSize) + n := 0 eof := false diff --git a/components/engine/daemon/logger/copier_test.go b/components/engine/daemon/logger/copier_test.go index 4210022dcd..a911a703e9 100644 --- a/components/engine/daemon/logger/copier_test.go +++ b/components/engine/daemon/logger/copier_test.go @@ -31,6 +31,25 @@ func (l *TestLoggerJSON) Close() error { return nil } func (l *TestLoggerJSON) Name() string { return "json" } +type TestSizedLoggerJSON struct { + *json.Encoder + mu sync.Mutex +} + +func (l *TestSizedLoggerJSON) Log(m *Message) error { + l.mu.Lock() + defer l.mu.Unlock() + return l.Encode(m) +} + +func (*TestSizedLoggerJSON) Close() error { return nil } + +func (*TestSizedLoggerJSON) Name() string { return "sized-json" } + +func (*TestSizedLoggerJSON) BufSize() int { + return 32 * 1024 +} + func TestCopier(t *testing.T) { stdoutLine := "Line that thinks that it is log line from docker stdout" stderrLine := "Line that thinks that it is log line from docker stderr" @@ -104,10 +123,9 @@ func TestCopier(t *testing.T) { // TestCopierLongLines tests long lines without line breaks func TestCopierLongLines(t *testing.T) { - // Long lines (should be split at "bufSize") - const bufSize = 16 * 1024 - stdoutLongLine := strings.Repeat("a", bufSize) - stderrLongLine := strings.Repeat("b", bufSize) + // Long lines (should be split at "defaultBufSize") + stdoutLongLine := strings.Repeat("a", defaultBufSize) + stderrLongLine := strings.Repeat("b", defaultBufSize) stdoutTrailingLine := "stdout trailing line" stderrTrailingLine := "stderr trailing line" @@ -205,6 +223,41 @@ func TestCopierSlow(t *testing.T) { } } +func TestCopierWithSized(t *testing.T) { + var jsonBuf bytes.Buffer + expectedMsgs := 2 + sizedLogger := &TestSizedLoggerJSON{Encoder: json.NewEncoder(&jsonBuf)} + logbuf := bytes.NewBufferString(strings.Repeat(".", sizedLogger.BufSize()*expectedMsgs)) + c := NewCopier(map[string]io.Reader{"stdout": logbuf}, sizedLogger) + + c.Run() + // Wait for Copier to finish writing to the buffered logger. + c.Wait() + c.Close() + + recvdMsgs := 0 + dec := json.NewDecoder(&jsonBuf) + for { + var msg Message + if err := dec.Decode(&msg); err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + if msg.Source != "stdout" { + t.Fatalf("Wrong Source: %q, should be %q", msg.Source, "stdout") + } + if len(msg.Line) != sizedLogger.BufSize() { + t.Fatalf("Line was not of expected max length %d, was %d", sizedLogger.BufSize(), len(msg.Line)) + } + recvdMsgs++ + } + if recvdMsgs != expectedMsgs { + t.Fatalf("expected to receive %d messages, actually received %d", expectedMsgs, recvdMsgs) + } +} + type BenchmarkLoggerDummy struct { } diff --git a/components/engine/daemon/logger/logger.go b/components/engine/daemon/logger/logger.go index a9d1e7640b..9aa45aabdf 100644 --- a/components/engine/daemon/logger/logger.go +++ b/components/engine/daemon/logger/logger.go @@ -81,6 +81,13 @@ type Logger interface { Close() error } +// SizedLogger is the interface for logging drivers that can control +// the size of buffer used for their messages. +type SizedLogger interface { + Logger + BufSize() int +} + // ReadConfig is the configuration passed into ReadLogs. type ReadConfig struct { Since time.Time