From c2b1f7ca703e35b908b64e9ea3e21427030aff03 Mon Sep 17 00:00:00 2001 From: Liu Hua Date: Mon, 23 Oct 2017 21:00:22 +0800 Subject: [PATCH 1/7] Devicemapper: ignore Nodata errors when delete thin device if thin device is deteled and the metadata exists, you can not delete related containers. This patch ignore Nodata errors for thin device deletion Signed-off-by: Liu Hua Upstream-commit: 8451d03d8ef7457f82112179cd3e300c05a08d3d Component: engine --- components/engine/pkg/devicemapper/devmapper.go | 13 ++++++++++--- components/engine/pkg/devicemapper/devmapper_log.go | 3 +++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/components/engine/pkg/devicemapper/devmapper.go b/components/engine/pkg/devicemapper/devmapper.go index 48618765fb..c72992f7a1 100644 --- a/components/engine/pkg/devicemapper/devmapper.go +++ b/components/engine/pkg/devicemapper/devmapper.go @@ -67,12 +67,14 @@ var ( ErrBusy = errors.New("Device is Busy") ErrDeviceIDExists = errors.New("Device Id Exists") ErrEnxio = errors.New("No such device or address") + ErrEnoData = errors.New("No data available") ) var ( - dmSawBusy bool - dmSawExist bool - dmSawEnxio bool // No Such Device or Address + dmSawBusy bool + dmSawExist bool + dmSawEnxio bool // No Such Device or Address + dmSawEnoData bool // No data available ) type ( @@ -708,10 +710,15 @@ func DeleteDevice(poolName string, deviceID int) error { } dmSawBusy = false + dmSawEnoData = false if err := task.run(); err != nil { if dmSawBusy { return ErrBusy } + if dmSawEnoData { + logrus.Debugf("devicemapper: Device(id: %d) from pool(%s) does not exist", deviceID, poolName) + return nil + } return fmt.Errorf("devicemapper: Error running DeleteDevice %s", err) } return nil diff --git a/components/engine/pkg/devicemapper/devmapper_log.go b/components/engine/pkg/devicemapper/devmapper_log.go index f2ac7da87c..1da75101cf 100644 --- a/components/engine/pkg/devicemapper/devmapper_log.go +++ b/components/engine/pkg/devicemapper/devmapper_log.go @@ -55,6 +55,9 @@ func DevmapperLogCallback(level C.int, file *C.char, line, dmErrnoOrClass C.int, if strings.Contains(msg, "No such device or address") { dmSawEnxio = true } + if strings.Contains(msg, "No data available") { + dmSawEnoData = true + } } if dmLogger != nil { From 169f9cc018bce302fc43cefb8a1416ec61927212 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 2 Jan 2018 23:55:33 -0500 Subject: [PATCH 2/7] Add test case for 35333: Devicemapper: ignore Nodata errors when delete thin device This fix adds a test case for 35333: Devicemapper: ignore Nodata errors when delete thin device Signed-off-by: Yong Tang Upstream-commit: 7c6ef28042c20fdad23cd461ab49b9cfa0c757df Component: engine --- .../engine/integration/container/stop_test.go | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 components/engine/integration/container/stop_test.go diff --git a/components/engine/integration/container/stop_test.go b/components/engine/integration/container/stop_test.go new file mode 100644 index 0000000000..feecc6901f --- /dev/null +++ b/components/engine/integration/container/stop_test.go @@ -0,0 +1,74 @@ +package container + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/client" + "github.com/docker/docker/integration/util/request" + "github.com/gotestyourself/gotestyourself/icmd" + "github.com/gotestyourself/gotestyourself/poll" + "github.com/gotestyourself/gotestyourself/skip" + "github.com/stretchr/testify/require" +) + +func TestDeleteDevicemapper(t *testing.T) { + skip.IfCondition(t, testEnv.DaemonInfo.Driver != "devicemapper") + + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + foo, err := client.ContainerCreate(ctx, + &container.Config{ + Cmd: []string{"echo"}, + Image: "busybox", + }, + &container.HostConfig{}, + &network.NetworkingConfig{}, + "foo", + ) + require.NoError(t, err) + + err = client.ContainerStart(ctx, foo.ID, types.ContainerStartOptions{}) + require.NoError(t, err) + + inspect, err := client.ContainerInspect(ctx, foo.ID) + require.NoError(t, err) + + poll.WaitOn(t, containerIsStopped(ctx, client, foo.ID), poll.WithDelay(100*time.Millisecond)) + + deviceID := inspect.GraphDriver.Data["DeviceId"] + + // Find pool name from device name + deviceName := inspect.GraphDriver.Data["DeviceName"] + devicePrefix := deviceName[:strings.LastIndex(deviceName, "-")] + devicePool := fmt.Sprintf("/dev/mapper/%s-pool", devicePrefix) + + result := icmd.RunCommand("dmsetup", "message", devicePool, "0", fmt.Sprintf("delete %s", deviceID)) + result.Assert(t, icmd.Success) + + err = client.ContainerRemove(ctx, foo.ID, types.ContainerRemoveOptions{}) + require.NoError(t, err) +} + +func containerIsStopped(ctx context.Context, client client.APIClient, containerID string) func(log poll.LogT) poll.Result { + return func(log poll.LogT) poll.Result { + inspect, err := client.ContainerInspect(ctx, containerID) + + switch { + case err != nil: + return poll.Error(err) + case !inspect.State.Running: + return poll.Success() + default: + return poll.Continue("waiting for container to be stopped") + } + } +} From 02da0b92779679712246698d79cf858f2f1daadd Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 15 Jan 2018 11:50:30 +0100 Subject: [PATCH 3/7] Improve API version-middleware test Some improvements to the test; - Combine tests to reduce duplicated code - Add test-cases for empty version in request using the default version - Add test for valid versions in request actually setting the version Signed-off-by: Sebastiaan van Stijn Upstream-commit: 63906d8fae8164c8ef00e1edc957a90b22908bcf Component: engine --- .../api/server/middleware/version_test.go | 97 ++++++++----------- 1 file changed, 38 insertions(+), 59 deletions(-) diff --git a/components/engine/api/server/middleware/version_test.go b/components/engine/api/server/middleware/version_test.go index 9cf7cc247d..59df0675ad 100644 --- a/components/engine/api/server/middleware/version_test.go +++ b/components/engine/api/server/middleware/version_test.go @@ -4,7 +4,6 @@ import ( "net/http" "net/http/httptest" "runtime" - "strings" "testing" "github.com/docker/docker/api/server/httputils" @@ -12,37 +11,16 @@ import ( "golang.org/x/net/context" ) -func TestVersionMiddleware(t *testing.T) { +func TestVersionMiddlewareVersion(t *testing.T) { + defaultVersion := "1.10.0" + minVersion := "1.2.0" + expectedVersion := defaultVersion handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if httputils.VersionFromContext(ctx) == "" { - t.Fatal("Expected version, got empty string") - } + v := httputils.VersionFromContext(ctx) + assert.Equal(t, expectedVersion, v) return nil } - defaultVersion := "1.10.0" - minVersion := "1.2.0" - m := NewVersionMiddleware(defaultVersion, defaultVersion, minVersion) - h := m.WrapHandler(handler) - - req, _ := http.NewRequest("GET", "/containers/json", nil) - resp := httptest.NewRecorder() - ctx := context.Background() - if err := h(ctx, resp, req, map[string]string{}); err != nil { - t.Fatal(err) - } -} - -func TestVersionMiddlewareVersionTooOld(t *testing.T) { - handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if httputils.VersionFromContext(ctx) == "" { - t.Fatal("Expected version, got empty string") - } - return nil - } - - defaultVersion := "1.10.0" - minVersion := "1.2.0" m := NewVersionMiddleware(defaultVersion, defaultVersion, minVersion) h := m.WrapHandler(handler) @@ -50,44 +28,45 @@ func TestVersionMiddlewareVersionTooOld(t *testing.T) { resp := httptest.NewRecorder() ctx := context.Background() - vars := map[string]string{"version": "0.1"} - err := h(ctx, resp, req, vars) - - if !strings.Contains(err.Error(), "client version 0.1 is too old. Minimum supported API version is 1.2.0") { - t.Fatalf("Expected too old client error, got %v", err) + tests := []struct { + reqVersion string + expectedVersion string + errString string + }{ + { + expectedVersion: "1.10.0", + }, + { + reqVersion: "1.9.0", + expectedVersion: "1.9.0", + }, + { + reqVersion: "0.1", + errString: "client version 0.1 is too old. Minimum supported API version is 1.2.0, please upgrade your client to a newer version", + }, + { + reqVersion: "9999.9999", + errString: "client version 9999.9999 is too new. Maximum supported API version is 1.10.0", + }, } -} -func TestVersionMiddlewareVersionTooNew(t *testing.T) { - handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if httputils.VersionFromContext(ctx) == "" { - t.Fatal("Expected version, got empty string") + for _, test := range tests { + expectedVersion = test.expectedVersion + + err := h(ctx, resp, req, map[string]string{"version": test.reqVersion}) + + if test.errString != "" { + assert.EqualError(t, err, test.errString) + } else { + assert.NoError(t, err) } - return nil - } - - defaultVersion := "1.10.0" - minVersion := "1.2.0" - m := NewVersionMiddleware(defaultVersion, defaultVersion, minVersion) - h := m.WrapHandler(handler) - - req, _ := http.NewRequest("GET", "/containers/json", nil) - resp := httptest.NewRecorder() - ctx := context.Background() - - vars := map[string]string{"version": "9999.9999"} - err := h(ctx, resp, req, vars) - - if !strings.Contains(err.Error(), "client version 9999.9999 is too new. Maximum supported API version is 1.10.0") { - t.Fatalf("Expected too new client error, got %v", err) } } func TestVersionMiddlewareWithErrorsReturnsHeaders(t *testing.T) { handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if httputils.VersionFromContext(ctx) == "" { - t.Fatal("Expected version, got empty string") - } + v := httputils.VersionFromContext(ctx) + assert.NotEmpty(t, v) return nil } @@ -102,8 +81,8 @@ func TestVersionMiddlewareWithErrorsReturnsHeaders(t *testing.T) { vars := map[string]string{"version": "0.1"} err := h(ctx, resp, req, vars) - assert.Error(t, err) + hdr := resp.Result().Header assert.Contains(t, hdr.Get("Server"), "Docker/"+defaultVersion) assert.Contains(t, hdr.Get("Server"), runtime.GOOS) From 9dd65d097b12384d7d465fe5846ab49da0b3f59a Mon Sep 17 00:00:00 2001 From: bonczj Date: Tue, 26 Sep 2017 10:22:10 -0400 Subject: [PATCH 4/7] Added tag log option to json-logger Fixes #19803 Updated the json-logger to utilize the common log option 'tag' that can define container/image information to include as part of logging. When the 'tag' log option is not included, there is no change to the log content via the json-logger. When the 'tag' log option is included, the tag will be parsed as a template and the result will be stored within each log entry as the attribute 'tag'. Update: Removing test added to integration_cli as those have been deprecated. Update: Using proper test calls (require and assert) in jsonfilelog_test.go based on review. Update: Added new unit test configs for logs with tag. Updated unit test error checking. Update: Cleanup check in jsonlogbytes_test.go to match pending changes in PR #34946. Update: Merging to correct conflicts from PR #34946. Signed-off-by: bonczj Upstream-commit: 5f50f4f511cd84e79bf005817af346b1764df27f Component: engine --- .../daemon/logger/jsonfilelog/jsonfilelog.go | 14 +++++- .../logger/jsonfilelog/jsonfilelog_test.go | 45 ++++++++++++++++++- .../logger/jsonfilelog/jsonlog/jsonlog.go | 3 ++ .../jsonfilelog/jsonlog/jsonlogbytes.go | 10 +++++ .../jsonfilelog/jsonlog/jsonlogbytes_test.go | 2 + .../daemon/logger/jsonfilelog/read_test.go | 2 +- 6 files changed, 72 insertions(+), 4 deletions(-) diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go index 7aa92f3d37..19505db3fc 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go @@ -27,6 +27,7 @@ type JSONFileLogger struct { closed bool writer *loggerutils.LogFile readers map[*logger.LogWatcher]struct{} // stores the active log followers + tag string // tag values requested by the user to log } func init() { @@ -61,6 +62,12 @@ func New(info logger.Info) (logger.Logger, error) { } } + // no default template. only use a tag if the user asked for it + tag, err := loggerutils.ParseLogTag(info, "") + if err != nil { + return nil, err + } + var extra []byte attrs, err := info.ExtraAttributes(nil) if err != nil { @@ -76,7 +83,7 @@ func New(info logger.Info) (logger.Logger, error) { buf := bytes.NewBuffer(nil) marshalFunc := func(msg *logger.Message) ([]byte, error) { - if err := marshalMessage(msg, extra, buf); err != nil { + if err := marshalMessage(msg, extra, buf, tag); err != nil { return nil, err } b := buf.Bytes() @@ -92,6 +99,7 @@ func New(info logger.Info) (logger.Logger, error) { return &JSONFileLogger{ writer: writer, readers: make(map[*logger.LogWatcher]struct{}), + tag: tag, }, nil } @@ -103,7 +111,7 @@ func (l *JSONFileLogger) Log(msg *logger.Message) error { return err } -func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error { +func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer, tag string) error { logLine := msg.Line if !msg.Partial { logLine = append(msg.Line, '\n') @@ -113,6 +121,7 @@ func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffe Stream: msg.Source, Created: msg.Timestamp, RawAttrs: extra, + Tag: tag, }).MarshalJSONBuf(buf) if err != nil { return errors.Wrap(err, "error writing log message to buffer") @@ -130,6 +139,7 @@ func ValidateLogOpt(cfg map[string]string) error { case "labels": case "env": case "env-regex": + case "tag": default: return fmt.Errorf("unknown log opt '%s' for json-file log driver", key) } diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go index 893c054669..e1fd46e7de 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go @@ -57,6 +57,49 @@ func TestJSONFileLogger(t *testing.T) { } } +func TestJSONFileLoggerWithTags(t *testing.T) { + cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657" + cname := "test-container" + tmp, err := ioutil.TempDir("", "docker-logger-") + + require.NoError(t, err) + + defer os.RemoveAll(tmp) + filename := filepath.Join(tmp, "container.log") + l, err := New(logger.Info{ + Config: map[string]string{ + "tag": "{{.ID}}/{{.Name}}", // first 12 characters of ContainerID and full ContainerName + }, + ContainerID: cid, + ContainerName: cname, + LogPath: filename, + }) + + require.NoError(t, err) + defer l.Close() + + err = l.Log(&logger.Message{Line: []byte("line1"), Source: "src1"}) + require.NoError(t, err) + + err = l.Log(&logger.Message{Line: []byte("line2"), Source: "src2"}) + require.NoError(t, err) + + err = l.Log(&logger.Message{Line: []byte("line3"), Source: "src3"}) + require.NoError(t, err) + + res, err := ioutil.ReadFile(filename) + require.NoError(t, err) + + expected := `{"log":"line1\n","stream":"src1","tag":"a7317399f3f8/test-container","time":"0001-01-01T00:00:00Z"} +{"log":"line2\n","stream":"src2","tag":"a7317399f3f8/test-container","time":"0001-01-01T00:00:00Z"} +{"log":"line3\n","stream":"src3","tag":"a7317399f3f8/test-container","time":"0001-01-01T00:00:00Z"} +` + + if string(res) != expected { + t.Fatalf("Wrong log content: %q, expected %q", res, expected) + } +} + func BenchmarkJSONFileLoggerLog(b *testing.B) { tmp := fs.NewDir(b, "bench-jsonfilelog") defer tmp.Remove() @@ -82,7 +125,7 @@ func BenchmarkJSONFileLoggerLog(b *testing.B) { } buf := bytes.NewBuffer(nil) - require.NoError(b, marshalMessage(msg, nil, buf)) + require.NoError(b, marshalMessage(msg, nil, buf, "")) b.SetBytes(int64(buf.Len())) b.ResetTimer() diff --git a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlog.go b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlog.go index 549e355855..3121a8539f 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlog.go @@ -14,6 +14,8 @@ type JSONLog struct { Created time.Time `json:"time"` // Attrs is the list of extra attributes provided by the user Attrs map[string]string `json:"attrs,omitempty"` + // Tags requested the operator + Tag string `json:"tag,omitempty"` } // Reset all fields to their zero value. @@ -22,4 +24,5 @@ func (jl *JSONLog) Reset() { jl.Stream = "" jl.Created = time.Time{} jl.Attrs = make(map[string]string) + jl.Tag = "" } diff --git a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes.go b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes.go index 37604ae549..00e2d00f50 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes.go @@ -12,6 +12,7 @@ type JSONLogs struct { Log []byte `json:"log,omitempty"` Stream string `json:"stream,omitempty"` Created time.Time `json:"time"` + Tag string `json:"tag,omitempty"` // json-encoded bytes RawAttrs json.RawMessage `json:"attrs,omitempty"` @@ -37,6 +38,15 @@ func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error { buf.WriteString(`"stream":`) ffjsonWriteJSONBytesAsString(buf, []byte(mj.Stream)) } + if len(mj.Tag) > 0 { + if first { + first = false + } else { + buf.WriteString(`,`) + } + buf.WriteString(`"tag":`) + ffjsonWriteJSONBytesAsString(buf, []byte(mj.Tag)) + } if len(mj.RawAttrs) > 0 { if first { first = false diff --git a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go index 4645cd4faa..cb506a3006 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go @@ -29,6 +29,8 @@ func TestJSONLogsMarshalJSONBuf(t *testing.T) { {Log: []byte{0x7F}}: `^{\"log\":\"\x7f\",\"time\":`, // with raw attributes {Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":`, + // with Tag set + {Log: []byte("A log line with tag"), Tag: "test-tag"}: `^{\"log\":\"A log line with tag\",\"tag\":\"test-tag\",\"time\":`, } for jsonLog, expression := range logs { var buf bytes.Buffer diff --git a/components/engine/daemon/logger/jsonfilelog/read_test.go b/components/engine/daemon/logger/jsonfilelog/read_test.go index 599fdf9336..ddc1265f03 100644 --- a/components/engine/daemon/logger/jsonfilelog/read_test.go +++ b/components/engine/daemon/logger/jsonfilelog/read_test.go @@ -35,7 +35,7 @@ func BenchmarkJSONFileLoggerReadLogs(b *testing.B) { } buf := bytes.NewBuffer(nil) - require.NoError(b, marshalMessage(msg, nil, buf)) + require.NoError(b, marshalMessage(msg, nil, buf, "")) b.SetBytes(int64(buf.Len())) b.ResetTimer() From 3cf8a0c4429a7b6bb6da2450fba3894898837059 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 8 Jan 2018 00:54:58 +0000 Subject: [PATCH 5/7] Carry 34248 Added tag log option to json-logger and use RawAttrs This fix carries PR 34248: Added tag log option to json-logger This fix changes to use RawAttrs based on review feedback. This fix fixes 19803, this fix closes 34248. Signed-off-by: Yong Tang Upstream-commit: e77267c5a682e2c5aaa32469f2c83c2479d57566 Component: engine --- .../daemon/logger/jsonfilelog/jsonfilelog.go | 17 ++++++++++------- .../logger/jsonfilelog/jsonfilelog_test.go | 14 ++++++-------- .../logger/jsonfilelog/jsonlog/jsonlog.go | 3 --- .../logger/jsonfilelog/jsonlog/jsonlogbytes.go | 10 ---------- .../jsonfilelog/jsonlog/jsonlogbytes_test.go | 2 +- .../daemon/logger/jsonfilelog/read_test.go | 2 +- 6 files changed, 18 insertions(+), 30 deletions(-) diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go index 19505db3fc..720a7c3b61 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go @@ -62,17 +62,21 @@ func New(info logger.Info) (logger.Logger, error) { } } + attrs, err := info.ExtraAttributes(nil) + if err != nil { + return nil, err + } + // no default template. only use a tag if the user asked for it tag, err := loggerutils.ParseLogTag(info, "") if err != nil { return nil, err } + if tag != "" { + attrs["tag"] = tag + } var extra []byte - attrs, err := info.ExtraAttributes(nil) - if err != nil { - return nil, err - } if len(attrs) > 0 { var err error extra, err = json.Marshal(attrs) @@ -83,7 +87,7 @@ func New(info logger.Info) (logger.Logger, error) { buf := bytes.NewBuffer(nil) marshalFunc := func(msg *logger.Message) ([]byte, error) { - if err := marshalMessage(msg, extra, buf, tag); err != nil { + if err := marshalMessage(msg, extra, buf); err != nil { return nil, err } b := buf.Bytes() @@ -111,7 +115,7 @@ func (l *JSONFileLogger) Log(msg *logger.Message) error { return err } -func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer, tag string) error { +func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error { logLine := msg.Line if !msg.Partial { logLine = append(msg.Line, '\n') @@ -121,7 +125,6 @@ func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffe Stream: msg.Source, Created: msg.Timestamp, RawAttrs: extra, - Tag: tag, }).MarshalJSONBuf(buf) if err != nil { return errors.Wrap(err, "error writing log message to buffer") diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go index e1fd46e7de..e988e862fd 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go @@ -14,6 +14,7 @@ import ( "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog" "github.com/gotestyourself/gotestyourself/fs" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -90,14 +91,11 @@ func TestJSONFileLoggerWithTags(t *testing.T) { res, err := ioutil.ReadFile(filename) require.NoError(t, err) - expected := `{"log":"line1\n","stream":"src1","tag":"a7317399f3f8/test-container","time":"0001-01-01T00:00:00Z"} -{"log":"line2\n","stream":"src2","tag":"a7317399f3f8/test-container","time":"0001-01-01T00:00:00Z"} -{"log":"line3\n","stream":"src3","tag":"a7317399f3f8/test-container","time":"0001-01-01T00:00:00Z"} + expected := `{"log":"line1\n","stream":"src1","attrs":{"tag":"a7317399f3f8/test-container"},"time":"0001-01-01T00:00:00Z"} +{"log":"line2\n","stream":"src2","attrs":{"tag":"a7317399f3f8/test-container"},"time":"0001-01-01T00:00:00Z"} +{"log":"line3\n","stream":"src3","attrs":{"tag":"a7317399f3f8/test-container"},"time":"0001-01-01T00:00:00Z"} ` - - if string(res) != expected { - t.Fatalf("Wrong log content: %q, expected %q", res, expected) - } + assert.Equal(t, expected, string(res)) } func BenchmarkJSONFileLoggerLog(b *testing.B) { @@ -125,7 +123,7 @@ func BenchmarkJSONFileLoggerLog(b *testing.B) { } buf := bytes.NewBuffer(nil) - require.NoError(b, marshalMessage(msg, nil, buf, "")) + require.NoError(b, marshalMessage(msg, nil, buf)) b.SetBytes(int64(buf.Len())) b.ResetTimer() diff --git a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlog.go b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlog.go index 3121a8539f..549e355855 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlog.go @@ -14,8 +14,6 @@ type JSONLog struct { Created time.Time `json:"time"` // Attrs is the list of extra attributes provided by the user Attrs map[string]string `json:"attrs,omitempty"` - // Tags requested the operator - Tag string `json:"tag,omitempty"` } // Reset all fields to their zero value. @@ -24,5 +22,4 @@ func (jl *JSONLog) Reset() { jl.Stream = "" jl.Created = time.Time{} jl.Attrs = make(map[string]string) - jl.Tag = "" } diff --git a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes.go b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes.go index 00e2d00f50..37604ae549 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes.go @@ -12,7 +12,6 @@ type JSONLogs struct { Log []byte `json:"log,omitempty"` Stream string `json:"stream,omitempty"` Created time.Time `json:"time"` - Tag string `json:"tag,omitempty"` // json-encoded bytes RawAttrs json.RawMessage `json:"attrs,omitempty"` @@ -38,15 +37,6 @@ func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error { buf.WriteString(`"stream":`) ffjsonWriteJSONBytesAsString(buf, []byte(mj.Stream)) } - if len(mj.Tag) > 0 { - if first { - first = false - } else { - buf.WriteString(`,`) - } - buf.WriteString(`"tag":`) - ffjsonWriteJSONBytesAsString(buf, []byte(mj.Tag)) - } if len(mj.RawAttrs) > 0 { if first { first = false diff --git a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go index cb506a3006..e52d32c037 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go @@ -30,7 +30,7 @@ func TestJSONLogsMarshalJSONBuf(t *testing.T) { // with raw attributes {Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":`, // with Tag set - {Log: []byte("A log line with tag"), Tag: "test-tag"}: `^{\"log\":\"A log line with tag\",\"tag\":\"test-tag\",\"time\":`, + {Log: []byte("A log line with tag"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line with tag\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":`, } for jsonLog, expression := range logs { var buf bytes.Buffer diff --git a/components/engine/daemon/logger/jsonfilelog/read_test.go b/components/engine/daemon/logger/jsonfilelog/read_test.go index ddc1265f03..599fdf9336 100644 --- a/components/engine/daemon/logger/jsonfilelog/read_test.go +++ b/components/engine/daemon/logger/jsonfilelog/read_test.go @@ -35,7 +35,7 @@ func BenchmarkJSONFileLoggerReadLogs(b *testing.B) { } buf := bytes.NewBuffer(nil) - require.NoError(b, marshalMessage(msg, nil, buf, "")) + require.NoError(b, marshalMessage(msg, nil, buf)) b.SetBytes(int64(buf.Len())) b.ResetTimer() From c2b247fce698f4661ba8dbbb57da26a2330dd7a4 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 15 Jan 2018 16:18:22 +0100 Subject: [PATCH 6/7] Move reload-related functions to reload.go Signed-off-by: Sebastiaan van Stijn Upstream-commit: 6121a8429b9d3a6d20e900c521c2f50fff5db406 Component: engine --- components/engine/daemon/daemon_unix.go | 46 ------------------ components/engine/daemon/daemon_windows.go | 6 --- components/engine/daemon/reload_unix.go | 56 ++++++++++++++++++++++ components/engine/daemon/reload_windows.go | 9 ++++ 4 files changed, 65 insertions(+), 52 deletions(-) create mode 100644 components/engine/daemon/reload_unix.go create mode 100644 components/engine/daemon/reload_windows.go diff --git a/components/engine/daemon/daemon_unix.go b/components/engine/daemon/daemon_unix.go index bf8741096c..74046147a1 100644 --- a/components/engine/daemon/daemon_unix.go +++ b/components/engine/daemon/daemon_unix.go @@ -4,7 +4,6 @@ package daemon import ( "bufio" - "bytes" "context" "fmt" "io/ioutil" @@ -680,51 +679,6 @@ func (daemon *Daemon) initRuntimes(runtimes map[string]types.Runtime) (err error return nil } -// reloadPlatform updates configuration with platform specific options -// and updates the passed attributes -func (daemon *Daemon) reloadPlatform(conf *config.Config, attributes map[string]string) error { - if err := conf.ValidatePlatformConfig(); err != nil { - return err - } - - if conf.IsValueSet("runtimes") { - // Always set the default one - conf.Runtimes[config.StockRuntimeName] = types.Runtime{Path: DefaultRuntimeBinary} - if err := daemon.initRuntimes(conf.Runtimes); err != nil { - return err - } - daemon.configStore.Runtimes = conf.Runtimes - } - - if conf.DefaultRuntime != "" { - daemon.configStore.DefaultRuntime = conf.DefaultRuntime - } - - if conf.IsValueSet("default-shm-size") { - daemon.configStore.ShmSize = conf.ShmSize - } - - if conf.IpcMode != "" { - daemon.configStore.IpcMode = conf.IpcMode - } - - // Update attributes - var runtimeList bytes.Buffer - for name, rt := range daemon.configStore.Runtimes { - if runtimeList.Len() > 0 { - runtimeList.WriteRune(' ') - } - runtimeList.WriteString(fmt.Sprintf("%s:%s", name, rt)) - } - - attributes["runtimes"] = runtimeList.String() - attributes["default-runtime"] = daemon.configStore.DefaultRuntime - attributes["default-shm-size"] = fmt.Sprintf("%d", daemon.configStore.ShmSize) - attributes["default-ipc-mode"] = daemon.configStore.IpcMode - - return nil -} - // verifyDaemonSettings performs validation of daemon config struct func verifyDaemonSettings(conf *config.Config) error { // Check for mutually incompatible config options diff --git a/components/engine/daemon/daemon_windows.go b/components/engine/daemon/daemon_windows.go index 77e12e908c..e3b2072479 100644 --- a/components/engine/daemon/daemon_windows.go +++ b/components/engine/daemon/daemon_windows.go @@ -207,12 +207,6 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes. return warnings, err } -// reloadPlatform updates configuration with platform specific options -// and updates the passed attributes -func (daemon *Daemon) reloadPlatform(config *config.Config, attributes map[string]string) error { - return nil -} - // verifyDaemonSettings performs validation of daemon config struct func verifyDaemonSettings(config *config.Config) error { return nil diff --git a/components/engine/daemon/reload_unix.go b/components/engine/daemon/reload_unix.go new file mode 100644 index 0000000000..cdc17452b9 --- /dev/null +++ b/components/engine/daemon/reload_unix.go @@ -0,0 +1,56 @@ +// +build linux freebsd + +package daemon + +import ( + "bytes" + "fmt" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/daemon/config" +) + +// reloadPlatform updates configuration with platform specific options +// and updates the passed attributes +func (daemon *Daemon) reloadPlatform(conf *config.Config, attributes map[string]string) error { + if err := conf.ValidatePlatformConfig(); err != nil { + return err + } + + if conf.IsValueSet("runtimes") { + // Always set the default one + conf.Runtimes[config.StockRuntimeName] = types.Runtime{Path: DefaultRuntimeBinary} + if err := daemon.initRuntimes(conf.Runtimes); err != nil { + return err + } + daemon.configStore.Runtimes = conf.Runtimes + } + + if conf.DefaultRuntime != "" { + daemon.configStore.DefaultRuntime = conf.DefaultRuntime + } + + if conf.IsValueSet("default-shm-size") { + daemon.configStore.ShmSize = conf.ShmSize + } + + if conf.IpcMode != "" { + daemon.configStore.IpcMode = conf.IpcMode + } + + // Update attributes + var runtimeList bytes.Buffer + for name, rt := range daemon.configStore.Runtimes { + if runtimeList.Len() > 0 { + runtimeList.WriteRune(' ') + } + runtimeList.WriteString(fmt.Sprintf("%s:%s", name, rt)) + } + + attributes["runtimes"] = runtimeList.String() + attributes["default-runtime"] = daemon.configStore.DefaultRuntime + attributes["default-shm-size"] = fmt.Sprintf("%d", daemon.configStore.ShmSize) + attributes["default-ipc-mode"] = daemon.configStore.IpcMode + + return nil +} diff --git a/components/engine/daemon/reload_windows.go b/components/engine/daemon/reload_windows.go new file mode 100644 index 0000000000..b7a4031eac --- /dev/null +++ b/components/engine/daemon/reload_windows.go @@ -0,0 +1,9 @@ +package daemon + +import "github.com/docker/docker/daemon/config" + +// reloadPlatform updates configuration with platform specific options +// and updates the passed attributes +func (daemon *Daemon) reloadPlatform(config *config.Config, attributes map[string]string) error { + return nil +} From 36e0e57cbe4728275b8c6da415ba4f2dba89e31d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 15 Jan 2018 17:11:05 +0100 Subject: [PATCH 7/7] Log active configuration when reloading When succesfully reloading the daemon configuration, print a message in the logs with the active configuration: INFO[2018-01-15T15:36:20.901688317Z] Got signal to reload configuration, reloading from: /etc/docker/daemon.json INFO[2018-01-14T02:23:48.782769942Z] Reloaded configuration: {"mtu":1500,"pidfile":"/var/run/docker.pid","data-root":"/var/lib/docker","exec-root":"/var/run/docker","group":"docker","deprecated-key-path":"/etc/docker/key.json","max-concurrent-downloads":3,"max-concurrent-uploads":5,"shutdown-timeout":15,"debug":true,"hosts":["unix:///var/run/docker.sock"],"log-level":"info","swarm-default-advertise-addr":"","metrics-addr":"","log-driver":"json-file","ip":"0.0.0.0","icc":true,"iptables":true,"ip-forward":true,"ip-masq":true,"userland-proxy":true,"disable-legacy-registry":true,"experimental":false,"network-control-plane-mtu":1500,"runtimes":{"runc":{"path":"docker-runc"}},"default-runtime":"runc","oom-score-adjust":-500,"default-shm-size":67108864,"default-ipc-mode":"shareable"} Signed-off-by: Sebastiaan van Stijn Upstream-commit: 8378dcf46d017c70df97d6f851e0196b113b422e Component: engine --- components/engine/daemon/config/config.go | 2 +- components/engine/daemon/reload.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/components/engine/daemon/config/config.go b/components/engine/daemon/config/config.go index e3ed530028..1466c0b71b 100644 --- a/components/engine/daemon/config/config.go +++ b/components/engine/daemon/config/config.go @@ -167,7 +167,7 @@ type CommonConfig struct { sync.Mutex // FIXME(vdemeester) This part is not that clear and is mainly dependent on cli flags // It should probably be handled outside this package. - ValuesSet map[string]interface{} + ValuesSet map[string]interface{} `json:"-"` Experimental bool `json:"experimental"` // Experimental indicates whether experimental features should be exposed or not diff --git a/components/engine/daemon/reload.go b/components/engine/daemon/reload.go index 0bbda29234..5a73c9ef92 100644 --- a/components/engine/daemon/reload.go +++ b/components/engine/daemon/reload.go @@ -27,11 +27,14 @@ func (daemon *Daemon) Reload(conf *config.Config) (err error) { attributes := map[string]string{} defer func() { + jsonString, _ := json.Marshal(daemon.configStore) + // we're unlocking here, because // LogDaemonEventWithAttributes() -> SystemInfo() -> GetAllRuntimes() // holds that lock too. daemon.configStore.Unlock() if err == nil { + logrus.Infof("Reloaded configuration: %s", jsonString) daemon.LogDaemonEventWithAttributes("reload", attributes) } }()