diff --git a/components/engine/api/server/middleware/version_test.go b/components/engine/api/server/middleware/version_test.go index 9cf7cc247d..59df0675ad 100644 --- a/components/engine/api/server/middleware/version_test.go +++ b/components/engine/api/server/middleware/version_test.go @@ -4,7 +4,6 @@ import ( "net/http" "net/http/httptest" "runtime" - "strings" "testing" "github.com/docker/docker/api/server/httputils" @@ -12,37 +11,16 @@ import ( "golang.org/x/net/context" ) -func TestVersionMiddleware(t *testing.T) { +func TestVersionMiddlewareVersion(t *testing.T) { + defaultVersion := "1.10.0" + minVersion := "1.2.0" + expectedVersion := defaultVersion handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if httputils.VersionFromContext(ctx) == "" { - t.Fatal("Expected version, got empty string") - } + v := httputils.VersionFromContext(ctx) + assert.Equal(t, expectedVersion, v) return nil } - defaultVersion := "1.10.0" - minVersion := "1.2.0" - m := NewVersionMiddleware(defaultVersion, defaultVersion, minVersion) - h := m.WrapHandler(handler) - - req, _ := http.NewRequest("GET", "/containers/json", nil) - resp := httptest.NewRecorder() - ctx := context.Background() - if err := h(ctx, resp, req, map[string]string{}); err != nil { - t.Fatal(err) - } -} - -func TestVersionMiddlewareVersionTooOld(t *testing.T) { - handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if httputils.VersionFromContext(ctx) == "" { - t.Fatal("Expected version, got empty string") - } - return nil - } - - defaultVersion := "1.10.0" - minVersion := "1.2.0" m := NewVersionMiddleware(defaultVersion, defaultVersion, minVersion) h := m.WrapHandler(handler) @@ -50,44 +28,45 @@ func TestVersionMiddlewareVersionTooOld(t *testing.T) { resp := httptest.NewRecorder() ctx := context.Background() - vars := map[string]string{"version": "0.1"} - err := h(ctx, resp, req, vars) - - if !strings.Contains(err.Error(), "client version 0.1 is too old. Minimum supported API version is 1.2.0") { - t.Fatalf("Expected too old client error, got %v", err) + tests := []struct { + reqVersion string + expectedVersion string + errString string + }{ + { + expectedVersion: "1.10.0", + }, + { + reqVersion: "1.9.0", + expectedVersion: "1.9.0", + }, + { + reqVersion: "0.1", + errString: "client version 0.1 is too old. Minimum supported API version is 1.2.0, please upgrade your client to a newer version", + }, + { + reqVersion: "9999.9999", + errString: "client version 9999.9999 is too new. Maximum supported API version is 1.10.0", + }, } -} -func TestVersionMiddlewareVersionTooNew(t *testing.T) { - handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if httputils.VersionFromContext(ctx) == "" { - t.Fatal("Expected version, got empty string") + for _, test := range tests { + expectedVersion = test.expectedVersion + + err := h(ctx, resp, req, map[string]string{"version": test.reqVersion}) + + if test.errString != "" { + assert.EqualError(t, err, test.errString) + } else { + assert.NoError(t, err) } - return nil - } - - defaultVersion := "1.10.0" - minVersion := "1.2.0" - m := NewVersionMiddleware(defaultVersion, defaultVersion, minVersion) - h := m.WrapHandler(handler) - - req, _ := http.NewRequest("GET", "/containers/json", nil) - resp := httptest.NewRecorder() - ctx := context.Background() - - vars := map[string]string{"version": "9999.9999"} - err := h(ctx, resp, req, vars) - - if !strings.Contains(err.Error(), "client version 9999.9999 is too new. Maximum supported API version is 1.10.0") { - t.Fatalf("Expected too new client error, got %v", err) } } func TestVersionMiddlewareWithErrorsReturnsHeaders(t *testing.T) { handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if httputils.VersionFromContext(ctx) == "" { - t.Fatal("Expected version, got empty string") - } + v := httputils.VersionFromContext(ctx) + assert.NotEmpty(t, v) return nil } @@ -102,8 +81,8 @@ func TestVersionMiddlewareWithErrorsReturnsHeaders(t *testing.T) { vars := map[string]string{"version": "0.1"} err := h(ctx, resp, req, vars) - assert.Error(t, err) + hdr := resp.Result().Header assert.Contains(t, hdr.Get("Server"), "Docker/"+defaultVersion) assert.Contains(t, hdr.Get("Server"), runtime.GOOS) diff --git a/components/engine/daemon/config/config.go b/components/engine/daemon/config/config.go index e3ed530028..1466c0b71b 100644 --- a/components/engine/daemon/config/config.go +++ b/components/engine/daemon/config/config.go @@ -167,7 +167,7 @@ type CommonConfig struct { sync.Mutex // FIXME(vdemeester) This part is not that clear and is mainly dependent on cli flags // It should probably be handled outside this package. - ValuesSet map[string]interface{} + ValuesSet map[string]interface{} `json:"-"` Experimental bool `json:"experimental"` // Experimental indicates whether experimental features should be exposed or not diff --git a/components/engine/daemon/daemon_unix.go b/components/engine/daemon/daemon_unix.go index bf8741096c..74046147a1 100644 --- a/components/engine/daemon/daemon_unix.go +++ b/components/engine/daemon/daemon_unix.go @@ -4,7 +4,6 @@ package daemon import ( "bufio" - "bytes" "context" "fmt" "io/ioutil" @@ -680,51 +679,6 @@ func (daemon *Daemon) initRuntimes(runtimes map[string]types.Runtime) (err error return nil } -// reloadPlatform updates configuration with platform specific options -// and updates the passed attributes -func (daemon *Daemon) reloadPlatform(conf *config.Config, attributes map[string]string) error { - if err := conf.ValidatePlatformConfig(); err != nil { - return err - } - - if conf.IsValueSet("runtimes") { - // Always set the default one - conf.Runtimes[config.StockRuntimeName] = types.Runtime{Path: DefaultRuntimeBinary} - if err := daemon.initRuntimes(conf.Runtimes); err != nil { - return err - } - daemon.configStore.Runtimes = conf.Runtimes - } - - if conf.DefaultRuntime != "" { - daemon.configStore.DefaultRuntime = conf.DefaultRuntime - } - - if conf.IsValueSet("default-shm-size") { - daemon.configStore.ShmSize = conf.ShmSize - } - - if conf.IpcMode != "" { - daemon.configStore.IpcMode = conf.IpcMode - } - - // Update attributes - var runtimeList bytes.Buffer - for name, rt := range daemon.configStore.Runtimes { - if runtimeList.Len() > 0 { - runtimeList.WriteRune(' ') - } - runtimeList.WriteString(fmt.Sprintf("%s:%s", name, rt)) - } - - attributes["runtimes"] = runtimeList.String() - attributes["default-runtime"] = daemon.configStore.DefaultRuntime - attributes["default-shm-size"] = fmt.Sprintf("%d", daemon.configStore.ShmSize) - attributes["default-ipc-mode"] = daemon.configStore.IpcMode - - return nil -} - // verifyDaemonSettings performs validation of daemon config struct func verifyDaemonSettings(conf *config.Config) error { // Check for mutually incompatible config options diff --git a/components/engine/daemon/daemon_windows.go b/components/engine/daemon/daemon_windows.go index 77e12e908c..e3b2072479 100644 --- a/components/engine/daemon/daemon_windows.go +++ b/components/engine/daemon/daemon_windows.go @@ -207,12 +207,6 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes. return warnings, err } -// reloadPlatform updates configuration with platform specific options -// and updates the passed attributes -func (daemon *Daemon) reloadPlatform(config *config.Config, attributes map[string]string) error { - return nil -} - // verifyDaemonSettings performs validation of daemon config struct func verifyDaemonSettings(config *config.Config) error { return nil diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go index 7aa92f3d37..720a7c3b61 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go @@ -27,6 +27,7 @@ type JSONFileLogger struct { closed bool writer *loggerutils.LogFile readers map[*logger.LogWatcher]struct{} // stores the active log followers + tag string // tag values requested by the user to log } func init() { @@ -61,11 +62,21 @@ func New(info logger.Info) (logger.Logger, error) { } } - var extra []byte attrs, err := info.ExtraAttributes(nil) if err != nil { return nil, err } + + // no default template. only use a tag if the user asked for it + tag, err := loggerutils.ParseLogTag(info, "") + if err != nil { + return nil, err + } + if tag != "" { + attrs["tag"] = tag + } + + var extra []byte if len(attrs) > 0 { var err error extra, err = json.Marshal(attrs) @@ -92,6 +103,7 @@ func New(info logger.Info) (logger.Logger, error) { return &JSONFileLogger{ writer: writer, readers: make(map[*logger.LogWatcher]struct{}), + tag: tag, }, nil } @@ -130,6 +142,7 @@ func ValidateLogOpt(cfg map[string]string) error { case "labels": case "env": case "env-regex": + case "tag": default: return fmt.Errorf("unknown log opt '%s' for json-file log driver", key) } diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go index 893c054669..e988e862fd 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go @@ -14,6 +14,7 @@ import ( "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog" "github.com/gotestyourself/gotestyourself/fs" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -57,6 +58,46 @@ func TestJSONFileLogger(t *testing.T) { } } +func TestJSONFileLoggerWithTags(t *testing.T) { + cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657" + cname := "test-container" + tmp, err := ioutil.TempDir("", "docker-logger-") + + require.NoError(t, err) + + defer os.RemoveAll(tmp) + filename := filepath.Join(tmp, "container.log") + l, err := New(logger.Info{ + Config: map[string]string{ + "tag": "{{.ID}}/{{.Name}}", // first 12 characters of ContainerID and full ContainerName + }, + ContainerID: cid, + ContainerName: cname, + LogPath: filename, + }) + + require.NoError(t, err) + defer l.Close() + + err = l.Log(&logger.Message{Line: []byte("line1"), Source: "src1"}) + require.NoError(t, err) + + err = l.Log(&logger.Message{Line: []byte("line2"), Source: "src2"}) + require.NoError(t, err) + + err = l.Log(&logger.Message{Line: []byte("line3"), Source: "src3"}) + require.NoError(t, err) + + res, err := ioutil.ReadFile(filename) + require.NoError(t, err) + + expected := `{"log":"line1\n","stream":"src1","attrs":{"tag":"a7317399f3f8/test-container"},"time":"0001-01-01T00:00:00Z"} +{"log":"line2\n","stream":"src2","attrs":{"tag":"a7317399f3f8/test-container"},"time":"0001-01-01T00:00:00Z"} +{"log":"line3\n","stream":"src3","attrs":{"tag":"a7317399f3f8/test-container"},"time":"0001-01-01T00:00:00Z"} +` + assert.Equal(t, expected, string(res)) +} + func BenchmarkJSONFileLoggerLog(b *testing.B) { tmp := fs.NewDir(b, "bench-jsonfilelog") defer tmp.Remove() diff --git a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go index 4645cd4faa..e52d32c037 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonlog/jsonlogbytes_test.go @@ -29,6 +29,8 @@ func TestJSONLogsMarshalJSONBuf(t *testing.T) { {Log: []byte{0x7F}}: `^{\"log\":\"\x7f\",\"time\":`, // with raw attributes {Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":`, + // with Tag set + {Log: []byte("A log line with tag"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line with tag\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":`, } for jsonLog, expression := range logs { var buf bytes.Buffer diff --git a/components/engine/daemon/reload.go b/components/engine/daemon/reload.go index 0bbda29234..5a73c9ef92 100644 --- a/components/engine/daemon/reload.go +++ b/components/engine/daemon/reload.go @@ -27,11 +27,14 @@ func (daemon *Daemon) Reload(conf *config.Config) (err error) { attributes := map[string]string{} defer func() { + jsonString, _ := json.Marshal(daemon.configStore) + // we're unlocking here, because // LogDaemonEventWithAttributes() -> SystemInfo() -> GetAllRuntimes() // holds that lock too. daemon.configStore.Unlock() if err == nil { + logrus.Infof("Reloaded configuration: %s", jsonString) daemon.LogDaemonEventWithAttributes("reload", attributes) } }() diff --git a/components/engine/daemon/reload_unix.go b/components/engine/daemon/reload_unix.go new file mode 100644 index 0000000000..cdc17452b9 --- /dev/null +++ b/components/engine/daemon/reload_unix.go @@ -0,0 +1,56 @@ +// +build linux freebsd + +package daemon + +import ( + "bytes" + "fmt" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/daemon/config" +) + +// reloadPlatform updates configuration with platform specific options +// and updates the passed attributes +func (daemon *Daemon) reloadPlatform(conf *config.Config, attributes map[string]string) error { + if err := conf.ValidatePlatformConfig(); err != nil { + return err + } + + if conf.IsValueSet("runtimes") { + // Always set the default one + conf.Runtimes[config.StockRuntimeName] = types.Runtime{Path: DefaultRuntimeBinary} + if err := daemon.initRuntimes(conf.Runtimes); err != nil { + return err + } + daemon.configStore.Runtimes = conf.Runtimes + } + + if conf.DefaultRuntime != "" { + daemon.configStore.DefaultRuntime = conf.DefaultRuntime + } + + if conf.IsValueSet("default-shm-size") { + daemon.configStore.ShmSize = conf.ShmSize + } + + if conf.IpcMode != "" { + daemon.configStore.IpcMode = conf.IpcMode + } + + // Update attributes + var runtimeList bytes.Buffer + for name, rt := range daemon.configStore.Runtimes { + if runtimeList.Len() > 0 { + runtimeList.WriteRune(' ') + } + runtimeList.WriteString(fmt.Sprintf("%s:%s", name, rt)) + } + + attributes["runtimes"] = runtimeList.String() + attributes["default-runtime"] = daemon.configStore.DefaultRuntime + attributes["default-shm-size"] = fmt.Sprintf("%d", daemon.configStore.ShmSize) + attributes["default-ipc-mode"] = daemon.configStore.IpcMode + + return nil +} diff --git a/components/engine/daemon/reload_windows.go b/components/engine/daemon/reload_windows.go new file mode 100644 index 0000000000..b7a4031eac --- /dev/null +++ b/components/engine/daemon/reload_windows.go @@ -0,0 +1,9 @@ +package daemon + +import "github.com/docker/docker/daemon/config" + +// reloadPlatform updates configuration with platform specific options +// and updates the passed attributes +func (daemon *Daemon) reloadPlatform(config *config.Config, attributes map[string]string) error { + return nil +} diff --git a/components/engine/integration/container/stop_test.go b/components/engine/integration/container/stop_test.go new file mode 100644 index 0000000000..feecc6901f --- /dev/null +++ b/components/engine/integration/container/stop_test.go @@ -0,0 +1,74 @@ +package container + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/client" + "github.com/docker/docker/integration/util/request" + "github.com/gotestyourself/gotestyourself/icmd" + "github.com/gotestyourself/gotestyourself/poll" + "github.com/gotestyourself/gotestyourself/skip" + "github.com/stretchr/testify/require" +) + +func TestDeleteDevicemapper(t *testing.T) { + skip.IfCondition(t, testEnv.DaemonInfo.Driver != "devicemapper") + + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + foo, err := client.ContainerCreate(ctx, + &container.Config{ + Cmd: []string{"echo"}, + Image: "busybox", + }, + &container.HostConfig{}, + &network.NetworkingConfig{}, + "foo", + ) + require.NoError(t, err) + + err = client.ContainerStart(ctx, foo.ID, types.ContainerStartOptions{}) + require.NoError(t, err) + + inspect, err := client.ContainerInspect(ctx, foo.ID) + require.NoError(t, err) + + poll.WaitOn(t, containerIsStopped(ctx, client, foo.ID), poll.WithDelay(100*time.Millisecond)) + + deviceID := inspect.GraphDriver.Data["DeviceId"] + + // Find pool name from device name + deviceName := inspect.GraphDriver.Data["DeviceName"] + devicePrefix := deviceName[:strings.LastIndex(deviceName, "-")] + devicePool := fmt.Sprintf("/dev/mapper/%s-pool", devicePrefix) + + result := icmd.RunCommand("dmsetup", "message", devicePool, "0", fmt.Sprintf("delete %s", deviceID)) + result.Assert(t, icmd.Success) + + err = client.ContainerRemove(ctx, foo.ID, types.ContainerRemoveOptions{}) + require.NoError(t, err) +} + +func containerIsStopped(ctx context.Context, client client.APIClient, containerID string) func(log poll.LogT) poll.Result { + return func(log poll.LogT) poll.Result { + inspect, err := client.ContainerInspect(ctx, containerID) + + switch { + case err != nil: + return poll.Error(err) + case !inspect.State.Running: + return poll.Success() + default: + return poll.Continue("waiting for container to be stopped") + } + } +} diff --git a/components/engine/pkg/devicemapper/devmapper.go b/components/engine/pkg/devicemapper/devmapper.go index 48618765fb..c72992f7a1 100644 --- a/components/engine/pkg/devicemapper/devmapper.go +++ b/components/engine/pkg/devicemapper/devmapper.go @@ -67,12 +67,14 @@ var ( ErrBusy = errors.New("Device is Busy") ErrDeviceIDExists = errors.New("Device Id Exists") ErrEnxio = errors.New("No such device or address") + ErrEnoData = errors.New("No data available") ) var ( - dmSawBusy bool - dmSawExist bool - dmSawEnxio bool // No Such Device or Address + dmSawBusy bool + dmSawExist bool + dmSawEnxio bool // No Such Device or Address + dmSawEnoData bool // No data available ) type ( @@ -708,10 +710,15 @@ func DeleteDevice(poolName string, deviceID int) error { } dmSawBusy = false + dmSawEnoData = false if err := task.run(); err != nil { if dmSawBusy { return ErrBusy } + if dmSawEnoData { + logrus.Debugf("devicemapper: Device(id: %d) from pool(%s) does not exist", deviceID, poolName) + return nil + } return fmt.Errorf("devicemapper: Error running DeleteDevice %s", err) } return nil diff --git a/components/engine/pkg/devicemapper/devmapper_log.go b/components/engine/pkg/devicemapper/devmapper_log.go index f2ac7da87c..1da75101cf 100644 --- a/components/engine/pkg/devicemapper/devmapper_log.go +++ b/components/engine/pkg/devicemapper/devmapper_log.go @@ -55,6 +55,9 @@ func DevmapperLogCallback(level C.int, file *C.char, line, dmErrnoOrClass C.int, if strings.Contains(msg, "No such device or address") { dmSawEnxio = true } + if strings.Contains(msg, "No data available") { + dmSawEnoData = true + } } if dmLogger != nil {