From b2b2305fac0ac5f8363fc308cdebb048ef27522e Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Thu, 17 Aug 2017 14:48:11 +0300 Subject: [PATCH 1/5] daemon.setMounts(): copy slice in place It does not make sense to copy a slice element by element, then discard the source one. Let's do copy in place instead which is way more efficient. Signed-off-by: Kir Kolyshkin Upstream-commit: d4c94e83ca58b5ffd99840a8e3cbf22f1ab331c0 Component: engine --- components/engine/daemon/oci_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/daemon/oci_linux.go b/components/engine/daemon/oci_linux.go index 62e7c7f1d6..4d891c1f17 100644 --- a/components/engine/daemon/oci_linux.go +++ b/components/engine/daemon/oci_linux.go @@ -533,7 +533,7 @@ func setMounts(daemon *Daemon, s *specs.Spec, c *container.Container, mounts []c // - /dev/shm, in case IpcMode is none. // While at it, also // - set size for /dev/shm from shmsize. - var defaultMounts []specs.Mount + defaultMounts := s.Mounts[:0] _, mountDev := userMounts["/dev"] for _, m := range s.Mounts { if _, ok := userMounts[m.Destination]; ok { From 52ce230729722f5b851ebf4cb9bcaccfec2baeff Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 9 May 2018 21:14:54 -0400 Subject: [PATCH 2/5] Add metrics for log failures/partials Signed-off-by: Brian Goff Upstream-commit: 673dd28e7e2fb6e40e4c56823900fdb152b9c53c Component: engine --- components/engine/daemon/logger/copier.go | 4 ++++ components/engine/daemon/logger/metrics.go | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 components/engine/daemon/logger/metrics.go diff --git a/components/engine/daemon/logger/copier.go b/components/engine/daemon/logger/copier.go index ae86777f33..e24272fa6d 100644 --- a/components/engine/daemon/logger/copier.go +++ b/components/engine/daemon/logger/copier.go @@ -81,6 +81,7 @@ func (c *Copier) copySrc(name string, src io.Reader) { read, err := src.Read(buf[n:upto]) if err != nil { if err != io.EOF { + logReadsFailedCount.Inc(1) logrus.Errorf("Error scanning log stream: %s", err) return } @@ -120,6 +121,7 @@ func (c *Copier) copySrc(name string, src io.Reader) { } if logErr := c.dst.Log(msg); logErr != nil { + logWritesFailedCount.Inc(1) logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr) } } @@ -143,6 +145,7 @@ func (c *Copier) copySrc(name string, src io.Reader) { partialid = stringid.GenerateRandomID() ordinal = 1 firstPartial = false + totalPartialLogs.Inc(1) } else { msg.Timestamp = partialTS } @@ -151,6 +154,7 @@ func (c *Copier) copySrc(name string, src io.Reader) { hasMorePartial = true if logErr := c.dst.Log(msg); logErr != nil { + logWritesFailedCount.Inc(1) logrus.Errorf("Failed to log msg %q for logger %s: %s", msg.Line, c.dst.Name(), logErr) } p = 0 diff --git a/components/engine/daemon/logger/metrics.go b/components/engine/daemon/logger/metrics.go new file mode 100644 index 0000000000..b7dfd38ec2 --- /dev/null +++ b/components/engine/daemon/logger/metrics.go @@ -0,0 +1,21 @@ +package logger // import "github.com/docker/docker/daemon/logger" + +import ( + "github.com/docker/go-metrics" +) + +var ( + logWritesFailedCount metrics.Counter + logReadsFailedCount metrics.Counter + totalPartialLogs metrics.Counter +) + +func init() { + loggerMetrics := metrics.NewNamespace("logger", "", nil) + + logWritesFailedCount = loggerMetrics.NewCounter("log_write_operations_failed", "Number of log write operations that failed") + logReadsFailedCount = loggerMetrics.NewCounter("log_read_operations_failed", "Number of log reads from container stdio that failed") + totalPartialLogs = loggerMetrics.NewCounter("log_entries_size_greater_than_buffer", "Number of log entries which are larger than the log buffer") + + metrics.Register(loggerMetrics) +} From 5da69b8b391f039c1ec38a64e478cd015d37f2e0 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 11 May 2018 13:40:46 +0200 Subject: [PATCH 3/5] overlay: do not ignore invalid storage-driver options The overlay storage driver currently does not support any option, but was silently ignoring any option that was passed. This patch verifies that no options are passed, and if they are passed will produce an error. Before this change: dockerd --storage-driver=overlay --storage-opt dm.thinp_percent=95 INFO[2018-05-11T11:40:40.996597152Z] libcontainerd: started new docker-containerd process pid=256 .... INFO[2018-05-11T11:40:41.135392535Z] Daemon has completed initialization INFO[2018-05-11T11:40:41.141035093Z] API listen on /var/run/docker.sock After this change: dockerd --storage-driver=overlay --storage-opt dm.thinp_percent=95 INFO[2018-05-11T11:39:21.632610319Z] libcontainerd: started new docker-containerd process pid=233 .... Error starting daemon: error initializing graphdriver: overlay: unknown option dm.thinp_percent Signed-off-by: Sebastiaan van Stijn Upstream-commit: 30f15d2bdc551870464d1cd024a92341cf1ae4aa Component: engine --- .../daemon/graphdriver/overlay/overlay.go | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/components/engine/daemon/graphdriver/overlay/overlay.go b/components/engine/daemon/graphdriver/overlay/overlay.go index 2e0bec5bc4..6932e4df81 100644 --- a/components/engine/daemon/graphdriver/overlay/overlay.go +++ b/components/engine/daemon/graphdriver/overlay/overlay.go @@ -12,6 +12,7 @@ import ( "path" "path/filepath" "strconv" + "strings" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/daemon/graphdriver/copy" @@ -22,6 +23,7 @@ import ( "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/locker" "github.com/docker/docker/pkg/mount" + "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/system" "github.com/opencontainers/selinux/go-selinux/label" "github.com/sirupsen/logrus" @@ -95,6 +97,8 @@ func (d *naiveDiffDriverWithApply) ApplyDiff(id, parent string, diff io.Reader) // of that. This means all child images share file (but not directory) // data with the parent. +type overlayOptions struct{} + // Driver contains information about the home directory and the list of active mounts that are created using this driver. type Driver struct { home string @@ -115,6 +119,10 @@ func init() { // If an overlay filesystem is not supported over an existing filesystem then // error graphdriver.ErrIncompatibleFS is returned. func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { + _, err := parseOptions(options) + if err != nil { + return nil, err + } if err := supportsOverlay(); err != nil { return nil, graphdriver.ErrNotSupported @@ -176,6 +184,22 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return NaiveDiffDriverWithApply(d, uidMaps, gidMaps), nil } +func parseOptions(options []string) (*overlayOptions, error) { + o := &overlayOptions{} + for _, option := range options { + key, _, err := parsers.ParseKeyValueOpt(option) + if err != nil { + return nil, err + } + key = strings.ToLower(key) + switch key { + default: + return nil, fmt.Errorf("overlay: unknown option %s", key) + } + } + return o, nil +} + func supportsOverlay() error { // We can try to modprobe overlay first before looking at // proc/filesystems for when overlay is supported From 4ca879c24161b1cc204544adb31b8dc16e04b2e9 Mon Sep 17 00:00:00 2001 From: Abhinandan Prativadi Date: Fri, 11 May 2018 14:37:01 -0700 Subject: [PATCH 4/5] Fixing network inspect for swarm Signed-off-by: Abhinandan Prativadi Upstream-commit: a059d6f4f5eb5fdb319b6f715a9bb50eb5d6173d Component: engine --- .../engine/daemon/cluster/convert/network.go | 12 ++++++------ .../engine/integration/network/inspect_test.go | 16 +++++++++++++--- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/components/engine/daemon/cluster/convert/network.go b/components/engine/daemon/cluster/convert/network.go index 44f5f56466..34660fc4ff 100644 --- a/components/engine/daemon/cluster/convert/network.go +++ b/components/engine/daemon/cluster/convert/network.go @@ -140,13 +140,13 @@ func swarmPortConfigToAPIPortConfig(portConfig *swarmapi.PortConfig) types.PortC func BasicNetworkFromGRPC(n swarmapi.Network) basictypes.NetworkResource { spec := n.Spec var ipam networktypes.IPAM - if spec.IPAM != nil { - if spec.IPAM.Driver != nil { - ipam.Driver = spec.IPAM.Driver.Name - ipam.Options = spec.IPAM.Driver.Options + if n.IPAM != nil { + if n.IPAM.Driver != nil { + ipam.Driver = n.IPAM.Driver.Name + ipam.Options = n.IPAM.Driver.Options } - ipam.Config = make([]networktypes.IPAMConfig, 0, len(spec.IPAM.Configs)) - for _, ic := range spec.IPAM.Configs { + ipam.Config = make([]networktypes.IPAMConfig, 0, len(n.IPAM.Configs)) + for _, ic := range n.IPAM.Configs { ipamConfig := networktypes.IPAMConfig{ Subnet: ic.Subnet, IPRange: ic.Range, diff --git a/components/engine/integration/network/inspect_test.go b/components/engine/integration/network/inspect_test.go index b728da9d23..e88d739fd9 100644 --- a/components/engine/integration/network/inspect_test.go +++ b/components/engine/integration/network/inspect_test.go @@ -162,9 +162,19 @@ func noTasks(client client.ServiceAPIClient) func(log poll.LogT) poll.Result { // Check to see if Service and Tasks info are part of the inspect verbose response func validNetworkVerbose(network types.NetworkResource, service string, instances uint64) bool { if service, ok := network.Services[service]; ok { - if len(service.Tasks) == int(instances) { - return true + if len(service.Tasks) != int(instances) { + return false } } - return false + + if network.IPAM.Config == nil { + return false + } + + for _, cfg := range network.IPAM.Config { + if cfg.Gateway == "" || cfg.Subnet == "" { + return false + } + } + return true } From 49316401a9b49f82de02d0a3b785fe341beeb67e Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 13 May 2018 22:24:04 +0200 Subject: [PATCH 5/5] Fix missing plugin name in message This message was missing the name of the plugin, resulting in plugin_test.go:92: timeout hit after 30s: plugin %!q(MISSING) exists On failing tests. Signed-off-by: Sebastiaan van Stijn Upstream-commit: b526964584d213074de7c5573a3373a58699c181 Component: engine --- components/engine/internal/test/daemon/plugin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/internal/test/daemon/plugin.go b/components/engine/internal/test/daemon/plugin.go index fad9727161..9a7cc345ea 100644 --- a/components/engine/internal/test/daemon/plugin.go +++ b/components/engine/internal/test/daemon/plugin.go @@ -38,7 +38,7 @@ func (d *Daemon) PluginIsNotPresent(name string) func(poll.LogT) poll.Result { if err != nil { return poll.Error(err) } - return poll.Continue("plugin %q exists") + return poll.Continue("plugin %q exists", name) }) }