From 74f7c76b5b21cf19154f3935b4b69bb491c3bd25 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 13 Nov 2017 13:31:28 -0800 Subject: [PATCH] container: protect health monitor channel While this code was likely called from a single thread before, we have now seen panics, indicating that it could be called in parallel. This change adds a mutex to protect opening and closing of the channel. There may be another root cause associated with this panic, such as something that led to the calling of this in parallel, as this code is old and we had seen this condition until recently. This fix is by no means a permanent fix. Typically, bugs like this indicate misplaced channel ownership. In idiomatic uses, the channel should have a particular "owner" that coordinates sending and closure. In this case, the owner of the channel is unclear, so it gets opened lazily. Synchronizing this access is a decent solution, but a refactor may yield better results. Signed-off-by: Stephen J Day (cherry picked from commit 5b55747a523671fa6e626848060460a48d058451) Signed-off-by: Andrew Hsu --- components/engine/container/health.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/components/engine/container/health.go b/components/engine/container/health.go index 5919008d27..75365fec6b 100644 --- a/components/engine/container/health.go +++ b/components/engine/container/health.go @@ -1,6 +1,8 @@ package container import ( + "sync" + "github.com/docker/docker/api/types" "github.com/sirupsen/logrus" ) @@ -9,6 +11,7 @@ import ( type Health struct { types.Health stop chan struct{} // Write struct{} to stop the monitor + mu sync.Mutex } // String returns a human-readable description of the health-check state @@ -26,9 +29,12 @@ func (s *Health) String() string { } } -// OpenMonitorChannel creates and returns a new monitor channel. If there already is one, -// it returns nil. +// OpenMonitorChannel creates and returns a new monitor channel. If there +// already is one, it returns nil. func (s *Health) OpenMonitorChannel() chan struct{} { + s.mu.Lock() + defer s.mu.Unlock() + if s.stop == nil { logrus.Debug("OpenMonitorChannel") s.stop = make(chan struct{}) @@ -39,6 +45,9 @@ func (s *Health) OpenMonitorChannel() chan struct{} { // CloseMonitorChannel closes any existing monitor channel. func (s *Health) CloseMonitorChannel() { + s.mu.Lock() + defer s.mu.Unlock() + if s.stop != nil { logrus.Debug("CloseMonitorChannel: waiting for probe to stop") close(s.stop)