From a2b8d3a9e825a94696f3fbf36619ab4dfa3886d8 Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Thu, 16 Nov 2017 17:09:16 +1100 Subject: [PATCH 1/5] pkg: devmapper: dynamically load dm_task_deferred_remove dm_task_deferred_remove is not supported by all distributions, due to out-dated versions of devicemapper. However, in the case where the devicemapper library was updated without rebuilding Docker (which can happen in some distributions) then we should attempt to dynamically load the relevant object rather than try to link to it. This can only be done if Docker was built dynamically, for obvious reasons. In order to avoid having issues arise when dlsym(3) was unnecessary, gate the whole dlsym(3) logic behind a buildflag that we disable by default (libdm_dlsym_deferred_remove). Signed-off-by: Aleksa Sarai Upstream-commit: 98fe4bd8f1e35f8e498e268f653a43cbfa31e751 Component: engine --- components/engine/hack/make.sh | 12 +- ...mapper_wrapper_dynamic_deferred_remove.go} | 10 +- ...r_wrapper_dynamic_dlsym_deferred_remove.go | 128 ++++++++++++++++++ .../devmapper_wrapper_no_deferred_remove.go | 6 +- 4 files changed, 149 insertions(+), 7 deletions(-) rename components/engine/pkg/devicemapper/{devmapper_wrapper_deferred_remove.go => devmapper_wrapper_dynamic_deferred_remove.go} (79%) create mode 100644 components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_dlsym_deferred_remove.go diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index 99c51a7b62..be90bc019a 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -103,6 +103,12 @@ if [ ! "$GOPATH" ]; then exit 1 fi +# Adds $1_$2 to DOCKER_BUILDTAGS unless it already +# contains a word starting from $1_ +add_buildtag() { + [[ " $DOCKER_BUILDTAGS" == *" $1_"* ]] || DOCKER_BUILDTAGS+=" $1_$2" +} + if ${PKG_CONFIG} 'libsystemd >= 209' 2> /dev/null ; then DOCKER_BUILDTAGS+=" journald" elif ${PKG_CONFIG} 'libsystemd-journal' 2> /dev/null ; then @@ -118,12 +124,14 @@ if \ fi # test whether "libdevmapper.h" is new enough to support deferred remove -# functionality. +# functionality. We favour libdm_dlsym_deferred_remove over +# libdm_no_deferred_remove in dynamic cases because the binary could be shipped +# with a newer libdevmapper than the one it was built wih. if \ command -v gcc &> /dev/null \ && ! ( echo -e '#include \nint main() { dm_task_deferred_remove(NULL); }'| gcc -xc - -o /dev/null $(pkg-config --libs devmapper) &> /dev/null ) \ ; then - DOCKER_BUILDTAGS+=' libdm_no_deferred_remove' + add_buildtag libdm dlsym_deferred_remove fi # Use these flags when compiling the tests and final binary diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper_deferred_remove.go b/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_deferred_remove.go similarity index 79% rename from components/engine/pkg/devicemapper/devmapper_wrapper_deferred_remove.go rename to components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_deferred_remove.go index 2db5c799c4..3d3021c4e1 100644 --- a/components/engine/pkg/devicemapper/devmapper_wrapper_deferred_remove.go +++ b/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_deferred_remove.go @@ -1,11 +1,15 @@ -// +build linux,cgo,!libdm_no_deferred_remove +// +build linux,cgo,!static_build +// +build !libdm_dlsym_deferred_remove,!libdm_no_deferred_remove package devicemapper // import "github.com/docker/docker/pkg/devicemapper" -// #include +/* +#include +*/ import "C" -// LibraryDeferredRemovalSupport tells if the feature is enabled in the build +// LibraryDeferredRemovalSupport tells if the feature is supported by the +// current Docker invocation. const LibraryDeferredRemovalSupport = true func dmTaskDeferredRemoveFct(task *cdmTask) int { diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_dlsym_deferred_remove.go b/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_dlsym_deferred_remove.go new file mode 100644 index 0000000000..5dfb369f1f --- /dev/null +++ b/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_dlsym_deferred_remove.go @@ -0,0 +1,128 @@ +// +build linux,cgo,!static_build +// +build libdm_dlsym_deferred_remove,!libdm_no_deferred_remove + +package devicemapper + +/* +#cgo LDFLAGS: -ldl +#include +#include +#include + +// Yes, I know this looks scary. In order to be able to fill our own internal +// dm_info with deferred_remove we need to have a struct definition that is +// correct (regardless of the version of libdm that was used to compile it). To +// this end, we define struct_backport_dm_info. This code comes from lvm2, and +// I have verified that the structure has only ever had elements *appended* to +// it (since 2001). +// +// It is also important that this structure be _larger_ than the dm_info that +// libdevmapper expected. Otherwise libdm might try to write to memory it +// shouldn't (they don't have a "known size" API). +struct backport_dm_info { + int exists; + int suspended; + int live_table; + int inactive_table; + int32_t open_count; + uint32_t event_nr; + uint32_t major; + uint32_t minor; + int read_only; + + int32_t target_count; + + int deferred_remove; + int internal_suspend; + + // Padding, purely for our own safety. This is to avoid cases where libdm + // was updated underneath us and we call into dm_task_get_info() with too + // small of a buffer. + char _[512]; +}; + +// We have to wrap this in CGo, because Go really doesn't like function pointers. +int call_dm_task_deferred_remove(void *fn, struct dm_task *task) +{ + int (*_dm_task_deferred_remove)(struct dm_task *task) = fn; + return _dm_task_deferred_remove(task); +} +*/ +import "C" + +import ( + "unsafe" + + "github.com/sirupsen/logrus" +) + +// dm_task_deferred_remove is not supported by all distributions, due to +// out-dated versions of devicemapper. However, in the case where the +// devicemapper library was updated without rebuilding Docker (which can happen +// in some distributions) then we should attempt to dynamically load the +// relevant object rather than try to link to it. + +// dmTaskDeferredRemoveFct is a "bound" version of dm_task_deferred_remove. +// It is nil if dm_task_deferred_remove was not found in the libdevmapper that +// is currently loaded. +var dmTaskDeferredRemovePtr unsafe.Pointer + +// LibraryDeferredRemovalSupport tells if the feature is supported by the +// current Docker invocation. This value is fixed during init. +var LibraryDeferredRemovalSupport bool + +func init() { + // Clear any errors. + var err *C.char + C.dlerror() + + // The symbol we want to fetch. + symName := C.CString("dm_task_deferred_remove") + defer C.free(unsafe.Pointer(symName)) + + // See if we can find dm_task_deferred_remove. Since we already are linked + // to libdevmapper, we can search our own address space (rather than trying + // to guess what libdevmapper is called). We use NULL here, as RTLD_DEFAULT + // is not available in CGO (even if you set _GNU_SOURCE for some reason). + // The semantics are identical on glibc. + sym := C.dlsym(nil, symName) + err = C.dlerror() + if err != nil { + logrus.Debugf("devmapper: could not load dm_task_deferred_remove: %s", C.GoString(err)) + return + } + + logrus.Debugf("devmapper: found dm_task_deferred_remove at %x", uintptr(sym)) + dmTaskDeferredRemovePtr = sym + LibraryDeferredRemovalSupport = true +} + +func dmTaskDeferredRemoveFct(task *cdmTask) int { + sym := dmTaskDeferredRemovePtr + if sym == nil || !LibraryDeferredRemovalSupport { + return -1 + } + return int(C.call_dm_task_deferred_remove(sym, (*C.struct_dm_task)(task))) +} + +func dmTaskGetInfoWithDeferredFct(task *cdmTask, info *Info) int { + if !LibraryDeferredRemovalSupport { + return -1 + } + + Cinfo := C.struct_backport_dm_info{} + defer func() { + info.Exists = int(Cinfo.exists) + info.Suspended = int(Cinfo.suspended) + info.LiveTable = int(Cinfo.live_table) + info.InactiveTable = int(Cinfo.inactive_table) + info.OpenCount = int32(Cinfo.open_count) + info.EventNr = uint32(Cinfo.event_nr) + info.Major = uint32(Cinfo.major) + info.Minor = uint32(Cinfo.minor) + info.ReadOnly = int(Cinfo.read_only) + info.TargetCount = int32(Cinfo.target_count) + info.DeferredRemove = int(Cinfo.deferred_remove) + }() + return int(C.dm_task_get_info((*C.struct_dm_task)(task), (*C.struct_dm_info)(unsafe.Pointer(&Cinfo)))) +} diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go b/components/engine/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go index 33eabe593b..8889f0f46f 100644 --- a/components/engine/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go +++ b/components/engine/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go @@ -1,8 +1,10 @@ -// +build linux,cgo,libdm_no_deferred_remove +// +build linux,cgo +// +build !libdm_dlsym_deferred_remove,libdm_no_deferred_remove package devicemapper // import "github.com/docker/docker/pkg/devicemapper" -// LibraryDeferredRemovalSupport tells if the feature is enabled in the build +// LibraryDeferredRemovalSupport tells if the feature is supported by the +// current Docker invocation. const LibraryDeferredRemovalSupport = false func dmTaskDeferredRemoveFct(task *cdmTask) int { From baa55da752ec6c290e0b60dea838945bd9e82e20 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 10 Apr 2018 16:29:48 +0200 Subject: [PATCH 2/5] =?UTF-8?q?Move=20integration-cli=20daemon=20package?= =?UTF-8?q?=20to=20internal/test=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … and do not use the `docker` cli in it. One of the reason of this move is to not make `integration` package using legacy `integration-cli` package. Next move will be to support swarm within this package *and* provide some helper function using the api (compared to the one using cli in `integration-cli/daemon` package). Signed-off-by: Vincent Demeester Upstream-commit: f0d277fe84a72b29c0d2d541c20d5a9c4d7e4884 Component: engine --- .../engine/integration-cli/check_test.go | 7 +- .../engine/integration-cli/daemon/daemon.go | 605 +---------------- .../container/daemon_linux_test.go | 4 +- .../integration/container/export_test.go | 4 +- .../container/mounts_linux_test.go | 4 +- .../integration/container/restart_test.go | 4 +- .../integration/network/inspect_test.go | 1 - .../integration/network/macvlan_test.go | 14 +- .../integration/plugin/authz/main_test.go | 8 +- .../integration/plugin/logging/main_test.go | 2 - .../plugin/logging/validation_test.go | 4 +- .../integration/plugin/volumes/main_test.go | 2 - .../integration/plugin/volumes/mounts_test.go | 4 +- .../engine/integration/secret/main_test.go | 2 - .../engine/integration/service/main_test.go | 2 - .../system/cgroupdriver_systemd_test.go | 4 +- .../engine/internal/test/daemon/daemon.go | 618 ++++++++++++++++++ .../test}/daemon/daemon_unix.go | 7 +- .../test}/daemon/daemon_windows.go | 5 +- components/engine/internal/test/daemon/ops.go | 13 + 20 files changed, 677 insertions(+), 637 deletions(-) create mode 100644 components/engine/internal/test/daemon/daemon.go rename components/engine/{integration-cli => internal/test}/daemon/daemon_unix.go (78%) rename components/engine/{integration-cli => internal/test}/daemon/daemon_windows.go (74%) create mode 100644 components/engine/internal/test/daemon/ops.go diff --git a/components/engine/integration-cli/check_test.go b/components/engine/integration-cli/check_test.go index 18d3d3780a..a455893d8c 100644 --- a/components/engine/integration-cli/check_test.go +++ b/components/engine/integration-cli/check_test.go @@ -21,6 +21,7 @@ import ( "github.com/docker/docker/integration-cli/environment" "github.com/docker/docker/integration-cli/fixtures/plugin" "github.com/docker/docker/integration-cli/registry" + testdaemon "github.com/docker/docker/internal/test/daemon" ienv "github.com/docker/docker/internal/test/environment" "github.com/docker/docker/pkg/reexec" "github.com/go-check/check" @@ -100,7 +101,7 @@ func (s *DockerSuite) OnTimeout(c *check.C) { daemonPid := int(rawPid) if daemonPid > 0 { - daemon.SignalDaemonDump(daemonPid) + testdaemon.SignalDaemonDump(daemonPid) } } @@ -285,7 +286,7 @@ func (s *DockerDaemonSuite) TearDownTest(c *check.C) { } func (s *DockerDaemonSuite) TearDownSuite(c *check.C) { - filepath.Walk(daemon.SockRoot, func(path string, fi os.FileInfo, err error) error { + filepath.Walk(testdaemon.SockRoot, func(path string, fi os.FileInfo, err error) error { if err != nil { // ignore errors here // not cleaning up sockets is not really an error @@ -296,7 +297,7 @@ func (s *DockerDaemonSuite) TearDownSuite(c *check.C) { } return nil }) - os.RemoveAll(daemon.SockRoot) + os.RemoveAll(testdaemon.SockRoot) } const defaultSwarmPort = 2477 diff --git a/components/engine/integration-cli/daemon/daemon.go b/components/engine/integration-cli/daemon/daemon.go index 9ca54236f4..4ef1dafe79 100644 --- a/components/engine/integration-cli/daemon/daemon.go +++ b/components/engine/integration-cli/daemon/daemon.go @@ -1,33 +1,17 @@ package daemon // import "github.com/docker/docker/integration-cli/daemon" import ( - "encoding/json" "fmt" - "io" - "io/ioutil" - "net/http" - "os" "os/exec" - "path/filepath" - "strconv" "strings" "time" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/events" - "github.com/docker/docker/client" "github.com/docker/docker/integration-cli/checker" - "github.com/docker/docker/integration-cli/request" - "github.com/docker/docker/opts" - "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/stringid" - "github.com/docker/go-connections/sockets" - "github.com/docker/go-connections/tlsconfig" + "github.com/docker/docker/internal/test/daemon" "github.com/go-check/check" "github.com/gotestyourself/gotestyourself/assert" "github.com/gotestyourself/gotestyourself/icmd" "github.com/pkg/errors" - "golang.org/x/net/context" ) type testingT interface { @@ -40,32 +24,10 @@ type logT interface { Logf(string, ...interface{}) } -// SockRoot holds the path of the default docker integration daemon socket -var SockRoot = filepath.Join(os.TempDir(), "docker-integration") - -var errDaemonNotStarted = errors.New("daemon not started") - // Daemon represents a Docker daemon for the testing framework. type Daemon struct { - GlobalFlags []string - Root string - Folder string - Wait chan error - UseDefaultHost bool - UseDefaultTLSHost bool - - id string - logFile *os.File - stdin io.WriteCloser - stdout, stderr io.ReadCloser - cmd *exec.Cmd - storageDriver string - userlandProxy bool - execRoot string - experimental bool - dockerBinary string - dockerdBinary string - log logT + *daemon.Daemon + dockerBinary string } // Config holds docker daemon integration configuration @@ -73,504 +35,24 @@ type Config struct { Experimental bool } -type clientConfig struct { - transport *http.Transport - scheme string - addr string -} - // New returns a Daemon instance to be used for testing. // This will create a directory such as d123456789 in the folder specified by $DOCKER_INTEGRATION_DAEMON_DEST or $DEST. // The daemon will not automatically start. func New(t testingT, dockerBinary string, dockerdBinary string, config Config) *Daemon { - dest := os.Getenv("DOCKER_INTEGRATION_DAEMON_DEST") - if dest == "" { - dest = os.Getenv("DEST") + ops := []func(*daemon.Daemon){ + daemon.WithDockerdBinary(dockerdBinary), } - if dest == "" { - t.Fatalf("Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable") - } - - if err := os.MkdirAll(SockRoot, 0700); err != nil { - t.Fatalf("could not create daemon socket root") - } - - id := fmt.Sprintf("d%s", stringid.TruncateID(stringid.GenerateRandomID())) - dir := filepath.Join(dest, id) - daemonFolder, err := filepath.Abs(dir) - if err != nil { - t.Fatalf("Could not make %q an absolute path", dir) - } - daemonRoot := filepath.Join(daemonFolder, "root") - - if err := os.MkdirAll(daemonRoot, 0755); err != nil { - t.Fatalf("Could not create daemon root %q", dir) - } - - userlandProxy := true - if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { - if val, err := strconv.ParseBool(env); err != nil { - userlandProxy = val - } + if config.Experimental { + ops = append(ops, daemon.WithExperimental) } + d := daemon.New(t, ops...) return &Daemon{ - id: id, - Folder: daemonFolder, - Root: daemonRoot, - storageDriver: os.Getenv("DOCKER_GRAPHDRIVER"), - userlandProxy: userlandProxy, - execRoot: filepath.Join(os.TempDir(), "docker-execroot", id), - dockerBinary: dockerBinary, - dockerdBinary: dockerdBinary, - experimental: config.Experimental, - log: t, + Daemon: d, + dockerBinary: dockerBinary, } } -// RootDir returns the root directory of the daemon. -func (d *Daemon) RootDir() string { - return d.Root -} - -// ID returns the generated id of the daemon -func (d *Daemon) ID() string { - return d.id -} - -// StorageDriver returns the configured storage driver of the daemon -func (d *Daemon) StorageDriver() string { - return d.storageDriver -} - -// CleanupExecRoot cleans the daemon exec root (network namespaces, ...) -func (d *Daemon) CleanupExecRoot(c *check.C) { - cleanupExecRoot(c, d.execRoot) -} - -func (d *Daemon) getClientConfig() (*clientConfig, error) { - var ( - transport *http.Transport - scheme string - addr string - proto string - ) - if d.UseDefaultTLSHost { - option := &tlsconfig.Options{ - CAFile: "fixtures/https/ca.pem", - CertFile: "fixtures/https/client-cert.pem", - KeyFile: "fixtures/https/client-key.pem", - } - tlsConfig, err := tlsconfig.Client(*option) - if err != nil { - return nil, err - } - transport = &http.Transport{ - TLSClientConfig: tlsConfig, - } - addr = fmt.Sprintf("%s:%d", opts.DefaultHTTPHost, opts.DefaultTLSHTTPPort) - scheme = "https" - proto = "tcp" - } else if d.UseDefaultHost { - addr = opts.DefaultUnixSocket - proto = "unix" - scheme = "http" - transport = &http.Transport{} - } else { - addr = d.sockPath() - proto = "unix" - scheme = "http" - transport = &http.Transport{} - } - - if err := sockets.ConfigureTransport(transport, proto, addr); err != nil { - return nil, err - } - transport.DisableKeepAlives = true - - return &clientConfig{ - transport: transport, - scheme: scheme, - addr: addr, - }, nil -} - -// Start starts the daemon and return once it is ready to receive requests. -func (d *Daemon) Start(t testingT, args ...string) { - if err := d.StartWithError(args...); err != nil { - t.Fatalf("Error starting daemon with arguments: %v", args) - } -} - -// StartWithError starts the daemon and return once it is ready to receive requests. -// It returns an error in case it couldn't start. -func (d *Daemon) StartWithError(args ...string) error { - logFile, err := os.OpenFile(filepath.Join(d.Folder, "docker.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return errors.Wrapf(err, "[%s] Could not create %s/docker.log", d.id, d.Folder) - } - - return d.StartWithLogFile(logFile, args...) -} - -// StartWithLogFile will start the daemon and attach its streams to a given file. -func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { - dockerdBinary, err := exec.LookPath(d.dockerdBinary) - if err != nil { - return errors.Wrapf(err, "[%s] could not find docker binary in $PATH", d.id) - } - args := append(d.GlobalFlags, - "--containerd", "/var/run/docker/containerd/docker-containerd.sock", - "--data-root", d.Root, - "--exec-root", d.execRoot, - "--pidfile", fmt.Sprintf("%s/docker.pid", d.Folder), - fmt.Sprintf("--userland-proxy=%t", d.userlandProxy), - ) - if d.experimental { - args = append(args, "--experimental", "--init") - } - if !(d.UseDefaultHost || d.UseDefaultTLSHost) { - args = append(args, []string{"--host", d.Sock()}...) - } - if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { - args = append(args, []string{"--userns-remap", root}...) - } - - // If we don't explicitly set the log-level or debug flag(-D) then - // turn on debug mode - foundLog := false - foundSd := false - for _, a := range providedArgs { - if strings.Contains(a, "--log-level") || strings.Contains(a, "-D") || strings.Contains(a, "--debug") { - foundLog = true - } - if strings.Contains(a, "--storage-driver") { - foundSd = true - } - } - if !foundLog { - args = append(args, "--debug") - } - if d.storageDriver != "" && !foundSd { - args = append(args, "--storage-driver", d.storageDriver) - } - - args = append(args, providedArgs...) - d.cmd = exec.Command(dockerdBinary, args...) - d.cmd.Env = append(os.Environ(), "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE=1") - d.cmd.Stdout = out - d.cmd.Stderr = out - d.logFile = out - - if err := d.cmd.Start(); err != nil { - return errors.Errorf("[%s] could not start daemon container: %v", d.id, err) - } - - wait := make(chan error) - - go func() { - wait <- d.cmd.Wait() - d.log.Logf("[%s] exiting daemon", d.id) - close(wait) - }() - - d.Wait = wait - - tick := time.Tick(500 * time.Millisecond) - // make sure daemon is ready to receive requests - startTime := time.Now().Unix() - for { - d.log.Logf("[%s] waiting for daemon to start", d.id) - if time.Now().Unix()-startTime > 5 { - // After 5 seconds, give up - return errors.Errorf("[%s] Daemon exited and never started", d.id) - } - select { - case <-time.After(2 * time.Second): - return errors.Errorf("[%s] timeout: daemon does not respond", d.id) - case <-tick: - clientConfig, err := d.getClientConfig() - if err != nil { - return err - } - - client := &http.Client{ - Transport: clientConfig.transport, - } - - req, err := http.NewRequest("GET", "/_ping", nil) - if err != nil { - return errors.Wrapf(err, "[%s] could not create new request", d.id) - } - req.URL.Host = clientConfig.addr - req.URL.Scheme = clientConfig.scheme - resp, err := client.Do(req) - if err != nil { - continue - } - resp.Body.Close() - if resp.StatusCode != http.StatusOK { - d.log.Logf("[%s] received status != 200 OK: %s\n", d.id, resp.Status) - } - d.log.Logf("[%s] daemon started\n", d.id) - d.Root, err = d.queryRootDir() - if err != nil { - return errors.Errorf("[%s] error querying daemon for root directory: %v", d.id, err) - } - return nil - case <-d.Wait: - return errors.Errorf("[%s] Daemon exited during startup", d.id) - } - } -} - -// StartWithBusybox will first start the daemon with Daemon.Start() -// then save the busybox image from the main daemon and load it into this Daemon instance. -func (d *Daemon) StartWithBusybox(t testingT, arg ...string) { - d.Start(t, arg...) - d.LoadBusybox(t) -} - -// Kill will send a SIGKILL to the daemon -func (d *Daemon) Kill() error { - if d.cmd == nil || d.Wait == nil { - return errDaemonNotStarted - } - - defer func() { - d.logFile.Close() - d.cmd = nil - }() - - if err := d.cmd.Process.Kill(); err != nil { - return err - } - - return os.Remove(fmt.Sprintf("%s/docker.pid", d.Folder)) -} - -// Pid returns the pid of the daemon -func (d *Daemon) Pid() int { - return d.cmd.Process.Pid -} - -// Interrupt stops the daemon by sending it an Interrupt signal -func (d *Daemon) Interrupt() error { - return d.Signal(os.Interrupt) -} - -// Signal sends the specified signal to the daemon if running -func (d *Daemon) Signal(signal os.Signal) error { - if d.cmd == nil || d.Wait == nil { - return errDaemonNotStarted - } - return d.cmd.Process.Signal(signal) -} - -// DumpStackAndQuit sends SIGQUIT to the daemon, which triggers it to dump its -// stack to its log file and exit -// This is used primarily for gathering debug information on test timeout -func (d *Daemon) DumpStackAndQuit() { - if d.cmd == nil || d.cmd.Process == nil { - return - } - SignalDaemonDump(d.cmd.Process.Pid) -} - -// Stop will send a SIGINT every second and wait for the daemon to stop. -// If it times out, a SIGKILL is sent. -// Stop will not delete the daemon directory. If a purged daemon is needed, -// instantiate a new one with NewDaemon. -// If an error occurs while starting the daemon, the test will fail. -func (d *Daemon) Stop(t testingT) { - err := d.StopWithError() - if err != nil { - if err != errDaemonNotStarted { - t.Fatalf("Error while stopping the daemon %s : %v", d.id, err) - } else { - t.Logf("Daemon %s is not started", d.id) - } - } -} - -// StopWithError will send a SIGINT every second and wait for the daemon to stop. -// If it timeouts, a SIGKILL is sent. -// Stop will not delete the daemon directory. If a purged daemon is needed, -// instantiate a new one with NewDaemon. -func (d *Daemon) StopWithError() error { - if d.cmd == nil || d.Wait == nil { - return errDaemonNotStarted - } - - defer func() { - d.logFile.Close() - d.cmd = nil - }() - - i := 1 - tick := time.Tick(time.Second) - - if err := d.cmd.Process.Signal(os.Interrupt); err != nil { - if strings.Contains(err.Error(), "os: process already finished") { - return errDaemonNotStarted - } - return errors.Errorf("could not send signal: %v", err) - } -out1: - for { - select { - case err := <-d.Wait: - return err - case <-time.After(20 * time.Second): - // time for stopping jobs and run onShutdown hooks - d.log.Logf("[%s] daemon started", d.id) - break out1 - } - } - -out2: - for { - select { - case err := <-d.Wait: - return err - case <-tick: - i++ - if i > 5 { - d.log.Logf("tried to interrupt daemon for %d times, now try to kill it", i) - break out2 - } - d.log.Logf("Attempt #%d: daemon is still running with pid %d", i, d.cmd.Process.Pid) - if err := d.cmd.Process.Signal(os.Interrupt); err != nil { - return errors.Errorf("could not send signal: %v", err) - } - } - } - - if err := d.cmd.Process.Kill(); err != nil { - d.log.Logf("Could not kill daemon: %v", err) - return err - } - - d.cmd.Wait() - - return os.Remove(fmt.Sprintf("%s/docker.pid", d.Folder)) -} - -// Restart will restart the daemon by first stopping it and the starting it. -// If an error occurs while starting the daemon, the test will fail. -func (d *Daemon) Restart(t testingT, args ...string) { - d.Stop(t) - d.handleUserns() - d.Start(t, args...) -} - -// RestartWithError will restart the daemon by first stopping it and then starting it. -func (d *Daemon) RestartWithError(arg ...string) error { - if err := d.StopWithError(); err != nil { - return err - } - d.handleUserns() - return d.StartWithError(arg...) -} - -func (d *Daemon) handleUserns() { - // in the case of tests running a user namespace-enabled daemon, we have resolved - // d.Root to be the actual final path of the graph dir after the "uid.gid" of - // remapped root is added--we need to subtract it from the path before calling - // start or else we will continue making subdirectories rather than truly restarting - // with the same location/root: - if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { - d.Root = filepath.Dir(d.Root) - } -} - -// LoadBusybox image into the daemon -func (d *Daemon) LoadBusybox(t testingT) { - clientHost, err := client.NewEnvClient() - assert.NilError(t, err, "failed to create client") - defer clientHost.Close() - - ctx := context.Background() - reader, err := clientHost.ImageSave(ctx, []string{"busybox:latest"}) - assert.NilError(t, err, "failed to download busybox") - defer reader.Close() - - client, err := d.NewClient() - assert.NilError(t, err, "failed to create client") - defer client.Close() - - resp, err := client.ImageLoad(ctx, reader, true) - assert.NilError(t, err, "failed to load busybox") - defer resp.Body.Close() -} - -func (d *Daemon) queryRootDir() (string, error) { - // update daemon root by asking /info endpoint (to support user - // namespaced daemon with root remapped uid.gid directory) - clientConfig, err := d.getClientConfig() - if err != nil { - return "", err - } - - client := &http.Client{ - Transport: clientConfig.transport, - } - - req, err := http.NewRequest("GET", "/info", nil) - if err != nil { - return "", err - } - req.Header.Set("Content-Type", "application/json") - req.URL.Host = clientConfig.addr - req.URL.Scheme = clientConfig.scheme - - resp, err := client.Do(req) - if err != nil { - return "", err - } - body := ioutils.NewReadCloserWrapper(resp.Body, func() error { - return resp.Body.Close() - }) - - type Info struct { - DockerRootDir string - } - var b []byte - var i Info - b, err = request.ReadBody(body) - if err == nil && resp.StatusCode == http.StatusOK { - // read the docker root dir - if err = json.Unmarshal(b, &i); err == nil { - return i.DockerRootDir, nil - } - } - return "", err -} - -// Sock returns the socket path of the daemon -func (d *Daemon) Sock() string { - return fmt.Sprintf("unix://" + d.sockPath()) -} - -func (d *Daemon) sockPath() string { - return filepath.Join(SockRoot, d.id+".sock") -} - -// WaitRun waits for a container to be running for 10s -func (d *Daemon) WaitRun(contID string) error { - args := []string{"--host", d.Sock()} - return WaitInspectWithArgs(d.dockerBinary, contID, "{{.State.Running}}", "true", 10*time.Second, args...) -} - -// Info returns the info struct for this daemon -func (d *Daemon) Info(t assert.TestingT) types.Info { - apiclient, err := client.NewClientWithOpts(client.WithHost((d.Sock()))) - assert.NilError(t, err) - info, err := apiclient.Info(context.Background()) - assert.NilError(t, err) - return info -} - // Cmd executes a docker CLI command against this daemon. // Example: d.Cmd("version") will run docker -H unix://path/to/unix.sock version func (d *Daemon) Cmd(args ...string) (string, error) { @@ -594,11 +76,6 @@ func (d *Daemon) PrependHostArg(args []string) []string { return append([]string{"--host", d.Sock()}, args...) } -// LogFileName returns the path the daemon's log file -func (d *Daemon) LogFileName() string { - return d.logFile.Name() -} - // GetIDByName returns the ID of an object (container, volume, …) given its name func (d *Daemon) GetIDByName(name string) (string, error) { return d.inspectFieldWithError(name, "Id") @@ -616,11 +93,6 @@ func (d *Daemon) ActiveContainers() (ids []string) { return } -// ReadLogFile returns the content of the daemon log file -func (d *Daemon) ReadLogFile() ([]byte, error) { - return ioutil.ReadFile(d.logFile.Name()) -} - // InspectField returns the field filter by 'filter' func (d *Daemon) InspectField(name, filter string) (string, error) { return d.inspectFilter(name, filter) @@ -672,59 +144,10 @@ func (d *Daemon) CheckActiveContainerCount(c *check.C) (interface{}, check.Comme return len(strings.Split(strings.TrimSpace(out), "\n")), check.Commentf("output: %q", string(out)) } -// ReloadConfig asks the daemon to reload its configuration -func (d *Daemon) ReloadConfig() error { - if d.cmd == nil || d.cmd.Process == nil { - return errors.New("daemon is not running") - } - - errCh := make(chan error) - started := make(chan struct{}) - go func() { - _, body, err := request.DoOnHost(d.Sock(), "/events", request.Method(http.MethodGet)) - close(started) - if err != nil { - errCh <- err - } - defer body.Close() - dec := json.NewDecoder(body) - for { - var e events.Message - if err := dec.Decode(&e); err != nil { - errCh <- err - return - } - if e.Type != events.DaemonEventType { - continue - } - if e.Action != "reload" { - continue - } - close(errCh) // notify that we are done - return - } - }() - - <-started - if err := signalDaemonReload(d.cmd.Process.Pid); err != nil { - return errors.Errorf("error signaling daemon reload: %v", err) - } - select { - case err := <-errCh: - if err != nil { - return errors.Errorf("error waiting for daemon reload event: %v", err) - } - case <-time.After(30 * time.Second): - return errors.New("timeout waiting for daemon reload event") - } - return nil -} - -// NewClient creates new client based on daemon's socket path -func (d *Daemon) NewClient() (*client.Client, error) { - return client.NewClientWithOpts( - client.FromEnv, - client.WithHost(d.Sock())) +// WaitRun waits for a container to be running for 10s +func (d *Daemon) WaitRun(contID string) error { + args := []string{"--host", d.Sock()} + return WaitInspectWithArgs(d.dockerBinary, contID, "{{.State.Running}}", "true", 10*time.Second, args...) } // WaitInspectWithArgs waits for the specified expression to be equals to the specified expected string in the given time. diff --git a/components/engine/integration/container/daemon_linux_test.go b/components/engine/integration/container/daemon_linux_test.go index c2920502e3..abcf9b0d06 100644 --- a/components/engine/integration/container/daemon_linux_test.go +++ b/components/engine/integration/container/daemon_linux_test.go @@ -9,8 +9,8 @@ import ( "testing" "github.com/docker/docker/api/types" - "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/internal/test/daemon" "github.com/gotestyourself/gotestyourself/assert" "github.com/gotestyourself/gotestyourself/skip" "golang.org/x/sys/unix" @@ -30,7 +30,7 @@ func TestContainerStartOnDaemonRestart(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon(), "cannot start daemon on remote test run") t.Parallel() - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t, "--iptables=false") defer d.Stop(t) diff --git a/components/engine/integration/container/export_test.go b/components/engine/integration/container/export_test.go index f7f0295ce5..272feff75d 100644 --- a/components/engine/integration/container/export_test.go +++ b/components/engine/integration/container/export_test.go @@ -9,9 +9,9 @@ import ( "github.com/docker/docker/api/types" containerTypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" + "github.com/docker/docker/internal/test/daemon" "github.com/docker/docker/pkg/jsonmessage" "github.com/gotestyourself/gotestyourself/assert" is "github.com/gotestyourself/gotestyourself/assert/cmp" @@ -62,7 +62,7 @@ func TestExportContainerAfterDaemonRestart(t *testing.T) { skip.If(t, testEnv.DaemonInfo.OSType != "linux") skip.If(t, testEnv.IsRemoteDaemon()) - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) client, err := d.NewClient() assert.NilError(t, err) diff --git a/components/engine/integration/container/mounts_linux_test.go b/components/engine/integration/container/mounts_linux_test.go index cdde06a9ef..2674a2901f 100644 --- a/components/engine/integration/container/mounts_linux_test.go +++ b/components/engine/integration/container/mounts_linux_test.go @@ -12,8 +12,8 @@ import ( "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" - "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration/internal/request" + "github.com/docker/docker/internal/test/daemon" "github.com/docker/docker/pkg/stdcopy" "github.com/docker/docker/pkg/system" "github.com/gotestyourself/gotestyourself/assert" @@ -25,7 +25,7 @@ import ( func TestContainerShmNoLeak(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon(), "cannot start daemon on remote test run") t.Parallel() - d := daemon.New(t, "docker", "dockerd", daemon.Config{}) + d := daemon.New(t) client, err := d.NewClient() if err != nil { t.Fatal(err) diff --git a/components/engine/integration/container/restart_test.go b/components/engine/integration/container/restart_test.go index 7a2576e219..d6deea730e 100644 --- a/components/engine/integration/container/restart_test.go +++ b/components/engine/integration/container/restart_test.go @@ -8,7 +8,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/internal/test/daemon" "github.com/gotestyourself/gotestyourself/skip" ) @@ -55,7 +55,7 @@ func TestDaemonRestartKillContainers(t *testing.T) { t.Parallel() - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) client, err := d.NewClient() if err != nil { t.Fatal(err) diff --git a/components/engine/integration/network/inspect_test.go b/components/engine/integration/network/inspect_test.go index c27b3529d3..3f23c61ae2 100644 --- a/components/engine/integration/network/inspect_test.go +++ b/components/engine/integration/network/inspect_test.go @@ -15,7 +15,6 @@ import ( ) const defaultSwarmPort = 2477 -const dockerdBinary = "dockerd" func TestInspectNetwork(t *testing.T) { defer setupTest(t)() diff --git a/components/engine/integration/network/macvlan_test.go b/components/engine/integration/network/macvlan_test.go index 46945ada28..146f67ee57 100644 --- a/components/engine/integration/network/macvlan_test.go +++ b/components/engine/integration/network/macvlan_test.go @@ -9,8 +9,8 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" - "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/internal/test/daemon" "github.com/docker/docker/pkg/parsers/kernel" "github.com/gotestyourself/gotestyourself/assert" "github.com/gotestyourself/gotestyourself/assert/cmp" @@ -24,7 +24,7 @@ func TestDockerNetworkMacvlanPersistance(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon()) skip.If(t, !macvlanKernelSupport(), "Kernel doesn't support macvlan") - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t) defer d.Stop(t) @@ -53,7 +53,7 @@ func TestDockerNetworkMacvlanOverlapParent(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon()) skip.If(t, !macvlanKernelSupport(), "Kernel doesn't support macvlan") - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t) defer d.Stop(t) @@ -95,7 +95,7 @@ func TestDockerNetworkMacvlanSubinterface(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon()) skip.If(t, !macvlanKernelSupport(), "Kernel doesn't support macvlan") - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t) defer d.Stop(t) @@ -131,7 +131,7 @@ func TestDockerNetworkMacvlanBridgeNilParent(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon()) skip.If(t, !macvlanKernelSupport(), "Kernel doesn't support macvlan") - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t) defer d.Stop(t) client, err := d.NewClient() @@ -157,7 +157,7 @@ func TestDockerNetworkMacvlanBridgeInternal(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon()) skip.If(t, !macvlanKernelSupport(), "Kernel doesn't support macvlan") - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t) defer d.Stop(t) client, err := d.NewClient() @@ -191,7 +191,7 @@ func TestDockerNetworkMacvlanMultiSubnet(t *testing.T) { skip.If(t, !macvlanKernelSupport(), "Kernel doesn't support macvlan") t.Skip("Temporarily skipping while investigating sporadic v6 CI issues") - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t) defer d.Stop(t) client, err := d.NewClient() diff --git a/components/engine/integration/plugin/authz/main_test.go b/components/engine/integration/plugin/authz/main_test.go index 3d5f406b92..1958ce376a 100644 --- a/components/engine/integration/plugin/authz/main_test.go +++ b/components/engine/integration/plugin/authz/main_test.go @@ -12,7 +12,7 @@ import ( "strings" "testing" - "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/internal/test/daemon" "github.com/docker/docker/internal/test/environment" "github.com/docker/docker/pkg/authorization" "github.com/docker/docker/pkg/plugins" @@ -25,8 +25,6 @@ var ( server *httptest.Server ) -const dockerdBinary = "dockerd" - func TestMain(m *testing.M) { var err error testEnv, err = environment.New() @@ -52,9 +50,7 @@ func setupTest(t *testing.T) func() { skip.IfCondition(t, testEnv.IsRemoteDaemon(), "cannot run daemon when remote daemon") environment.ProtectAll(t, testEnv) - d = daemon.New(t, "", dockerdBinary, daemon.Config{ - Experimental: testEnv.DaemonInfo.ExperimentalBuild, - }) + d = daemon.New(t, daemon.WithExperimental) return func() { if d != nil { diff --git a/components/engine/integration/plugin/logging/main_test.go b/components/engine/integration/plugin/logging/main_test.go index b2446ca8c8..e1292a5718 100644 --- a/components/engine/integration/plugin/logging/main_test.go +++ b/components/engine/integration/plugin/logging/main_test.go @@ -12,8 +12,6 @@ var ( testEnv *environment.Execution ) -const dockerdBinary = "dockerd" - func TestMain(m *testing.M) { var err error testEnv, err = environment.New() diff --git a/components/engine/integration/plugin/logging/validation_test.go b/components/engine/integration/plugin/logging/validation_test.go index f0d6e9e8b2..7f67aea75c 100644 --- a/components/engine/integration/plugin/logging/validation_test.go +++ b/components/engine/integration/plugin/logging/validation_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/docker/docker/api/types" - "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/internal/test/daemon" "github.com/gotestyourself/gotestyourself/assert" "github.com/gotestyourself/gotestyourself/skip" ) @@ -17,7 +17,7 @@ func TestDaemonStartWithLogOpt(t *testing.T) { skip.IfCondition(t, testEnv.IsRemoteDaemon(), "cannot run daemon when remote daemon") t.Parallel() - d := daemon.New(t, "", dockerdBinary, daemon.Config{}) + d := daemon.New(t) d.Start(t, "--iptables=false") defer d.Stop(t) diff --git a/components/engine/integration/plugin/volumes/main_test.go b/components/engine/integration/plugin/volumes/main_test.go index e3aa260f8d..5a5678d9c4 100644 --- a/components/engine/integration/plugin/volumes/main_test.go +++ b/components/engine/integration/plugin/volumes/main_test.go @@ -12,8 +12,6 @@ var ( testEnv *environment.Execution ) -const dockerdBinary = "dockerd" - func TestMain(m *testing.M) { var err error testEnv, err = environment.New() diff --git a/components/engine/integration/plugin/volumes/mounts_test.go b/components/engine/integration/plugin/volumes/mounts_test.go index 2c464f6607..d57fcfd36f 100644 --- a/components/engine/integration/plugin/volumes/mounts_test.go +++ b/components/engine/integration/plugin/volumes/mounts_test.go @@ -7,8 +7,8 @@ import ( "testing" "github.com/docker/docker/api/types" - "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration-cli/fixtures/plugin" + "github.com/docker/docker/internal/test/daemon" "github.com/gotestyourself/gotestyourself/assert" ) @@ -17,7 +17,7 @@ import ( func TestPluginWithDevMounts(t *testing.T) { t.Parallel() - d := daemon.New(t, "", dockerdBinary, daemon.Config{}) + d := daemon.New(t) d.Start(t, "--iptables=false") defer d.Stop(t) diff --git a/components/engine/integration/secret/main_test.go b/components/engine/integration/secret/main_test.go index 4a3d4c75e9..abd4eef9f0 100644 --- a/components/engine/integration/secret/main_test.go +++ b/components/engine/integration/secret/main_test.go @@ -10,8 +10,6 @@ import ( var testEnv *environment.Execution -const dockerdBinary = "dockerd" - func TestMain(m *testing.M) { var err error testEnv, err = environment.New() diff --git a/components/engine/integration/service/main_test.go b/components/engine/integration/service/main_test.go index d887403e51..28fd19df4d 100644 --- a/components/engine/integration/service/main_test.go +++ b/components/engine/integration/service/main_test.go @@ -10,8 +10,6 @@ import ( var testEnv *environment.Execution -const dockerdBinary = "dockerd" - func TestMain(m *testing.M) { var err error testEnv, err = environment.New() diff --git a/components/engine/integration/system/cgroupdriver_systemd_test.go b/components/engine/integration/system/cgroupdriver_systemd_test.go index c1f864a1c7..12e716c66c 100644 --- a/components/engine/integration/system/cgroupdriver_systemd_test.go +++ b/components/engine/integration/system/cgroupdriver_systemd_test.go @@ -7,7 +7,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/internal/test/daemon" "github.com/gotestyourself/gotestyourself/assert" ) @@ -33,7 +33,7 @@ func TestCgroupDriverSystemdMemoryLimit(t *testing.T) { t.Skip("systemd not available") } - d := daemon.New(t, "docker", "dockerd", daemon.Config{}) + d := daemon.New(t) client, err := d.NewClient() assert.NilError(t, err) d.StartWithBusybox(t, "--exec-opt", "native.cgroupdriver=systemd", "--iptables=false") diff --git a/components/engine/internal/test/daemon/daemon.go b/components/engine/internal/test/daemon/daemon.go new file mode 100644 index 0000000000..739e134659 --- /dev/null +++ b/components/engine/internal/test/daemon/daemon.go @@ -0,0 +1,618 @@ +package daemon // import "github.com/docker/docker/internal/test/daemon" + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/events" + "github.com/docker/docker/client" + "github.com/docker/docker/integration-cli/request" + "github.com/docker/docker/opts" + "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/stringid" + "github.com/docker/go-connections/sockets" + "github.com/docker/go-connections/tlsconfig" + "github.com/gotestyourself/gotestyourself/assert" + "github.com/pkg/errors" +) + +type testingT interface { + assert.TestingT + logT + Fatalf(string, ...interface{}) +} + +type logT interface { + Logf(string, ...interface{}) +} + +const defaultDockerdBinary = "dockerd" + +var errDaemonNotStarted = errors.New("daemon not started") + +// SockRoot holds the path of the default docker integration daemon socket +var SockRoot = filepath.Join(os.TempDir(), "docker-integration") + +type clientConfig struct { + transport *http.Transport + scheme string + addr string +} + +// Daemon represents a Docker daemon for the testing framework +type Daemon struct { + GlobalFlags []string + Root string + Folder string + Wait chan error + UseDefaultHost bool + UseDefaultTLSHost bool + + id string + logFile *os.File + cmd *exec.Cmd + storageDriver string + userlandProxy bool + execRoot string + experimental bool + dockerdBinary string + log logT +} + +// New returns a Daemon instance to be used for testing. +// This will create a directory such as d123456789 in the folder specified by $DOCKER_INTEGRATION_DAEMON_DEST or $DEST. +// The daemon will not automatically start. +func New(t testingT, ops ...func(*Daemon)) *Daemon { + dest := os.Getenv("DOCKER_INTEGRATION_DAEMON_DEST") + if dest == "" { + dest = os.Getenv("DEST") + } + assert.Check(t, dest != "", "Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable") + + storageDriver := os.Getenv("DOCKER_GRAPHDRIVER") + + assert.NilError(t, os.MkdirAll(SockRoot, 0700), "could not create daemon socket root") + + id := fmt.Sprintf("d%s", stringid.TruncateID(stringid.GenerateRandomID())) + dir := filepath.Join(dest, id) + daemonFolder, err := filepath.Abs(dir) + assert.NilError(t, err, "Could not make %q an absolute path", dir) + daemonRoot := filepath.Join(daemonFolder, "root") + + assert.NilError(t, os.MkdirAll(daemonRoot, 0755), "Could not create daemon root %q", dir) + + userlandProxy := true + if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { + if val, err := strconv.ParseBool(env); err != nil { + userlandProxy = val + } + } + d := &Daemon{ + id: id, + Folder: daemonFolder, + Root: daemonRoot, + storageDriver: storageDriver, + userlandProxy: userlandProxy, + execRoot: filepath.Join(os.TempDir(), "docker-execroot", id), + dockerdBinary: defaultDockerdBinary, + log: t, + } + + for _, op := range ops { + op(d) + } + + return d +} + +// RootDir returns the root directory of the daemon. +func (d *Daemon) RootDir() string { + return d.Root +} + +// ID returns the generated id of the daemon +func (d *Daemon) ID() string { + return d.id +} + +// StorageDriver returns the configured storage driver of the daemon +func (d *Daemon) StorageDriver() string { + return d.storageDriver +} + +// Sock returns the socket path of the daemon +func (d *Daemon) Sock() string { + return fmt.Sprintf("unix://" + d.sockPath()) +} + +func (d *Daemon) sockPath() string { + return filepath.Join(SockRoot, d.id+".sock") +} + +// LogFileName returns the path the daemon's log file +func (d *Daemon) LogFileName() string { + return d.logFile.Name() +} + +// ReadLogFile returns the content of the daemon log file +func (d *Daemon) ReadLogFile() ([]byte, error) { + return ioutil.ReadFile(d.logFile.Name()) +} + +// NewClient creates new client based on daemon's socket path +func (d *Daemon) NewClient() (*client.Client, error) { + return client.NewClientWithOpts( + client.FromEnv, + client.WithHost(d.Sock())) +} + +// CleanupExecRoot cleans the daemon exec root (network namespaces, ...) +func (d *Daemon) CleanupExecRoot(t testingT) { + cleanupExecRoot(t, d.execRoot) +} + +// Start starts the daemon and return once it is ready to receive requests. +func (d *Daemon) Start(t testingT, args ...string) { + if err := d.StartWithError(args...); err != nil { + t.Fatalf("Error starting daemon with arguments: %v", args) + } +} + +// StartWithError starts the daemon and return once it is ready to receive requests. +// It returns an error in case it couldn't start. +func (d *Daemon) StartWithError(args ...string) error { + logFile, err := os.OpenFile(filepath.Join(d.Folder, "docker.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return errors.Wrapf(err, "[%s] Could not create %s/docker.log", d.id, d.Folder) + } + + return d.StartWithLogFile(logFile, args...) +} + +// StartWithLogFile will start the daemon and attach its streams to a given file. +func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { + dockerdBinary, err := exec.LookPath(d.dockerdBinary) + if err != nil { + return errors.Wrapf(err, "[%s] could not find docker binary in $PATH", d.id) + } + args := append(d.GlobalFlags, + "--containerd", "/var/run/docker/containerd/docker-containerd.sock", + "--data-root", d.Root, + "--exec-root", d.execRoot, + "--pidfile", fmt.Sprintf("%s/docker.pid", d.Folder), + fmt.Sprintf("--userland-proxy=%t", d.userlandProxy), + ) + if d.experimental { + args = append(args, "--experimental", "--init") + } + if !(d.UseDefaultHost || d.UseDefaultTLSHost) { + args = append(args, []string{"--host", d.Sock()}...) + } + if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { + args = append(args, []string{"--userns-remap", root}...) + } + + // If we don't explicitly set the log-level or debug flag(-D) then + // turn on debug mode + foundLog := false + foundSd := false + for _, a := range providedArgs { + if strings.Contains(a, "--log-level") || strings.Contains(a, "-D") || strings.Contains(a, "--debug") { + foundLog = true + } + if strings.Contains(a, "--storage-driver") { + foundSd = true + } + } + if !foundLog { + args = append(args, "--debug") + } + if d.storageDriver != "" && !foundSd { + args = append(args, "--storage-driver", d.storageDriver) + } + + args = append(args, providedArgs...) + d.cmd = exec.Command(dockerdBinary, args...) + d.cmd.Env = append(os.Environ(), "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE=1") + d.cmd.Stdout = out + d.cmd.Stderr = out + d.logFile = out + + if err := d.cmd.Start(); err != nil { + return errors.Errorf("[%s] could not start daemon container: %v", d.id, err) + } + + wait := make(chan error) + + go func() { + wait <- d.cmd.Wait() + d.log.Logf("[%s] exiting daemon", d.id) + close(wait) + }() + + d.Wait = wait + + tick := time.Tick(500 * time.Millisecond) + // make sure daemon is ready to receive requests + startTime := time.Now().Unix() + for { + d.log.Logf("[%s] waiting for daemon to start", d.id) + if time.Now().Unix()-startTime > 5 { + // After 5 seconds, give up + return errors.Errorf("[%s] Daemon exited and never started", d.id) + } + select { + case <-time.After(2 * time.Second): + return errors.Errorf("[%s] timeout: daemon does not respond", d.id) + case <-tick: + clientConfig, err := d.getClientConfig() + if err != nil { + return err + } + + client := &http.Client{ + Transport: clientConfig.transport, + } + + req, err := http.NewRequest("GET", "/_ping", nil) + if err != nil { + return errors.Wrapf(err, "[%s] could not create new request", d.id) + } + req.URL.Host = clientConfig.addr + req.URL.Scheme = clientConfig.scheme + resp, err := client.Do(req) + if err != nil { + continue + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + d.log.Logf("[%s] received status != 200 OK: %s\n", d.id, resp.Status) + } + d.log.Logf("[%s] daemon started\n", d.id) + d.Root, err = d.queryRootDir() + if err != nil { + return errors.Errorf("[%s] error querying daemon for root directory: %v", d.id, err) + } + return nil + case <-d.Wait: + return errors.Errorf("[%s] Daemon exited during startup", d.id) + } + } +} + +// StartWithBusybox will first start the daemon with Daemon.Start() +// then save the busybox image from the main daemon and load it into this Daemon instance. +func (d *Daemon) StartWithBusybox(t testingT, arg ...string) { + d.Start(t, arg...) + d.LoadBusybox(t) +} + +// Kill will send a SIGKILL to the daemon +func (d *Daemon) Kill() error { + if d.cmd == nil || d.Wait == nil { + return errDaemonNotStarted + } + + defer func() { + d.logFile.Close() + d.cmd = nil + }() + + if err := d.cmd.Process.Kill(); err != nil { + return err + } + + return os.Remove(fmt.Sprintf("%s/docker.pid", d.Folder)) +} + +// Pid returns the pid of the daemon +func (d *Daemon) Pid() int { + return d.cmd.Process.Pid +} + +// Interrupt stops the daemon by sending it an Interrupt signal +func (d *Daemon) Interrupt() error { + return d.Signal(os.Interrupt) +} + +// Signal sends the specified signal to the daemon if running +func (d *Daemon) Signal(signal os.Signal) error { + if d.cmd == nil || d.Wait == nil { + return errDaemonNotStarted + } + return d.cmd.Process.Signal(signal) +} + +// DumpStackAndQuit sends SIGQUIT to the daemon, which triggers it to dump its +// stack to its log file and exit +// This is used primarily for gathering debug information on test timeout +func (d *Daemon) DumpStackAndQuit() { + if d.cmd == nil || d.cmd.Process == nil { + return + } + SignalDaemonDump(d.cmd.Process.Pid) +} + +// Stop will send a SIGINT every second and wait for the daemon to stop. +// If it times out, a SIGKILL is sent. +// Stop will not delete the daemon directory. If a purged daemon is needed, +// instantiate a new one with NewDaemon. +// If an error occurs while starting the daemon, the test will fail. +func (d *Daemon) Stop(t testingT) { + err := d.StopWithError() + if err != nil { + if err != errDaemonNotStarted { + t.Fatalf("Error while stopping the daemon %s : %v", d.id, err) + } else { + t.Logf("Daemon %s is not started", d.id) + } + } +} + +// StopWithError will send a SIGINT every second and wait for the daemon to stop. +// If it timeouts, a SIGKILL is sent. +// Stop will not delete the daemon directory. If a purged daemon is needed, +// instantiate a new one with NewDaemon. +func (d *Daemon) StopWithError() error { + if d.cmd == nil || d.Wait == nil { + return errDaemonNotStarted + } + + defer func() { + d.logFile.Close() + d.cmd = nil + }() + + i := 1 + tick := time.Tick(time.Second) + + if err := d.cmd.Process.Signal(os.Interrupt); err != nil { + if strings.Contains(err.Error(), "os: process already finished") { + return errDaemonNotStarted + } + return errors.Errorf("could not send signal: %v", err) + } +out1: + for { + select { + case err := <-d.Wait: + return err + case <-time.After(20 * time.Second): + // time for stopping jobs and run onShutdown hooks + d.log.Logf("[%s] daemon started", d.id) + break out1 + } + } + +out2: + for { + select { + case err := <-d.Wait: + return err + case <-tick: + i++ + if i > 5 { + d.log.Logf("tried to interrupt daemon for %d times, now try to kill it", i) + break out2 + } + d.log.Logf("Attempt #%d: daemon is still running with pid %d", i, d.cmd.Process.Pid) + if err := d.cmd.Process.Signal(os.Interrupt); err != nil { + return errors.Errorf("could not send signal: %v", err) + } + } + } + + if err := d.cmd.Process.Kill(); err != nil { + d.log.Logf("Could not kill daemon: %v", err) + return err + } + + d.cmd.Wait() + + return os.Remove(fmt.Sprintf("%s/docker.pid", d.Folder)) +} + +// Restart will restart the daemon by first stopping it and the starting it. +// If an error occurs while starting the daemon, the test will fail. +func (d *Daemon) Restart(t testingT, args ...string) { + d.Stop(t) + d.handleUserns() + d.Start(t, args...) +} + +// RestartWithError will restart the daemon by first stopping it and then starting it. +func (d *Daemon) RestartWithError(arg ...string) error { + if err := d.StopWithError(); err != nil { + return err + } + d.handleUserns() + return d.StartWithError(arg...) +} + +func (d *Daemon) handleUserns() { + // in the case of tests running a user namespace-enabled daemon, we have resolved + // d.Root to be the actual final path of the graph dir after the "uid.gid" of + // remapped root is added--we need to subtract it from the path before calling + // start or else we will continue making subdirectories rather than truly restarting + // with the same location/root: + if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { + d.Root = filepath.Dir(d.Root) + } +} + +// ReloadConfig asks the daemon to reload its configuration +func (d *Daemon) ReloadConfig() error { + if d.cmd == nil || d.cmd.Process == nil { + return errors.New("daemon is not running") + } + + errCh := make(chan error) + started := make(chan struct{}) + go func() { + _, body, err := request.DoOnHost(d.Sock(), "/events", request.Method(http.MethodGet)) + close(started) + if err != nil { + errCh <- err + } + defer body.Close() + dec := json.NewDecoder(body) + for { + var e events.Message + if err := dec.Decode(&e); err != nil { + errCh <- err + return + } + if e.Type != events.DaemonEventType { + continue + } + if e.Action != "reload" { + continue + } + close(errCh) // notify that we are done + return + } + }() + + <-started + if err := signalDaemonReload(d.cmd.Process.Pid); err != nil { + return errors.Errorf("error signaling daemon reload: %v", err) + } + select { + case err := <-errCh: + if err != nil { + return errors.Errorf("error waiting for daemon reload event: %v", err) + } + case <-time.After(30 * time.Second): + return errors.New("timeout waiting for daemon reload event") + } + return nil +} + +// LoadBusybox image into the daemon +func (d *Daemon) LoadBusybox(t assert.TestingT) { + clientHost, err := client.NewEnvClient() + assert.NilError(t, err, "failed to create client") + defer clientHost.Close() + + ctx := context.Background() + reader, err := clientHost.ImageSave(ctx, []string{"busybox:latest"}) + assert.NilError(t, err, "failed to download busybox") + defer reader.Close() + + client, err := d.NewClient() + assert.NilError(t, err, "failed to create client") + defer client.Close() + + resp, err := client.ImageLoad(ctx, reader, true) + assert.NilError(t, err, "failed to load busybox") + defer resp.Body.Close() +} + +func (d *Daemon) getClientConfig() (*clientConfig, error) { + var ( + transport *http.Transport + scheme string + addr string + proto string + ) + if d.UseDefaultTLSHost { + option := &tlsconfig.Options{ + CAFile: "fixtures/https/ca.pem", + CertFile: "fixtures/https/client-cert.pem", + KeyFile: "fixtures/https/client-key.pem", + } + tlsConfig, err := tlsconfig.Client(*option) + if err != nil { + return nil, err + } + transport = &http.Transport{ + TLSClientConfig: tlsConfig, + } + addr = fmt.Sprintf("%s:%d", opts.DefaultHTTPHost, opts.DefaultTLSHTTPPort) + scheme = "https" + proto = "tcp" + } else if d.UseDefaultHost { + addr = opts.DefaultUnixSocket + proto = "unix" + scheme = "http" + transport = &http.Transport{} + } else { + addr = d.sockPath() + proto = "unix" + scheme = "http" + transport = &http.Transport{} + } + + if err := sockets.ConfigureTransport(transport, proto, addr); err != nil { + return nil, err + } + transport.DisableKeepAlives = true + + return &clientConfig{ + transport: transport, + scheme: scheme, + addr: addr, + }, nil +} + +func (d *Daemon) queryRootDir() (string, error) { + // update daemon root by asking /info endpoint (to support user + // namespaced daemon with root remapped uid.gid directory) + clientConfig, err := d.getClientConfig() + if err != nil { + return "", err + } + + client := &http.Client{ + Transport: clientConfig.transport, + } + + req, err := http.NewRequest("GET", "/info", nil) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + req.URL.Host = clientConfig.addr + req.URL.Scheme = clientConfig.scheme + + resp, err := client.Do(req) + if err != nil { + return "", err + } + body := ioutils.NewReadCloserWrapper(resp.Body, func() error { + return resp.Body.Close() + }) + + type Info struct { + DockerRootDir string + } + var b []byte + var i Info + b, err = request.ReadBody(body) + if err == nil && resp.StatusCode == http.StatusOK { + // read the docker root dir + if err = json.Unmarshal(b, &i); err == nil { + return i.DockerRootDir, nil + } + } + return "", err +} + +// Info returns the info struct for this daemon +func (d *Daemon) Info(t assert.TestingT) types.Info { + apiclient, err := client.NewClientWithOpts(client.WithHost((d.Sock()))) + assert.NilError(t, err) + info, err := apiclient.Info(context.Background()) + assert.NilError(t, err) + return info +} diff --git a/components/engine/integration-cli/daemon/daemon_unix.go b/components/engine/internal/test/daemon/daemon_unix.go similarity index 78% rename from components/engine/integration-cli/daemon/daemon_unix.go rename to components/engine/internal/test/daemon/daemon_unix.go index 97548c3103..c0aa26c9f8 100644 --- a/components/engine/integration-cli/daemon/daemon_unix.go +++ b/components/engine/internal/test/daemon/daemon_unix.go @@ -1,16 +1,15 @@ // +build !windows -package daemon // import "github.com/docker/docker/integration-cli/daemon" +package daemon // import "github.com/docker/docker/internal/test/daemon" import ( "os" "path/filepath" - "github.com/go-check/check" "golang.org/x/sys/unix" ) -func cleanupExecRoot(c *check.C, execRoot string) { +func cleanupExecRoot(t testingT, execRoot string) { // Cleanup network namespaces in the exec root of this // daemon because this exec root is specific to this // daemon instance and has no chance of getting @@ -19,7 +18,7 @@ func cleanupExecRoot(c *check.C, execRoot string) { netnsPath := filepath.Join(execRoot, "netns") filepath.Walk(netnsPath, func(path string, info os.FileInfo, err error) error { if err := unix.Unmount(path, unix.MNT_FORCE); err != nil { - c.Logf("unmount of %s failed: %v", path, err) + t.Logf("unmount of %s failed: %v", path, err) } os.Remove(path) return nil diff --git a/components/engine/integration-cli/daemon/daemon_windows.go b/components/engine/internal/test/daemon/daemon_windows.go similarity index 74% rename from components/engine/integration-cli/daemon/daemon_windows.go rename to components/engine/internal/test/daemon/daemon_windows.go index 63f6b03239..8ec554fbc4 100644 --- a/components/engine/integration-cli/daemon/daemon_windows.go +++ b/components/engine/internal/test/daemon/daemon_windows.go @@ -1,10 +1,9 @@ -package daemon // import "github.com/docker/docker/integration-cli/daemon" +package daemon // import "github.com/docker/docker/internal/test/daemon" import ( "fmt" "strconv" - "github.com/go-check/check" "golang.org/x/sys/windows" ) @@ -22,5 +21,5 @@ func signalDaemonReload(pid int) error { return fmt.Errorf("daemon reload not supported") } -func cleanupExecRoot(c *check.C, execRoot string) { +func cleanupExecRoot(t testingT, execRoot string) { } diff --git a/components/engine/internal/test/daemon/ops.go b/components/engine/internal/test/daemon/ops.go new file mode 100644 index 0000000000..0c49749414 --- /dev/null +++ b/components/engine/internal/test/daemon/ops.go @@ -0,0 +1,13 @@ +package daemon + +// WithExperimental sets the daemon in experimental mode +func WithExperimental(d *Daemon) { + d.experimental = true +} + +// WithDockerdBinary sets the dockerd binary to the specified one +func WithDockerdBinary(dockerdBinary string) func(*Daemon) { + return func(d *Daemon) { + d.dockerdBinary = dockerdBinary + } +} From c4fdce16ba74f51c31a00195f68d690fbaf8449d Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 2 Apr 2018 16:23:08 -0400 Subject: [PATCH 3/5] Don't make container mount unbindable Signed-off-by: Michael Crosby Upstream-commit: 4c000662feb3c8e3d63cbcb044a47f627cd9bb45 Component: engine --- .../daemon/container_operations_unix.go | 13 +--- .../container/mounts_linux_test.go | 74 ------------------- 2 files changed, 1 insertion(+), 86 deletions(-) diff --git a/components/engine/daemon/container_operations_unix.go b/components/engine/daemon/container_operations_unix.go index 4e92b6392e..bc7ee45233 100644 --- a/components/engine/daemon/container_operations_unix.go +++ b/components/engine/daemon/container_operations_unix.go @@ -293,7 +293,6 @@ func (daemon *Daemon) createSecretsDir(c *container.Container) error { if err := mount.Mount("tmpfs", dir, "tmpfs", "nodev,nosuid,noexec,"+tmpfsOwnership); err != nil { return errors.Wrap(err, "unable to setup secret mount") } - return nil } @@ -400,15 +399,5 @@ func (daemon *Daemon) setupContainerMountsRoot(c *container.Container) error { if err != nil { return err } - - if err := idtools.MkdirAllAndChown(p, 0700, daemon.idMappings.RootPair()); err != nil { - return err - } - - if err := mount.MakeUnbindable(p); err != nil { - // Setting unbindable is a precaution and is not neccessary for correct operation. - // Do not error out if this fails. - logrus.WithError(err).WithField("resource", p).WithField("container", c.ID).Warn("Error setting container resource mounts to unbindable, this may cause mount leakages, preventing removal of this container.") - } - return nil + return idtools.MkdirAllAndChown(p, 0700, daemon.idMappings.RootPair()) } diff --git a/components/engine/integration/container/mounts_linux_test.go b/components/engine/integration/container/mounts_linux_test.go index 2674a2901f..38aa98355a 100644 --- a/components/engine/integration/container/mounts_linux_test.go +++ b/components/engine/integration/container/mounts_linux_test.go @@ -1,7 +1,6 @@ package container // import "github.com/docker/docker/integration/container" import ( - "bytes" "context" "fmt" "path/filepath" @@ -13,8 +12,6 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/docker/docker/integration/internal/request" - "github.com/docker/docker/internal/test/daemon" - "github.com/docker/docker/pkg/stdcopy" "github.com/docker/docker/pkg/system" "github.com/gotestyourself/gotestyourself/assert" is "github.com/gotestyourself/gotestyourself/assert/cmp" @@ -22,77 +19,6 @@ import ( "github.com/gotestyourself/gotestyourself/skip" ) -func TestContainerShmNoLeak(t *testing.T) { - skip.If(t, testEnv.IsRemoteDaemon(), "cannot start daemon on remote test run") - t.Parallel() - d := daemon.New(t) - client, err := d.NewClient() - if err != nil { - t.Fatal(err) - } - d.StartWithBusybox(t, "--iptables=false") - defer d.Stop(t) - - ctx := context.Background() - cfg := container.Config{ - Image: "busybox", - Cmd: []string{"top"}, - } - - ctr, err := client.ContainerCreate(ctx, &cfg, nil, nil, "") - if err != nil { - t.Fatal(err) - } - defer client.ContainerRemove(ctx, ctr.ID, types.ContainerRemoveOptions{Force: true}) - - if err := client.ContainerStart(ctx, ctr.ID, types.ContainerStartOptions{}); err != nil { - t.Fatal(err) - } - - // this should recursively bind mount everything in the test daemons root - // except of course we are hoping that the previous containers /dev/shm mount did not leak into this new container - hc := container.HostConfig{ - Mounts: []mount.Mount{ - { - Type: mount.TypeBind, - Source: d.Root, - Target: "/testdaemonroot", - }, - }, - } - cfg.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("mount | grep testdaemonroot | grep containers | grep %s", ctr.ID)} - cfg.AttachStdout = true - cfg.AttachStderr = true - ctrLeak, err := client.ContainerCreate(ctx, &cfg, &hc, nil, "") - if err != nil { - t.Fatal(err) - } - - attach, err := client.ContainerAttach(ctx, ctrLeak.ID, types.ContainerAttachOptions{ - Stream: true, - Stdout: true, - Stderr: true, - }) - if err != nil { - t.Fatal(err) - } - - if err := client.ContainerStart(ctx, ctrLeak.ID, types.ContainerStartOptions{}); err != nil { - t.Fatal(err) - } - - buf := bytes.NewBuffer(nil) - - if _, err := stdcopy.StdCopy(buf, buf, attach.Reader); err != nil { - t.Fatal(err) - } - - out := bytes.TrimSpace(buf.Bytes()) - if !bytes.Equal(out, []byte{}) { - t.Fatalf("mount leaked: %s", string(out)) - } -} - func TestContainerNetworkMountsNoChown(t *testing.T) { // chown only applies to Linux bind mounted volumes; must be same host to verify skip.If(t, testEnv.DaemonInfo.OSType != "linux" || testEnv.IsRemoteDaemon()) From 59714fb17498a901b4f4fcb273bd84db7d22f5d3 Mon Sep 17 00:00:00 2001 From: Anda Xu Date: Tue, 10 Apr 2018 16:54:39 -0700 Subject: [PATCH 4/5] remove the retries for service update Signed-off-by: Anda Xu Upstream-commit: 7380935331f0c35315003578258f6c1f47c1a586 Component: engine --- .../engine/integration-cli/daemon/daemon_swarm.go | 15 --------------- .../docker_cli_service_update_test.go | 8 ++++---- .../integration-cli/docker_cli_swarm_test.go | 4 ++-- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/components/engine/integration-cli/daemon/daemon_swarm.go b/components/engine/integration-cli/daemon/daemon_swarm.go index be0ddef99e..d4ba6de537 100644 --- a/components/engine/integration-cli/daemon/daemon_swarm.go +++ b/components/engine/integration-cli/daemon/daemon_swarm.go @@ -644,18 +644,3 @@ func (d *Swarm) CheckLeader(c *check.C) (interface{}, check.CommentInterface) { } return fmt.Errorf("no leader"), check.Commentf("could not find leader") } - -// CmdRetryOutOfSequence tries the specified command against the current daemon for 10 times -func (d *Swarm) CmdRetryOutOfSequence(args ...string) (string, error) { - for i := 0; ; i++ { - out, err := d.Cmd(args...) - if err != nil { - if strings.Contains(out, "update out of sequence") { - if i < 10 { - continue - } - } - } - return out, err - } -} diff --git a/components/engine/integration-cli/docker_cli_service_update_test.go b/components/engine/integration-cli/docker_cli_service_update_test.go index 7b590a0f4e..a281327afe 100644 --- a/components/engine/integration-cli/docker_cli_service_update_test.go +++ b/components/engine/integration-cli/docker_cli_service_update_test.go @@ -69,7 +69,7 @@ func (s *DockerSwarmSuite) TestServiceUpdateSecrets(c *check.C) { c.Assert(err, checker.IsNil, check.Commentf(out)) // add secret - out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "test", "--secret-add", fmt.Sprintf("source=%s,target=%s", testName, testTarget)) + out, err = d.Cmd("service", "update", "--detach", "test", "--secret-add", fmt.Sprintf("source=%s,target=%s", testName, testTarget)) c.Assert(err, checker.IsNil, check.Commentf(out)) out, err = d.Cmd("service", "inspect", "--format", "{{ json .Spec.TaskTemplate.ContainerSpec.Secrets }}", serviceName) @@ -84,7 +84,7 @@ func (s *DockerSwarmSuite) TestServiceUpdateSecrets(c *check.C) { c.Assert(refs[0].File.Name, checker.Equals, testTarget) // remove - out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "test", "--secret-rm", testName) + out, err = d.Cmd("service", "update", "--detach", "test", "--secret-rm", testName) c.Assert(err, checker.IsNil, check.Commentf(out)) out, err = d.Cmd("service", "inspect", "--format", "{{ json .Spec.TaskTemplate.ContainerSpec.Secrets }}", serviceName) @@ -111,7 +111,7 @@ func (s *DockerSwarmSuite) TestServiceUpdateConfigs(c *check.C) { c.Assert(err, checker.IsNil, check.Commentf(out)) // add config - out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "test", "--config-add", fmt.Sprintf("source=%s,target=%s", testName, testTarget)) + out, err = d.Cmd("service", "update", "--detach", "test", "--config-add", fmt.Sprintf("source=%s,target=%s", testName, testTarget)) c.Assert(err, checker.IsNil, check.Commentf(out)) out, err = d.Cmd("service", "inspect", "--format", "{{ json .Spec.TaskTemplate.ContainerSpec.Configs }}", serviceName) @@ -126,7 +126,7 @@ func (s *DockerSwarmSuite) TestServiceUpdateConfigs(c *check.C) { c.Assert(refs[0].File.Name, checker.Equals, testTarget) // remove - out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "test", "--config-rm", testName) + out, err = d.Cmd("service", "update", "--detach", "test", "--config-rm", testName) c.Assert(err, checker.IsNil, check.Commentf(out)) out, err = d.Cmd("service", "inspect", "--format", "{{ json .Spec.TaskTemplate.ContainerSpec.Configs }}", serviceName) diff --git a/components/engine/integration-cli/docker_cli_swarm_test.go b/components/engine/integration-cli/docker_cli_swarm_test.go index 4b3358255f..ccd3b08ea4 100644 --- a/components/engine/integration-cli/docker_cli_swarm_test.go +++ b/components/engine/integration-cli/docker_cli_swarm_test.go @@ -283,10 +283,10 @@ func (s *DockerSwarmSuite) TestSwarmPublishAdd(c *check.C) { out, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", name) c.Assert(err, checker.IsNil) - out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", name) + out, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", name) c.Assert(err, checker.IsNil) - out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", "--publish-add", "80:20", name) + out, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", "--publish-add", "80:20", name) c.Assert(err, checker.NotNil) out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.EndpointSpec.Ports }}", name) From 4059b74760cdc8a7cccb713d24aaebff1fe90e7c Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Wed, 11 Apr 2018 12:10:17 +0200 Subject: [PATCH 5/5] Make internal/test/daemon.Daemon swarm aware This remove the daemon.Swarm construction by make the new test Daemon struct aware of swarm. Signed-off-by: Vincent Demeester Upstream-commit: 83d18cf4e3e84055f7034816eed2a10c04e777ca Component: engine --- .../engine/integration-cli/check_test.go | 22 +- .../engine/integration-cli/daemon/daemon.go | 21 +- .../integration-cli/daemon/daemon_swarm.go | 494 +----------------- .../integration-cli/daemon_swarm_hack_test.go | 4 +- .../docker_api_swarm_node_test.go | 13 +- .../docker_api_swarm_service_test.go | 19 +- .../integration-cli/docker_api_swarm_test.go | 240 +++++---- .../docker_cli_prune_unix_test.go | 2 +- .../docker_cli_service_logs_test.go | 2 +- .../integration-cli/docker_cli_swarm_test.go | 67 ++- .../integration/internal/swarm/service.go | 32 +- .../engine/internal/test/daemon/config.go | 66 +++ .../engine/internal/test/daemon/daemon.go | 38 +- .../engine/internal/test/daemon/node.go | 69 +++ components/engine/internal/test/daemon/ops.go | 14 + .../engine/internal/test/daemon/secret.go | 68 +++ .../engine/internal/test/daemon/service.go | 108 ++++ .../engine/internal/test/daemon/swarm.go | 139 +++++ 18 files changed, 721 insertions(+), 697 deletions(-) create mode 100644 components/engine/internal/test/daemon/config.go create mode 100644 components/engine/internal/test/daemon/node.go create mode 100644 components/engine/internal/test/daemon/secret.go create mode 100644 components/engine/internal/test/daemon/service.go create mode 100644 components/engine/internal/test/daemon/swarm.go diff --git a/components/engine/integration-cli/check_test.go b/components/engine/integration-cli/check_test.go index a455893d8c..0cae1503f8 100644 --- a/components/engine/integration-cli/check_test.go +++ b/components/engine/integration-cli/check_test.go @@ -311,7 +311,7 @@ func init() { type DockerSwarmSuite struct { server *httptest.Server ds *DockerSuite - daemons []*daemon.Swarm + daemons []*daemon.Daemon daemonsLock sync.Mutex // protect access to daemons portIndex int } @@ -328,14 +328,10 @@ func (s *DockerSwarmSuite) SetUpTest(c *check.C) { testRequires(c, DaemonIsLinux, SameHostDaemon) } -func (s *DockerSwarmSuite) AddDaemon(c *check.C, joinSwarm, manager bool) *daemon.Swarm { - d := &daemon.Swarm{ - Daemon: daemon.New(c, dockerBinary, dockerdBinary, daemon.Config{ - Experimental: testEnv.DaemonInfo.ExperimentalBuild, - }), - Port: defaultSwarmPort + s.portIndex, - } - d.ListenAddr = fmt.Sprintf("0.0.0.0:%d", d.Port) +func (s *DockerSwarmSuite) AddDaemon(c *check.C, joinSwarm, manager bool) *daemon.Daemon { + d := daemon.New(c, dockerBinary, dockerdBinary, daemon.Config{ + Experimental: testEnv.DaemonInfo.ExperimentalBuild, + }, testdaemon.WithSwarmPort(defaultSwarmPort+s.portIndex)) args := []string{"--iptables=false", "--swarm-default-advertise-addr=lo"} // avoid networking conflicts d.StartWithBusybox(c, args...) @@ -346,12 +342,12 @@ func (s *DockerSwarmSuite) AddDaemon(c *check.C, joinSwarm, manager bool) *daemo if manager { token = tokens.Manager } - c.Assert(d.Join(swarm.JoinRequest{ - RemoteAddrs: []string{s.daemons[0].ListenAddr}, + d.SwarmJoin(c, swarm.JoinRequest{ + RemoteAddrs: []string{s.daemons[0].SwarmListenAddr()}, JoinToken: token, - }), check.IsNil) + }) } else { - c.Assert(d.Init(swarm.InitRequest{}), check.IsNil) + d.SwarmInit(c, swarm.InitRequest{}) } } diff --git a/components/engine/integration-cli/daemon/daemon.go b/components/engine/integration-cli/daemon/daemon.go index 4ef1dafe79..0ec9e892b2 100644 --- a/components/engine/integration-cli/daemon/daemon.go +++ b/components/engine/integration-cli/daemon/daemon.go @@ -38,10 +38,8 @@ type Config struct { // New returns a Daemon instance to be used for testing. // This will create a directory such as d123456789 in the folder specified by $DOCKER_INTEGRATION_DAEMON_DEST or $DEST. // The daemon will not automatically start. -func New(t testingT, dockerBinary string, dockerdBinary string, config Config) *Daemon { - ops := []func(*daemon.Daemon){ - daemon.WithDockerdBinary(dockerdBinary), - } +func New(t testingT, dockerBinary string, dockerdBinary string, config Config, ops ...func(*daemon.Daemon)) *Daemon { + ops = append(ops, daemon.WithDockerdBinary(dockerdBinary)) if config.Experimental { ops = append(ops, daemon.WithExperimental) } @@ -150,6 +148,21 @@ func (d *Daemon) WaitRun(contID string) error { return WaitInspectWithArgs(d.dockerBinary, contID, "{{.State.Running}}", "true", 10*time.Second, args...) } +// CmdRetryOutOfSequence tries the specified command against the current daemon for 10 times +func (d *Daemon) CmdRetryOutOfSequence(args ...string) (string, error) { + for i := 0; ; i++ { + out, err := d.Cmd(args...) + if err != nil { + if strings.Contains(out, "update out of sequence") { + if i < 10 { + continue + } + } + } + return out, err + } +} + // WaitInspectWithArgs waits for the specified expression to be equals to the specified expected string in the given time. // Deprecated: use cli.WaitCmd instead func WaitInspectWithArgs(dockerBinary, name, expr, expected string, timeout time.Duration, arg ...string) error { diff --git a/components/engine/integration-cli/daemon/daemon_swarm.go b/components/engine/integration-cli/daemon/daemon_swarm.go index be0ddef99e..cf75724a32 100644 --- a/components/engine/integration-cli/daemon/daemon_swarm.go +++ b/components/engine/integration-cli/daemon/daemon_swarm.go @@ -3,7 +3,6 @@ package daemon // import "github.com/docker/docker/integration-cli/daemon" import ( "fmt" "strings" - "time" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" @@ -12,178 +11,12 @@ import ( "github.com/docker/docker/integration-cli/checker" "github.com/go-check/check" "github.com/gotestyourself/gotestyourself/assert" - "github.com/pkg/errors" "golang.org/x/net/context" ) -// Swarm is a test daemon with helpers for participating in a swarm. -type Swarm struct { - *Daemon - swarm.Info - Port int - ListenAddr string -} - -// Init initializes a new swarm cluster. -func (d *Swarm) Init(req swarm.InitRequest) error { - if req.ListenAddr == "" { - req.ListenAddr = d.ListenAddr - } - cli, err := d.NewClient() - if err != nil { - return fmt.Errorf("initializing swarm: failed to create client %v", err) - } - defer cli.Close() - _, err = cli.SwarmInit(context.Background(), req) - if err != nil { - return fmt.Errorf("initializing swarm: %v", err) - } - info, err := d.SwarmInfo() - if err != nil { - return err - } - d.Info = info - return nil -} - -// Join joins a daemon to an existing cluster. -func (d *Swarm) Join(req swarm.JoinRequest) error { - if req.ListenAddr == "" { - req.ListenAddr = d.ListenAddr - } - cli, err := d.NewClient() - if err != nil { - return fmt.Errorf("joining swarm: failed to create client %v", err) - } - defer cli.Close() - err = cli.SwarmJoin(context.Background(), req) - if err != nil { - return fmt.Errorf("joining swarm: %v", err) - } - info, err := d.SwarmInfo() - if err != nil { - return err - } - d.Info = info - return nil -} - -// Leave forces daemon to leave current cluster. -func (d *Swarm) Leave(force bool) error { - cli, err := d.NewClient() - if err != nil { - return fmt.Errorf("leaving swarm: failed to create client %v", err) - } - defer cli.Close() - err = cli.SwarmLeave(context.Background(), force) - if err != nil { - err = fmt.Errorf("leaving swarm: %v", err) - } - return err -} - -// SwarmInfo returns the swarm information of the daemon -func (d *Swarm) SwarmInfo() (swarm.Info, error) { - cli, err := d.NewClient() - if err != nil { - return swarm.Info{}, fmt.Errorf("get swarm info: %v", err) - } - - info, err := cli.Info(context.Background()) - if err != nil { - return swarm.Info{}, fmt.Errorf("get swarm info: %v", err) - } - - return info.Swarm, nil -} - -// Unlock tries to unlock a locked swarm -func (d *Swarm) Unlock(req swarm.UnlockRequest) error { - cli, err := d.NewClient() - if err != nil { - return fmt.Errorf("unlocking swarm: failed to create client %v", err) - } - defer cli.Close() - err = cli.SwarmUnlock(context.Background(), req) - if err != nil { - err = errors.Wrap(err, "unlocking swarm") - } - return err -} - -// ServiceConstructor defines a swarm service constructor function -type ServiceConstructor func(*swarm.Service) - -// NodeConstructor defines a swarm node constructor -type NodeConstructor func(*swarm.Node) - -// SecretConstructor defines a swarm secret constructor -type SecretConstructor func(*swarm.Secret) - -// ConfigConstructor defines a swarm config constructor -type ConfigConstructor func(*swarm.Config) - -// SpecConstructor defines a swarm spec constructor -type SpecConstructor func(*swarm.Spec) - -// CreateServiceWithOptions creates a swarm service given the specified service constructors -// and auth config -func (d *Swarm) CreateServiceWithOptions(c *check.C, opts types.ServiceCreateOptions, f ...ServiceConstructor) string { - var service swarm.Service - for _, fn := range f { - fn(&service) - } - - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - res, err := cli.ServiceCreate(ctx, service.Spec, opts) - c.Assert(err, checker.IsNil) - return res.ID -} - -// CreateService creates a swarm service given the specified service constructor -func (d *Swarm) CreateService(c *check.C, f ...ServiceConstructor) string { - return d.CreateServiceWithOptions(c, types.ServiceCreateOptions{}, f...) -} - -// GetService returns the swarm service corresponding to the specified id -func (d *Swarm) GetService(c *check.C, id string) *swarm.Service { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - service, _, err := cli.ServiceInspectWithRaw(context.Background(), id, types.ServiceInspectOptions{}) - c.Assert(err, checker.IsNil) - return &service -} - -// GetServiceTasks returns the swarm tasks for the specified service -func (d *Swarm) GetServiceTasks(c *check.C, service string) []swarm.Task { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - filterArgs := filters.NewArgs() - filterArgs.Add("desired-state", "running") - filterArgs.Add("service", service) - - options := types.TaskListOptions{ - Filters: filterArgs, - } - - tasks, err := cli.TaskList(context.Background(), options) - c.Assert(err, checker.IsNil) - return tasks -} - // CheckServiceTasksInState returns the number of tasks with a matching state, // and optional message substring. -func (d *Swarm) CheckServiceTasksInState(service string, state swarm.TaskState, message string) func(*check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckServiceTasksInState(service string, state swarm.TaskState, message string) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { tasks := d.GetServiceTasks(c, service) var count int @@ -200,7 +33,7 @@ func (d *Swarm) CheckServiceTasksInState(service string, state swarm.TaskState, // CheckServiceTasksInStateWithError returns the number of tasks with a matching state, // and optional message substring. -func (d *Swarm) CheckServiceTasksInStateWithError(service string, state swarm.TaskState, errorMessage string) func(*check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckServiceTasksInStateWithError(service string, state swarm.TaskState, errorMessage string) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { tasks := d.GetServiceTasks(c, service) var count int @@ -216,12 +49,12 @@ func (d *Swarm) CheckServiceTasksInStateWithError(service string, state swarm.Ta } // CheckServiceRunningTasks returns the number of running tasks for the specified service -func (d *Swarm) CheckServiceRunningTasks(service string) func(*check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckServiceRunningTasks(service string) func(*check.C) (interface{}, check.CommentInterface) { return d.CheckServiceTasksInState(service, swarm.TaskStateRunning, "") } // CheckServiceUpdateState returns the current update state for the specified service -func (d *Swarm) CheckServiceUpdateState(service string) func(*check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckServiceUpdateState(service string) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { service := d.GetService(c, service) if service.UpdateStatus == nil { @@ -232,7 +65,7 @@ func (d *Swarm) CheckServiceUpdateState(service string) func(*check.C) (interfac } // CheckPluginRunning returns the runtime state of the plugin -func (d *Swarm) CheckPluginRunning(plugin string) func(c *check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckPluginRunning(plugin string) func(c *check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { apiclient, err := d.NewClient() assert.NilError(c, err) @@ -246,7 +79,7 @@ func (d *Swarm) CheckPluginRunning(plugin string) func(c *check.C) (interface{}, } // CheckPluginImage returns the runtime state of the plugin -func (d *Swarm) CheckPluginImage(plugin string) func(c *check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckPluginImage(plugin string) func(c *check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { apiclient, err := d.NewClient() assert.NilError(c, err) @@ -260,7 +93,7 @@ func (d *Swarm) CheckPluginImage(plugin string) func(c *check.C) (interface{}, c } // CheckServiceTasks returns the number of tasks for the specified service -func (d *Swarm) CheckServiceTasks(service string) func(*check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckServiceTasks(service string) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { tasks := d.GetServiceTasks(c, service) return len(tasks), nil @@ -268,7 +101,7 @@ func (d *Swarm) CheckServiceTasks(service string) func(*check.C) (interface{}, c } // CheckRunningTaskNetworks returns the number of times each network is referenced from a task. -func (d *Swarm) CheckRunningTaskNetworks(c *check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckRunningTaskNetworks(c *check.C) (interface{}, check.CommentInterface) { cli, err := d.NewClient() c.Assert(err, checker.IsNil) defer cli.Close() @@ -293,7 +126,7 @@ func (d *Swarm) CheckRunningTaskNetworks(c *check.C) (interface{}, check.Comment } // CheckRunningTaskImages returns the times each image is running as a task. -func (d *Swarm) CheckRunningTaskImages(c *check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckRunningTaskImages(c *check.C) (interface{}, check.CommentInterface) { cli, err := d.NewClient() c.Assert(err, checker.IsNil) defer cli.Close() @@ -318,7 +151,7 @@ func (d *Swarm) CheckRunningTaskImages(c *check.C) (interface{}, check.CommentIn } // CheckNodeReadyCount returns the number of ready node on the swarm -func (d *Swarm) CheckNodeReadyCount(c *check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckNodeReadyCount(c *check.C) (interface{}, check.CommentInterface) { nodes := d.ListNodes(c) var readyCount int for _, node := range nodes { @@ -329,303 +162,21 @@ func (d *Swarm) CheckNodeReadyCount(c *check.C) (interface{}, check.CommentInter return readyCount, nil } -// GetTask returns the swarm task identified by the specified id -func (d *Swarm) GetTask(c *check.C, id string) swarm.Task { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - task, _, err := cli.TaskInspectWithRaw(context.Background(), id) - c.Assert(err, checker.IsNil) - return task -} - -// UpdateService updates a swarm service with the specified service constructor -func (d *Swarm) UpdateService(c *check.C, service *swarm.Service, f ...ServiceConstructor) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - for _, fn := range f { - fn(service) - } - - _, err = cli.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, types.ServiceUpdateOptions{}) - c.Assert(err, checker.IsNil) -} - -// RemoveService removes the specified service -func (d *Swarm) RemoveService(c *check.C, id string) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - err = cli.ServiceRemove(context.Background(), id) - c.Assert(err, checker.IsNil) -} - -// GetNode returns a swarm node identified by the specified id -func (d *Swarm) GetNode(c *check.C, id string) *swarm.Node { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - node, _, err := cli.NodeInspectWithRaw(context.Background(), id) - c.Assert(err, checker.IsNil) - c.Assert(node.ID, checker.Equals, id) - return &node -} - -// RemoveNode removes the specified node -func (d *Swarm) RemoveNode(c *check.C, id string, force bool) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - options := types.NodeRemoveOptions{ - Force: force, - } - err = cli.NodeRemove(context.Background(), id, options) - c.Assert(err, checker.IsNil) -} - -// UpdateNode updates a swarm node with the specified node constructor -func (d *Swarm) UpdateNode(c *check.C, id string, f ...NodeConstructor) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - for i := 0; ; i++ { - node := d.GetNode(c, id) - for _, fn := range f { - fn(node) - } - - err = cli.NodeUpdate(context.Background(), node.ID, node.Version, node.Spec) - if i < 10 && err != nil && strings.Contains(err.Error(), "update out of sequence") { - time.Sleep(100 * time.Millisecond) - continue - } - c.Assert(err, checker.IsNil) - return - } -} - -// ListNodes returns the list of the current swarm nodes -func (d *Swarm) ListNodes(c *check.C) []swarm.Node { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{}) - c.Assert(err, checker.IsNil) - - return nodes -} - -// ListServices returns the list of the current swarm services -func (d *Swarm) ListServices(c *check.C) []swarm.Service { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{}) - c.Assert(err, checker.IsNil) - return services -} - -// CreateSecret creates a secret given the specified spec -func (d *Swarm) CreateSecret(c *check.C, secretSpec swarm.SecretSpec) string { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - scr, err := cli.SecretCreate(context.Background(), secretSpec) - c.Assert(err, checker.IsNil) - - return scr.ID -} - -// ListSecrets returns the list of the current swarm secrets -func (d *Swarm) ListSecrets(c *check.C) []swarm.Secret { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - secrets, err := cli.SecretList(context.Background(), types.SecretListOptions{}) - c.Assert(err, checker.IsNil) - return secrets -} - -// GetSecret returns a swarm secret identified by the specified id -func (d *Swarm) GetSecret(c *check.C, id string) *swarm.Secret { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - secret, _, err := cli.SecretInspectWithRaw(context.Background(), id) - c.Assert(err, checker.IsNil) - return &secret -} - -// DeleteSecret removes the swarm secret identified by the specified id -func (d *Swarm) DeleteSecret(c *check.C, id string) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - err = cli.SecretRemove(context.Background(), id) - c.Assert(err, checker.IsNil) -} - -// UpdateSecret updates the swarm secret identified by the specified id -// Currently, only label update is supported. -func (d *Swarm) UpdateSecret(c *check.C, id string, f ...SecretConstructor) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - secret := d.GetSecret(c, id) - for _, fn := range f { - fn(secret) - } - - err = cli.SecretUpdate(context.Background(), secret.ID, secret.Version, secret.Spec) - - c.Assert(err, checker.IsNil) -} - -// CreateConfig creates a config given the specified spec -func (d *Swarm) CreateConfig(c *check.C, configSpec swarm.ConfigSpec) string { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - scr, err := cli.ConfigCreate(context.Background(), configSpec) - c.Assert(err, checker.IsNil) - return scr.ID -} - -// ListConfigs returns the list of the current swarm configs -func (d *Swarm) ListConfigs(c *check.C) []swarm.Config { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - configs, err := cli.ConfigList(context.Background(), types.ConfigListOptions{}) - c.Assert(err, checker.IsNil) - return configs -} - -// GetConfig returns a swarm config identified by the specified id -func (d *Swarm) GetConfig(c *check.C, id string) *swarm.Config { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - config, _, err := cli.ConfigInspectWithRaw(context.Background(), id) - c.Assert(err, checker.IsNil) - return &config -} - -// DeleteConfig removes the swarm config identified by the specified id -func (d *Swarm) DeleteConfig(c *check.C, id string) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - err = cli.ConfigRemove(context.Background(), id) - c.Assert(err, checker.IsNil) -} - -// UpdateConfig updates the swarm config identified by the specified id -// Currently, only label update is supported. -func (d *Swarm) UpdateConfig(c *check.C, id string, f ...ConfigConstructor) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - config := d.GetConfig(c, id) - for _, fn := range f { - fn(config) - } - - err = cli.ConfigUpdate(context.Background(), config.ID, config.Version, config.Spec) - c.Assert(err, checker.IsNil) -} - -// GetSwarm returns the current swarm object -func (d *Swarm) GetSwarm(c *check.C) swarm.Swarm { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - sw, err := cli.SwarmInspect(context.Background()) - c.Assert(err, checker.IsNil) - return sw -} - -// UpdateSwarm updates the current swarm object with the specified spec constructors -func (d *Swarm) UpdateSwarm(c *check.C, f ...SpecConstructor) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - sw := d.GetSwarm(c) - for _, fn := range f { - fn(&sw.Spec) - } - - err = cli.SwarmUpdate(context.Background(), sw.Version, sw.Spec, swarm.UpdateFlags{}) - c.Assert(err, checker.IsNil) -} - -// RotateTokens update the swarm to rotate tokens -func (d *Swarm) RotateTokens(c *check.C) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - sw, err := cli.SwarmInspect(context.Background()) - c.Assert(err, checker.IsNil) - - flags := swarm.UpdateFlags{ - RotateManagerToken: true, - RotateWorkerToken: true, - } - - err = cli.SwarmUpdate(context.Background(), sw.Version, sw.Spec, flags) - c.Assert(err, checker.IsNil) -} - -// JoinTokens returns the current swarm join tokens -func (d *Swarm) JoinTokens(c *check.C) swarm.JoinTokens { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - sw, err := cli.SwarmInspect(context.Background()) - c.Assert(err, checker.IsNil) - return sw.JoinTokens -} - // CheckLocalNodeState returns the current swarm node state -func (d *Swarm) CheckLocalNodeState(c *check.C) (interface{}, check.CommentInterface) { - info, err := d.SwarmInfo() - c.Assert(err, checker.IsNil) +func (d *Daemon) CheckLocalNodeState(c *check.C) (interface{}, check.CommentInterface) { + info := d.SwarmInfo(c) return info.LocalNodeState, nil } // CheckControlAvailable returns the current swarm control available -func (d *Swarm) CheckControlAvailable(c *check.C) (interface{}, check.CommentInterface) { - info, err := d.SwarmInfo() - c.Assert(err, checker.IsNil) +func (d *Daemon) CheckControlAvailable(c *check.C) (interface{}, check.CommentInterface) { + info := d.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) return info.ControlAvailable, nil } // CheckLeader returns whether there is a leader on the swarm or not -func (d *Swarm) CheckLeader(c *check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckLeader(c *check.C) (interface{}, check.CommentInterface) { cli, err := d.NewClient() c.Assert(err, checker.IsNil) defer cli.Close() @@ -644,18 +195,3 @@ func (d *Swarm) CheckLeader(c *check.C) (interface{}, check.CommentInterface) { } return fmt.Errorf("no leader"), check.Commentf("could not find leader") } - -// CmdRetryOutOfSequence tries the specified command against the current daemon for 10 times -func (d *Swarm) CmdRetryOutOfSequence(args ...string) (string, error) { - for i := 0; ; i++ { - out, err := d.Cmd(args...) - if err != nil { - if strings.Contains(out, "update out of sequence") { - if i < 10 { - continue - } - } - } - return out, err - } -} diff --git a/components/engine/integration-cli/daemon_swarm_hack_test.go b/components/engine/integration-cli/daemon_swarm_hack_test.go index e1fb333f82..7a23e84bfc 100644 --- a/components/engine/integration-cli/daemon_swarm_hack_test.go +++ b/components/engine/integration-cli/daemon_swarm_hack_test.go @@ -5,11 +5,11 @@ import ( "github.com/go-check/check" ) -func (s *DockerSwarmSuite) getDaemon(c *check.C, nodeID string) *daemon.Swarm { +func (s *DockerSwarmSuite) getDaemon(c *check.C, nodeID string) *daemon.Daemon { s.daemonsLock.Lock() defer s.daemonsLock.Unlock() for _, d := range s.daemons { - if d.NodeID == nodeID { + if d.NodeID() == nodeID { return d } } diff --git a/components/engine/integration-cli/docker_api_swarm_node_test.go b/components/engine/integration-cli/docker_api_swarm_node_test.go index 98f80552cf..6f939ea460 100644 --- a/components/engine/integration-cli/docker_api_swarm_node_test.go +++ b/components/engine/integration-cli/docker_api_swarm_node_test.go @@ -21,8 +21,8 @@ func (s *DockerSwarmSuite) TestAPISwarmListNodes(c *check.C) { loop0: for _, n := range nodes { - for _, d := range []*daemon.Swarm{d1, d2, d3} { - if n.ID == d.NodeID { + for _, d := range []*daemon.Daemon{d1, d2, d3} { + if n.ID == d.NodeID() { continue loop0 } } @@ -53,8 +53,7 @@ func (s *DockerSwarmSuite) TestAPISwarmNodeRemove(c *check.C) { c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes)) // Getting the info so we can take the NodeID - d2Info, err := d2.SwarmInfo() - c.Assert(err, checker.IsNil) + d2Info := d2.SwarmInfo(c) // forceful removal of d2 should work d1.RemoveNode(c, d2Info.NodeID, true) @@ -88,14 +87,14 @@ func (s *DockerSwarmSuite) TestAPISwarmNodeDrainPause(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount), checker.Equals, instances) // drain d2, all containers should move to d1 - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Availability = swarm.NodeAvailabilityDrain }) waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances) waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 0) // set d2 back to active - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Availability = swarm.NodeAvailabilityActive }) @@ -115,7 +114,7 @@ func (s *DockerSwarmSuite) TestAPISwarmNodeDrainPause(c *check.C) { d2ContainerCount := len(d2.ActiveContainers()) // set d2 to paused, scale service up, only d1 gets new tasks - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Availability = swarm.NodeAvailabilityPause }) diff --git a/components/engine/integration-cli/docker_api_swarm_service_test.go b/components/engine/integration-cli/docker_api_swarm_service_test.go index 845453d757..f715265f63 100644 --- a/components/engine/integration-cli/docker_api_swarm_service_test.go +++ b/components/engine/integration-cli/docker_api_swarm_service_test.go @@ -15,12 +15,13 @@ import ( "github.com/docker/docker/integration-cli/checker" "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration-cli/fixtures/plugin" + testdaemon "github.com/docker/docker/internal/test/daemon" "github.com/go-check/check" "golang.org/x/net/context" "golang.org/x/sys/unix" ) -func setPortConfig(portConfig []swarm.PortConfig) daemon.ServiceConstructor { +func setPortConfig(portConfig []swarm.PortConfig) testdaemon.ServiceConstructor { return func(s *swarm.Service) { if s.Spec.EndpointSpec == nil { s.Spec.EndpointSpec = &swarm.EndpointSpec{} @@ -140,7 +141,7 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesCreateGlobal(c *check.C) { func (s *DockerSwarmSuite) TestAPISwarmServicesUpdate(c *check.C) { const nodeCount = 3 - var daemons [nodeCount]*daemon.Swarm + var daemons [nodeCount]*daemon.Daemon for i := 0; i < nodeCount; i++ { daemons[i] = s.AddDaemon(c, true, i == 0) } @@ -309,7 +310,7 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateStartFirst(c *check.C) { func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *check.C) { const nodeCount = 3 - var daemons [nodeCount]*daemon.Swarm + var daemons [nodeCount]*daemon.Daemon for i := 0; i < nodeCount; i++ { daemons[i] = s.AddDaemon(c, true, i == 0) } @@ -349,7 +350,7 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *check.C) { func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *check.C) { const nodeCount = 3 - var daemons [nodeCount]*daemon.Swarm + var daemons [nodeCount]*daemon.Daemon for i := 0; i < nodeCount; i++ { daemons[i] = s.AddDaemon(c, true, i == 0) } @@ -401,7 +402,7 @@ func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *check.C) { func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *check.C) { const nodeCount = 3 - var daemons [nodeCount]*daemon.Swarm + var daemons [nodeCount]*daemon.Daemon for i := 0; i < nodeCount; i++ { daemons[i] = s.AddDaemon(c, true, i == 0) } @@ -496,7 +497,7 @@ func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *check.C) { func (s *DockerSwarmSuite) TestAPISwarmServicePlacementPrefs(c *check.C) { const nodeCount = 3 - var daemons [nodeCount]*daemon.Swarm + var daemons [nodeCount]*daemon.Daemon for i := 0; i < nodeCount; i++ { daemons[i] = s.AddDaemon(c, true, i == 0) } @@ -551,9 +552,9 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) - getContainers := func() map[string]*daemon.Swarm { - m := make(map[string]*daemon.Swarm) - for _, d := range []*daemon.Swarm{d1, d2, d3} { + getContainers := func() map[string]*daemon.Daemon { + m := make(map[string]*daemon.Daemon) + for _, d := range []*daemon.Daemon{d1, d2, d3} { for _, id := range d.ActiveContainers() { m[id] = d } diff --git a/components/engine/integration-cli/docker_api_swarm_test.go b/components/engine/integration-cli/docker_api_swarm_test.go index 2ba69acdb8..69fcdd3e82 100644 --- a/components/engine/integration-cli/docker_api_swarm_test.go +++ b/components/engine/integration-cli/docker_api_swarm_test.go @@ -23,6 +23,7 @@ import ( "github.com/docker/docker/integration-cli/checker" "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration-cli/request" + testdaemon "github.com/docker/docker/internal/test/daemon" "github.com/docker/swarmkit/ca" "github.com/go-check/check" "github.com/gotestyourself/gotestyourself/assert" @@ -35,30 +36,30 @@ var defaultReconciliationTimeout = 30 * time.Second func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) { // todo: should find a better way to verify that components are running than /info d1 := s.AddDaemon(c, true, true) - info, err := d1.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d1.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.True) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) c.Assert(info.Cluster.RootRotationInProgress, checker.False) d2 := s.AddDaemon(c, true, false) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.False) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) // Leaving cluster - c.Assert(d2.Leave(false), checker.IsNil) + c.Assert(d2.SwarmLeave(false), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.False) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) - c.Assert(d2.Join(swarm.JoinRequest{JoinToken: d1.JoinTokens(c).Worker, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil) + d2.SwarmJoin(c, swarm.JoinRequest{ + ListenAddr: d1.SwarmListenAddr(), + JoinToken: d1.JoinTokens(c).Worker, + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.False) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) @@ -69,93 +70,100 @@ func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) { d1.Start(c) d2.Start(c) - info, err = d1.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d1.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.True) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.False) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) } func (s *DockerSwarmSuite) TestAPISwarmJoinToken(c *check.C) { d1 := s.AddDaemon(c, false, false) - c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) + d1.SwarmInit(c, swarm.InitRequest{}) // todo: error message differs depending if some components of token are valid d2 := s.AddDaemon(c, false, false) - err := d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.ListenAddr}}) + c2 := d2.NewClientT(c) + err := c2.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "join token is necessary") - info, err := d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) - err = d2.Join(swarm.JoinRequest{JoinToken: "foobaz", RemoteAddrs: []string{d1.ListenAddr}}) + err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), + JoinToken: "foobaz", + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "invalid join token") - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) workerToken := d1.JoinTokens(c).Worker - c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + d2.SwarmJoin(c, swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), + JoinToken: workerToken, + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) - c.Assert(d2.Leave(false), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + c.Assert(d2.SwarmLeave(false), checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) // change tokens d1.RotateTokens(c) - err = d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}) + err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), + JoinToken: workerToken, + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "join token is necessary") - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) workerToken = d1.JoinTokens(c).Worker - c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + d2.SwarmJoin(c, swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.SwarmListenAddr()}}) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) - c.Assert(d2.Leave(false), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + c.Assert(d2.SwarmLeave(false), checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) // change spec, don't change tokens d1.UpdateSwarm(c, func(s *swarm.Spec) {}) - err = d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.ListenAddr}}) + err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "join token is necessary") - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) - c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + d2.SwarmJoin(c, swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.SwarmListenAddr()}}) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) - c.Assert(d2.Leave(false), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + c.Assert(d2.SwarmLeave(false), checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) } func (s *DockerSwarmSuite) TestUpdateSwarmAddExternalCA(c *check.C) { d1 := s.AddDaemon(c, false, false) - c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) + d1.SwarmInit(c, swarm.InitRequest{}) d1.UpdateSwarm(c, func(s *swarm.Spec) { s.CAConfig.ExternalCAs = []*swarm.ExternalCA{ { @@ -169,8 +177,7 @@ func (s *DockerSwarmSuite) TestUpdateSwarmAddExternalCA(c *check.C) { }, } }) - info, err := d1.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d1.SwarmInfo(c) c.Assert(info.Cluster.Spec.CAConfig.ExternalCAs, checker.HasLen, 2) c.Assert(info.Cluster.Spec.CAConfig.ExternalCAs[0].CACert, checker.Equals, "") c.Assert(info.Cluster.Spec.CAConfig.ExternalCAs[1].CACert, checker.Equals, "cacert") @@ -182,28 +189,32 @@ func (s *DockerSwarmSuite) TestAPISwarmCAHash(c *check.C) { splitToken := strings.Split(d1.JoinTokens(c).Worker, "-") splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e" replacementToken := strings.Join(splitToken, "-") - err := d2.Join(swarm.JoinRequest{JoinToken: replacementToken, RemoteAddrs: []string{d1.ListenAddr}}) + c2 := d2.NewClientT(c) + err := c2.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), + JoinToken: replacementToken, + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "remote CA does not match fingerprint") } func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) { d1 := s.AddDaemon(c, false, false) - c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) + d1.SwarmInit(c, swarm.InitRequest{}) d2 := s.AddDaemon(c, true, false) - info, err := d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d2.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.False) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Role = swarm.NodeRoleManager }) waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.True) - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Role = swarm.NodeRoleWorker }) @@ -228,7 +239,7 @@ func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) { }, checker.Equals, "swarm-worker") // Demoting last node should fail - node := d1.GetNode(c, d1.NodeID) + node := d1.GetNode(c, d1.NodeID()) node.Spec.Role = swarm.NodeRoleWorker url := fmt.Sprintf("/nodes/%s/update?version=%d", node.ID, node.Version.Index) res, body, err := request.DoOnHost(d1.Sock(), url, request.Method("POST"), request.JSONBody(node.Spec)) @@ -246,13 +257,12 @@ func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) { if !strings.Contains(string(b), "last manager of the swarm") { c.Assert(string(b), checker.Contains, "this would result in a loss of quorum") } - info, err = d1.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d1.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) c.Assert(info.ControlAvailable, checker.True) // Promote already demoted node - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Role = swarm.NodeRoleManager }) @@ -278,7 +288,7 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaderProxy(c *check.C) { // 3 services should be started now, because the requests were proxied to leader // query each node and make sure it returns 3 services - for _, d := range []*daemon.Swarm{d1, d2, d3} { + for _, d := range []*daemon.Daemon{d1, d2, d3} { services := d.ListServices(c) c.Assert(services, checker.HasLen, 3) } @@ -291,23 +301,23 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) { d3 := s.AddDaemon(c, true, true) // assert that the first node we made is the leader, and the other two are followers - c.Assert(d1.GetNode(c, d1.NodeID).ManagerStatus.Leader, checker.True) - c.Assert(d1.GetNode(c, d2.NodeID).ManagerStatus.Leader, checker.False) - c.Assert(d1.GetNode(c, d3.NodeID).ManagerStatus.Leader, checker.False) + c.Assert(d1.GetNode(c, d1.NodeID()).ManagerStatus.Leader, checker.True) + c.Assert(d1.GetNode(c, d2.NodeID()).ManagerStatus.Leader, checker.False) + c.Assert(d1.GetNode(c, d3.NodeID()).ManagerStatus.Leader, checker.False) d1.Stop(c) var ( - leader *daemon.Swarm // keep track of leader - followers []*daemon.Swarm // keep track of followers + leader *daemon.Daemon // keep track of leader + followers []*daemon.Daemon // keep track of followers ) - checkLeader := func(nodes ...*daemon.Swarm) checkF { + checkLeader := func(nodes ...*daemon.Daemon) checkF { return func(c *check.C) (interface{}, check.CommentInterface) { // clear these out before each run leader = nil followers = nil for _, d := range nodes { - if d.GetNode(c, d.NodeID).ManagerStatus.Leader { + if d.GetNode(c, d.NodeID()).ManagerStatus.Leader { leader = d } else { followers = append(followers, d) @@ -344,7 +354,7 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) { c.Assert(leader, checker.NotNil) c.Assert(followers, checker.HasLen, 2) // and that after we added d1 back, the leader hasn't changed - c.Assert(leader.NodeID, checker.Equals, stableleader.NodeID) + c.Assert(leader.NodeID(), checker.Equals, stableleader.NodeID()) } func (s *DockerSwarmSuite) TestAPISwarmRaftQuorum(c *check.C) { @@ -400,8 +410,8 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaveRemovesContainer(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances+1) - c.Assert(d.Leave(false), checker.NotNil) - c.Assert(d.Leave(true), checker.IsNil) + c.Assert(d.SwarmLeave(false), checker.NotNil) + c.Assert(d.SwarmLeave(true), checker.IsNil) waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) @@ -420,17 +430,18 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaveOnPendingJoin(c *check.C) { c.Assert(err, checker.IsNil) id = strings.TrimSpace(id) - err = d2.Join(swarm.JoinRequest{ + c2 := d2.NewClientT(c) + err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), RemoteAddrs: []string{"123.123.123.123:1234"}, }) c.Assert(err, check.NotNil) c.Assert(err.Error(), checker.Contains, "Timeout was reached") - info, err := d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStatePending) - c.Assert(d2.Leave(true), checker.IsNil) + c.Assert(d2.SwarmLeave(true), checker.IsNil) waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 1) @@ -443,7 +454,9 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaveOnPendingJoin(c *check.C) { func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) { testRequires(c, Network) d := s.AddDaemon(c, false, false) - err := d.Join(swarm.JoinRequest{ + client := d.NewClientT(c) + err := client.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d.SwarmListenAddr(), RemoteAddrs: []string{"123.123.123.123:1234"}, }) c.Assert(err, check.NotNil) @@ -454,8 +467,7 @@ func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) { d.Stop(c) d.Start(c) - info, err := d.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) } @@ -539,7 +551,7 @@ func (s *DockerSwarmSuite) TestAPISwarmForceNewCluster(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount), checker.Equals, instances) // drain d2, all containers should move to d1 - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Availability = swarm.NodeAvailabilityDrain }) waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances) @@ -547,16 +559,15 @@ func (s *DockerSwarmSuite) TestAPISwarmForceNewCluster(c *check.C) { d2.Stop(c) - c.Assert(d1.Init(swarm.InitRequest{ + d1.SwarmInit(c, swarm.InitRequest{ ForceNewCluster: true, Spec: swarm.Spec{}, - }), checker.IsNil) + }) waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances) d3 := s.AddDaemon(c, true, true) - info, err := d3.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d3.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.True) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) @@ -622,7 +633,7 @@ func serviceForUpdate(s *swarm.Service) { s.Spec.Name = "updatetest" } -func setInstances(replicas int) daemon.ServiceConstructor { +func setInstances(replicas int) testdaemon.ServiceConstructor { ureplicas := uint64(replicas) return func(s *swarm.Service) { s.Spec.Mode = swarm.ServiceMode{ @@ -633,7 +644,7 @@ func setInstances(replicas int) daemon.ServiceConstructor { } } -func setUpdateOrder(order string) daemon.ServiceConstructor { +func setUpdateOrder(order string) testdaemon.ServiceConstructor { return func(s *swarm.Service) { if s.Spec.UpdateConfig == nil { s.Spec.UpdateConfig = &swarm.UpdateConfig{} @@ -642,7 +653,7 @@ func setUpdateOrder(order string) daemon.ServiceConstructor { } } -func setRollbackOrder(order string) daemon.ServiceConstructor { +func setRollbackOrder(order string) testdaemon.ServiceConstructor { return func(s *swarm.Service) { if s.Spec.RollbackConfig == nil { s.Spec.RollbackConfig = &swarm.UpdateConfig{} @@ -651,7 +662,7 @@ func setRollbackOrder(order string) daemon.ServiceConstructor { } } -func setImage(image string) daemon.ServiceConstructor { +func setImage(image string) testdaemon.ServiceConstructor { return func(s *swarm.Service) { if s.Spec.TaskTemplate.ContainerSpec == nil { s.Spec.TaskTemplate.ContainerSpec = &swarm.ContainerSpec{} @@ -660,25 +671,25 @@ func setImage(image string) daemon.ServiceConstructor { } } -func setFailureAction(failureAction string) daemon.ServiceConstructor { +func setFailureAction(failureAction string) testdaemon.ServiceConstructor { return func(s *swarm.Service) { s.Spec.UpdateConfig.FailureAction = failureAction } } -func setMaxFailureRatio(maxFailureRatio float32) daemon.ServiceConstructor { +func setMaxFailureRatio(maxFailureRatio float32) testdaemon.ServiceConstructor { return func(s *swarm.Service) { s.Spec.UpdateConfig.MaxFailureRatio = maxFailureRatio } } -func setParallelism(parallelism uint64) daemon.ServiceConstructor { +func setParallelism(parallelism uint64) testdaemon.ServiceConstructor { return func(s *swarm.Service) { s.Spec.UpdateConfig.Parallelism = parallelism } } -func setConstraints(constraints []string) daemon.ServiceConstructor { +func setConstraints(constraints []string) testdaemon.ServiceConstructor { return func(s *swarm.Service) { if s.Spec.TaskTemplate.Placement == nil { s.Spec.TaskTemplate.Placement = &swarm.Placement{} @@ -687,7 +698,7 @@ func setConstraints(constraints []string) daemon.ServiceConstructor { } } -func setPlacementPrefs(prefs []swarm.PlacementPreference) daemon.ServiceConstructor { +func setPlacementPrefs(prefs []swarm.PlacementPreference) testdaemon.ServiceConstructor { return func(s *swarm.Service) { if s.Spec.TaskTemplate.Placement == nil { s.Spec.TaskTemplate.Placement = &swarm.Placement{} @@ -702,18 +713,19 @@ func setGlobalMode(s *swarm.Service) { } } -func checkClusterHealth(c *check.C, cl []*daemon.Swarm, managerCount, workerCount int) { +func checkClusterHealth(c *check.C, cl []*daemon.Daemon, managerCount, workerCount int) { var totalMCount, totalWCount int for _, d := range cl { var ( info swarm.Info - err error ) // check info in a waitAndAssert, because if the cluster doesn't have a leader, `info` will return an error checkInfo := func(c *check.C) (interface{}, check.CommentInterface) { - info, err = d.SwarmInfo() + client := d.NewClientT(c) + daemonInfo, err := client.Info(context.Background()) + info = daemonInfo.Swarm return err, check.Commentf("cluster not ready in time") } waitAndAssert(c, defaultReconciliationTimeout, checkInfo, checker.IsNil) @@ -733,7 +745,7 @@ func checkClusterHealth(c *check.C, cl []*daemon.Swarm, managerCount, workerCoun } nn := d.GetNode(c, n.ID) n = *nn - return n.Status.State == swarm.NodeStateReady, check.Commentf("state of node %s, reported by %s", n.ID, d.Info.NodeID) + return n.Status.State == swarm.NodeStateReady, check.Commentf("state of node %s, reported by %s", n.ID, d.NodeID()) } waitAndAssert(c, defaultReconciliationTimeout, waitReady, checker.True) @@ -743,18 +755,18 @@ func checkClusterHealth(c *check.C, cl []*daemon.Swarm, managerCount, workerCoun } nn := d.GetNode(c, n.ID) n = *nn - return n.Spec.Availability == swarm.NodeAvailabilityActive, check.Commentf("availability of node %s, reported by %s", n.ID, d.Info.NodeID) + return n.Spec.Availability == swarm.NodeAvailabilityActive, check.Commentf("availability of node %s, reported by %s", n.ID, d.NodeID()) } waitAndAssert(c, defaultReconciliationTimeout, waitActive, checker.True) if n.Spec.Role == swarm.NodeRoleManager { - c.Assert(n.ManagerStatus, checker.NotNil, check.Commentf("manager status of node %s (manager), reported by %s", n.ID, d.Info.NodeID)) + c.Assert(n.ManagerStatus, checker.NotNil, check.Commentf("manager status of node %s (manager), reported by %s", n.ID, d.NodeID())) if n.ManagerStatus.Leader { leaderFound = true } mCount++ } else { - c.Assert(n.ManagerStatus, checker.IsNil, check.Commentf("manager status of node %s (worker), reported by %s", n.ID, d.Info.NodeID)) + c.Assert(n.ManagerStatus, checker.IsNil, check.Commentf("manager status of node %s (worker), reported by %s", n.ID, d.NodeID())) wCount++ } } @@ -769,11 +781,10 @@ func checkClusterHealth(c *check.C, cl []*daemon.Swarm, managerCount, workerCoun func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) { mCount, wCount := 5, 1 - var nodes []*daemon.Swarm + var nodes []*daemon.Daemon for i := 0; i < mCount; i++ { manager := s.AddDaemon(c, true, true) - info, err := manager.SwarmInfo() - c.Assert(err, checker.IsNil) + info := manager.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.True) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) nodes = append(nodes, manager) @@ -781,8 +792,7 @@ func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) { for i := 0; i < wCount; i++ { worker := s.AddDaemon(c, true, false) - info, err := worker.SwarmInfo() - c.Assert(err, checker.IsNil) + info := worker.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.False) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) nodes = append(nodes, worker) @@ -795,7 +805,7 @@ func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) { errs := make(chan error, len(nodes)) for _, d := range nodes { - go func(daemon *daemon.Swarm) { + go func(daemon *daemon.Daemon) { defer wg.Done() if err := daemon.StopWithError(); err != nil { errs <- err @@ -820,7 +830,7 @@ func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) { errs := make(chan error, len(nodes)) for _, d := range nodes { - go func(daemon *daemon.Swarm) { + go func(daemon *daemon.Daemon) { defer wg.Done() if err := daemon.StartWithError("--iptables=false"); err != nil { errs <- err @@ -859,7 +869,7 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateWithName(c *check.C) { // Unlocking an unlocked swarm results in an error func (s *DockerSwarmSuite) TestAPISwarmUnlockNotLocked(c *check.C) { d := s.AddDaemon(c, true, true) - err := d.Unlock(swarm.UnlockRequest{UnlockKey: "wrong-key"}) + err := d.SwarmUnlock(swarm.UnlockRequest{UnlockKey: "wrong-key"}) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "swarm is not locked") } @@ -870,7 +880,10 @@ func (s *DockerSwarmSuite) TestAPISwarmErrorHandling(c *check.C) { c.Assert(err, checker.IsNil) defer ln.Close() d := s.AddDaemon(c, false, false) - err = d.Init(swarm.InitRequest{}) + client := d.NewClientT(c) + _, err = client.SwarmInit(context.Background(), swarm.InitRequest{ + ListenAddr: d.SwarmListenAddr(), + }) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "address already in use") } @@ -940,13 +953,13 @@ func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) { m := s.AddDaemon(c, true, true) w := s.AddDaemon(c, true, false) - info, err := m.SwarmInfo() - c.Assert(err, checker.IsNil) + info := m.SwarmInfo(c) currentTrustRoot := info.Cluster.TLSInfo.TrustRoot // rotate multiple times for i := 0; i < 4; i++ { + var err error var cert, key []byte if i%2 != 0 { cert, _, key, err = initca.New(&csr.CertificateRequest{ @@ -966,8 +979,7 @@ func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) { // poll to make sure update succeeds var clusterTLSInfo swarm.TLSInfo for j := 0; j < 18; j++ { - info, err := m.SwarmInfo() - c.Assert(err, checker.IsNil) + info := m.SwarmInfo(c) // the desired CA cert and key is always redacted c.Assert(info.Cluster.Spec.CAConfig.SigningCAKey, checker.Equals, "") @@ -989,8 +1001,8 @@ func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) { // could take another second or two for the nodes to trust the new roots after they've all gotten // new TLS certificates for j := 0; j < 18; j++ { - mInfo := m.GetNode(c, m.NodeID).Description.TLSInfo - wInfo := m.GetNode(c, w.NodeID).Description.TLSInfo + mInfo := m.GetNode(c, m.NodeID()).Description.TLSInfo + wInfo := m.GetNode(c, w.NodeID()).Description.TLSInfo if mInfo.TrustRoot == clusterTLSInfo.TrustRoot && wInfo.TrustRoot == clusterTLSInfo.TrustRoot { break @@ -1000,8 +1012,8 @@ func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) { time.Sleep(250 * time.Millisecond) } - c.Assert(m.GetNode(c, m.NodeID).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo) - c.Assert(m.GetNode(c, w.NodeID).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo) + c.Assert(m.GetNode(c, m.NodeID()).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo) + c.Assert(m.GetNode(c, w.NodeID()).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo) currentTrustRoot = clusterTLSInfo.TrustRoot } } diff --git a/components/engine/integration-cli/docker_cli_prune_unix_test.go b/components/engine/integration-cli/docker_cli_prune_unix_test.go index 763a72de16..bc5bdc835e 100644 --- a/components/engine/integration-cli/docker_cli_prune_unix_test.go +++ b/components/engine/integration-cli/docker_cli_prune_unix_test.go @@ -16,7 +16,7 @@ import ( "github.com/go-check/check" ) -func pruneNetworkAndVerify(c *check.C, d *daemon.Swarm, kept, pruned []string) { +func pruneNetworkAndVerify(c *check.C, d *daemon.Daemon, kept, pruned []string) { _, err := d.Cmd("network", "prune", "--force") c.Assert(err, checker.IsNil) diff --git a/components/engine/integration-cli/docker_cli_service_logs_test.go b/components/engine/integration-cli/docker_cli_service_logs_test.go index b3f58271f1..c26a7455a5 100644 --- a/components/engine/integration-cli/docker_cli_service_logs_test.go +++ b/components/engine/integration-cli/docker_cli_service_logs_test.go @@ -53,7 +53,7 @@ func (s *DockerSwarmSuite) TestServiceLogs(c *check.C) { // countLogLines returns a closure that can be used with waitAndAssert to // verify that a minimum number of expected container log messages have been // output. -func countLogLines(d *daemon.Swarm, name string) func(*check.C) (interface{}, check.CommentInterface) { +func countLogLines(d *daemon.Daemon, name string) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { result := icmd.RunCmd(d.Command("service", "logs", "-t", "--raw", name)) result.Assert(c, icmd.Expected{}) diff --git a/components/engine/integration-cli/docker_cli_swarm_test.go b/components/engine/integration-cli/docker_cli_swarm_test.go index 4b3358255f..9a45664cba 100644 --- a/components/engine/integration-cli/docker_cli_swarm_test.go +++ b/components/engine/integration-cli/docker_cli_swarm_test.go @@ -57,7 +57,7 @@ func (s *DockerSwarmSuite) TestSwarmUpdate(c *check.C) { // passing an external CA (this is without starting a root rotation) does not fail cli.Docker(cli.Args("swarm", "update", "--external-ca", "protocol=cfssl,url=https://something.org", "--external-ca", "protocol=cfssl,url=https://somethingelse.org,cacert=fixtures/https/ca.pem"), - cli.Daemon(d.Daemon)).Assert(c, icmd.Success) + cli.Daemon(d)).Assert(c, icmd.Success) expected, err := ioutil.ReadFile("fixtures/https/ca.pem") c.Assert(err, checker.IsNil) @@ -73,7 +73,7 @@ func (s *DockerSwarmSuite) TestSwarmUpdate(c *check.C) { result := cli.Docker(cli.Args("swarm", "update", "--external-ca", fmt.Sprintf("protocol=cfssl,url=https://something.org,cacert=%s", tempFile.Path())), - cli.Daemon(d.Daemon)) + cli.Daemon(d)) result.Assert(c, icmd.Expected{ ExitCode: 125, Err: "must be in PEM format", @@ -94,7 +94,7 @@ func (s *DockerSwarmSuite) TestSwarmInit(c *check.C) { result := cli.Docker(cli.Args("swarm", "init", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s", "--external-ca", fmt.Sprintf("protocol=cfssl,url=https://somethingelse.org,cacert=%s", tempFile.Path())), - cli.Daemon(d.Daemon)) + cli.Daemon(d)) result.Assert(c, icmd.Expected{ ExitCode: 125, Err: "must be in PEM format", @@ -103,7 +103,7 @@ func (s *DockerSwarmSuite) TestSwarmInit(c *check.C) { cli.Docker(cli.Args("swarm", "init", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s", "--external-ca", "protocol=cfssl,url=https://something.org", "--external-ca", "protocol=cfssl,url=https://somethingelse.org,cacert=fixtures/https/ca.pem"), - cli.Daemon(d.Daemon)).Assert(c, icmd.Success) + cli.Daemon(d)).Assert(c, icmd.Success) expected, err := ioutil.ReadFile("fixtures/https/ca.pem") c.Assert(err, checker.IsNil) @@ -115,8 +115,8 @@ func (s *DockerSwarmSuite) TestSwarmInit(c *check.C) { c.Assert(spec.CAConfig.ExternalCAs[0].CACert, checker.Equals, "") c.Assert(spec.CAConfig.ExternalCAs[1].CACert, checker.Equals, string(expected)) - c.Assert(d.Leave(true), checker.IsNil) - cli.Docker(cli.Args("swarm", "init"), cli.Daemon(d.Daemon)).Assert(c, icmd.Success) + c.Assert(d.SwarmLeave(true), checker.IsNil) + cli.Docker(cli.Args("swarm", "init"), cli.Daemon(d)).Assert(c, icmd.Success) spec = getSpec() c.Assert(spec.CAConfig.NodeCertExpiry, checker.Equals, 90*24*time.Hour) @@ -126,12 +126,12 @@ func (s *DockerSwarmSuite) TestSwarmInit(c *check.C) { func (s *DockerSwarmSuite) TestSwarmInitIPv6(c *check.C) { testRequires(c, IPv6) d1 := s.AddDaemon(c, false, false) - cli.Docker(cli.Args("swarm", "init", "--listen-add", "::1"), cli.Daemon(d1.Daemon)).Assert(c, icmd.Success) + cli.Docker(cli.Args("swarm", "init", "--listen-add", "::1"), cli.Daemon(d1)).Assert(c, icmd.Success) d2 := s.AddDaemon(c, false, false) - cli.Docker(cli.Args("swarm", "join", "::1"), cli.Daemon(d2.Daemon)).Assert(c, icmd.Success) + cli.Docker(cli.Args("swarm", "join", "::1"), cli.Daemon(d2)).Assert(c, icmd.Success) - out := cli.Docker(cli.Args("info"), cli.Daemon(d2.Daemon)).Assert(c, icmd.Success).Combined() + out := cli.Docker(cli.Args("info"), cli.Daemon(d2)).Assert(c, icmd.Success).Combined() c.Assert(out, checker.Contains, "Swarm: active") } @@ -145,13 +145,12 @@ func (s *DockerSwarmSuite) TestSwarmInitUnspecifiedAdvertiseAddr(c *check.C) { func (s *DockerSwarmSuite) TestSwarmIncompatibleDaemon(c *check.C) { // init swarm mode and stop a daemon d := s.AddDaemon(c, true, true) - info, err := d.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) d.Stop(c) // start a daemon with --cluster-store and --cluster-advertise - err = d.StartWithError("--cluster-store=consul://consuladdr:consulport/some/path", "--cluster-advertise=1.1.1.1:2375") + err := d.StartWithError("--cluster-store=consul://consuladdr:consulport/some/path", "--cluster-advertise=1.1.1.1:2375") c.Assert(err, checker.NotNil) content, err := d.ReadLogFile() c.Assert(err, checker.IsNil) @@ -426,7 +425,7 @@ func (s *DockerSwarmSuite) TestOverlayAttachableOnSwarmLeave(c *check.C) { c.Assert(err, checker.IsNil, check.Commentf(out)) // Leave the swarm - err = d.Leave(true) + err = d.SwarmLeave(true) c.Assert(err, checker.IsNil) // Check the container is disconnected @@ -989,13 +988,12 @@ func (s *DockerSwarmSuite) TestDNSConfigUpdate(c *check.C) { c.Assert(strings.TrimSpace(out), checker.Equals, "{[1.2.3.4] [example.com] [timeout:3]}") } -func getNodeStatus(c *check.C, d *daemon.Swarm) swarm.LocalNodeState { - info, err := d.SwarmInfo() - c.Assert(err, checker.IsNil) +func getNodeStatus(c *check.C, d *daemon.Daemon) swarm.LocalNodeState { + info := d.SwarmInfo(c) return info.LocalNodeState } -func checkKeyIsEncrypted(d *daemon.Swarm) func(*check.C) (interface{}, check.CommentInterface) { +func checkKeyIsEncrypted(d *daemon.Daemon) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { keyBytes, err := ioutil.ReadFile(filepath.Join(d.Folder, "root", "swarm", "certificates", "swarm-node.key")) if err != nil { @@ -1011,7 +1009,7 @@ func checkKeyIsEncrypted(d *daemon.Swarm) func(*check.C) (interface{}, check.Com } } -func checkSwarmLockedToUnlocked(c *check.C, d *daemon.Swarm, unlockKey string) { +func checkSwarmLockedToUnlocked(c *check.C, d *daemon.Daemon, unlockKey string) { // Wait for the PEM file to become unencrypted waitAndAssert(c, defaultReconciliationTimeout, checkKeyIsEncrypted(d), checker.Equals, false) @@ -1019,7 +1017,7 @@ func checkSwarmLockedToUnlocked(c *check.C, d *daemon.Swarm, unlockKey string) { c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive) } -func checkSwarmUnlockedToLocked(c *check.C, d *daemon.Swarm) { +func checkSwarmUnlockedToLocked(c *check.C, d *daemon.Daemon) { // Wait for the PEM file to become encrypted waitAndAssert(c, defaultReconciliationTimeout, checkKeyIsEncrypted(d), checker.Equals, true) @@ -1117,8 +1115,7 @@ func (s *DockerSwarmSuite) TestSwarmLeaveLocked(c *check.C) { // It starts off locked d.Restart(c, "--swarm-default-advertise-addr=lo") - info, err := d.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateLocked) outs, _ = d.Cmd("node", "ls") @@ -1132,15 +1129,13 @@ func (s *DockerSwarmSuite) TestSwarmLeaveLocked(c *check.C) { outs, err = d.Cmd("swarm", "leave", "--force") c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) - info, err = d.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) outs, err = d.Cmd("swarm", "init") c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) - info, err = d.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) } @@ -1176,7 +1171,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) { c.Assert(outs, checker.Equals, unlockKey+"\n") // The ones that got the cluster update should be set to locked - for _, d := range []*daemon.Swarm{d1, d3} { + for _, d := range []*daemon.Daemon{d1, d3} { checkSwarmUnlockedToLocked(c, d) cmd := d.Command("swarm", "unlock") @@ -1197,7 +1192,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) { c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) // the ones that got the update are now set to unlocked - for _, d := range []*daemon.Swarm{d1, d3} { + for _, d := range []*daemon.Daemon{d1, d3} { checkSwarmLockedToUnlocked(c, d, unlockKey) } @@ -1247,7 +1242,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) { c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive) // promote worker - outs, err = d1.Cmd("node", "promote", d2.Info.NodeID) + outs, err = d1.Cmd("node", "promote", d2.NodeID()) c.Assert(err, checker.IsNil) c.Assert(outs, checker.Contains, "promoted to a manager in the swarm") @@ -1255,7 +1250,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) { d3 := s.AddDaemon(c, true, true) // both new nodes are locked - for _, d := range []*daemon.Swarm{d2, d3} { + for _, d := range []*daemon.Daemon{d2, d3} { checkSwarmUnlockedToLocked(c, d) cmd := d.Command("swarm", "unlock") @@ -1265,7 +1260,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) { } // demote manager back to worker - workers are not locked - outs, err = d1.Cmd("node", "demote", d3.Info.NodeID) + outs, err = d1.Cmd("node", "demote", d3.NodeID()) c.Assert(err, checker.IsNil) c.Assert(outs, checker.Contains, "demoted in the swarm") @@ -1409,7 +1404,7 @@ func (s *DockerSwarmSuite) TestSwarmClusterRotateUnlockKey(c *check.C) { d2.Restart(c) d3.Restart(c) - for _, d := range []*daemon.Swarm{d2, d3} { + for _, d := range []*daemon.Daemon{d2, d3} { c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked) outs, _ := d.Cmd("node", "ls") @@ -1521,7 +1516,7 @@ func (s *DockerSwarmSuite) TestSwarmManagerAddress(c *check.C) { d3 := s.AddDaemon(c, true, false) // Manager Addresses will always show Node 1's address - expectedOutput := fmt.Sprintf("Manager Addresses:\n 127.0.0.1:%d\n", d1.Port) + expectedOutput := fmt.Sprintf("Manager Addresses:\n 127.0.0.1:%d\n", d1.SwarmPort) out, err := d1.Cmd("info") c.Assert(err, checker.IsNil) @@ -1641,7 +1636,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinWithDrain(c *check.C) { d1 := s.AddDaemon(c, false, false) - out, err = d1.Cmd("swarm", "join", "--availability=drain", "--token", token, d.ListenAddr) + out, err = d1.Cmd("swarm", "join", "--availability=drain", "--token", token, d.SwarmListenAddr()) c.Assert(err, checker.IsNil) c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") @@ -1835,7 +1830,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinLeave(c *check.C) { // Verify that back to back join/leave does not cause panics d1 := s.AddDaemon(c, false, false) for i := 0; i < 10; i++ { - out, err = d1.Cmd("swarm", "join", "--token", token, d.ListenAddr) + out, err = d1.Cmd("swarm", "join", "--token", token, d.SwarmListenAddr()) c.Assert(err, checker.IsNil) c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") @@ -1846,7 +1841,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinLeave(c *check.C) { const defaultRetryCount = 10 -func waitForEvent(c *check.C, d *daemon.Swarm, since string, filter string, event string, retry int) string { +func waitForEvent(c *check.C, d *daemon.Daemon, since string, filter string, event string, retry int) string { if retry < 1 { c.Fatalf("retry count %d is invalid. It should be no less than 1", retry) return "" @@ -1982,7 +1977,7 @@ func (s *DockerSwarmSuite) TestSwarmClusterEventsNode(c *check.C) { s.AddDaemon(c, true, true) d3 := s.AddDaemon(c, true, true) - d3ID := d3.NodeID + d3ID := d3.NodeID() waitForEvent(c, d1, "0", "-f scope=swarm", "node create "+d3ID, defaultRetryCount) t1 := daemonUnixTime(c) diff --git a/components/engine/integration/internal/swarm/service.go b/components/engine/integration/internal/swarm/service.go index 28d367c660..031995f87a 100644 --- a/components/engine/integration/internal/swarm/service.go +++ b/components/engine/integration/internal/swarm/service.go @@ -2,7 +2,6 @@ package swarm import ( "context" - "fmt" "runtime" "testing" "time" @@ -11,18 +10,13 @@ import ( "github.com/docker/docker/api/types/filters" swarmtypes "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/client" - "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/internal/test/daemon" "github.com/docker/docker/internal/test/environment" "github.com/gotestyourself/gotestyourself/assert" "github.com/gotestyourself/gotestyourself/poll" "github.com/gotestyourself/gotestyourself/skip" ) -const ( - dockerdBinary = "dockerd" - defaultSwarmPort = 2477 -) - // ServicePoll tweaks the pollSettings for `service` func ServicePoll(config *poll.Settings) { // Override the default pollSettings for `service` resource here ... @@ -55,23 +49,17 @@ func ContainerPoll(config *poll.Settings) { } // NewSwarm creates a swarm daemon for testing -func NewSwarm(t *testing.T, testEnv *environment.Execution) *daemon.Swarm { +func NewSwarm(t *testing.T, testEnv *environment.Execution, ops ...func(*daemon.Daemon)) *daemon.Daemon { skip.IfCondition(t, testEnv.IsRemoteDaemon()) - d := &daemon.Swarm{ - Daemon: daemon.New(t, "", dockerdBinary, daemon.Config{ - Experimental: testEnv.DaemonInfo.ExperimentalBuild, - }), - // TODO: better method of finding an unused port - Port: defaultSwarmPort, + if testEnv.DaemonInfo.ExperimentalBuild { + ops = append(ops, daemon.WithExperimental) } - // TODO: move to a NewSwarm constructor - d.ListenAddr = fmt.Sprintf("0.0.0.0:%d", d.Port) - + d := daemon.New(t, ops...) // avoid networking conflicts args := []string{"--iptables=false", "--swarm-default-advertise-addr=lo"} d.StartWithBusybox(t, args...) - assert.NilError(t, d.Init(swarmtypes.InitRequest{})) + d.SwarmInit(t, swarmtypes.InitRequest{}) return d } @@ -79,7 +67,7 @@ func NewSwarm(t *testing.T, testEnv *environment.Execution) *daemon.Swarm { type ServiceSpecOpt func(*swarmtypes.ServiceSpec) // CreateService creates a service on the passed in swarm daemon. -func CreateService(t *testing.T, d *daemon.Swarm, opts ...ServiceSpecOpt) string { +func CreateService(t *testing.T, d *daemon.Daemon, opts ...ServiceSpecOpt) string { spec := defaultServiceSpec() for _, o := range opts { o(&spec) @@ -151,7 +139,7 @@ func ServiceWithName(name string) ServiceSpecOpt { } // GetRunningTasks gets the list of running tasks for a service -func GetRunningTasks(t *testing.T, d *daemon.Swarm, serviceID string) []swarmtypes.Task { +func GetRunningTasks(t *testing.T, d *daemon.Daemon, serviceID string) []swarmtypes.Task { client := GetClient(t, d) filterArgs := filters.NewArgs() @@ -167,7 +155,7 @@ func GetRunningTasks(t *testing.T, d *daemon.Swarm, serviceID string) []swarmtyp } // ExecTask runs the passed in exec config on the given task -func ExecTask(t *testing.T, d *daemon.Swarm, task swarmtypes.Task, config types.ExecConfig) types.HijackedResponse { +func ExecTask(t *testing.T, d *daemon.Daemon, task swarmtypes.Task, config types.ExecConfig) types.HijackedResponse { client := GetClient(t, d) ctx := context.Background() @@ -187,7 +175,7 @@ func ensureContainerSpec(spec *swarmtypes.ServiceSpec) { } // GetClient creates a new client for the passed in swarm daemon. -func GetClient(t *testing.T, d *daemon.Swarm) client.APIClient { +func GetClient(t *testing.T, d *daemon.Daemon) client.APIClient { client, err := client.NewClientWithOpts(client.WithHost((d.Sock()))) assert.NilError(t, err) return client diff --git a/components/engine/internal/test/daemon/config.go b/components/engine/internal/test/daemon/config.go new file mode 100644 index 0000000000..4ecc41b514 --- /dev/null +++ b/components/engine/internal/test/daemon/config.go @@ -0,0 +1,66 @@ +package daemon + +import ( + "context" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/swarm" + "github.com/gotestyourself/gotestyourself/assert" +) + +// ConfigConstructor defines a swarm config constructor +type ConfigConstructor func(*swarm.Config) + +// CreateConfig creates a config given the specified spec +func (d *Daemon) CreateConfig(t assert.TestingT, configSpec swarm.ConfigSpec) string { + cli := d.NewClientT(t) + defer cli.Close() + + scr, err := cli.ConfigCreate(context.Background(), configSpec) + assert.NilError(t, err) + return scr.ID +} + +// ListConfigs returns the list of the current swarm configs +func (d *Daemon) ListConfigs(t assert.TestingT) []swarm.Config { + cli := d.NewClientT(t) + defer cli.Close() + + configs, err := cli.ConfigList(context.Background(), types.ConfigListOptions{}) + assert.NilError(t, err) + return configs +} + +// GetConfig returns a swarm config identified by the specified id +func (d *Daemon) GetConfig(t assert.TestingT, id string) *swarm.Config { + cli := d.NewClientT(t) + defer cli.Close() + + config, _, err := cli.ConfigInspectWithRaw(context.Background(), id) + assert.NilError(t, err) + return &config +} + +// DeleteConfig removes the swarm config identified by the specified id +func (d *Daemon) DeleteConfig(t assert.TestingT, id string) { + cli := d.NewClientT(t) + defer cli.Close() + + err := cli.ConfigRemove(context.Background(), id) + assert.NilError(t, err) +} + +// UpdateConfig updates the swarm config identified by the specified id +// Currently, only label update is supported. +func (d *Daemon) UpdateConfig(t assert.TestingT, id string, f ...ConfigConstructor) { + cli := d.NewClientT(t) + defer cli.Close() + + config := d.GetConfig(t, id) + for _, fn := range f { + fn(config) + } + + err := cli.ConfigUpdate(context.Background(), config.ID, config.Version, config.Spec) + assert.NilError(t, err) +} diff --git a/components/engine/internal/test/daemon/daemon.go b/components/engine/internal/test/daemon/daemon.go index 739e134659..7324503278 100644 --- a/components/engine/internal/test/daemon/daemon.go +++ b/components/engine/internal/test/daemon/daemon.go @@ -67,6 +67,13 @@ type Daemon struct { experimental bool dockerdBinary string log logT + + // swarm related field + swarmListenAddr string + SwarmPort int // FIXME(vdemeester) should probably not be exported + + // cached information + CachedInfo types.Info } // New returns a Daemon instance to be used for testing. @@ -98,14 +105,16 @@ func New(t testingT, ops ...func(*Daemon)) *Daemon { } } d := &Daemon{ - id: id, - Folder: daemonFolder, - Root: daemonRoot, - storageDriver: storageDriver, - userlandProxy: userlandProxy, - execRoot: filepath.Join(os.TempDir(), "docker-execroot", id), - dockerdBinary: defaultDockerdBinary, - log: t, + id: id, + Folder: daemonFolder, + Root: daemonRoot, + storageDriver: storageDriver, + userlandProxy: userlandProxy, + execRoot: filepath.Join(os.TempDir(), "docker-execroot", id), + dockerdBinary: defaultDockerdBinary, + swarmListenAddr: defaultSwarmListenAddr, + SwarmPort: defaultSwarmPort, + log: t, } for _, op := range ops { @@ -150,12 +159,23 @@ func (d *Daemon) ReadLogFile() ([]byte, error) { } // NewClient creates new client based on daemon's socket path +// FIXME(vdemeester): replace NewClient with NewClientT func (d *Daemon) NewClient() (*client.Client, error) { return client.NewClientWithOpts( client.FromEnv, client.WithHost(d.Sock())) } +// NewClientT creates new client based on daemon's socket path +// FIXME(vdemeester): replace NewClient with NewClientT +func (d *Daemon) NewClientT(t assert.TestingT) *client.Client { + c, err := client.NewClientWithOpts( + client.FromEnv, + client.WithHost(d.Sock())) + assert.NilError(t, err, "cannot create daemon client") + return c +} + // CleanupExecRoot cleans the daemon exec root (network namespaces, ...) func (d *Daemon) CleanupExecRoot(t testingT) { cleanupExecRoot(t, d.execRoot) @@ -610,7 +630,7 @@ func (d *Daemon) queryRootDir() (string, error) { // Info returns the info struct for this daemon func (d *Daemon) Info(t assert.TestingT) types.Info { - apiclient, err := client.NewClientWithOpts(client.WithHost((d.Sock()))) + apiclient, err := d.NewClient() assert.NilError(t, err) info, err := apiclient.Info(context.Background()) assert.NilError(t, err) diff --git a/components/engine/internal/test/daemon/node.go b/components/engine/internal/test/daemon/node.go new file mode 100644 index 0000000000..9955208b47 --- /dev/null +++ b/components/engine/internal/test/daemon/node.go @@ -0,0 +1,69 @@ +package daemon + +import ( + "context" + "strings" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/swarm" + "github.com/gotestyourself/gotestyourself/assert" +) + +// NodeConstructor defines a swarm node constructor +type NodeConstructor func(*swarm.Node) + +// GetNode returns a swarm node identified by the specified id +func (d *Daemon) GetNode(t assert.TestingT, id string) *swarm.Node { + cli := d.NewClientT(t) + defer cli.Close() + + node, _, err := cli.NodeInspectWithRaw(context.Background(), id) + assert.NilError(t, err) + assert.Check(t, node.ID == id) + return &node +} + +// RemoveNode removes the specified node +func (d *Daemon) RemoveNode(t assert.TestingT, id string, force bool) { + cli := d.NewClientT(t) + defer cli.Close() + + options := types.NodeRemoveOptions{ + Force: force, + } + err := cli.NodeRemove(context.Background(), id, options) + assert.NilError(t, err) +} + +// UpdateNode updates a swarm node with the specified node constructor +func (d *Daemon) UpdateNode(t assert.TestingT, id string, f ...NodeConstructor) { + cli := d.NewClientT(t) + defer cli.Close() + + for i := 0; ; i++ { + node := d.GetNode(t, id) + for _, fn := range f { + fn(node) + } + + err := cli.NodeUpdate(context.Background(), node.ID, node.Version, node.Spec) + if i < 10 && err != nil && strings.Contains(err.Error(), "update out of sequence") { + time.Sleep(100 * time.Millisecond) + continue + } + assert.NilError(t, err) + return + } +} + +// ListNodes returns the list of the current swarm nodes +func (d *Daemon) ListNodes(t assert.TestingT) []swarm.Node { + cli := d.NewClientT(t) + defer cli.Close() + + nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{}) + assert.NilError(t, err) + + return nodes +} diff --git a/components/engine/internal/test/daemon/ops.go b/components/engine/internal/test/daemon/ops.go index 0c49749414..0176c19d93 100644 --- a/components/engine/internal/test/daemon/ops.go +++ b/components/engine/internal/test/daemon/ops.go @@ -11,3 +11,17 @@ func WithDockerdBinary(dockerdBinary string) func(*Daemon) { d.dockerdBinary = dockerdBinary } } + +// WithSwarmPort sets the swarm port to use for swarm mode +func WithSwarmPort(port int) func(*Daemon) { + return func(d *Daemon) { + d.SwarmPort = port + } +} + +// WithSwarmListenAddr sets the swarm listen addr to use for swarm mode +func WithSwarmListenAddr(listenAddr string) func(*Daemon) { + return func(d *Daemon) { + d.swarmListenAddr = listenAddr + } +} diff --git a/components/engine/internal/test/daemon/secret.go b/components/engine/internal/test/daemon/secret.go new file mode 100644 index 0000000000..075aedc2e0 --- /dev/null +++ b/components/engine/internal/test/daemon/secret.go @@ -0,0 +1,68 @@ +package daemon + +import ( + "context" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/swarm" + "github.com/gotestyourself/gotestyourself/assert" +) + +// SecretConstructor defines a swarm secret constructor +type SecretConstructor func(*swarm.Secret) + +// CreateSecret creates a secret given the specified spec +func (d *Daemon) CreateSecret(t assert.TestingT, secretSpec swarm.SecretSpec) string { + cli := d.NewClientT(t) + defer cli.Close() + + scr, err := cli.SecretCreate(context.Background(), secretSpec) + assert.NilError(t, err) + + return scr.ID +} + +// ListSecrets returns the list of the current swarm secrets +func (d *Daemon) ListSecrets(t assert.TestingT) []swarm.Secret { + cli := d.NewClientT(t) + defer cli.Close() + + secrets, err := cli.SecretList(context.Background(), types.SecretListOptions{}) + assert.NilError(t, err) + return secrets +} + +// GetSecret returns a swarm secret identified by the specified id +func (d *Daemon) GetSecret(t assert.TestingT, id string) *swarm.Secret { + cli := d.NewClientT(t) + defer cli.Close() + + secret, _, err := cli.SecretInspectWithRaw(context.Background(), id) + assert.NilError(t, err) + return &secret +} + +// DeleteSecret removes the swarm secret identified by the specified id +func (d *Daemon) DeleteSecret(t assert.TestingT, id string) { + cli := d.NewClientT(t) + defer cli.Close() + + err := cli.SecretRemove(context.Background(), id) + assert.NilError(t, err) +} + +// UpdateSecret updates the swarm secret identified by the specified id +// Currently, only label update is supported. +func (d *Daemon) UpdateSecret(t assert.TestingT, id string, f ...SecretConstructor) { + cli := d.NewClientT(t) + defer cli.Close() + + secret := d.GetSecret(t, id) + for _, fn := range f { + fn(secret) + } + + err := cli.SecretUpdate(context.Background(), secret.ID, secret.Version, secret.Spec) + + assert.NilError(t, err) +} diff --git a/components/engine/internal/test/daemon/service.go b/components/engine/internal/test/daemon/service.go new file mode 100644 index 0000000000..26e6004ae1 --- /dev/null +++ b/components/engine/internal/test/daemon/service.go @@ -0,0 +1,108 @@ +package daemon + +import ( + "context" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/swarm" + "github.com/gotestyourself/gotestyourself/assert" +) + +// ServiceConstructor defines a swarm service constructor function +type ServiceConstructor func(*swarm.Service) + +// CreateServiceWithOptions creates a swarm service given the specified service constructors +// and auth config +func (d *Daemon) CreateServiceWithOptions(t assert.TestingT, opts types.ServiceCreateOptions, f ...ServiceConstructor) string { + var service swarm.Service + for _, fn := range f { + fn(&service) + } + + cli := d.NewClientT(t) + defer cli.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + res, err := cli.ServiceCreate(ctx, service.Spec, opts) + assert.NilError(t, err) + return res.ID +} + +// CreateService creates a swarm service given the specified service constructor +func (d *Daemon) CreateService(t assert.TestingT, f ...ServiceConstructor) string { + return d.CreateServiceWithOptions(t, types.ServiceCreateOptions{}, f...) +} + +// GetService returns the swarm service corresponding to the specified id +func (d *Daemon) GetService(t assert.TestingT, id string) *swarm.Service { + cli := d.NewClientT(t) + defer cli.Close() + + service, _, err := cli.ServiceInspectWithRaw(context.Background(), id, types.ServiceInspectOptions{}) + assert.NilError(t, err) + return &service +} + +// GetServiceTasks returns the swarm tasks for the specified service +func (d *Daemon) GetServiceTasks(t assert.TestingT, service string) []swarm.Task { + cli := d.NewClientT(t) + defer cli.Close() + + filterArgs := filters.NewArgs() + filterArgs.Add("desired-state", "running") + filterArgs.Add("service", service) + + options := types.TaskListOptions{ + Filters: filterArgs, + } + + tasks, err := cli.TaskList(context.Background(), options) + assert.NilError(t, err) + return tasks +} + +// UpdateService updates a swarm service with the specified service constructor +func (d *Daemon) UpdateService(t assert.TestingT, service *swarm.Service, f ...ServiceConstructor) { + cli := d.NewClientT(t) + defer cli.Close() + + for _, fn := range f { + fn(service) + } + + _, err := cli.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, types.ServiceUpdateOptions{}) + assert.NilError(t, err) +} + +// RemoveService removes the specified service +func (d *Daemon) RemoveService(t assert.TestingT, id string) { + cli := d.NewClientT(t) + defer cli.Close() + + err := cli.ServiceRemove(context.Background(), id) + assert.NilError(t, err) +} + +// ListServices returns the list of the current swarm services +func (d *Daemon) ListServices(t assert.TestingT) []swarm.Service { + cli := d.NewClientT(t) + defer cli.Close() + + services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{}) + assert.NilError(t, err) + return services +} + +// GetTask returns the swarm task identified by the specified id +func (d *Daemon) GetTask(t assert.TestingT, id string) swarm.Task { + cli := d.NewClientT(t) + defer cli.Close() + + task, _, err := cli.TaskInspectWithRaw(context.Background(), id) + assert.NilError(t, err) + return task +} diff --git a/components/engine/internal/test/daemon/swarm.go b/components/engine/internal/test/daemon/swarm.go new file mode 100644 index 0000000000..f0cf373468 --- /dev/null +++ b/components/engine/internal/test/daemon/swarm.go @@ -0,0 +1,139 @@ +package daemon + +import ( + "context" + "fmt" + + "github.com/docker/docker/api/types/swarm" + "github.com/gotestyourself/gotestyourself/assert" + "github.com/pkg/errors" +) + +const ( + defaultSwarmPort = 2477 + defaultSwarmListenAddr = "0.0.0.0" +) + +// SpecConstructor defines a swarm spec constructor +type SpecConstructor func(*swarm.Spec) + +// SwarmListenAddr returns the listen-addr used for the daemon +func (d *Daemon) SwarmListenAddr() string { + return fmt.Sprintf("%s:%d", d.swarmListenAddr, d.SwarmPort) +} + +// NodeID returns the swarm mode node ID +func (d *Daemon) NodeID() string { + return d.CachedInfo.Swarm.NodeID +} + +// SwarmInit initializes a new swarm cluster. +func (d *Daemon) SwarmInit(t assert.TestingT, req swarm.InitRequest) { + if req.ListenAddr == "" { + req.ListenAddr = fmt.Sprintf("%s:%d", d.swarmListenAddr, d.SwarmPort) + } + cli := d.NewClientT(t) + defer cli.Close() + _, err := cli.SwarmInit(context.Background(), req) + assert.NilError(t, err, "initializing swarm") + d.CachedInfo = d.Info(t) +} + +// SwarmJoin joins a daemon to an existing cluster. +func (d *Daemon) SwarmJoin(t assert.TestingT, req swarm.JoinRequest) { + if req.ListenAddr == "" { + req.ListenAddr = fmt.Sprintf("%s:%d", d.swarmListenAddr, d.SwarmPort) + } + cli := d.NewClientT(t) + defer cli.Close() + err := cli.SwarmJoin(context.Background(), req) + assert.NilError(t, err, "initializing swarm") + d.CachedInfo = d.Info(t) +} + +// SwarmLeave forces daemon to leave current cluster. +func (d *Daemon) SwarmLeave(force bool) error { + cli, err := d.NewClient() + if err != nil { + return fmt.Errorf("leaving swarm: failed to create client %v", err) + } + defer cli.Close() + err = cli.SwarmLeave(context.Background(), force) + if err != nil { + err = fmt.Errorf("leaving swarm: %v", err) + } + return err +} + +// SwarmInfo returns the swarm information of the daemon +func (d *Daemon) SwarmInfo(t assert.TestingT) swarm.Info { + cli := d.NewClientT(t) + info, err := cli.Info(context.Background()) + assert.NilError(t, err, "get swarm info") + return info.Swarm +} + +// SwarmUnlock tries to unlock a locked swarm +func (d *Daemon) SwarmUnlock(req swarm.UnlockRequest) error { + cli, err := d.NewClient() + if err != nil { + return fmt.Errorf("unlocking swarm: failed to create client %v", err) + } + defer cli.Close() + err = cli.SwarmUnlock(context.Background(), req) + if err != nil { + err = errors.Wrap(err, "unlocking swarm") + } + return err +} + +// GetSwarm returns the current swarm object +func (d *Daemon) GetSwarm(t assert.TestingT) swarm.Swarm { + cli := d.NewClientT(t) + defer cli.Close() + + sw, err := cli.SwarmInspect(context.Background()) + assert.NilError(t, err) + return sw +} + +// UpdateSwarm updates the current swarm object with the specified spec constructors +func (d *Daemon) UpdateSwarm(t assert.TestingT, f ...SpecConstructor) { + cli := d.NewClientT(t) + defer cli.Close() + + sw := d.GetSwarm(t) + for _, fn := range f { + fn(&sw.Spec) + } + + err := cli.SwarmUpdate(context.Background(), sw.Version, sw.Spec, swarm.UpdateFlags{}) + assert.NilError(t, err) +} + +// RotateTokens update the swarm to rotate tokens +func (d *Daemon) RotateTokens(t assert.TestingT) { + cli := d.NewClientT(t) + defer cli.Close() + + sw, err := cli.SwarmInspect(context.Background()) + assert.NilError(t, err) + + flags := swarm.UpdateFlags{ + RotateManagerToken: true, + RotateWorkerToken: true, + } + + err = cli.SwarmUpdate(context.Background(), sw.Version, sw.Spec, flags) + assert.NilError(t, err) +} + +// JoinTokens returns the current swarm join tokens +func (d *Daemon) JoinTokens(t assert.TestingT) swarm.JoinTokens { + cli := d.NewClientT(t) + defer cli.Close() + + sw, err := cli.SwarmInspect(context.Background()) + assert.NilError(t, err) + return sw.JoinTokens +}