diff --git a/components/engine/daemon/container_operations_unix.go b/components/engine/daemon/container_operations_unix.go index 4e92b6392e..bc7ee45233 100644 --- a/components/engine/daemon/container_operations_unix.go +++ b/components/engine/daemon/container_operations_unix.go @@ -293,7 +293,6 @@ func (daemon *Daemon) createSecretsDir(c *container.Container) error { if err := mount.Mount("tmpfs", dir, "tmpfs", "nodev,nosuid,noexec,"+tmpfsOwnership); err != nil { return errors.Wrap(err, "unable to setup secret mount") } - return nil } @@ -400,15 +399,5 @@ func (daemon *Daemon) setupContainerMountsRoot(c *container.Container) error { if err != nil { return err } - - if err := idtools.MkdirAllAndChown(p, 0700, daemon.idMappings.RootPair()); err != nil { - return err - } - - if err := mount.MakeUnbindable(p); err != nil { - // Setting unbindable is a precaution and is not neccessary for correct operation. - // Do not error out if this fails. - logrus.WithError(err).WithField("resource", p).WithField("container", c.ID).Warn("Error setting container resource mounts to unbindable, this may cause mount leakages, preventing removal of this container.") - } - return nil + return idtools.MkdirAllAndChown(p, 0700, daemon.idMappings.RootPair()) } diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index 99c51a7b62..be90bc019a 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -103,6 +103,12 @@ if [ ! "$GOPATH" ]; then exit 1 fi +# Adds $1_$2 to DOCKER_BUILDTAGS unless it already +# contains a word starting from $1_ +add_buildtag() { + [[ " $DOCKER_BUILDTAGS" == *" $1_"* ]] || DOCKER_BUILDTAGS+=" $1_$2" +} + if ${PKG_CONFIG} 'libsystemd >= 209' 2> /dev/null ; then DOCKER_BUILDTAGS+=" journald" elif ${PKG_CONFIG} 'libsystemd-journal' 2> /dev/null ; then @@ -118,12 +124,14 @@ if \ fi # test whether "libdevmapper.h" is new enough to support deferred remove -# functionality. +# functionality. We favour libdm_dlsym_deferred_remove over +# libdm_no_deferred_remove in dynamic cases because the binary could be shipped +# with a newer libdevmapper than the one it was built wih. if \ command -v gcc &> /dev/null \ && ! ( echo -e '#include \nint main() { dm_task_deferred_remove(NULL); }'| gcc -xc - -o /dev/null $(pkg-config --libs devmapper) &> /dev/null ) \ ; then - DOCKER_BUILDTAGS+=' libdm_no_deferred_remove' + add_buildtag libdm dlsym_deferred_remove fi # Use these flags when compiling the tests and final binary diff --git a/components/engine/integration-cli/check_test.go b/components/engine/integration-cli/check_test.go index 18d3d3780a..0cae1503f8 100644 --- a/components/engine/integration-cli/check_test.go +++ b/components/engine/integration-cli/check_test.go @@ -21,6 +21,7 @@ import ( "github.com/docker/docker/integration-cli/environment" "github.com/docker/docker/integration-cli/fixtures/plugin" "github.com/docker/docker/integration-cli/registry" + testdaemon "github.com/docker/docker/internal/test/daemon" ienv "github.com/docker/docker/internal/test/environment" "github.com/docker/docker/pkg/reexec" "github.com/go-check/check" @@ -100,7 +101,7 @@ func (s *DockerSuite) OnTimeout(c *check.C) { daemonPid := int(rawPid) if daemonPid > 0 { - daemon.SignalDaemonDump(daemonPid) + testdaemon.SignalDaemonDump(daemonPid) } } @@ -285,7 +286,7 @@ func (s *DockerDaemonSuite) TearDownTest(c *check.C) { } func (s *DockerDaemonSuite) TearDownSuite(c *check.C) { - filepath.Walk(daemon.SockRoot, func(path string, fi os.FileInfo, err error) error { + filepath.Walk(testdaemon.SockRoot, func(path string, fi os.FileInfo, err error) error { if err != nil { // ignore errors here // not cleaning up sockets is not really an error @@ -296,7 +297,7 @@ func (s *DockerDaemonSuite) TearDownSuite(c *check.C) { } return nil }) - os.RemoveAll(daemon.SockRoot) + os.RemoveAll(testdaemon.SockRoot) } const defaultSwarmPort = 2477 @@ -310,7 +311,7 @@ func init() { type DockerSwarmSuite struct { server *httptest.Server ds *DockerSuite - daemons []*daemon.Swarm + daemons []*daemon.Daemon daemonsLock sync.Mutex // protect access to daemons portIndex int } @@ -327,14 +328,10 @@ func (s *DockerSwarmSuite) SetUpTest(c *check.C) { testRequires(c, DaemonIsLinux, SameHostDaemon) } -func (s *DockerSwarmSuite) AddDaemon(c *check.C, joinSwarm, manager bool) *daemon.Swarm { - d := &daemon.Swarm{ - Daemon: daemon.New(c, dockerBinary, dockerdBinary, daemon.Config{ - Experimental: testEnv.DaemonInfo.ExperimentalBuild, - }), - Port: defaultSwarmPort + s.portIndex, - } - d.ListenAddr = fmt.Sprintf("0.0.0.0:%d", d.Port) +func (s *DockerSwarmSuite) AddDaemon(c *check.C, joinSwarm, manager bool) *daemon.Daemon { + d := daemon.New(c, dockerBinary, dockerdBinary, daemon.Config{ + Experimental: testEnv.DaemonInfo.ExperimentalBuild, + }, testdaemon.WithSwarmPort(defaultSwarmPort+s.portIndex)) args := []string{"--iptables=false", "--swarm-default-advertise-addr=lo"} // avoid networking conflicts d.StartWithBusybox(c, args...) @@ -345,12 +342,12 @@ func (s *DockerSwarmSuite) AddDaemon(c *check.C, joinSwarm, manager bool) *daemo if manager { token = tokens.Manager } - c.Assert(d.Join(swarm.JoinRequest{ - RemoteAddrs: []string{s.daemons[0].ListenAddr}, + d.SwarmJoin(c, swarm.JoinRequest{ + RemoteAddrs: []string{s.daemons[0].SwarmListenAddr()}, JoinToken: token, - }), check.IsNil) + }) } else { - c.Assert(d.Init(swarm.InitRequest{}), check.IsNil) + d.SwarmInit(c, swarm.InitRequest{}) } } diff --git a/components/engine/integration-cli/daemon/daemon.go b/components/engine/integration-cli/daemon/daemon.go index 9ca54236f4..0ec9e892b2 100644 --- a/components/engine/integration-cli/daemon/daemon.go +++ b/components/engine/integration-cli/daemon/daemon.go @@ -1,33 +1,17 @@ package daemon // import "github.com/docker/docker/integration-cli/daemon" import ( - "encoding/json" "fmt" - "io" - "io/ioutil" - "net/http" - "os" "os/exec" - "path/filepath" - "strconv" "strings" "time" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/events" - "github.com/docker/docker/client" "github.com/docker/docker/integration-cli/checker" - "github.com/docker/docker/integration-cli/request" - "github.com/docker/docker/opts" - "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/stringid" - "github.com/docker/go-connections/sockets" - "github.com/docker/go-connections/tlsconfig" + "github.com/docker/docker/internal/test/daemon" "github.com/go-check/check" "github.com/gotestyourself/gotestyourself/assert" "github.com/gotestyourself/gotestyourself/icmd" "github.com/pkg/errors" - "golang.org/x/net/context" ) type testingT interface { @@ -40,32 +24,10 @@ type logT interface { Logf(string, ...interface{}) } -// SockRoot holds the path of the default docker integration daemon socket -var SockRoot = filepath.Join(os.TempDir(), "docker-integration") - -var errDaemonNotStarted = errors.New("daemon not started") - // Daemon represents a Docker daemon for the testing framework. type Daemon struct { - GlobalFlags []string - Root string - Folder string - Wait chan error - UseDefaultHost bool - UseDefaultTLSHost bool - - id string - logFile *os.File - stdin io.WriteCloser - stdout, stderr io.ReadCloser - cmd *exec.Cmd - storageDriver string - userlandProxy bool - execRoot string - experimental bool - dockerBinary string - dockerdBinary string - log logT + *daemon.Daemon + dockerBinary string } // Config holds docker daemon integration configuration @@ -73,504 +35,22 @@ type Config struct { Experimental bool } -type clientConfig struct { - transport *http.Transport - scheme string - addr string -} - // New returns a Daemon instance to be used for testing. // This will create a directory such as d123456789 in the folder specified by $DOCKER_INTEGRATION_DAEMON_DEST or $DEST. // The daemon will not automatically start. -func New(t testingT, dockerBinary string, dockerdBinary string, config Config) *Daemon { - dest := os.Getenv("DOCKER_INTEGRATION_DAEMON_DEST") - if dest == "" { - dest = os.Getenv("DEST") - } - if dest == "" { - t.Fatalf("Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable") - } - - if err := os.MkdirAll(SockRoot, 0700); err != nil { - t.Fatalf("could not create daemon socket root") - } - - id := fmt.Sprintf("d%s", stringid.TruncateID(stringid.GenerateRandomID())) - dir := filepath.Join(dest, id) - daemonFolder, err := filepath.Abs(dir) - if err != nil { - t.Fatalf("Could not make %q an absolute path", dir) - } - daemonRoot := filepath.Join(daemonFolder, "root") - - if err := os.MkdirAll(daemonRoot, 0755); err != nil { - t.Fatalf("Could not create daemon root %q", dir) - } - - userlandProxy := true - if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { - if val, err := strconv.ParseBool(env); err != nil { - userlandProxy = val - } +func New(t testingT, dockerBinary string, dockerdBinary string, config Config, ops ...func(*daemon.Daemon)) *Daemon { + ops = append(ops, daemon.WithDockerdBinary(dockerdBinary)) + if config.Experimental { + ops = append(ops, daemon.WithExperimental) } + d := daemon.New(t, ops...) return &Daemon{ - id: id, - Folder: daemonFolder, - Root: daemonRoot, - storageDriver: os.Getenv("DOCKER_GRAPHDRIVER"), - userlandProxy: userlandProxy, - execRoot: filepath.Join(os.TempDir(), "docker-execroot", id), - dockerBinary: dockerBinary, - dockerdBinary: dockerdBinary, - experimental: config.Experimental, - log: t, + Daemon: d, + dockerBinary: dockerBinary, } } -// RootDir returns the root directory of the daemon. -func (d *Daemon) RootDir() string { - return d.Root -} - -// ID returns the generated id of the daemon -func (d *Daemon) ID() string { - return d.id -} - -// StorageDriver returns the configured storage driver of the daemon -func (d *Daemon) StorageDriver() string { - return d.storageDriver -} - -// CleanupExecRoot cleans the daemon exec root (network namespaces, ...) -func (d *Daemon) CleanupExecRoot(c *check.C) { - cleanupExecRoot(c, d.execRoot) -} - -func (d *Daemon) getClientConfig() (*clientConfig, error) { - var ( - transport *http.Transport - scheme string - addr string - proto string - ) - if d.UseDefaultTLSHost { - option := &tlsconfig.Options{ - CAFile: "fixtures/https/ca.pem", - CertFile: "fixtures/https/client-cert.pem", - KeyFile: "fixtures/https/client-key.pem", - } - tlsConfig, err := tlsconfig.Client(*option) - if err != nil { - return nil, err - } - transport = &http.Transport{ - TLSClientConfig: tlsConfig, - } - addr = fmt.Sprintf("%s:%d", opts.DefaultHTTPHost, opts.DefaultTLSHTTPPort) - scheme = "https" - proto = "tcp" - } else if d.UseDefaultHost { - addr = opts.DefaultUnixSocket - proto = "unix" - scheme = "http" - transport = &http.Transport{} - } else { - addr = d.sockPath() - proto = "unix" - scheme = "http" - transport = &http.Transport{} - } - - if err := sockets.ConfigureTransport(transport, proto, addr); err != nil { - return nil, err - } - transport.DisableKeepAlives = true - - return &clientConfig{ - transport: transport, - scheme: scheme, - addr: addr, - }, nil -} - -// Start starts the daemon and return once it is ready to receive requests. -func (d *Daemon) Start(t testingT, args ...string) { - if err := d.StartWithError(args...); err != nil { - t.Fatalf("Error starting daemon with arguments: %v", args) - } -} - -// StartWithError starts the daemon and return once it is ready to receive requests. -// It returns an error in case it couldn't start. -func (d *Daemon) StartWithError(args ...string) error { - logFile, err := os.OpenFile(filepath.Join(d.Folder, "docker.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return errors.Wrapf(err, "[%s] Could not create %s/docker.log", d.id, d.Folder) - } - - return d.StartWithLogFile(logFile, args...) -} - -// StartWithLogFile will start the daemon and attach its streams to a given file. -func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { - dockerdBinary, err := exec.LookPath(d.dockerdBinary) - if err != nil { - return errors.Wrapf(err, "[%s] could not find docker binary in $PATH", d.id) - } - args := append(d.GlobalFlags, - "--containerd", "/var/run/docker/containerd/docker-containerd.sock", - "--data-root", d.Root, - "--exec-root", d.execRoot, - "--pidfile", fmt.Sprintf("%s/docker.pid", d.Folder), - fmt.Sprintf("--userland-proxy=%t", d.userlandProxy), - ) - if d.experimental { - args = append(args, "--experimental", "--init") - } - if !(d.UseDefaultHost || d.UseDefaultTLSHost) { - args = append(args, []string{"--host", d.Sock()}...) - } - if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { - args = append(args, []string{"--userns-remap", root}...) - } - - // If we don't explicitly set the log-level or debug flag(-D) then - // turn on debug mode - foundLog := false - foundSd := false - for _, a := range providedArgs { - if strings.Contains(a, "--log-level") || strings.Contains(a, "-D") || strings.Contains(a, "--debug") { - foundLog = true - } - if strings.Contains(a, "--storage-driver") { - foundSd = true - } - } - if !foundLog { - args = append(args, "--debug") - } - if d.storageDriver != "" && !foundSd { - args = append(args, "--storage-driver", d.storageDriver) - } - - args = append(args, providedArgs...) - d.cmd = exec.Command(dockerdBinary, args...) - d.cmd.Env = append(os.Environ(), "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE=1") - d.cmd.Stdout = out - d.cmd.Stderr = out - d.logFile = out - - if err := d.cmd.Start(); err != nil { - return errors.Errorf("[%s] could not start daemon container: %v", d.id, err) - } - - wait := make(chan error) - - go func() { - wait <- d.cmd.Wait() - d.log.Logf("[%s] exiting daemon", d.id) - close(wait) - }() - - d.Wait = wait - - tick := time.Tick(500 * time.Millisecond) - // make sure daemon is ready to receive requests - startTime := time.Now().Unix() - for { - d.log.Logf("[%s] waiting for daemon to start", d.id) - if time.Now().Unix()-startTime > 5 { - // After 5 seconds, give up - return errors.Errorf("[%s] Daemon exited and never started", d.id) - } - select { - case <-time.After(2 * time.Second): - return errors.Errorf("[%s] timeout: daemon does not respond", d.id) - case <-tick: - clientConfig, err := d.getClientConfig() - if err != nil { - return err - } - - client := &http.Client{ - Transport: clientConfig.transport, - } - - req, err := http.NewRequest("GET", "/_ping", nil) - if err != nil { - return errors.Wrapf(err, "[%s] could not create new request", d.id) - } - req.URL.Host = clientConfig.addr - req.URL.Scheme = clientConfig.scheme - resp, err := client.Do(req) - if err != nil { - continue - } - resp.Body.Close() - if resp.StatusCode != http.StatusOK { - d.log.Logf("[%s] received status != 200 OK: %s\n", d.id, resp.Status) - } - d.log.Logf("[%s] daemon started\n", d.id) - d.Root, err = d.queryRootDir() - if err != nil { - return errors.Errorf("[%s] error querying daemon for root directory: %v", d.id, err) - } - return nil - case <-d.Wait: - return errors.Errorf("[%s] Daemon exited during startup", d.id) - } - } -} - -// StartWithBusybox will first start the daemon with Daemon.Start() -// then save the busybox image from the main daemon and load it into this Daemon instance. -func (d *Daemon) StartWithBusybox(t testingT, arg ...string) { - d.Start(t, arg...) - d.LoadBusybox(t) -} - -// Kill will send a SIGKILL to the daemon -func (d *Daemon) Kill() error { - if d.cmd == nil || d.Wait == nil { - return errDaemonNotStarted - } - - defer func() { - d.logFile.Close() - d.cmd = nil - }() - - if err := d.cmd.Process.Kill(); err != nil { - return err - } - - return os.Remove(fmt.Sprintf("%s/docker.pid", d.Folder)) -} - -// Pid returns the pid of the daemon -func (d *Daemon) Pid() int { - return d.cmd.Process.Pid -} - -// Interrupt stops the daemon by sending it an Interrupt signal -func (d *Daemon) Interrupt() error { - return d.Signal(os.Interrupt) -} - -// Signal sends the specified signal to the daemon if running -func (d *Daemon) Signal(signal os.Signal) error { - if d.cmd == nil || d.Wait == nil { - return errDaemonNotStarted - } - return d.cmd.Process.Signal(signal) -} - -// DumpStackAndQuit sends SIGQUIT to the daemon, which triggers it to dump its -// stack to its log file and exit -// This is used primarily for gathering debug information on test timeout -func (d *Daemon) DumpStackAndQuit() { - if d.cmd == nil || d.cmd.Process == nil { - return - } - SignalDaemonDump(d.cmd.Process.Pid) -} - -// Stop will send a SIGINT every second and wait for the daemon to stop. -// If it times out, a SIGKILL is sent. -// Stop will not delete the daemon directory. If a purged daemon is needed, -// instantiate a new one with NewDaemon. -// If an error occurs while starting the daemon, the test will fail. -func (d *Daemon) Stop(t testingT) { - err := d.StopWithError() - if err != nil { - if err != errDaemonNotStarted { - t.Fatalf("Error while stopping the daemon %s : %v", d.id, err) - } else { - t.Logf("Daemon %s is not started", d.id) - } - } -} - -// StopWithError will send a SIGINT every second and wait for the daemon to stop. -// If it timeouts, a SIGKILL is sent. -// Stop will not delete the daemon directory. If a purged daemon is needed, -// instantiate a new one with NewDaemon. -func (d *Daemon) StopWithError() error { - if d.cmd == nil || d.Wait == nil { - return errDaemonNotStarted - } - - defer func() { - d.logFile.Close() - d.cmd = nil - }() - - i := 1 - tick := time.Tick(time.Second) - - if err := d.cmd.Process.Signal(os.Interrupt); err != nil { - if strings.Contains(err.Error(), "os: process already finished") { - return errDaemonNotStarted - } - return errors.Errorf("could not send signal: %v", err) - } -out1: - for { - select { - case err := <-d.Wait: - return err - case <-time.After(20 * time.Second): - // time for stopping jobs and run onShutdown hooks - d.log.Logf("[%s] daemon started", d.id) - break out1 - } - } - -out2: - for { - select { - case err := <-d.Wait: - return err - case <-tick: - i++ - if i > 5 { - d.log.Logf("tried to interrupt daemon for %d times, now try to kill it", i) - break out2 - } - d.log.Logf("Attempt #%d: daemon is still running with pid %d", i, d.cmd.Process.Pid) - if err := d.cmd.Process.Signal(os.Interrupt); err != nil { - return errors.Errorf("could not send signal: %v", err) - } - } - } - - if err := d.cmd.Process.Kill(); err != nil { - d.log.Logf("Could not kill daemon: %v", err) - return err - } - - d.cmd.Wait() - - return os.Remove(fmt.Sprintf("%s/docker.pid", d.Folder)) -} - -// Restart will restart the daemon by first stopping it and the starting it. -// If an error occurs while starting the daemon, the test will fail. -func (d *Daemon) Restart(t testingT, args ...string) { - d.Stop(t) - d.handleUserns() - d.Start(t, args...) -} - -// RestartWithError will restart the daemon by first stopping it and then starting it. -func (d *Daemon) RestartWithError(arg ...string) error { - if err := d.StopWithError(); err != nil { - return err - } - d.handleUserns() - return d.StartWithError(arg...) -} - -func (d *Daemon) handleUserns() { - // in the case of tests running a user namespace-enabled daemon, we have resolved - // d.Root to be the actual final path of the graph dir after the "uid.gid" of - // remapped root is added--we need to subtract it from the path before calling - // start or else we will continue making subdirectories rather than truly restarting - // with the same location/root: - if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { - d.Root = filepath.Dir(d.Root) - } -} - -// LoadBusybox image into the daemon -func (d *Daemon) LoadBusybox(t testingT) { - clientHost, err := client.NewEnvClient() - assert.NilError(t, err, "failed to create client") - defer clientHost.Close() - - ctx := context.Background() - reader, err := clientHost.ImageSave(ctx, []string{"busybox:latest"}) - assert.NilError(t, err, "failed to download busybox") - defer reader.Close() - - client, err := d.NewClient() - assert.NilError(t, err, "failed to create client") - defer client.Close() - - resp, err := client.ImageLoad(ctx, reader, true) - assert.NilError(t, err, "failed to load busybox") - defer resp.Body.Close() -} - -func (d *Daemon) queryRootDir() (string, error) { - // update daemon root by asking /info endpoint (to support user - // namespaced daemon with root remapped uid.gid directory) - clientConfig, err := d.getClientConfig() - if err != nil { - return "", err - } - - client := &http.Client{ - Transport: clientConfig.transport, - } - - req, err := http.NewRequest("GET", "/info", nil) - if err != nil { - return "", err - } - req.Header.Set("Content-Type", "application/json") - req.URL.Host = clientConfig.addr - req.URL.Scheme = clientConfig.scheme - - resp, err := client.Do(req) - if err != nil { - return "", err - } - body := ioutils.NewReadCloserWrapper(resp.Body, func() error { - return resp.Body.Close() - }) - - type Info struct { - DockerRootDir string - } - var b []byte - var i Info - b, err = request.ReadBody(body) - if err == nil && resp.StatusCode == http.StatusOK { - // read the docker root dir - if err = json.Unmarshal(b, &i); err == nil { - return i.DockerRootDir, nil - } - } - return "", err -} - -// Sock returns the socket path of the daemon -func (d *Daemon) Sock() string { - return fmt.Sprintf("unix://" + d.sockPath()) -} - -func (d *Daemon) sockPath() string { - return filepath.Join(SockRoot, d.id+".sock") -} - -// WaitRun waits for a container to be running for 10s -func (d *Daemon) WaitRun(contID string) error { - args := []string{"--host", d.Sock()} - return WaitInspectWithArgs(d.dockerBinary, contID, "{{.State.Running}}", "true", 10*time.Second, args...) -} - -// Info returns the info struct for this daemon -func (d *Daemon) Info(t assert.TestingT) types.Info { - apiclient, err := client.NewClientWithOpts(client.WithHost((d.Sock()))) - assert.NilError(t, err) - info, err := apiclient.Info(context.Background()) - assert.NilError(t, err) - return info -} - // Cmd executes a docker CLI command against this daemon. // Example: d.Cmd("version") will run docker -H unix://path/to/unix.sock version func (d *Daemon) Cmd(args ...string) (string, error) { @@ -594,11 +74,6 @@ func (d *Daemon) PrependHostArg(args []string) []string { return append([]string{"--host", d.Sock()}, args...) } -// LogFileName returns the path the daemon's log file -func (d *Daemon) LogFileName() string { - return d.logFile.Name() -} - // GetIDByName returns the ID of an object (container, volume, …) given its name func (d *Daemon) GetIDByName(name string) (string, error) { return d.inspectFieldWithError(name, "Id") @@ -616,11 +91,6 @@ func (d *Daemon) ActiveContainers() (ids []string) { return } -// ReadLogFile returns the content of the daemon log file -func (d *Daemon) ReadLogFile() ([]byte, error) { - return ioutil.ReadFile(d.logFile.Name()) -} - // InspectField returns the field filter by 'filter' func (d *Daemon) InspectField(name, filter string) (string, error) { return d.inspectFilter(name, filter) @@ -672,59 +142,25 @@ func (d *Daemon) CheckActiveContainerCount(c *check.C) (interface{}, check.Comme return len(strings.Split(strings.TrimSpace(out), "\n")), check.Commentf("output: %q", string(out)) } -// ReloadConfig asks the daemon to reload its configuration -func (d *Daemon) ReloadConfig() error { - if d.cmd == nil || d.cmd.Process == nil { - return errors.New("daemon is not running") - } - - errCh := make(chan error) - started := make(chan struct{}) - go func() { - _, body, err := request.DoOnHost(d.Sock(), "/events", request.Method(http.MethodGet)) - close(started) - if err != nil { - errCh <- err - } - defer body.Close() - dec := json.NewDecoder(body) - for { - var e events.Message - if err := dec.Decode(&e); err != nil { - errCh <- err - return - } - if e.Type != events.DaemonEventType { - continue - } - if e.Action != "reload" { - continue - } - close(errCh) // notify that we are done - return - } - }() - - <-started - if err := signalDaemonReload(d.cmd.Process.Pid); err != nil { - return errors.Errorf("error signaling daemon reload: %v", err) - } - select { - case err := <-errCh: - if err != nil { - return errors.Errorf("error waiting for daemon reload event: %v", err) - } - case <-time.After(30 * time.Second): - return errors.New("timeout waiting for daemon reload event") - } - return nil +// WaitRun waits for a container to be running for 10s +func (d *Daemon) WaitRun(contID string) error { + args := []string{"--host", d.Sock()} + return WaitInspectWithArgs(d.dockerBinary, contID, "{{.State.Running}}", "true", 10*time.Second, args...) } -// NewClient creates new client based on daemon's socket path -func (d *Daemon) NewClient() (*client.Client, error) { - return client.NewClientWithOpts( - client.FromEnv, - client.WithHost(d.Sock())) +// CmdRetryOutOfSequence tries the specified command against the current daemon for 10 times +func (d *Daemon) CmdRetryOutOfSequence(args ...string) (string, error) { + for i := 0; ; i++ { + out, err := d.Cmd(args...) + if err != nil { + if strings.Contains(out, "update out of sequence") { + if i < 10 { + continue + } + } + } + return out, err + } } // WaitInspectWithArgs waits for the specified expression to be equals to the specified expected string in the given time. diff --git a/components/engine/integration-cli/daemon/daemon_swarm.go b/components/engine/integration-cli/daemon/daemon_swarm.go index be0ddef99e..cf75724a32 100644 --- a/components/engine/integration-cli/daemon/daemon_swarm.go +++ b/components/engine/integration-cli/daemon/daemon_swarm.go @@ -3,7 +3,6 @@ package daemon // import "github.com/docker/docker/integration-cli/daemon" import ( "fmt" "strings" - "time" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" @@ -12,178 +11,12 @@ import ( "github.com/docker/docker/integration-cli/checker" "github.com/go-check/check" "github.com/gotestyourself/gotestyourself/assert" - "github.com/pkg/errors" "golang.org/x/net/context" ) -// Swarm is a test daemon with helpers for participating in a swarm. -type Swarm struct { - *Daemon - swarm.Info - Port int - ListenAddr string -} - -// Init initializes a new swarm cluster. -func (d *Swarm) Init(req swarm.InitRequest) error { - if req.ListenAddr == "" { - req.ListenAddr = d.ListenAddr - } - cli, err := d.NewClient() - if err != nil { - return fmt.Errorf("initializing swarm: failed to create client %v", err) - } - defer cli.Close() - _, err = cli.SwarmInit(context.Background(), req) - if err != nil { - return fmt.Errorf("initializing swarm: %v", err) - } - info, err := d.SwarmInfo() - if err != nil { - return err - } - d.Info = info - return nil -} - -// Join joins a daemon to an existing cluster. -func (d *Swarm) Join(req swarm.JoinRequest) error { - if req.ListenAddr == "" { - req.ListenAddr = d.ListenAddr - } - cli, err := d.NewClient() - if err != nil { - return fmt.Errorf("joining swarm: failed to create client %v", err) - } - defer cli.Close() - err = cli.SwarmJoin(context.Background(), req) - if err != nil { - return fmt.Errorf("joining swarm: %v", err) - } - info, err := d.SwarmInfo() - if err != nil { - return err - } - d.Info = info - return nil -} - -// Leave forces daemon to leave current cluster. -func (d *Swarm) Leave(force bool) error { - cli, err := d.NewClient() - if err != nil { - return fmt.Errorf("leaving swarm: failed to create client %v", err) - } - defer cli.Close() - err = cli.SwarmLeave(context.Background(), force) - if err != nil { - err = fmt.Errorf("leaving swarm: %v", err) - } - return err -} - -// SwarmInfo returns the swarm information of the daemon -func (d *Swarm) SwarmInfo() (swarm.Info, error) { - cli, err := d.NewClient() - if err != nil { - return swarm.Info{}, fmt.Errorf("get swarm info: %v", err) - } - - info, err := cli.Info(context.Background()) - if err != nil { - return swarm.Info{}, fmt.Errorf("get swarm info: %v", err) - } - - return info.Swarm, nil -} - -// Unlock tries to unlock a locked swarm -func (d *Swarm) Unlock(req swarm.UnlockRequest) error { - cli, err := d.NewClient() - if err != nil { - return fmt.Errorf("unlocking swarm: failed to create client %v", err) - } - defer cli.Close() - err = cli.SwarmUnlock(context.Background(), req) - if err != nil { - err = errors.Wrap(err, "unlocking swarm") - } - return err -} - -// ServiceConstructor defines a swarm service constructor function -type ServiceConstructor func(*swarm.Service) - -// NodeConstructor defines a swarm node constructor -type NodeConstructor func(*swarm.Node) - -// SecretConstructor defines a swarm secret constructor -type SecretConstructor func(*swarm.Secret) - -// ConfigConstructor defines a swarm config constructor -type ConfigConstructor func(*swarm.Config) - -// SpecConstructor defines a swarm spec constructor -type SpecConstructor func(*swarm.Spec) - -// CreateServiceWithOptions creates a swarm service given the specified service constructors -// and auth config -func (d *Swarm) CreateServiceWithOptions(c *check.C, opts types.ServiceCreateOptions, f ...ServiceConstructor) string { - var service swarm.Service - for _, fn := range f { - fn(&service) - } - - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - res, err := cli.ServiceCreate(ctx, service.Spec, opts) - c.Assert(err, checker.IsNil) - return res.ID -} - -// CreateService creates a swarm service given the specified service constructor -func (d *Swarm) CreateService(c *check.C, f ...ServiceConstructor) string { - return d.CreateServiceWithOptions(c, types.ServiceCreateOptions{}, f...) -} - -// GetService returns the swarm service corresponding to the specified id -func (d *Swarm) GetService(c *check.C, id string) *swarm.Service { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - service, _, err := cli.ServiceInspectWithRaw(context.Background(), id, types.ServiceInspectOptions{}) - c.Assert(err, checker.IsNil) - return &service -} - -// GetServiceTasks returns the swarm tasks for the specified service -func (d *Swarm) GetServiceTasks(c *check.C, service string) []swarm.Task { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - filterArgs := filters.NewArgs() - filterArgs.Add("desired-state", "running") - filterArgs.Add("service", service) - - options := types.TaskListOptions{ - Filters: filterArgs, - } - - tasks, err := cli.TaskList(context.Background(), options) - c.Assert(err, checker.IsNil) - return tasks -} - // CheckServiceTasksInState returns the number of tasks with a matching state, // and optional message substring. -func (d *Swarm) CheckServiceTasksInState(service string, state swarm.TaskState, message string) func(*check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckServiceTasksInState(service string, state swarm.TaskState, message string) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { tasks := d.GetServiceTasks(c, service) var count int @@ -200,7 +33,7 @@ func (d *Swarm) CheckServiceTasksInState(service string, state swarm.TaskState, // CheckServiceTasksInStateWithError returns the number of tasks with a matching state, // and optional message substring. -func (d *Swarm) CheckServiceTasksInStateWithError(service string, state swarm.TaskState, errorMessage string) func(*check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckServiceTasksInStateWithError(service string, state swarm.TaskState, errorMessage string) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { tasks := d.GetServiceTasks(c, service) var count int @@ -216,12 +49,12 @@ func (d *Swarm) CheckServiceTasksInStateWithError(service string, state swarm.Ta } // CheckServiceRunningTasks returns the number of running tasks for the specified service -func (d *Swarm) CheckServiceRunningTasks(service string) func(*check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckServiceRunningTasks(service string) func(*check.C) (interface{}, check.CommentInterface) { return d.CheckServiceTasksInState(service, swarm.TaskStateRunning, "") } // CheckServiceUpdateState returns the current update state for the specified service -func (d *Swarm) CheckServiceUpdateState(service string) func(*check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckServiceUpdateState(service string) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { service := d.GetService(c, service) if service.UpdateStatus == nil { @@ -232,7 +65,7 @@ func (d *Swarm) CheckServiceUpdateState(service string) func(*check.C) (interfac } // CheckPluginRunning returns the runtime state of the plugin -func (d *Swarm) CheckPluginRunning(plugin string) func(c *check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckPluginRunning(plugin string) func(c *check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { apiclient, err := d.NewClient() assert.NilError(c, err) @@ -246,7 +79,7 @@ func (d *Swarm) CheckPluginRunning(plugin string) func(c *check.C) (interface{}, } // CheckPluginImage returns the runtime state of the plugin -func (d *Swarm) CheckPluginImage(plugin string) func(c *check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckPluginImage(plugin string) func(c *check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { apiclient, err := d.NewClient() assert.NilError(c, err) @@ -260,7 +93,7 @@ func (d *Swarm) CheckPluginImage(plugin string) func(c *check.C) (interface{}, c } // CheckServiceTasks returns the number of tasks for the specified service -func (d *Swarm) CheckServiceTasks(service string) func(*check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckServiceTasks(service string) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { tasks := d.GetServiceTasks(c, service) return len(tasks), nil @@ -268,7 +101,7 @@ func (d *Swarm) CheckServiceTasks(service string) func(*check.C) (interface{}, c } // CheckRunningTaskNetworks returns the number of times each network is referenced from a task. -func (d *Swarm) CheckRunningTaskNetworks(c *check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckRunningTaskNetworks(c *check.C) (interface{}, check.CommentInterface) { cli, err := d.NewClient() c.Assert(err, checker.IsNil) defer cli.Close() @@ -293,7 +126,7 @@ func (d *Swarm) CheckRunningTaskNetworks(c *check.C) (interface{}, check.Comment } // CheckRunningTaskImages returns the times each image is running as a task. -func (d *Swarm) CheckRunningTaskImages(c *check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckRunningTaskImages(c *check.C) (interface{}, check.CommentInterface) { cli, err := d.NewClient() c.Assert(err, checker.IsNil) defer cli.Close() @@ -318,7 +151,7 @@ func (d *Swarm) CheckRunningTaskImages(c *check.C) (interface{}, check.CommentIn } // CheckNodeReadyCount returns the number of ready node on the swarm -func (d *Swarm) CheckNodeReadyCount(c *check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckNodeReadyCount(c *check.C) (interface{}, check.CommentInterface) { nodes := d.ListNodes(c) var readyCount int for _, node := range nodes { @@ -329,303 +162,21 @@ func (d *Swarm) CheckNodeReadyCount(c *check.C) (interface{}, check.CommentInter return readyCount, nil } -// GetTask returns the swarm task identified by the specified id -func (d *Swarm) GetTask(c *check.C, id string) swarm.Task { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - task, _, err := cli.TaskInspectWithRaw(context.Background(), id) - c.Assert(err, checker.IsNil) - return task -} - -// UpdateService updates a swarm service with the specified service constructor -func (d *Swarm) UpdateService(c *check.C, service *swarm.Service, f ...ServiceConstructor) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - for _, fn := range f { - fn(service) - } - - _, err = cli.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, types.ServiceUpdateOptions{}) - c.Assert(err, checker.IsNil) -} - -// RemoveService removes the specified service -func (d *Swarm) RemoveService(c *check.C, id string) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - err = cli.ServiceRemove(context.Background(), id) - c.Assert(err, checker.IsNil) -} - -// GetNode returns a swarm node identified by the specified id -func (d *Swarm) GetNode(c *check.C, id string) *swarm.Node { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - node, _, err := cli.NodeInspectWithRaw(context.Background(), id) - c.Assert(err, checker.IsNil) - c.Assert(node.ID, checker.Equals, id) - return &node -} - -// RemoveNode removes the specified node -func (d *Swarm) RemoveNode(c *check.C, id string, force bool) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - options := types.NodeRemoveOptions{ - Force: force, - } - err = cli.NodeRemove(context.Background(), id, options) - c.Assert(err, checker.IsNil) -} - -// UpdateNode updates a swarm node with the specified node constructor -func (d *Swarm) UpdateNode(c *check.C, id string, f ...NodeConstructor) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - for i := 0; ; i++ { - node := d.GetNode(c, id) - for _, fn := range f { - fn(node) - } - - err = cli.NodeUpdate(context.Background(), node.ID, node.Version, node.Spec) - if i < 10 && err != nil && strings.Contains(err.Error(), "update out of sequence") { - time.Sleep(100 * time.Millisecond) - continue - } - c.Assert(err, checker.IsNil) - return - } -} - -// ListNodes returns the list of the current swarm nodes -func (d *Swarm) ListNodes(c *check.C) []swarm.Node { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{}) - c.Assert(err, checker.IsNil) - - return nodes -} - -// ListServices returns the list of the current swarm services -func (d *Swarm) ListServices(c *check.C) []swarm.Service { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{}) - c.Assert(err, checker.IsNil) - return services -} - -// CreateSecret creates a secret given the specified spec -func (d *Swarm) CreateSecret(c *check.C, secretSpec swarm.SecretSpec) string { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - scr, err := cli.SecretCreate(context.Background(), secretSpec) - c.Assert(err, checker.IsNil) - - return scr.ID -} - -// ListSecrets returns the list of the current swarm secrets -func (d *Swarm) ListSecrets(c *check.C) []swarm.Secret { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - secrets, err := cli.SecretList(context.Background(), types.SecretListOptions{}) - c.Assert(err, checker.IsNil) - return secrets -} - -// GetSecret returns a swarm secret identified by the specified id -func (d *Swarm) GetSecret(c *check.C, id string) *swarm.Secret { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - secret, _, err := cli.SecretInspectWithRaw(context.Background(), id) - c.Assert(err, checker.IsNil) - return &secret -} - -// DeleteSecret removes the swarm secret identified by the specified id -func (d *Swarm) DeleteSecret(c *check.C, id string) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - err = cli.SecretRemove(context.Background(), id) - c.Assert(err, checker.IsNil) -} - -// UpdateSecret updates the swarm secret identified by the specified id -// Currently, only label update is supported. -func (d *Swarm) UpdateSecret(c *check.C, id string, f ...SecretConstructor) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - secret := d.GetSecret(c, id) - for _, fn := range f { - fn(secret) - } - - err = cli.SecretUpdate(context.Background(), secret.ID, secret.Version, secret.Spec) - - c.Assert(err, checker.IsNil) -} - -// CreateConfig creates a config given the specified spec -func (d *Swarm) CreateConfig(c *check.C, configSpec swarm.ConfigSpec) string { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - scr, err := cli.ConfigCreate(context.Background(), configSpec) - c.Assert(err, checker.IsNil) - return scr.ID -} - -// ListConfigs returns the list of the current swarm configs -func (d *Swarm) ListConfigs(c *check.C) []swarm.Config { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - configs, err := cli.ConfigList(context.Background(), types.ConfigListOptions{}) - c.Assert(err, checker.IsNil) - return configs -} - -// GetConfig returns a swarm config identified by the specified id -func (d *Swarm) GetConfig(c *check.C, id string) *swarm.Config { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - config, _, err := cli.ConfigInspectWithRaw(context.Background(), id) - c.Assert(err, checker.IsNil) - return &config -} - -// DeleteConfig removes the swarm config identified by the specified id -func (d *Swarm) DeleteConfig(c *check.C, id string) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - err = cli.ConfigRemove(context.Background(), id) - c.Assert(err, checker.IsNil) -} - -// UpdateConfig updates the swarm config identified by the specified id -// Currently, only label update is supported. -func (d *Swarm) UpdateConfig(c *check.C, id string, f ...ConfigConstructor) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - config := d.GetConfig(c, id) - for _, fn := range f { - fn(config) - } - - err = cli.ConfigUpdate(context.Background(), config.ID, config.Version, config.Spec) - c.Assert(err, checker.IsNil) -} - -// GetSwarm returns the current swarm object -func (d *Swarm) GetSwarm(c *check.C) swarm.Swarm { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - sw, err := cli.SwarmInspect(context.Background()) - c.Assert(err, checker.IsNil) - return sw -} - -// UpdateSwarm updates the current swarm object with the specified spec constructors -func (d *Swarm) UpdateSwarm(c *check.C, f ...SpecConstructor) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - sw := d.GetSwarm(c) - for _, fn := range f { - fn(&sw.Spec) - } - - err = cli.SwarmUpdate(context.Background(), sw.Version, sw.Spec, swarm.UpdateFlags{}) - c.Assert(err, checker.IsNil) -} - -// RotateTokens update the swarm to rotate tokens -func (d *Swarm) RotateTokens(c *check.C) { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - sw, err := cli.SwarmInspect(context.Background()) - c.Assert(err, checker.IsNil) - - flags := swarm.UpdateFlags{ - RotateManagerToken: true, - RotateWorkerToken: true, - } - - err = cli.SwarmUpdate(context.Background(), sw.Version, sw.Spec, flags) - c.Assert(err, checker.IsNil) -} - -// JoinTokens returns the current swarm join tokens -func (d *Swarm) JoinTokens(c *check.C) swarm.JoinTokens { - cli, err := d.NewClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - sw, err := cli.SwarmInspect(context.Background()) - c.Assert(err, checker.IsNil) - return sw.JoinTokens -} - // CheckLocalNodeState returns the current swarm node state -func (d *Swarm) CheckLocalNodeState(c *check.C) (interface{}, check.CommentInterface) { - info, err := d.SwarmInfo() - c.Assert(err, checker.IsNil) +func (d *Daemon) CheckLocalNodeState(c *check.C) (interface{}, check.CommentInterface) { + info := d.SwarmInfo(c) return info.LocalNodeState, nil } // CheckControlAvailable returns the current swarm control available -func (d *Swarm) CheckControlAvailable(c *check.C) (interface{}, check.CommentInterface) { - info, err := d.SwarmInfo() - c.Assert(err, checker.IsNil) +func (d *Daemon) CheckControlAvailable(c *check.C) (interface{}, check.CommentInterface) { + info := d.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) return info.ControlAvailable, nil } // CheckLeader returns whether there is a leader on the swarm or not -func (d *Swarm) CheckLeader(c *check.C) (interface{}, check.CommentInterface) { +func (d *Daemon) CheckLeader(c *check.C) (interface{}, check.CommentInterface) { cli, err := d.NewClient() c.Assert(err, checker.IsNil) defer cli.Close() @@ -644,18 +195,3 @@ func (d *Swarm) CheckLeader(c *check.C) (interface{}, check.CommentInterface) { } return fmt.Errorf("no leader"), check.Commentf("could not find leader") } - -// CmdRetryOutOfSequence tries the specified command against the current daemon for 10 times -func (d *Swarm) CmdRetryOutOfSequence(args ...string) (string, error) { - for i := 0; ; i++ { - out, err := d.Cmd(args...) - if err != nil { - if strings.Contains(out, "update out of sequence") { - if i < 10 { - continue - } - } - } - return out, err - } -} diff --git a/components/engine/integration-cli/daemon_swarm_hack_test.go b/components/engine/integration-cli/daemon_swarm_hack_test.go index e1fb333f82..7a23e84bfc 100644 --- a/components/engine/integration-cli/daemon_swarm_hack_test.go +++ b/components/engine/integration-cli/daemon_swarm_hack_test.go @@ -5,11 +5,11 @@ import ( "github.com/go-check/check" ) -func (s *DockerSwarmSuite) getDaemon(c *check.C, nodeID string) *daemon.Swarm { +func (s *DockerSwarmSuite) getDaemon(c *check.C, nodeID string) *daemon.Daemon { s.daemonsLock.Lock() defer s.daemonsLock.Unlock() for _, d := range s.daemons { - if d.NodeID == nodeID { + if d.NodeID() == nodeID { return d } } diff --git a/components/engine/integration-cli/docker_api_swarm_node_test.go b/components/engine/integration-cli/docker_api_swarm_node_test.go index 98f80552cf..6f939ea460 100644 --- a/components/engine/integration-cli/docker_api_swarm_node_test.go +++ b/components/engine/integration-cli/docker_api_swarm_node_test.go @@ -21,8 +21,8 @@ func (s *DockerSwarmSuite) TestAPISwarmListNodes(c *check.C) { loop0: for _, n := range nodes { - for _, d := range []*daemon.Swarm{d1, d2, d3} { - if n.ID == d.NodeID { + for _, d := range []*daemon.Daemon{d1, d2, d3} { + if n.ID == d.NodeID() { continue loop0 } } @@ -53,8 +53,7 @@ func (s *DockerSwarmSuite) TestAPISwarmNodeRemove(c *check.C) { c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes)) // Getting the info so we can take the NodeID - d2Info, err := d2.SwarmInfo() - c.Assert(err, checker.IsNil) + d2Info := d2.SwarmInfo(c) // forceful removal of d2 should work d1.RemoveNode(c, d2Info.NodeID, true) @@ -88,14 +87,14 @@ func (s *DockerSwarmSuite) TestAPISwarmNodeDrainPause(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount), checker.Equals, instances) // drain d2, all containers should move to d1 - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Availability = swarm.NodeAvailabilityDrain }) waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances) waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 0) // set d2 back to active - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Availability = swarm.NodeAvailabilityActive }) @@ -115,7 +114,7 @@ func (s *DockerSwarmSuite) TestAPISwarmNodeDrainPause(c *check.C) { d2ContainerCount := len(d2.ActiveContainers()) // set d2 to paused, scale service up, only d1 gets new tasks - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Availability = swarm.NodeAvailabilityPause }) diff --git a/components/engine/integration-cli/docker_api_swarm_service_test.go b/components/engine/integration-cli/docker_api_swarm_service_test.go index 845453d757..f715265f63 100644 --- a/components/engine/integration-cli/docker_api_swarm_service_test.go +++ b/components/engine/integration-cli/docker_api_swarm_service_test.go @@ -15,12 +15,13 @@ import ( "github.com/docker/docker/integration-cli/checker" "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration-cli/fixtures/plugin" + testdaemon "github.com/docker/docker/internal/test/daemon" "github.com/go-check/check" "golang.org/x/net/context" "golang.org/x/sys/unix" ) -func setPortConfig(portConfig []swarm.PortConfig) daemon.ServiceConstructor { +func setPortConfig(portConfig []swarm.PortConfig) testdaemon.ServiceConstructor { return func(s *swarm.Service) { if s.Spec.EndpointSpec == nil { s.Spec.EndpointSpec = &swarm.EndpointSpec{} @@ -140,7 +141,7 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesCreateGlobal(c *check.C) { func (s *DockerSwarmSuite) TestAPISwarmServicesUpdate(c *check.C) { const nodeCount = 3 - var daemons [nodeCount]*daemon.Swarm + var daemons [nodeCount]*daemon.Daemon for i := 0; i < nodeCount; i++ { daemons[i] = s.AddDaemon(c, true, i == 0) } @@ -309,7 +310,7 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateStartFirst(c *check.C) { func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *check.C) { const nodeCount = 3 - var daemons [nodeCount]*daemon.Swarm + var daemons [nodeCount]*daemon.Daemon for i := 0; i < nodeCount; i++ { daemons[i] = s.AddDaemon(c, true, i == 0) } @@ -349,7 +350,7 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *check.C) { func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *check.C) { const nodeCount = 3 - var daemons [nodeCount]*daemon.Swarm + var daemons [nodeCount]*daemon.Daemon for i := 0; i < nodeCount; i++ { daemons[i] = s.AddDaemon(c, true, i == 0) } @@ -401,7 +402,7 @@ func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *check.C) { func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *check.C) { const nodeCount = 3 - var daemons [nodeCount]*daemon.Swarm + var daemons [nodeCount]*daemon.Daemon for i := 0; i < nodeCount; i++ { daemons[i] = s.AddDaemon(c, true, i == 0) } @@ -496,7 +497,7 @@ func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *check.C) { func (s *DockerSwarmSuite) TestAPISwarmServicePlacementPrefs(c *check.C) { const nodeCount = 3 - var daemons [nodeCount]*daemon.Swarm + var daemons [nodeCount]*daemon.Daemon for i := 0; i < nodeCount; i++ { daemons[i] = s.AddDaemon(c, true, i == 0) } @@ -551,9 +552,9 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) - getContainers := func() map[string]*daemon.Swarm { - m := make(map[string]*daemon.Swarm) - for _, d := range []*daemon.Swarm{d1, d2, d3} { + getContainers := func() map[string]*daemon.Daemon { + m := make(map[string]*daemon.Daemon) + for _, d := range []*daemon.Daemon{d1, d2, d3} { for _, id := range d.ActiveContainers() { m[id] = d } diff --git a/components/engine/integration-cli/docker_api_swarm_test.go b/components/engine/integration-cli/docker_api_swarm_test.go index 2ba69acdb8..69fcdd3e82 100644 --- a/components/engine/integration-cli/docker_api_swarm_test.go +++ b/components/engine/integration-cli/docker_api_swarm_test.go @@ -23,6 +23,7 @@ import ( "github.com/docker/docker/integration-cli/checker" "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration-cli/request" + testdaemon "github.com/docker/docker/internal/test/daemon" "github.com/docker/swarmkit/ca" "github.com/go-check/check" "github.com/gotestyourself/gotestyourself/assert" @@ -35,30 +36,30 @@ var defaultReconciliationTimeout = 30 * time.Second func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) { // todo: should find a better way to verify that components are running than /info d1 := s.AddDaemon(c, true, true) - info, err := d1.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d1.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.True) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) c.Assert(info.Cluster.RootRotationInProgress, checker.False) d2 := s.AddDaemon(c, true, false) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.False) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) // Leaving cluster - c.Assert(d2.Leave(false), checker.IsNil) + c.Assert(d2.SwarmLeave(false), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.False) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) - c.Assert(d2.Join(swarm.JoinRequest{JoinToken: d1.JoinTokens(c).Worker, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil) + d2.SwarmJoin(c, swarm.JoinRequest{ + ListenAddr: d1.SwarmListenAddr(), + JoinToken: d1.JoinTokens(c).Worker, + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.False) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) @@ -69,93 +70,100 @@ func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) { d1.Start(c) d2.Start(c) - info, err = d1.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d1.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.True) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.False) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) } func (s *DockerSwarmSuite) TestAPISwarmJoinToken(c *check.C) { d1 := s.AddDaemon(c, false, false) - c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) + d1.SwarmInit(c, swarm.InitRequest{}) // todo: error message differs depending if some components of token are valid d2 := s.AddDaemon(c, false, false) - err := d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.ListenAddr}}) + c2 := d2.NewClientT(c) + err := c2.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "join token is necessary") - info, err := d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) - err = d2.Join(swarm.JoinRequest{JoinToken: "foobaz", RemoteAddrs: []string{d1.ListenAddr}}) + err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), + JoinToken: "foobaz", + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "invalid join token") - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) workerToken := d1.JoinTokens(c).Worker - c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + d2.SwarmJoin(c, swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), + JoinToken: workerToken, + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) - c.Assert(d2.Leave(false), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + c.Assert(d2.SwarmLeave(false), checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) // change tokens d1.RotateTokens(c) - err = d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}) + err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), + JoinToken: workerToken, + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "join token is necessary") - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) workerToken = d1.JoinTokens(c).Worker - c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + d2.SwarmJoin(c, swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.SwarmListenAddr()}}) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) - c.Assert(d2.Leave(false), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + c.Assert(d2.SwarmLeave(false), checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) // change spec, don't change tokens d1.UpdateSwarm(c, func(s *swarm.Spec) {}) - err = d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.ListenAddr}}) + err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "join token is necessary") - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) - c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.ListenAddr}}), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + d2.SwarmJoin(c, swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.SwarmListenAddr()}}) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) - c.Assert(d2.Leave(false), checker.IsNil) - info, err = d2.SwarmInfo() - c.Assert(err, checker.IsNil) + c.Assert(d2.SwarmLeave(false), checker.IsNil) + info = d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) } func (s *DockerSwarmSuite) TestUpdateSwarmAddExternalCA(c *check.C) { d1 := s.AddDaemon(c, false, false) - c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) + d1.SwarmInit(c, swarm.InitRequest{}) d1.UpdateSwarm(c, func(s *swarm.Spec) { s.CAConfig.ExternalCAs = []*swarm.ExternalCA{ { @@ -169,8 +177,7 @@ func (s *DockerSwarmSuite) TestUpdateSwarmAddExternalCA(c *check.C) { }, } }) - info, err := d1.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d1.SwarmInfo(c) c.Assert(info.Cluster.Spec.CAConfig.ExternalCAs, checker.HasLen, 2) c.Assert(info.Cluster.Spec.CAConfig.ExternalCAs[0].CACert, checker.Equals, "") c.Assert(info.Cluster.Spec.CAConfig.ExternalCAs[1].CACert, checker.Equals, "cacert") @@ -182,28 +189,32 @@ func (s *DockerSwarmSuite) TestAPISwarmCAHash(c *check.C) { splitToken := strings.Split(d1.JoinTokens(c).Worker, "-") splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e" replacementToken := strings.Join(splitToken, "-") - err := d2.Join(swarm.JoinRequest{JoinToken: replacementToken, RemoteAddrs: []string{d1.ListenAddr}}) + c2 := d2.NewClientT(c) + err := c2.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), + JoinToken: replacementToken, + RemoteAddrs: []string{d1.SwarmListenAddr()}, + }) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "remote CA does not match fingerprint") } func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) { d1 := s.AddDaemon(c, false, false) - c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) + d1.SwarmInit(c, swarm.InitRequest{}) d2 := s.AddDaemon(c, true, false) - info, err := d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d2.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.False) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Role = swarm.NodeRoleManager }) waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.True) - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Role = swarm.NodeRoleWorker }) @@ -228,7 +239,7 @@ func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) { }, checker.Equals, "swarm-worker") // Demoting last node should fail - node := d1.GetNode(c, d1.NodeID) + node := d1.GetNode(c, d1.NodeID()) node.Spec.Role = swarm.NodeRoleWorker url := fmt.Sprintf("/nodes/%s/update?version=%d", node.ID, node.Version.Index) res, body, err := request.DoOnHost(d1.Sock(), url, request.Method("POST"), request.JSONBody(node.Spec)) @@ -246,13 +257,12 @@ func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) { if !strings.Contains(string(b), "last manager of the swarm") { c.Assert(string(b), checker.Contains, "this would result in a loss of quorum") } - info, err = d1.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d1.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) c.Assert(info.ControlAvailable, checker.True) // Promote already demoted node - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Role = swarm.NodeRoleManager }) @@ -278,7 +288,7 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaderProxy(c *check.C) { // 3 services should be started now, because the requests were proxied to leader // query each node and make sure it returns 3 services - for _, d := range []*daemon.Swarm{d1, d2, d3} { + for _, d := range []*daemon.Daemon{d1, d2, d3} { services := d.ListServices(c) c.Assert(services, checker.HasLen, 3) } @@ -291,23 +301,23 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) { d3 := s.AddDaemon(c, true, true) // assert that the first node we made is the leader, and the other two are followers - c.Assert(d1.GetNode(c, d1.NodeID).ManagerStatus.Leader, checker.True) - c.Assert(d1.GetNode(c, d2.NodeID).ManagerStatus.Leader, checker.False) - c.Assert(d1.GetNode(c, d3.NodeID).ManagerStatus.Leader, checker.False) + c.Assert(d1.GetNode(c, d1.NodeID()).ManagerStatus.Leader, checker.True) + c.Assert(d1.GetNode(c, d2.NodeID()).ManagerStatus.Leader, checker.False) + c.Assert(d1.GetNode(c, d3.NodeID()).ManagerStatus.Leader, checker.False) d1.Stop(c) var ( - leader *daemon.Swarm // keep track of leader - followers []*daemon.Swarm // keep track of followers + leader *daemon.Daemon // keep track of leader + followers []*daemon.Daemon // keep track of followers ) - checkLeader := func(nodes ...*daemon.Swarm) checkF { + checkLeader := func(nodes ...*daemon.Daemon) checkF { return func(c *check.C) (interface{}, check.CommentInterface) { // clear these out before each run leader = nil followers = nil for _, d := range nodes { - if d.GetNode(c, d.NodeID).ManagerStatus.Leader { + if d.GetNode(c, d.NodeID()).ManagerStatus.Leader { leader = d } else { followers = append(followers, d) @@ -344,7 +354,7 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) { c.Assert(leader, checker.NotNil) c.Assert(followers, checker.HasLen, 2) // and that after we added d1 back, the leader hasn't changed - c.Assert(leader.NodeID, checker.Equals, stableleader.NodeID) + c.Assert(leader.NodeID(), checker.Equals, stableleader.NodeID()) } func (s *DockerSwarmSuite) TestAPISwarmRaftQuorum(c *check.C) { @@ -400,8 +410,8 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaveRemovesContainer(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances+1) - c.Assert(d.Leave(false), checker.NotNil) - c.Assert(d.Leave(true), checker.IsNil) + c.Assert(d.SwarmLeave(false), checker.NotNil) + c.Assert(d.SwarmLeave(true), checker.IsNil) waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) @@ -420,17 +430,18 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaveOnPendingJoin(c *check.C) { c.Assert(err, checker.IsNil) id = strings.TrimSpace(id) - err = d2.Join(swarm.JoinRequest{ + c2 := d2.NewClientT(c) + err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d2.SwarmListenAddr(), RemoteAddrs: []string{"123.123.123.123:1234"}, }) c.Assert(err, check.NotNil) c.Assert(err.Error(), checker.Contains, "Timeout was reached") - info, err := d2.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d2.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStatePending) - c.Assert(d2.Leave(true), checker.IsNil) + c.Assert(d2.SwarmLeave(true), checker.IsNil) waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 1) @@ -443,7 +454,9 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaveOnPendingJoin(c *check.C) { func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) { testRequires(c, Network) d := s.AddDaemon(c, false, false) - err := d.Join(swarm.JoinRequest{ + client := d.NewClientT(c) + err := client.SwarmJoin(context.Background(), swarm.JoinRequest{ + ListenAddr: d.SwarmListenAddr(), RemoteAddrs: []string{"123.123.123.123:1234"}, }) c.Assert(err, check.NotNil) @@ -454,8 +467,7 @@ func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) { d.Stop(c) d.Start(c) - info, err := d.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) } @@ -539,7 +551,7 @@ func (s *DockerSwarmSuite) TestAPISwarmForceNewCluster(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount), checker.Equals, instances) // drain d2, all containers should move to d1 - d1.UpdateNode(c, d2.NodeID, func(n *swarm.Node) { + d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { n.Spec.Availability = swarm.NodeAvailabilityDrain }) waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances) @@ -547,16 +559,15 @@ func (s *DockerSwarmSuite) TestAPISwarmForceNewCluster(c *check.C) { d2.Stop(c) - c.Assert(d1.Init(swarm.InitRequest{ + d1.SwarmInit(c, swarm.InitRequest{ ForceNewCluster: true, Spec: swarm.Spec{}, - }), checker.IsNil) + }) waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances) d3 := s.AddDaemon(c, true, true) - info, err := d3.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d3.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.True) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) @@ -622,7 +633,7 @@ func serviceForUpdate(s *swarm.Service) { s.Spec.Name = "updatetest" } -func setInstances(replicas int) daemon.ServiceConstructor { +func setInstances(replicas int) testdaemon.ServiceConstructor { ureplicas := uint64(replicas) return func(s *swarm.Service) { s.Spec.Mode = swarm.ServiceMode{ @@ -633,7 +644,7 @@ func setInstances(replicas int) daemon.ServiceConstructor { } } -func setUpdateOrder(order string) daemon.ServiceConstructor { +func setUpdateOrder(order string) testdaemon.ServiceConstructor { return func(s *swarm.Service) { if s.Spec.UpdateConfig == nil { s.Spec.UpdateConfig = &swarm.UpdateConfig{} @@ -642,7 +653,7 @@ func setUpdateOrder(order string) daemon.ServiceConstructor { } } -func setRollbackOrder(order string) daemon.ServiceConstructor { +func setRollbackOrder(order string) testdaemon.ServiceConstructor { return func(s *swarm.Service) { if s.Spec.RollbackConfig == nil { s.Spec.RollbackConfig = &swarm.UpdateConfig{} @@ -651,7 +662,7 @@ func setRollbackOrder(order string) daemon.ServiceConstructor { } } -func setImage(image string) daemon.ServiceConstructor { +func setImage(image string) testdaemon.ServiceConstructor { return func(s *swarm.Service) { if s.Spec.TaskTemplate.ContainerSpec == nil { s.Spec.TaskTemplate.ContainerSpec = &swarm.ContainerSpec{} @@ -660,25 +671,25 @@ func setImage(image string) daemon.ServiceConstructor { } } -func setFailureAction(failureAction string) daemon.ServiceConstructor { +func setFailureAction(failureAction string) testdaemon.ServiceConstructor { return func(s *swarm.Service) { s.Spec.UpdateConfig.FailureAction = failureAction } } -func setMaxFailureRatio(maxFailureRatio float32) daemon.ServiceConstructor { +func setMaxFailureRatio(maxFailureRatio float32) testdaemon.ServiceConstructor { return func(s *swarm.Service) { s.Spec.UpdateConfig.MaxFailureRatio = maxFailureRatio } } -func setParallelism(parallelism uint64) daemon.ServiceConstructor { +func setParallelism(parallelism uint64) testdaemon.ServiceConstructor { return func(s *swarm.Service) { s.Spec.UpdateConfig.Parallelism = parallelism } } -func setConstraints(constraints []string) daemon.ServiceConstructor { +func setConstraints(constraints []string) testdaemon.ServiceConstructor { return func(s *swarm.Service) { if s.Spec.TaskTemplate.Placement == nil { s.Spec.TaskTemplate.Placement = &swarm.Placement{} @@ -687,7 +698,7 @@ func setConstraints(constraints []string) daemon.ServiceConstructor { } } -func setPlacementPrefs(prefs []swarm.PlacementPreference) daemon.ServiceConstructor { +func setPlacementPrefs(prefs []swarm.PlacementPreference) testdaemon.ServiceConstructor { return func(s *swarm.Service) { if s.Spec.TaskTemplate.Placement == nil { s.Spec.TaskTemplate.Placement = &swarm.Placement{} @@ -702,18 +713,19 @@ func setGlobalMode(s *swarm.Service) { } } -func checkClusterHealth(c *check.C, cl []*daemon.Swarm, managerCount, workerCount int) { +func checkClusterHealth(c *check.C, cl []*daemon.Daemon, managerCount, workerCount int) { var totalMCount, totalWCount int for _, d := range cl { var ( info swarm.Info - err error ) // check info in a waitAndAssert, because if the cluster doesn't have a leader, `info` will return an error checkInfo := func(c *check.C) (interface{}, check.CommentInterface) { - info, err = d.SwarmInfo() + client := d.NewClientT(c) + daemonInfo, err := client.Info(context.Background()) + info = daemonInfo.Swarm return err, check.Commentf("cluster not ready in time") } waitAndAssert(c, defaultReconciliationTimeout, checkInfo, checker.IsNil) @@ -733,7 +745,7 @@ func checkClusterHealth(c *check.C, cl []*daemon.Swarm, managerCount, workerCoun } nn := d.GetNode(c, n.ID) n = *nn - return n.Status.State == swarm.NodeStateReady, check.Commentf("state of node %s, reported by %s", n.ID, d.Info.NodeID) + return n.Status.State == swarm.NodeStateReady, check.Commentf("state of node %s, reported by %s", n.ID, d.NodeID()) } waitAndAssert(c, defaultReconciliationTimeout, waitReady, checker.True) @@ -743,18 +755,18 @@ func checkClusterHealth(c *check.C, cl []*daemon.Swarm, managerCount, workerCoun } nn := d.GetNode(c, n.ID) n = *nn - return n.Spec.Availability == swarm.NodeAvailabilityActive, check.Commentf("availability of node %s, reported by %s", n.ID, d.Info.NodeID) + return n.Spec.Availability == swarm.NodeAvailabilityActive, check.Commentf("availability of node %s, reported by %s", n.ID, d.NodeID()) } waitAndAssert(c, defaultReconciliationTimeout, waitActive, checker.True) if n.Spec.Role == swarm.NodeRoleManager { - c.Assert(n.ManagerStatus, checker.NotNil, check.Commentf("manager status of node %s (manager), reported by %s", n.ID, d.Info.NodeID)) + c.Assert(n.ManagerStatus, checker.NotNil, check.Commentf("manager status of node %s (manager), reported by %s", n.ID, d.NodeID())) if n.ManagerStatus.Leader { leaderFound = true } mCount++ } else { - c.Assert(n.ManagerStatus, checker.IsNil, check.Commentf("manager status of node %s (worker), reported by %s", n.ID, d.Info.NodeID)) + c.Assert(n.ManagerStatus, checker.IsNil, check.Commentf("manager status of node %s (worker), reported by %s", n.ID, d.NodeID())) wCount++ } } @@ -769,11 +781,10 @@ func checkClusterHealth(c *check.C, cl []*daemon.Swarm, managerCount, workerCoun func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) { mCount, wCount := 5, 1 - var nodes []*daemon.Swarm + var nodes []*daemon.Daemon for i := 0; i < mCount; i++ { manager := s.AddDaemon(c, true, true) - info, err := manager.SwarmInfo() - c.Assert(err, checker.IsNil) + info := manager.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.True) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) nodes = append(nodes, manager) @@ -781,8 +792,7 @@ func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) { for i := 0; i < wCount; i++ { worker := s.AddDaemon(c, true, false) - info, err := worker.SwarmInfo() - c.Assert(err, checker.IsNil) + info := worker.SwarmInfo(c) c.Assert(info.ControlAvailable, checker.False) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) nodes = append(nodes, worker) @@ -795,7 +805,7 @@ func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) { errs := make(chan error, len(nodes)) for _, d := range nodes { - go func(daemon *daemon.Swarm) { + go func(daemon *daemon.Daemon) { defer wg.Done() if err := daemon.StopWithError(); err != nil { errs <- err @@ -820,7 +830,7 @@ func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) { errs := make(chan error, len(nodes)) for _, d := range nodes { - go func(daemon *daemon.Swarm) { + go func(daemon *daemon.Daemon) { defer wg.Done() if err := daemon.StartWithError("--iptables=false"); err != nil { errs <- err @@ -859,7 +869,7 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateWithName(c *check.C) { // Unlocking an unlocked swarm results in an error func (s *DockerSwarmSuite) TestAPISwarmUnlockNotLocked(c *check.C) { d := s.AddDaemon(c, true, true) - err := d.Unlock(swarm.UnlockRequest{UnlockKey: "wrong-key"}) + err := d.SwarmUnlock(swarm.UnlockRequest{UnlockKey: "wrong-key"}) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "swarm is not locked") } @@ -870,7 +880,10 @@ func (s *DockerSwarmSuite) TestAPISwarmErrorHandling(c *check.C) { c.Assert(err, checker.IsNil) defer ln.Close() d := s.AddDaemon(c, false, false) - err = d.Init(swarm.InitRequest{}) + client := d.NewClientT(c) + _, err = client.SwarmInit(context.Background(), swarm.InitRequest{ + ListenAddr: d.SwarmListenAddr(), + }) c.Assert(err, checker.NotNil) c.Assert(err.Error(), checker.Contains, "address already in use") } @@ -940,13 +953,13 @@ func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) { m := s.AddDaemon(c, true, true) w := s.AddDaemon(c, true, false) - info, err := m.SwarmInfo() - c.Assert(err, checker.IsNil) + info := m.SwarmInfo(c) currentTrustRoot := info.Cluster.TLSInfo.TrustRoot // rotate multiple times for i := 0; i < 4; i++ { + var err error var cert, key []byte if i%2 != 0 { cert, _, key, err = initca.New(&csr.CertificateRequest{ @@ -966,8 +979,7 @@ func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) { // poll to make sure update succeeds var clusterTLSInfo swarm.TLSInfo for j := 0; j < 18; j++ { - info, err := m.SwarmInfo() - c.Assert(err, checker.IsNil) + info := m.SwarmInfo(c) // the desired CA cert and key is always redacted c.Assert(info.Cluster.Spec.CAConfig.SigningCAKey, checker.Equals, "") @@ -989,8 +1001,8 @@ func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) { // could take another second or two for the nodes to trust the new roots after they've all gotten // new TLS certificates for j := 0; j < 18; j++ { - mInfo := m.GetNode(c, m.NodeID).Description.TLSInfo - wInfo := m.GetNode(c, w.NodeID).Description.TLSInfo + mInfo := m.GetNode(c, m.NodeID()).Description.TLSInfo + wInfo := m.GetNode(c, w.NodeID()).Description.TLSInfo if mInfo.TrustRoot == clusterTLSInfo.TrustRoot && wInfo.TrustRoot == clusterTLSInfo.TrustRoot { break @@ -1000,8 +1012,8 @@ func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) { time.Sleep(250 * time.Millisecond) } - c.Assert(m.GetNode(c, m.NodeID).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo) - c.Assert(m.GetNode(c, w.NodeID).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo) + c.Assert(m.GetNode(c, m.NodeID()).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo) + c.Assert(m.GetNode(c, w.NodeID()).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo) currentTrustRoot = clusterTLSInfo.TrustRoot } } diff --git a/components/engine/integration-cli/docker_cli_prune_unix_test.go b/components/engine/integration-cli/docker_cli_prune_unix_test.go index 763a72de16..bc5bdc835e 100644 --- a/components/engine/integration-cli/docker_cli_prune_unix_test.go +++ b/components/engine/integration-cli/docker_cli_prune_unix_test.go @@ -16,7 +16,7 @@ import ( "github.com/go-check/check" ) -func pruneNetworkAndVerify(c *check.C, d *daemon.Swarm, kept, pruned []string) { +func pruneNetworkAndVerify(c *check.C, d *daemon.Daemon, kept, pruned []string) { _, err := d.Cmd("network", "prune", "--force") c.Assert(err, checker.IsNil) diff --git a/components/engine/integration-cli/docker_cli_service_logs_test.go b/components/engine/integration-cli/docker_cli_service_logs_test.go index b3f58271f1..c26a7455a5 100644 --- a/components/engine/integration-cli/docker_cli_service_logs_test.go +++ b/components/engine/integration-cli/docker_cli_service_logs_test.go @@ -53,7 +53,7 @@ func (s *DockerSwarmSuite) TestServiceLogs(c *check.C) { // countLogLines returns a closure that can be used with waitAndAssert to // verify that a minimum number of expected container log messages have been // output. -func countLogLines(d *daemon.Swarm, name string) func(*check.C) (interface{}, check.CommentInterface) { +func countLogLines(d *daemon.Daemon, name string) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { result := icmd.RunCmd(d.Command("service", "logs", "-t", "--raw", name)) result.Assert(c, icmd.Expected{}) diff --git a/components/engine/integration-cli/docker_cli_service_update_test.go b/components/engine/integration-cli/docker_cli_service_update_test.go index 7b590a0f4e..a281327afe 100644 --- a/components/engine/integration-cli/docker_cli_service_update_test.go +++ b/components/engine/integration-cli/docker_cli_service_update_test.go @@ -69,7 +69,7 @@ func (s *DockerSwarmSuite) TestServiceUpdateSecrets(c *check.C) { c.Assert(err, checker.IsNil, check.Commentf(out)) // add secret - out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "test", "--secret-add", fmt.Sprintf("source=%s,target=%s", testName, testTarget)) + out, err = d.Cmd("service", "update", "--detach", "test", "--secret-add", fmt.Sprintf("source=%s,target=%s", testName, testTarget)) c.Assert(err, checker.IsNil, check.Commentf(out)) out, err = d.Cmd("service", "inspect", "--format", "{{ json .Spec.TaskTemplate.ContainerSpec.Secrets }}", serviceName) @@ -84,7 +84,7 @@ func (s *DockerSwarmSuite) TestServiceUpdateSecrets(c *check.C) { c.Assert(refs[0].File.Name, checker.Equals, testTarget) // remove - out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "test", "--secret-rm", testName) + out, err = d.Cmd("service", "update", "--detach", "test", "--secret-rm", testName) c.Assert(err, checker.IsNil, check.Commentf(out)) out, err = d.Cmd("service", "inspect", "--format", "{{ json .Spec.TaskTemplate.ContainerSpec.Secrets }}", serviceName) @@ -111,7 +111,7 @@ func (s *DockerSwarmSuite) TestServiceUpdateConfigs(c *check.C) { c.Assert(err, checker.IsNil, check.Commentf(out)) // add config - out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "test", "--config-add", fmt.Sprintf("source=%s,target=%s", testName, testTarget)) + out, err = d.Cmd("service", "update", "--detach", "test", "--config-add", fmt.Sprintf("source=%s,target=%s", testName, testTarget)) c.Assert(err, checker.IsNil, check.Commentf(out)) out, err = d.Cmd("service", "inspect", "--format", "{{ json .Spec.TaskTemplate.ContainerSpec.Configs }}", serviceName) @@ -126,7 +126,7 @@ func (s *DockerSwarmSuite) TestServiceUpdateConfigs(c *check.C) { c.Assert(refs[0].File.Name, checker.Equals, testTarget) // remove - out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "test", "--config-rm", testName) + out, err = d.Cmd("service", "update", "--detach", "test", "--config-rm", testName) c.Assert(err, checker.IsNil, check.Commentf(out)) out, err = d.Cmd("service", "inspect", "--format", "{{ json .Spec.TaskTemplate.ContainerSpec.Configs }}", serviceName) diff --git a/components/engine/integration-cli/docker_cli_swarm_test.go b/components/engine/integration-cli/docker_cli_swarm_test.go index 4b3358255f..0d11555574 100644 --- a/components/engine/integration-cli/docker_cli_swarm_test.go +++ b/components/engine/integration-cli/docker_cli_swarm_test.go @@ -57,7 +57,7 @@ func (s *DockerSwarmSuite) TestSwarmUpdate(c *check.C) { // passing an external CA (this is without starting a root rotation) does not fail cli.Docker(cli.Args("swarm", "update", "--external-ca", "protocol=cfssl,url=https://something.org", "--external-ca", "protocol=cfssl,url=https://somethingelse.org,cacert=fixtures/https/ca.pem"), - cli.Daemon(d.Daemon)).Assert(c, icmd.Success) + cli.Daemon(d)).Assert(c, icmd.Success) expected, err := ioutil.ReadFile("fixtures/https/ca.pem") c.Assert(err, checker.IsNil) @@ -73,7 +73,7 @@ func (s *DockerSwarmSuite) TestSwarmUpdate(c *check.C) { result := cli.Docker(cli.Args("swarm", "update", "--external-ca", fmt.Sprintf("protocol=cfssl,url=https://something.org,cacert=%s", tempFile.Path())), - cli.Daemon(d.Daemon)) + cli.Daemon(d)) result.Assert(c, icmd.Expected{ ExitCode: 125, Err: "must be in PEM format", @@ -94,7 +94,7 @@ func (s *DockerSwarmSuite) TestSwarmInit(c *check.C) { result := cli.Docker(cli.Args("swarm", "init", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s", "--external-ca", fmt.Sprintf("protocol=cfssl,url=https://somethingelse.org,cacert=%s", tempFile.Path())), - cli.Daemon(d.Daemon)) + cli.Daemon(d)) result.Assert(c, icmd.Expected{ ExitCode: 125, Err: "must be in PEM format", @@ -103,7 +103,7 @@ func (s *DockerSwarmSuite) TestSwarmInit(c *check.C) { cli.Docker(cli.Args("swarm", "init", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s", "--external-ca", "protocol=cfssl,url=https://something.org", "--external-ca", "protocol=cfssl,url=https://somethingelse.org,cacert=fixtures/https/ca.pem"), - cli.Daemon(d.Daemon)).Assert(c, icmd.Success) + cli.Daemon(d)).Assert(c, icmd.Success) expected, err := ioutil.ReadFile("fixtures/https/ca.pem") c.Assert(err, checker.IsNil) @@ -115,8 +115,8 @@ func (s *DockerSwarmSuite) TestSwarmInit(c *check.C) { c.Assert(spec.CAConfig.ExternalCAs[0].CACert, checker.Equals, "") c.Assert(spec.CAConfig.ExternalCAs[1].CACert, checker.Equals, string(expected)) - c.Assert(d.Leave(true), checker.IsNil) - cli.Docker(cli.Args("swarm", "init"), cli.Daemon(d.Daemon)).Assert(c, icmd.Success) + c.Assert(d.SwarmLeave(true), checker.IsNil) + cli.Docker(cli.Args("swarm", "init"), cli.Daemon(d)).Assert(c, icmd.Success) spec = getSpec() c.Assert(spec.CAConfig.NodeCertExpiry, checker.Equals, 90*24*time.Hour) @@ -126,12 +126,12 @@ func (s *DockerSwarmSuite) TestSwarmInit(c *check.C) { func (s *DockerSwarmSuite) TestSwarmInitIPv6(c *check.C) { testRequires(c, IPv6) d1 := s.AddDaemon(c, false, false) - cli.Docker(cli.Args("swarm", "init", "--listen-add", "::1"), cli.Daemon(d1.Daemon)).Assert(c, icmd.Success) + cli.Docker(cli.Args("swarm", "init", "--listen-add", "::1"), cli.Daemon(d1)).Assert(c, icmd.Success) d2 := s.AddDaemon(c, false, false) - cli.Docker(cli.Args("swarm", "join", "::1"), cli.Daemon(d2.Daemon)).Assert(c, icmd.Success) + cli.Docker(cli.Args("swarm", "join", "::1"), cli.Daemon(d2)).Assert(c, icmd.Success) - out := cli.Docker(cli.Args("info"), cli.Daemon(d2.Daemon)).Assert(c, icmd.Success).Combined() + out := cli.Docker(cli.Args("info"), cli.Daemon(d2)).Assert(c, icmd.Success).Combined() c.Assert(out, checker.Contains, "Swarm: active") } @@ -145,13 +145,12 @@ func (s *DockerSwarmSuite) TestSwarmInitUnspecifiedAdvertiseAddr(c *check.C) { func (s *DockerSwarmSuite) TestSwarmIncompatibleDaemon(c *check.C) { // init swarm mode and stop a daemon d := s.AddDaemon(c, true, true) - info, err := d.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) d.Stop(c) // start a daemon with --cluster-store and --cluster-advertise - err = d.StartWithError("--cluster-store=consul://consuladdr:consulport/some/path", "--cluster-advertise=1.1.1.1:2375") + err := d.StartWithError("--cluster-store=consul://consuladdr:consulport/some/path", "--cluster-advertise=1.1.1.1:2375") c.Assert(err, checker.NotNil) content, err := d.ReadLogFile() c.Assert(err, checker.IsNil) @@ -283,10 +282,10 @@ func (s *DockerSwarmSuite) TestSwarmPublishAdd(c *check.C) { out, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", name) c.Assert(err, checker.IsNil) - out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", name) + out, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", name) c.Assert(err, checker.IsNil) - out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", "--publish-add", "80:20", name) + out, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", "--publish-add", "80:20", name) c.Assert(err, checker.NotNil) out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.EndpointSpec.Ports }}", name) @@ -426,7 +425,7 @@ func (s *DockerSwarmSuite) TestOverlayAttachableOnSwarmLeave(c *check.C) { c.Assert(err, checker.IsNil, check.Commentf(out)) // Leave the swarm - err = d.Leave(true) + err = d.SwarmLeave(true) c.Assert(err, checker.IsNil) // Check the container is disconnected @@ -989,13 +988,12 @@ func (s *DockerSwarmSuite) TestDNSConfigUpdate(c *check.C) { c.Assert(strings.TrimSpace(out), checker.Equals, "{[1.2.3.4] [example.com] [timeout:3]}") } -func getNodeStatus(c *check.C, d *daemon.Swarm) swarm.LocalNodeState { - info, err := d.SwarmInfo() - c.Assert(err, checker.IsNil) +func getNodeStatus(c *check.C, d *daemon.Daemon) swarm.LocalNodeState { + info := d.SwarmInfo(c) return info.LocalNodeState } -func checkKeyIsEncrypted(d *daemon.Swarm) func(*check.C) (interface{}, check.CommentInterface) { +func checkKeyIsEncrypted(d *daemon.Daemon) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { keyBytes, err := ioutil.ReadFile(filepath.Join(d.Folder, "root", "swarm", "certificates", "swarm-node.key")) if err != nil { @@ -1011,7 +1009,7 @@ func checkKeyIsEncrypted(d *daemon.Swarm) func(*check.C) (interface{}, check.Com } } -func checkSwarmLockedToUnlocked(c *check.C, d *daemon.Swarm, unlockKey string) { +func checkSwarmLockedToUnlocked(c *check.C, d *daemon.Daemon, unlockKey string) { // Wait for the PEM file to become unencrypted waitAndAssert(c, defaultReconciliationTimeout, checkKeyIsEncrypted(d), checker.Equals, false) @@ -1019,7 +1017,7 @@ func checkSwarmLockedToUnlocked(c *check.C, d *daemon.Swarm, unlockKey string) { c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive) } -func checkSwarmUnlockedToLocked(c *check.C, d *daemon.Swarm) { +func checkSwarmUnlockedToLocked(c *check.C, d *daemon.Daemon) { // Wait for the PEM file to become encrypted waitAndAssert(c, defaultReconciliationTimeout, checkKeyIsEncrypted(d), checker.Equals, true) @@ -1117,8 +1115,7 @@ func (s *DockerSwarmSuite) TestSwarmLeaveLocked(c *check.C) { // It starts off locked d.Restart(c, "--swarm-default-advertise-addr=lo") - info, err := d.SwarmInfo() - c.Assert(err, checker.IsNil) + info := d.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateLocked) outs, _ = d.Cmd("node", "ls") @@ -1132,15 +1129,13 @@ func (s *DockerSwarmSuite) TestSwarmLeaveLocked(c *check.C) { outs, err = d.Cmd("swarm", "leave", "--force") c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) - info, err = d.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) outs, err = d.Cmd("swarm", "init") c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) - info, err = d.SwarmInfo() - c.Assert(err, checker.IsNil) + info = d.SwarmInfo(c) c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) } @@ -1176,7 +1171,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) { c.Assert(outs, checker.Equals, unlockKey+"\n") // The ones that got the cluster update should be set to locked - for _, d := range []*daemon.Swarm{d1, d3} { + for _, d := range []*daemon.Daemon{d1, d3} { checkSwarmUnlockedToLocked(c, d) cmd := d.Command("swarm", "unlock") @@ -1197,7 +1192,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) { c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) // the ones that got the update are now set to unlocked - for _, d := range []*daemon.Swarm{d1, d3} { + for _, d := range []*daemon.Daemon{d1, d3} { checkSwarmLockedToUnlocked(c, d, unlockKey) } @@ -1247,7 +1242,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) { c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive) // promote worker - outs, err = d1.Cmd("node", "promote", d2.Info.NodeID) + outs, err = d1.Cmd("node", "promote", d2.NodeID()) c.Assert(err, checker.IsNil) c.Assert(outs, checker.Contains, "promoted to a manager in the swarm") @@ -1255,7 +1250,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) { d3 := s.AddDaemon(c, true, true) // both new nodes are locked - for _, d := range []*daemon.Swarm{d2, d3} { + for _, d := range []*daemon.Daemon{d2, d3} { checkSwarmUnlockedToLocked(c, d) cmd := d.Command("swarm", "unlock") @@ -1265,7 +1260,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) { } // demote manager back to worker - workers are not locked - outs, err = d1.Cmd("node", "demote", d3.Info.NodeID) + outs, err = d1.Cmd("node", "demote", d3.NodeID()) c.Assert(err, checker.IsNil) c.Assert(outs, checker.Contains, "demoted in the swarm") @@ -1409,7 +1404,7 @@ func (s *DockerSwarmSuite) TestSwarmClusterRotateUnlockKey(c *check.C) { d2.Restart(c) d3.Restart(c) - for _, d := range []*daemon.Swarm{d2, d3} { + for _, d := range []*daemon.Daemon{d2, d3} { c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked) outs, _ := d.Cmd("node", "ls") @@ -1521,7 +1516,7 @@ func (s *DockerSwarmSuite) TestSwarmManagerAddress(c *check.C) { d3 := s.AddDaemon(c, true, false) // Manager Addresses will always show Node 1's address - expectedOutput := fmt.Sprintf("Manager Addresses:\n 127.0.0.1:%d\n", d1.Port) + expectedOutput := fmt.Sprintf("Manager Addresses:\n 127.0.0.1:%d\n", d1.SwarmPort) out, err := d1.Cmd("info") c.Assert(err, checker.IsNil) @@ -1641,7 +1636,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinWithDrain(c *check.C) { d1 := s.AddDaemon(c, false, false) - out, err = d1.Cmd("swarm", "join", "--availability=drain", "--token", token, d.ListenAddr) + out, err = d1.Cmd("swarm", "join", "--availability=drain", "--token", token, d.SwarmListenAddr()) c.Assert(err, checker.IsNil) c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") @@ -1835,7 +1830,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinLeave(c *check.C) { // Verify that back to back join/leave does not cause panics d1 := s.AddDaemon(c, false, false) for i := 0; i < 10; i++ { - out, err = d1.Cmd("swarm", "join", "--token", token, d.ListenAddr) + out, err = d1.Cmd("swarm", "join", "--token", token, d.SwarmListenAddr()) c.Assert(err, checker.IsNil) c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") @@ -1846,7 +1841,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinLeave(c *check.C) { const defaultRetryCount = 10 -func waitForEvent(c *check.C, d *daemon.Swarm, since string, filter string, event string, retry int) string { +func waitForEvent(c *check.C, d *daemon.Daemon, since string, filter string, event string, retry int) string { if retry < 1 { c.Fatalf("retry count %d is invalid. It should be no less than 1", retry) return "" @@ -1982,7 +1977,7 @@ func (s *DockerSwarmSuite) TestSwarmClusterEventsNode(c *check.C) { s.AddDaemon(c, true, true) d3 := s.AddDaemon(c, true, true) - d3ID := d3.NodeID + d3ID := d3.NodeID() waitForEvent(c, d1, "0", "-f scope=swarm", "node create "+d3ID, defaultRetryCount) t1 := daemonUnixTime(c) diff --git a/components/engine/integration/container/daemon_linux_test.go b/components/engine/integration/container/daemon_linux_test.go index c2920502e3..abcf9b0d06 100644 --- a/components/engine/integration/container/daemon_linux_test.go +++ b/components/engine/integration/container/daemon_linux_test.go @@ -9,8 +9,8 @@ import ( "testing" "github.com/docker/docker/api/types" - "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/internal/test/daemon" "github.com/gotestyourself/gotestyourself/assert" "github.com/gotestyourself/gotestyourself/skip" "golang.org/x/sys/unix" @@ -30,7 +30,7 @@ func TestContainerStartOnDaemonRestart(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon(), "cannot start daemon on remote test run") t.Parallel() - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t, "--iptables=false") defer d.Stop(t) diff --git a/components/engine/integration/container/export_test.go b/components/engine/integration/container/export_test.go index f7f0295ce5..272feff75d 100644 --- a/components/engine/integration/container/export_test.go +++ b/components/engine/integration/container/export_test.go @@ -9,9 +9,9 @@ import ( "github.com/docker/docker/api/types" containerTypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" + "github.com/docker/docker/internal/test/daemon" "github.com/docker/docker/pkg/jsonmessage" "github.com/gotestyourself/gotestyourself/assert" is "github.com/gotestyourself/gotestyourself/assert/cmp" @@ -62,7 +62,7 @@ func TestExportContainerAfterDaemonRestart(t *testing.T) { skip.If(t, testEnv.DaemonInfo.OSType != "linux") skip.If(t, testEnv.IsRemoteDaemon()) - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) client, err := d.NewClient() assert.NilError(t, err) diff --git a/components/engine/integration/container/mounts_linux_test.go b/components/engine/integration/container/mounts_linux_test.go index cdde06a9ef..38aa98355a 100644 --- a/components/engine/integration/container/mounts_linux_test.go +++ b/components/engine/integration/container/mounts_linux_test.go @@ -1,7 +1,6 @@ package container // import "github.com/docker/docker/integration/container" import ( - "bytes" "context" "fmt" "path/filepath" @@ -12,9 +11,7 @@ import ( "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" - "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration/internal/request" - "github.com/docker/docker/pkg/stdcopy" "github.com/docker/docker/pkg/system" "github.com/gotestyourself/gotestyourself/assert" is "github.com/gotestyourself/gotestyourself/assert/cmp" @@ -22,77 +19,6 @@ import ( "github.com/gotestyourself/gotestyourself/skip" ) -func TestContainerShmNoLeak(t *testing.T) { - skip.If(t, testEnv.IsRemoteDaemon(), "cannot start daemon on remote test run") - t.Parallel() - d := daemon.New(t, "docker", "dockerd", daemon.Config{}) - client, err := d.NewClient() - if err != nil { - t.Fatal(err) - } - d.StartWithBusybox(t, "--iptables=false") - defer d.Stop(t) - - ctx := context.Background() - cfg := container.Config{ - Image: "busybox", - Cmd: []string{"top"}, - } - - ctr, err := client.ContainerCreate(ctx, &cfg, nil, nil, "") - if err != nil { - t.Fatal(err) - } - defer client.ContainerRemove(ctx, ctr.ID, types.ContainerRemoveOptions{Force: true}) - - if err := client.ContainerStart(ctx, ctr.ID, types.ContainerStartOptions{}); err != nil { - t.Fatal(err) - } - - // this should recursively bind mount everything in the test daemons root - // except of course we are hoping that the previous containers /dev/shm mount did not leak into this new container - hc := container.HostConfig{ - Mounts: []mount.Mount{ - { - Type: mount.TypeBind, - Source: d.Root, - Target: "/testdaemonroot", - }, - }, - } - cfg.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("mount | grep testdaemonroot | grep containers | grep %s", ctr.ID)} - cfg.AttachStdout = true - cfg.AttachStderr = true - ctrLeak, err := client.ContainerCreate(ctx, &cfg, &hc, nil, "") - if err != nil { - t.Fatal(err) - } - - attach, err := client.ContainerAttach(ctx, ctrLeak.ID, types.ContainerAttachOptions{ - Stream: true, - Stdout: true, - Stderr: true, - }) - if err != nil { - t.Fatal(err) - } - - if err := client.ContainerStart(ctx, ctrLeak.ID, types.ContainerStartOptions{}); err != nil { - t.Fatal(err) - } - - buf := bytes.NewBuffer(nil) - - if _, err := stdcopy.StdCopy(buf, buf, attach.Reader); err != nil { - t.Fatal(err) - } - - out := bytes.TrimSpace(buf.Bytes()) - if !bytes.Equal(out, []byte{}) { - t.Fatalf("mount leaked: %s", string(out)) - } -} - func TestContainerNetworkMountsNoChown(t *testing.T) { // chown only applies to Linux bind mounted volumes; must be same host to verify skip.If(t, testEnv.DaemonInfo.OSType != "linux" || testEnv.IsRemoteDaemon()) diff --git a/components/engine/integration/container/restart_test.go b/components/engine/integration/container/restart_test.go index 7a2576e219..d6deea730e 100644 --- a/components/engine/integration/container/restart_test.go +++ b/components/engine/integration/container/restart_test.go @@ -8,7 +8,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/internal/test/daemon" "github.com/gotestyourself/gotestyourself/skip" ) @@ -55,7 +55,7 @@ func TestDaemonRestartKillContainers(t *testing.T) { t.Parallel() - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) client, err := d.NewClient() if err != nil { t.Fatal(err) diff --git a/components/engine/integration/internal/swarm/service.go b/components/engine/integration/internal/swarm/service.go index 28d367c660..031995f87a 100644 --- a/components/engine/integration/internal/swarm/service.go +++ b/components/engine/integration/internal/swarm/service.go @@ -2,7 +2,6 @@ package swarm import ( "context" - "fmt" "runtime" "testing" "time" @@ -11,18 +10,13 @@ import ( "github.com/docker/docker/api/types/filters" swarmtypes "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/client" - "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/internal/test/daemon" "github.com/docker/docker/internal/test/environment" "github.com/gotestyourself/gotestyourself/assert" "github.com/gotestyourself/gotestyourself/poll" "github.com/gotestyourself/gotestyourself/skip" ) -const ( - dockerdBinary = "dockerd" - defaultSwarmPort = 2477 -) - // ServicePoll tweaks the pollSettings for `service` func ServicePoll(config *poll.Settings) { // Override the default pollSettings for `service` resource here ... @@ -55,23 +49,17 @@ func ContainerPoll(config *poll.Settings) { } // NewSwarm creates a swarm daemon for testing -func NewSwarm(t *testing.T, testEnv *environment.Execution) *daemon.Swarm { +func NewSwarm(t *testing.T, testEnv *environment.Execution, ops ...func(*daemon.Daemon)) *daemon.Daemon { skip.IfCondition(t, testEnv.IsRemoteDaemon()) - d := &daemon.Swarm{ - Daemon: daemon.New(t, "", dockerdBinary, daemon.Config{ - Experimental: testEnv.DaemonInfo.ExperimentalBuild, - }), - // TODO: better method of finding an unused port - Port: defaultSwarmPort, + if testEnv.DaemonInfo.ExperimentalBuild { + ops = append(ops, daemon.WithExperimental) } - // TODO: move to a NewSwarm constructor - d.ListenAddr = fmt.Sprintf("0.0.0.0:%d", d.Port) - + d := daemon.New(t, ops...) // avoid networking conflicts args := []string{"--iptables=false", "--swarm-default-advertise-addr=lo"} d.StartWithBusybox(t, args...) - assert.NilError(t, d.Init(swarmtypes.InitRequest{})) + d.SwarmInit(t, swarmtypes.InitRequest{}) return d } @@ -79,7 +67,7 @@ func NewSwarm(t *testing.T, testEnv *environment.Execution) *daemon.Swarm { type ServiceSpecOpt func(*swarmtypes.ServiceSpec) // CreateService creates a service on the passed in swarm daemon. -func CreateService(t *testing.T, d *daemon.Swarm, opts ...ServiceSpecOpt) string { +func CreateService(t *testing.T, d *daemon.Daemon, opts ...ServiceSpecOpt) string { spec := defaultServiceSpec() for _, o := range opts { o(&spec) @@ -151,7 +139,7 @@ func ServiceWithName(name string) ServiceSpecOpt { } // GetRunningTasks gets the list of running tasks for a service -func GetRunningTasks(t *testing.T, d *daemon.Swarm, serviceID string) []swarmtypes.Task { +func GetRunningTasks(t *testing.T, d *daemon.Daemon, serviceID string) []swarmtypes.Task { client := GetClient(t, d) filterArgs := filters.NewArgs() @@ -167,7 +155,7 @@ func GetRunningTasks(t *testing.T, d *daemon.Swarm, serviceID string) []swarmtyp } // ExecTask runs the passed in exec config on the given task -func ExecTask(t *testing.T, d *daemon.Swarm, task swarmtypes.Task, config types.ExecConfig) types.HijackedResponse { +func ExecTask(t *testing.T, d *daemon.Daemon, task swarmtypes.Task, config types.ExecConfig) types.HijackedResponse { client := GetClient(t, d) ctx := context.Background() @@ -187,7 +175,7 @@ func ensureContainerSpec(spec *swarmtypes.ServiceSpec) { } // GetClient creates a new client for the passed in swarm daemon. -func GetClient(t *testing.T, d *daemon.Swarm) client.APIClient { +func GetClient(t *testing.T, d *daemon.Daemon) client.APIClient { client, err := client.NewClientWithOpts(client.WithHost((d.Sock()))) assert.NilError(t, err) return client diff --git a/components/engine/integration/network/inspect_test.go b/components/engine/integration/network/inspect_test.go index c27b3529d3..3f23c61ae2 100644 --- a/components/engine/integration/network/inspect_test.go +++ b/components/engine/integration/network/inspect_test.go @@ -15,7 +15,6 @@ import ( ) const defaultSwarmPort = 2477 -const dockerdBinary = "dockerd" func TestInspectNetwork(t *testing.T) { defer setupTest(t)() diff --git a/components/engine/integration/network/macvlan_test.go b/components/engine/integration/network/macvlan_test.go index 46945ada28..146f67ee57 100644 --- a/components/engine/integration/network/macvlan_test.go +++ b/components/engine/integration/network/macvlan_test.go @@ -9,8 +9,8 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" - "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/internal/test/daemon" "github.com/docker/docker/pkg/parsers/kernel" "github.com/gotestyourself/gotestyourself/assert" "github.com/gotestyourself/gotestyourself/assert/cmp" @@ -24,7 +24,7 @@ func TestDockerNetworkMacvlanPersistance(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon()) skip.If(t, !macvlanKernelSupport(), "Kernel doesn't support macvlan") - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t) defer d.Stop(t) @@ -53,7 +53,7 @@ func TestDockerNetworkMacvlanOverlapParent(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon()) skip.If(t, !macvlanKernelSupport(), "Kernel doesn't support macvlan") - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t) defer d.Stop(t) @@ -95,7 +95,7 @@ func TestDockerNetworkMacvlanSubinterface(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon()) skip.If(t, !macvlanKernelSupport(), "Kernel doesn't support macvlan") - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t) defer d.Stop(t) @@ -131,7 +131,7 @@ func TestDockerNetworkMacvlanBridgeNilParent(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon()) skip.If(t, !macvlanKernelSupport(), "Kernel doesn't support macvlan") - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t) defer d.Stop(t) client, err := d.NewClient() @@ -157,7 +157,7 @@ func TestDockerNetworkMacvlanBridgeInternal(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon()) skip.If(t, !macvlanKernelSupport(), "Kernel doesn't support macvlan") - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t) defer d.Stop(t) client, err := d.NewClient() @@ -191,7 +191,7 @@ func TestDockerNetworkMacvlanMultiSubnet(t *testing.T) { skip.If(t, !macvlanKernelSupport(), "Kernel doesn't support macvlan") t.Skip("Temporarily skipping while investigating sporadic v6 CI issues") - d := daemon.New(t, "", "dockerd", daemon.Config{}) + d := daemon.New(t) d.StartWithBusybox(t) defer d.Stop(t) client, err := d.NewClient() diff --git a/components/engine/integration/plugin/authz/main_test.go b/components/engine/integration/plugin/authz/main_test.go index 3d5f406b92..1958ce376a 100644 --- a/components/engine/integration/plugin/authz/main_test.go +++ b/components/engine/integration/plugin/authz/main_test.go @@ -12,7 +12,7 @@ import ( "strings" "testing" - "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/internal/test/daemon" "github.com/docker/docker/internal/test/environment" "github.com/docker/docker/pkg/authorization" "github.com/docker/docker/pkg/plugins" @@ -25,8 +25,6 @@ var ( server *httptest.Server ) -const dockerdBinary = "dockerd" - func TestMain(m *testing.M) { var err error testEnv, err = environment.New() @@ -52,9 +50,7 @@ func setupTest(t *testing.T) func() { skip.IfCondition(t, testEnv.IsRemoteDaemon(), "cannot run daemon when remote daemon") environment.ProtectAll(t, testEnv) - d = daemon.New(t, "", dockerdBinary, daemon.Config{ - Experimental: testEnv.DaemonInfo.ExperimentalBuild, - }) + d = daemon.New(t, daemon.WithExperimental) return func() { if d != nil { diff --git a/components/engine/integration/plugin/logging/main_test.go b/components/engine/integration/plugin/logging/main_test.go index b2446ca8c8..e1292a5718 100644 --- a/components/engine/integration/plugin/logging/main_test.go +++ b/components/engine/integration/plugin/logging/main_test.go @@ -12,8 +12,6 @@ var ( testEnv *environment.Execution ) -const dockerdBinary = "dockerd" - func TestMain(m *testing.M) { var err error testEnv, err = environment.New() diff --git a/components/engine/integration/plugin/logging/validation_test.go b/components/engine/integration/plugin/logging/validation_test.go index f0d6e9e8b2..7f67aea75c 100644 --- a/components/engine/integration/plugin/logging/validation_test.go +++ b/components/engine/integration/plugin/logging/validation_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/docker/docker/api/types" - "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/internal/test/daemon" "github.com/gotestyourself/gotestyourself/assert" "github.com/gotestyourself/gotestyourself/skip" ) @@ -17,7 +17,7 @@ func TestDaemonStartWithLogOpt(t *testing.T) { skip.IfCondition(t, testEnv.IsRemoteDaemon(), "cannot run daemon when remote daemon") t.Parallel() - d := daemon.New(t, "", dockerdBinary, daemon.Config{}) + d := daemon.New(t) d.Start(t, "--iptables=false") defer d.Stop(t) diff --git a/components/engine/integration/plugin/volumes/main_test.go b/components/engine/integration/plugin/volumes/main_test.go index e3aa260f8d..5a5678d9c4 100644 --- a/components/engine/integration/plugin/volumes/main_test.go +++ b/components/engine/integration/plugin/volumes/main_test.go @@ -12,8 +12,6 @@ var ( testEnv *environment.Execution ) -const dockerdBinary = "dockerd" - func TestMain(m *testing.M) { var err error testEnv, err = environment.New() diff --git a/components/engine/integration/plugin/volumes/mounts_test.go b/components/engine/integration/plugin/volumes/mounts_test.go index 2c464f6607..d57fcfd36f 100644 --- a/components/engine/integration/plugin/volumes/mounts_test.go +++ b/components/engine/integration/plugin/volumes/mounts_test.go @@ -7,8 +7,8 @@ import ( "testing" "github.com/docker/docker/api/types" - "github.com/docker/docker/integration-cli/daemon" "github.com/docker/docker/integration-cli/fixtures/plugin" + "github.com/docker/docker/internal/test/daemon" "github.com/gotestyourself/gotestyourself/assert" ) @@ -17,7 +17,7 @@ import ( func TestPluginWithDevMounts(t *testing.T) { t.Parallel() - d := daemon.New(t, "", dockerdBinary, daemon.Config{}) + d := daemon.New(t) d.Start(t, "--iptables=false") defer d.Stop(t) diff --git a/components/engine/integration/secret/main_test.go b/components/engine/integration/secret/main_test.go index 4a3d4c75e9..abd4eef9f0 100644 --- a/components/engine/integration/secret/main_test.go +++ b/components/engine/integration/secret/main_test.go @@ -10,8 +10,6 @@ import ( var testEnv *environment.Execution -const dockerdBinary = "dockerd" - func TestMain(m *testing.M) { var err error testEnv, err = environment.New() diff --git a/components/engine/integration/service/main_test.go b/components/engine/integration/service/main_test.go index d887403e51..28fd19df4d 100644 --- a/components/engine/integration/service/main_test.go +++ b/components/engine/integration/service/main_test.go @@ -10,8 +10,6 @@ import ( var testEnv *environment.Execution -const dockerdBinary = "dockerd" - func TestMain(m *testing.M) { var err error testEnv, err = environment.New() diff --git a/components/engine/integration/system/cgroupdriver_systemd_test.go b/components/engine/integration/system/cgroupdriver_systemd_test.go index c1f864a1c7..12e716c66c 100644 --- a/components/engine/integration/system/cgroupdriver_systemd_test.go +++ b/components/engine/integration/system/cgroupdriver_systemd_test.go @@ -7,7 +7,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" - "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/internal/test/daemon" "github.com/gotestyourself/gotestyourself/assert" ) @@ -33,7 +33,7 @@ func TestCgroupDriverSystemdMemoryLimit(t *testing.T) { t.Skip("systemd not available") } - d := daemon.New(t, "docker", "dockerd", daemon.Config{}) + d := daemon.New(t) client, err := d.NewClient() assert.NilError(t, err) d.StartWithBusybox(t, "--exec-opt", "native.cgroupdriver=systemd", "--iptables=false") diff --git a/components/engine/internal/test/daemon/config.go b/components/engine/internal/test/daemon/config.go new file mode 100644 index 0000000000..4ecc41b514 --- /dev/null +++ b/components/engine/internal/test/daemon/config.go @@ -0,0 +1,66 @@ +package daemon + +import ( + "context" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/swarm" + "github.com/gotestyourself/gotestyourself/assert" +) + +// ConfigConstructor defines a swarm config constructor +type ConfigConstructor func(*swarm.Config) + +// CreateConfig creates a config given the specified spec +func (d *Daemon) CreateConfig(t assert.TestingT, configSpec swarm.ConfigSpec) string { + cli := d.NewClientT(t) + defer cli.Close() + + scr, err := cli.ConfigCreate(context.Background(), configSpec) + assert.NilError(t, err) + return scr.ID +} + +// ListConfigs returns the list of the current swarm configs +func (d *Daemon) ListConfigs(t assert.TestingT) []swarm.Config { + cli := d.NewClientT(t) + defer cli.Close() + + configs, err := cli.ConfigList(context.Background(), types.ConfigListOptions{}) + assert.NilError(t, err) + return configs +} + +// GetConfig returns a swarm config identified by the specified id +func (d *Daemon) GetConfig(t assert.TestingT, id string) *swarm.Config { + cli := d.NewClientT(t) + defer cli.Close() + + config, _, err := cli.ConfigInspectWithRaw(context.Background(), id) + assert.NilError(t, err) + return &config +} + +// DeleteConfig removes the swarm config identified by the specified id +func (d *Daemon) DeleteConfig(t assert.TestingT, id string) { + cli := d.NewClientT(t) + defer cli.Close() + + err := cli.ConfigRemove(context.Background(), id) + assert.NilError(t, err) +} + +// UpdateConfig updates the swarm config identified by the specified id +// Currently, only label update is supported. +func (d *Daemon) UpdateConfig(t assert.TestingT, id string, f ...ConfigConstructor) { + cli := d.NewClientT(t) + defer cli.Close() + + config := d.GetConfig(t, id) + for _, fn := range f { + fn(config) + } + + err := cli.ConfigUpdate(context.Background(), config.ID, config.Version, config.Spec) + assert.NilError(t, err) +} diff --git a/components/engine/internal/test/daemon/daemon.go b/components/engine/internal/test/daemon/daemon.go new file mode 100644 index 0000000000..7324503278 --- /dev/null +++ b/components/engine/internal/test/daemon/daemon.go @@ -0,0 +1,638 @@ +package daemon // import "github.com/docker/docker/internal/test/daemon" + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/events" + "github.com/docker/docker/client" + "github.com/docker/docker/integration-cli/request" + "github.com/docker/docker/opts" + "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/stringid" + "github.com/docker/go-connections/sockets" + "github.com/docker/go-connections/tlsconfig" + "github.com/gotestyourself/gotestyourself/assert" + "github.com/pkg/errors" +) + +type testingT interface { + assert.TestingT + logT + Fatalf(string, ...interface{}) +} + +type logT interface { + Logf(string, ...interface{}) +} + +const defaultDockerdBinary = "dockerd" + +var errDaemonNotStarted = errors.New("daemon not started") + +// SockRoot holds the path of the default docker integration daemon socket +var SockRoot = filepath.Join(os.TempDir(), "docker-integration") + +type clientConfig struct { + transport *http.Transport + scheme string + addr string +} + +// Daemon represents a Docker daemon for the testing framework +type Daemon struct { + GlobalFlags []string + Root string + Folder string + Wait chan error + UseDefaultHost bool + UseDefaultTLSHost bool + + id string + logFile *os.File + cmd *exec.Cmd + storageDriver string + userlandProxy bool + execRoot string + experimental bool + dockerdBinary string + log logT + + // swarm related field + swarmListenAddr string + SwarmPort int // FIXME(vdemeester) should probably not be exported + + // cached information + CachedInfo types.Info +} + +// New returns a Daemon instance to be used for testing. +// This will create a directory such as d123456789 in the folder specified by $DOCKER_INTEGRATION_DAEMON_DEST or $DEST. +// The daemon will not automatically start. +func New(t testingT, ops ...func(*Daemon)) *Daemon { + dest := os.Getenv("DOCKER_INTEGRATION_DAEMON_DEST") + if dest == "" { + dest = os.Getenv("DEST") + } + assert.Check(t, dest != "", "Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable") + + storageDriver := os.Getenv("DOCKER_GRAPHDRIVER") + + assert.NilError(t, os.MkdirAll(SockRoot, 0700), "could not create daemon socket root") + + id := fmt.Sprintf("d%s", stringid.TruncateID(stringid.GenerateRandomID())) + dir := filepath.Join(dest, id) + daemonFolder, err := filepath.Abs(dir) + assert.NilError(t, err, "Could not make %q an absolute path", dir) + daemonRoot := filepath.Join(daemonFolder, "root") + + assert.NilError(t, os.MkdirAll(daemonRoot, 0755), "Could not create daemon root %q", dir) + + userlandProxy := true + if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { + if val, err := strconv.ParseBool(env); err != nil { + userlandProxy = val + } + } + d := &Daemon{ + id: id, + Folder: daemonFolder, + Root: daemonRoot, + storageDriver: storageDriver, + userlandProxy: userlandProxy, + execRoot: filepath.Join(os.TempDir(), "docker-execroot", id), + dockerdBinary: defaultDockerdBinary, + swarmListenAddr: defaultSwarmListenAddr, + SwarmPort: defaultSwarmPort, + log: t, + } + + for _, op := range ops { + op(d) + } + + return d +} + +// RootDir returns the root directory of the daemon. +func (d *Daemon) RootDir() string { + return d.Root +} + +// ID returns the generated id of the daemon +func (d *Daemon) ID() string { + return d.id +} + +// StorageDriver returns the configured storage driver of the daemon +func (d *Daemon) StorageDriver() string { + return d.storageDriver +} + +// Sock returns the socket path of the daemon +func (d *Daemon) Sock() string { + return fmt.Sprintf("unix://" + d.sockPath()) +} + +func (d *Daemon) sockPath() string { + return filepath.Join(SockRoot, d.id+".sock") +} + +// LogFileName returns the path the daemon's log file +func (d *Daemon) LogFileName() string { + return d.logFile.Name() +} + +// ReadLogFile returns the content of the daemon log file +func (d *Daemon) ReadLogFile() ([]byte, error) { + return ioutil.ReadFile(d.logFile.Name()) +} + +// NewClient creates new client based on daemon's socket path +// FIXME(vdemeester): replace NewClient with NewClientT +func (d *Daemon) NewClient() (*client.Client, error) { + return client.NewClientWithOpts( + client.FromEnv, + client.WithHost(d.Sock())) +} + +// NewClientT creates new client based on daemon's socket path +// FIXME(vdemeester): replace NewClient with NewClientT +func (d *Daemon) NewClientT(t assert.TestingT) *client.Client { + c, err := client.NewClientWithOpts( + client.FromEnv, + client.WithHost(d.Sock())) + assert.NilError(t, err, "cannot create daemon client") + return c +} + +// CleanupExecRoot cleans the daemon exec root (network namespaces, ...) +func (d *Daemon) CleanupExecRoot(t testingT) { + cleanupExecRoot(t, d.execRoot) +} + +// Start starts the daemon and return once it is ready to receive requests. +func (d *Daemon) Start(t testingT, args ...string) { + if err := d.StartWithError(args...); err != nil { + t.Fatalf("Error starting daemon with arguments: %v", args) + } +} + +// StartWithError starts the daemon and return once it is ready to receive requests. +// It returns an error in case it couldn't start. +func (d *Daemon) StartWithError(args ...string) error { + logFile, err := os.OpenFile(filepath.Join(d.Folder, "docker.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return errors.Wrapf(err, "[%s] Could not create %s/docker.log", d.id, d.Folder) + } + + return d.StartWithLogFile(logFile, args...) +} + +// StartWithLogFile will start the daemon and attach its streams to a given file. +func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { + dockerdBinary, err := exec.LookPath(d.dockerdBinary) + if err != nil { + return errors.Wrapf(err, "[%s] could not find docker binary in $PATH", d.id) + } + args := append(d.GlobalFlags, + "--containerd", "/var/run/docker/containerd/docker-containerd.sock", + "--data-root", d.Root, + "--exec-root", d.execRoot, + "--pidfile", fmt.Sprintf("%s/docker.pid", d.Folder), + fmt.Sprintf("--userland-proxy=%t", d.userlandProxy), + ) + if d.experimental { + args = append(args, "--experimental", "--init") + } + if !(d.UseDefaultHost || d.UseDefaultTLSHost) { + args = append(args, []string{"--host", d.Sock()}...) + } + if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { + args = append(args, []string{"--userns-remap", root}...) + } + + // If we don't explicitly set the log-level or debug flag(-D) then + // turn on debug mode + foundLog := false + foundSd := false + for _, a := range providedArgs { + if strings.Contains(a, "--log-level") || strings.Contains(a, "-D") || strings.Contains(a, "--debug") { + foundLog = true + } + if strings.Contains(a, "--storage-driver") { + foundSd = true + } + } + if !foundLog { + args = append(args, "--debug") + } + if d.storageDriver != "" && !foundSd { + args = append(args, "--storage-driver", d.storageDriver) + } + + args = append(args, providedArgs...) + d.cmd = exec.Command(dockerdBinary, args...) + d.cmd.Env = append(os.Environ(), "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE=1") + d.cmd.Stdout = out + d.cmd.Stderr = out + d.logFile = out + + if err := d.cmd.Start(); err != nil { + return errors.Errorf("[%s] could not start daemon container: %v", d.id, err) + } + + wait := make(chan error) + + go func() { + wait <- d.cmd.Wait() + d.log.Logf("[%s] exiting daemon", d.id) + close(wait) + }() + + d.Wait = wait + + tick := time.Tick(500 * time.Millisecond) + // make sure daemon is ready to receive requests + startTime := time.Now().Unix() + for { + d.log.Logf("[%s] waiting for daemon to start", d.id) + if time.Now().Unix()-startTime > 5 { + // After 5 seconds, give up + return errors.Errorf("[%s] Daemon exited and never started", d.id) + } + select { + case <-time.After(2 * time.Second): + return errors.Errorf("[%s] timeout: daemon does not respond", d.id) + case <-tick: + clientConfig, err := d.getClientConfig() + if err != nil { + return err + } + + client := &http.Client{ + Transport: clientConfig.transport, + } + + req, err := http.NewRequest("GET", "/_ping", nil) + if err != nil { + return errors.Wrapf(err, "[%s] could not create new request", d.id) + } + req.URL.Host = clientConfig.addr + req.URL.Scheme = clientConfig.scheme + resp, err := client.Do(req) + if err != nil { + continue + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + d.log.Logf("[%s] received status != 200 OK: %s\n", d.id, resp.Status) + } + d.log.Logf("[%s] daemon started\n", d.id) + d.Root, err = d.queryRootDir() + if err != nil { + return errors.Errorf("[%s] error querying daemon for root directory: %v", d.id, err) + } + return nil + case <-d.Wait: + return errors.Errorf("[%s] Daemon exited during startup", d.id) + } + } +} + +// StartWithBusybox will first start the daemon with Daemon.Start() +// then save the busybox image from the main daemon and load it into this Daemon instance. +func (d *Daemon) StartWithBusybox(t testingT, arg ...string) { + d.Start(t, arg...) + d.LoadBusybox(t) +} + +// Kill will send a SIGKILL to the daemon +func (d *Daemon) Kill() error { + if d.cmd == nil || d.Wait == nil { + return errDaemonNotStarted + } + + defer func() { + d.logFile.Close() + d.cmd = nil + }() + + if err := d.cmd.Process.Kill(); err != nil { + return err + } + + return os.Remove(fmt.Sprintf("%s/docker.pid", d.Folder)) +} + +// Pid returns the pid of the daemon +func (d *Daemon) Pid() int { + return d.cmd.Process.Pid +} + +// Interrupt stops the daemon by sending it an Interrupt signal +func (d *Daemon) Interrupt() error { + return d.Signal(os.Interrupt) +} + +// Signal sends the specified signal to the daemon if running +func (d *Daemon) Signal(signal os.Signal) error { + if d.cmd == nil || d.Wait == nil { + return errDaemonNotStarted + } + return d.cmd.Process.Signal(signal) +} + +// DumpStackAndQuit sends SIGQUIT to the daemon, which triggers it to dump its +// stack to its log file and exit +// This is used primarily for gathering debug information on test timeout +func (d *Daemon) DumpStackAndQuit() { + if d.cmd == nil || d.cmd.Process == nil { + return + } + SignalDaemonDump(d.cmd.Process.Pid) +} + +// Stop will send a SIGINT every second and wait for the daemon to stop. +// If it times out, a SIGKILL is sent. +// Stop will not delete the daemon directory. If a purged daemon is needed, +// instantiate a new one with NewDaemon. +// If an error occurs while starting the daemon, the test will fail. +func (d *Daemon) Stop(t testingT) { + err := d.StopWithError() + if err != nil { + if err != errDaemonNotStarted { + t.Fatalf("Error while stopping the daemon %s : %v", d.id, err) + } else { + t.Logf("Daemon %s is not started", d.id) + } + } +} + +// StopWithError will send a SIGINT every second and wait for the daemon to stop. +// If it timeouts, a SIGKILL is sent. +// Stop will not delete the daemon directory. If a purged daemon is needed, +// instantiate a new one with NewDaemon. +func (d *Daemon) StopWithError() error { + if d.cmd == nil || d.Wait == nil { + return errDaemonNotStarted + } + + defer func() { + d.logFile.Close() + d.cmd = nil + }() + + i := 1 + tick := time.Tick(time.Second) + + if err := d.cmd.Process.Signal(os.Interrupt); err != nil { + if strings.Contains(err.Error(), "os: process already finished") { + return errDaemonNotStarted + } + return errors.Errorf("could not send signal: %v", err) + } +out1: + for { + select { + case err := <-d.Wait: + return err + case <-time.After(20 * time.Second): + // time for stopping jobs and run onShutdown hooks + d.log.Logf("[%s] daemon started", d.id) + break out1 + } + } + +out2: + for { + select { + case err := <-d.Wait: + return err + case <-tick: + i++ + if i > 5 { + d.log.Logf("tried to interrupt daemon for %d times, now try to kill it", i) + break out2 + } + d.log.Logf("Attempt #%d: daemon is still running with pid %d", i, d.cmd.Process.Pid) + if err := d.cmd.Process.Signal(os.Interrupt); err != nil { + return errors.Errorf("could not send signal: %v", err) + } + } + } + + if err := d.cmd.Process.Kill(); err != nil { + d.log.Logf("Could not kill daemon: %v", err) + return err + } + + d.cmd.Wait() + + return os.Remove(fmt.Sprintf("%s/docker.pid", d.Folder)) +} + +// Restart will restart the daemon by first stopping it and the starting it. +// If an error occurs while starting the daemon, the test will fail. +func (d *Daemon) Restart(t testingT, args ...string) { + d.Stop(t) + d.handleUserns() + d.Start(t, args...) +} + +// RestartWithError will restart the daemon by first stopping it and then starting it. +func (d *Daemon) RestartWithError(arg ...string) error { + if err := d.StopWithError(); err != nil { + return err + } + d.handleUserns() + return d.StartWithError(arg...) +} + +func (d *Daemon) handleUserns() { + // in the case of tests running a user namespace-enabled daemon, we have resolved + // d.Root to be the actual final path of the graph dir after the "uid.gid" of + // remapped root is added--we need to subtract it from the path before calling + // start or else we will continue making subdirectories rather than truly restarting + // with the same location/root: + if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { + d.Root = filepath.Dir(d.Root) + } +} + +// ReloadConfig asks the daemon to reload its configuration +func (d *Daemon) ReloadConfig() error { + if d.cmd == nil || d.cmd.Process == nil { + return errors.New("daemon is not running") + } + + errCh := make(chan error) + started := make(chan struct{}) + go func() { + _, body, err := request.DoOnHost(d.Sock(), "/events", request.Method(http.MethodGet)) + close(started) + if err != nil { + errCh <- err + } + defer body.Close() + dec := json.NewDecoder(body) + for { + var e events.Message + if err := dec.Decode(&e); err != nil { + errCh <- err + return + } + if e.Type != events.DaemonEventType { + continue + } + if e.Action != "reload" { + continue + } + close(errCh) // notify that we are done + return + } + }() + + <-started + if err := signalDaemonReload(d.cmd.Process.Pid); err != nil { + return errors.Errorf("error signaling daemon reload: %v", err) + } + select { + case err := <-errCh: + if err != nil { + return errors.Errorf("error waiting for daemon reload event: %v", err) + } + case <-time.After(30 * time.Second): + return errors.New("timeout waiting for daemon reload event") + } + return nil +} + +// LoadBusybox image into the daemon +func (d *Daemon) LoadBusybox(t assert.TestingT) { + clientHost, err := client.NewEnvClient() + assert.NilError(t, err, "failed to create client") + defer clientHost.Close() + + ctx := context.Background() + reader, err := clientHost.ImageSave(ctx, []string{"busybox:latest"}) + assert.NilError(t, err, "failed to download busybox") + defer reader.Close() + + client, err := d.NewClient() + assert.NilError(t, err, "failed to create client") + defer client.Close() + + resp, err := client.ImageLoad(ctx, reader, true) + assert.NilError(t, err, "failed to load busybox") + defer resp.Body.Close() +} + +func (d *Daemon) getClientConfig() (*clientConfig, error) { + var ( + transport *http.Transport + scheme string + addr string + proto string + ) + if d.UseDefaultTLSHost { + option := &tlsconfig.Options{ + CAFile: "fixtures/https/ca.pem", + CertFile: "fixtures/https/client-cert.pem", + KeyFile: "fixtures/https/client-key.pem", + } + tlsConfig, err := tlsconfig.Client(*option) + if err != nil { + return nil, err + } + transport = &http.Transport{ + TLSClientConfig: tlsConfig, + } + addr = fmt.Sprintf("%s:%d", opts.DefaultHTTPHost, opts.DefaultTLSHTTPPort) + scheme = "https" + proto = "tcp" + } else if d.UseDefaultHost { + addr = opts.DefaultUnixSocket + proto = "unix" + scheme = "http" + transport = &http.Transport{} + } else { + addr = d.sockPath() + proto = "unix" + scheme = "http" + transport = &http.Transport{} + } + + if err := sockets.ConfigureTransport(transport, proto, addr); err != nil { + return nil, err + } + transport.DisableKeepAlives = true + + return &clientConfig{ + transport: transport, + scheme: scheme, + addr: addr, + }, nil +} + +func (d *Daemon) queryRootDir() (string, error) { + // update daemon root by asking /info endpoint (to support user + // namespaced daemon with root remapped uid.gid directory) + clientConfig, err := d.getClientConfig() + if err != nil { + return "", err + } + + client := &http.Client{ + Transport: clientConfig.transport, + } + + req, err := http.NewRequest("GET", "/info", nil) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + req.URL.Host = clientConfig.addr + req.URL.Scheme = clientConfig.scheme + + resp, err := client.Do(req) + if err != nil { + return "", err + } + body := ioutils.NewReadCloserWrapper(resp.Body, func() error { + return resp.Body.Close() + }) + + type Info struct { + DockerRootDir string + } + var b []byte + var i Info + b, err = request.ReadBody(body) + if err == nil && resp.StatusCode == http.StatusOK { + // read the docker root dir + if err = json.Unmarshal(b, &i); err == nil { + return i.DockerRootDir, nil + } + } + return "", err +} + +// Info returns the info struct for this daemon +func (d *Daemon) Info(t assert.TestingT) types.Info { + apiclient, err := d.NewClient() + assert.NilError(t, err) + info, err := apiclient.Info(context.Background()) + assert.NilError(t, err) + return info +} diff --git a/components/engine/integration-cli/daemon/daemon_unix.go b/components/engine/internal/test/daemon/daemon_unix.go similarity index 78% rename from components/engine/integration-cli/daemon/daemon_unix.go rename to components/engine/internal/test/daemon/daemon_unix.go index 97548c3103..c0aa26c9f8 100644 --- a/components/engine/integration-cli/daemon/daemon_unix.go +++ b/components/engine/internal/test/daemon/daemon_unix.go @@ -1,16 +1,15 @@ // +build !windows -package daemon // import "github.com/docker/docker/integration-cli/daemon" +package daemon // import "github.com/docker/docker/internal/test/daemon" import ( "os" "path/filepath" - "github.com/go-check/check" "golang.org/x/sys/unix" ) -func cleanupExecRoot(c *check.C, execRoot string) { +func cleanupExecRoot(t testingT, execRoot string) { // Cleanup network namespaces in the exec root of this // daemon because this exec root is specific to this // daemon instance and has no chance of getting @@ -19,7 +18,7 @@ func cleanupExecRoot(c *check.C, execRoot string) { netnsPath := filepath.Join(execRoot, "netns") filepath.Walk(netnsPath, func(path string, info os.FileInfo, err error) error { if err := unix.Unmount(path, unix.MNT_FORCE); err != nil { - c.Logf("unmount of %s failed: %v", path, err) + t.Logf("unmount of %s failed: %v", path, err) } os.Remove(path) return nil diff --git a/components/engine/integration-cli/daemon/daemon_windows.go b/components/engine/internal/test/daemon/daemon_windows.go similarity index 74% rename from components/engine/integration-cli/daemon/daemon_windows.go rename to components/engine/internal/test/daemon/daemon_windows.go index 63f6b03239..8ec554fbc4 100644 --- a/components/engine/integration-cli/daemon/daemon_windows.go +++ b/components/engine/internal/test/daemon/daemon_windows.go @@ -1,10 +1,9 @@ -package daemon // import "github.com/docker/docker/integration-cli/daemon" +package daemon // import "github.com/docker/docker/internal/test/daemon" import ( "fmt" "strconv" - "github.com/go-check/check" "golang.org/x/sys/windows" ) @@ -22,5 +21,5 @@ func signalDaemonReload(pid int) error { return fmt.Errorf("daemon reload not supported") } -func cleanupExecRoot(c *check.C, execRoot string) { +func cleanupExecRoot(t testingT, execRoot string) { } diff --git a/components/engine/internal/test/daemon/node.go b/components/engine/internal/test/daemon/node.go new file mode 100644 index 0000000000..9955208b47 --- /dev/null +++ b/components/engine/internal/test/daemon/node.go @@ -0,0 +1,69 @@ +package daemon + +import ( + "context" + "strings" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/swarm" + "github.com/gotestyourself/gotestyourself/assert" +) + +// NodeConstructor defines a swarm node constructor +type NodeConstructor func(*swarm.Node) + +// GetNode returns a swarm node identified by the specified id +func (d *Daemon) GetNode(t assert.TestingT, id string) *swarm.Node { + cli := d.NewClientT(t) + defer cli.Close() + + node, _, err := cli.NodeInspectWithRaw(context.Background(), id) + assert.NilError(t, err) + assert.Check(t, node.ID == id) + return &node +} + +// RemoveNode removes the specified node +func (d *Daemon) RemoveNode(t assert.TestingT, id string, force bool) { + cli := d.NewClientT(t) + defer cli.Close() + + options := types.NodeRemoveOptions{ + Force: force, + } + err := cli.NodeRemove(context.Background(), id, options) + assert.NilError(t, err) +} + +// UpdateNode updates a swarm node with the specified node constructor +func (d *Daemon) UpdateNode(t assert.TestingT, id string, f ...NodeConstructor) { + cli := d.NewClientT(t) + defer cli.Close() + + for i := 0; ; i++ { + node := d.GetNode(t, id) + for _, fn := range f { + fn(node) + } + + err := cli.NodeUpdate(context.Background(), node.ID, node.Version, node.Spec) + if i < 10 && err != nil && strings.Contains(err.Error(), "update out of sequence") { + time.Sleep(100 * time.Millisecond) + continue + } + assert.NilError(t, err) + return + } +} + +// ListNodes returns the list of the current swarm nodes +func (d *Daemon) ListNodes(t assert.TestingT) []swarm.Node { + cli := d.NewClientT(t) + defer cli.Close() + + nodes, err := cli.NodeList(context.Background(), types.NodeListOptions{}) + assert.NilError(t, err) + + return nodes +} diff --git a/components/engine/internal/test/daemon/ops.go b/components/engine/internal/test/daemon/ops.go new file mode 100644 index 0000000000..0176c19d93 --- /dev/null +++ b/components/engine/internal/test/daemon/ops.go @@ -0,0 +1,27 @@ +package daemon + +// WithExperimental sets the daemon in experimental mode +func WithExperimental(d *Daemon) { + d.experimental = true +} + +// WithDockerdBinary sets the dockerd binary to the specified one +func WithDockerdBinary(dockerdBinary string) func(*Daemon) { + return func(d *Daemon) { + d.dockerdBinary = dockerdBinary + } +} + +// WithSwarmPort sets the swarm port to use for swarm mode +func WithSwarmPort(port int) func(*Daemon) { + return func(d *Daemon) { + d.SwarmPort = port + } +} + +// WithSwarmListenAddr sets the swarm listen addr to use for swarm mode +func WithSwarmListenAddr(listenAddr string) func(*Daemon) { + return func(d *Daemon) { + d.swarmListenAddr = listenAddr + } +} diff --git a/components/engine/internal/test/daemon/secret.go b/components/engine/internal/test/daemon/secret.go new file mode 100644 index 0000000000..075aedc2e0 --- /dev/null +++ b/components/engine/internal/test/daemon/secret.go @@ -0,0 +1,68 @@ +package daemon + +import ( + "context" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/swarm" + "github.com/gotestyourself/gotestyourself/assert" +) + +// SecretConstructor defines a swarm secret constructor +type SecretConstructor func(*swarm.Secret) + +// CreateSecret creates a secret given the specified spec +func (d *Daemon) CreateSecret(t assert.TestingT, secretSpec swarm.SecretSpec) string { + cli := d.NewClientT(t) + defer cli.Close() + + scr, err := cli.SecretCreate(context.Background(), secretSpec) + assert.NilError(t, err) + + return scr.ID +} + +// ListSecrets returns the list of the current swarm secrets +func (d *Daemon) ListSecrets(t assert.TestingT) []swarm.Secret { + cli := d.NewClientT(t) + defer cli.Close() + + secrets, err := cli.SecretList(context.Background(), types.SecretListOptions{}) + assert.NilError(t, err) + return secrets +} + +// GetSecret returns a swarm secret identified by the specified id +func (d *Daemon) GetSecret(t assert.TestingT, id string) *swarm.Secret { + cli := d.NewClientT(t) + defer cli.Close() + + secret, _, err := cli.SecretInspectWithRaw(context.Background(), id) + assert.NilError(t, err) + return &secret +} + +// DeleteSecret removes the swarm secret identified by the specified id +func (d *Daemon) DeleteSecret(t assert.TestingT, id string) { + cli := d.NewClientT(t) + defer cli.Close() + + err := cli.SecretRemove(context.Background(), id) + assert.NilError(t, err) +} + +// UpdateSecret updates the swarm secret identified by the specified id +// Currently, only label update is supported. +func (d *Daemon) UpdateSecret(t assert.TestingT, id string, f ...SecretConstructor) { + cli := d.NewClientT(t) + defer cli.Close() + + secret := d.GetSecret(t, id) + for _, fn := range f { + fn(secret) + } + + err := cli.SecretUpdate(context.Background(), secret.ID, secret.Version, secret.Spec) + + assert.NilError(t, err) +} diff --git a/components/engine/internal/test/daemon/service.go b/components/engine/internal/test/daemon/service.go new file mode 100644 index 0000000000..26e6004ae1 --- /dev/null +++ b/components/engine/internal/test/daemon/service.go @@ -0,0 +1,108 @@ +package daemon + +import ( + "context" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/swarm" + "github.com/gotestyourself/gotestyourself/assert" +) + +// ServiceConstructor defines a swarm service constructor function +type ServiceConstructor func(*swarm.Service) + +// CreateServiceWithOptions creates a swarm service given the specified service constructors +// and auth config +func (d *Daemon) CreateServiceWithOptions(t assert.TestingT, opts types.ServiceCreateOptions, f ...ServiceConstructor) string { + var service swarm.Service + for _, fn := range f { + fn(&service) + } + + cli := d.NewClientT(t) + defer cli.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + res, err := cli.ServiceCreate(ctx, service.Spec, opts) + assert.NilError(t, err) + return res.ID +} + +// CreateService creates a swarm service given the specified service constructor +func (d *Daemon) CreateService(t assert.TestingT, f ...ServiceConstructor) string { + return d.CreateServiceWithOptions(t, types.ServiceCreateOptions{}, f...) +} + +// GetService returns the swarm service corresponding to the specified id +func (d *Daemon) GetService(t assert.TestingT, id string) *swarm.Service { + cli := d.NewClientT(t) + defer cli.Close() + + service, _, err := cli.ServiceInspectWithRaw(context.Background(), id, types.ServiceInspectOptions{}) + assert.NilError(t, err) + return &service +} + +// GetServiceTasks returns the swarm tasks for the specified service +func (d *Daemon) GetServiceTasks(t assert.TestingT, service string) []swarm.Task { + cli := d.NewClientT(t) + defer cli.Close() + + filterArgs := filters.NewArgs() + filterArgs.Add("desired-state", "running") + filterArgs.Add("service", service) + + options := types.TaskListOptions{ + Filters: filterArgs, + } + + tasks, err := cli.TaskList(context.Background(), options) + assert.NilError(t, err) + return tasks +} + +// UpdateService updates a swarm service with the specified service constructor +func (d *Daemon) UpdateService(t assert.TestingT, service *swarm.Service, f ...ServiceConstructor) { + cli := d.NewClientT(t) + defer cli.Close() + + for _, fn := range f { + fn(service) + } + + _, err := cli.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, types.ServiceUpdateOptions{}) + assert.NilError(t, err) +} + +// RemoveService removes the specified service +func (d *Daemon) RemoveService(t assert.TestingT, id string) { + cli := d.NewClientT(t) + defer cli.Close() + + err := cli.ServiceRemove(context.Background(), id) + assert.NilError(t, err) +} + +// ListServices returns the list of the current swarm services +func (d *Daemon) ListServices(t assert.TestingT) []swarm.Service { + cli := d.NewClientT(t) + defer cli.Close() + + services, err := cli.ServiceList(context.Background(), types.ServiceListOptions{}) + assert.NilError(t, err) + return services +} + +// GetTask returns the swarm task identified by the specified id +func (d *Daemon) GetTask(t assert.TestingT, id string) swarm.Task { + cli := d.NewClientT(t) + defer cli.Close() + + task, _, err := cli.TaskInspectWithRaw(context.Background(), id) + assert.NilError(t, err) + return task +} diff --git a/components/engine/internal/test/daemon/swarm.go b/components/engine/internal/test/daemon/swarm.go new file mode 100644 index 0000000000..f0cf373468 --- /dev/null +++ b/components/engine/internal/test/daemon/swarm.go @@ -0,0 +1,139 @@ +package daemon + +import ( + "context" + "fmt" + + "github.com/docker/docker/api/types/swarm" + "github.com/gotestyourself/gotestyourself/assert" + "github.com/pkg/errors" +) + +const ( + defaultSwarmPort = 2477 + defaultSwarmListenAddr = "0.0.0.0" +) + +// SpecConstructor defines a swarm spec constructor +type SpecConstructor func(*swarm.Spec) + +// SwarmListenAddr returns the listen-addr used for the daemon +func (d *Daemon) SwarmListenAddr() string { + return fmt.Sprintf("%s:%d", d.swarmListenAddr, d.SwarmPort) +} + +// NodeID returns the swarm mode node ID +func (d *Daemon) NodeID() string { + return d.CachedInfo.Swarm.NodeID +} + +// SwarmInit initializes a new swarm cluster. +func (d *Daemon) SwarmInit(t assert.TestingT, req swarm.InitRequest) { + if req.ListenAddr == "" { + req.ListenAddr = fmt.Sprintf("%s:%d", d.swarmListenAddr, d.SwarmPort) + } + cli := d.NewClientT(t) + defer cli.Close() + _, err := cli.SwarmInit(context.Background(), req) + assert.NilError(t, err, "initializing swarm") + d.CachedInfo = d.Info(t) +} + +// SwarmJoin joins a daemon to an existing cluster. +func (d *Daemon) SwarmJoin(t assert.TestingT, req swarm.JoinRequest) { + if req.ListenAddr == "" { + req.ListenAddr = fmt.Sprintf("%s:%d", d.swarmListenAddr, d.SwarmPort) + } + cli := d.NewClientT(t) + defer cli.Close() + err := cli.SwarmJoin(context.Background(), req) + assert.NilError(t, err, "initializing swarm") + d.CachedInfo = d.Info(t) +} + +// SwarmLeave forces daemon to leave current cluster. +func (d *Daemon) SwarmLeave(force bool) error { + cli, err := d.NewClient() + if err != nil { + return fmt.Errorf("leaving swarm: failed to create client %v", err) + } + defer cli.Close() + err = cli.SwarmLeave(context.Background(), force) + if err != nil { + err = fmt.Errorf("leaving swarm: %v", err) + } + return err +} + +// SwarmInfo returns the swarm information of the daemon +func (d *Daemon) SwarmInfo(t assert.TestingT) swarm.Info { + cli := d.NewClientT(t) + info, err := cli.Info(context.Background()) + assert.NilError(t, err, "get swarm info") + return info.Swarm +} + +// SwarmUnlock tries to unlock a locked swarm +func (d *Daemon) SwarmUnlock(req swarm.UnlockRequest) error { + cli, err := d.NewClient() + if err != nil { + return fmt.Errorf("unlocking swarm: failed to create client %v", err) + } + defer cli.Close() + err = cli.SwarmUnlock(context.Background(), req) + if err != nil { + err = errors.Wrap(err, "unlocking swarm") + } + return err +} + +// GetSwarm returns the current swarm object +func (d *Daemon) GetSwarm(t assert.TestingT) swarm.Swarm { + cli := d.NewClientT(t) + defer cli.Close() + + sw, err := cli.SwarmInspect(context.Background()) + assert.NilError(t, err) + return sw +} + +// UpdateSwarm updates the current swarm object with the specified spec constructors +func (d *Daemon) UpdateSwarm(t assert.TestingT, f ...SpecConstructor) { + cli := d.NewClientT(t) + defer cli.Close() + + sw := d.GetSwarm(t) + for _, fn := range f { + fn(&sw.Spec) + } + + err := cli.SwarmUpdate(context.Background(), sw.Version, sw.Spec, swarm.UpdateFlags{}) + assert.NilError(t, err) +} + +// RotateTokens update the swarm to rotate tokens +func (d *Daemon) RotateTokens(t assert.TestingT) { + cli := d.NewClientT(t) + defer cli.Close() + + sw, err := cli.SwarmInspect(context.Background()) + assert.NilError(t, err) + + flags := swarm.UpdateFlags{ + RotateManagerToken: true, + RotateWorkerToken: true, + } + + err = cli.SwarmUpdate(context.Background(), sw.Version, sw.Spec, flags) + assert.NilError(t, err) +} + +// JoinTokens returns the current swarm join tokens +func (d *Daemon) JoinTokens(t assert.TestingT) swarm.JoinTokens { + cli := d.NewClientT(t) + defer cli.Close() + + sw, err := cli.SwarmInspect(context.Background()) + assert.NilError(t, err) + return sw.JoinTokens +} diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper_deferred_remove.go b/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_deferred_remove.go similarity index 79% rename from components/engine/pkg/devicemapper/devmapper_wrapper_deferred_remove.go rename to components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_deferred_remove.go index 2db5c799c4..3d3021c4e1 100644 --- a/components/engine/pkg/devicemapper/devmapper_wrapper_deferred_remove.go +++ b/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_deferred_remove.go @@ -1,11 +1,15 @@ -// +build linux,cgo,!libdm_no_deferred_remove +// +build linux,cgo,!static_build +// +build !libdm_dlsym_deferred_remove,!libdm_no_deferred_remove package devicemapper // import "github.com/docker/docker/pkg/devicemapper" -// #include +/* +#include +*/ import "C" -// LibraryDeferredRemovalSupport tells if the feature is enabled in the build +// LibraryDeferredRemovalSupport tells if the feature is supported by the +// current Docker invocation. const LibraryDeferredRemovalSupport = true func dmTaskDeferredRemoveFct(task *cdmTask) int { diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_dlsym_deferred_remove.go b/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_dlsym_deferred_remove.go new file mode 100644 index 0000000000..5dfb369f1f --- /dev/null +++ b/components/engine/pkg/devicemapper/devmapper_wrapper_dynamic_dlsym_deferred_remove.go @@ -0,0 +1,128 @@ +// +build linux,cgo,!static_build +// +build libdm_dlsym_deferred_remove,!libdm_no_deferred_remove + +package devicemapper + +/* +#cgo LDFLAGS: -ldl +#include +#include +#include + +// Yes, I know this looks scary. In order to be able to fill our own internal +// dm_info with deferred_remove we need to have a struct definition that is +// correct (regardless of the version of libdm that was used to compile it). To +// this end, we define struct_backport_dm_info. This code comes from lvm2, and +// I have verified that the structure has only ever had elements *appended* to +// it (since 2001). +// +// It is also important that this structure be _larger_ than the dm_info that +// libdevmapper expected. Otherwise libdm might try to write to memory it +// shouldn't (they don't have a "known size" API). +struct backport_dm_info { + int exists; + int suspended; + int live_table; + int inactive_table; + int32_t open_count; + uint32_t event_nr; + uint32_t major; + uint32_t minor; + int read_only; + + int32_t target_count; + + int deferred_remove; + int internal_suspend; + + // Padding, purely for our own safety. This is to avoid cases where libdm + // was updated underneath us and we call into dm_task_get_info() with too + // small of a buffer. + char _[512]; +}; + +// We have to wrap this in CGo, because Go really doesn't like function pointers. +int call_dm_task_deferred_remove(void *fn, struct dm_task *task) +{ + int (*_dm_task_deferred_remove)(struct dm_task *task) = fn; + return _dm_task_deferred_remove(task); +} +*/ +import "C" + +import ( + "unsafe" + + "github.com/sirupsen/logrus" +) + +// dm_task_deferred_remove is not supported by all distributions, due to +// out-dated versions of devicemapper. However, in the case where the +// devicemapper library was updated without rebuilding Docker (which can happen +// in some distributions) then we should attempt to dynamically load the +// relevant object rather than try to link to it. + +// dmTaskDeferredRemoveFct is a "bound" version of dm_task_deferred_remove. +// It is nil if dm_task_deferred_remove was not found in the libdevmapper that +// is currently loaded. +var dmTaskDeferredRemovePtr unsafe.Pointer + +// LibraryDeferredRemovalSupport tells if the feature is supported by the +// current Docker invocation. This value is fixed during init. +var LibraryDeferredRemovalSupport bool + +func init() { + // Clear any errors. + var err *C.char + C.dlerror() + + // The symbol we want to fetch. + symName := C.CString("dm_task_deferred_remove") + defer C.free(unsafe.Pointer(symName)) + + // See if we can find dm_task_deferred_remove. Since we already are linked + // to libdevmapper, we can search our own address space (rather than trying + // to guess what libdevmapper is called). We use NULL here, as RTLD_DEFAULT + // is not available in CGO (even if you set _GNU_SOURCE for some reason). + // The semantics are identical on glibc. + sym := C.dlsym(nil, symName) + err = C.dlerror() + if err != nil { + logrus.Debugf("devmapper: could not load dm_task_deferred_remove: %s", C.GoString(err)) + return + } + + logrus.Debugf("devmapper: found dm_task_deferred_remove at %x", uintptr(sym)) + dmTaskDeferredRemovePtr = sym + LibraryDeferredRemovalSupport = true +} + +func dmTaskDeferredRemoveFct(task *cdmTask) int { + sym := dmTaskDeferredRemovePtr + if sym == nil || !LibraryDeferredRemovalSupport { + return -1 + } + return int(C.call_dm_task_deferred_remove(sym, (*C.struct_dm_task)(task))) +} + +func dmTaskGetInfoWithDeferredFct(task *cdmTask, info *Info) int { + if !LibraryDeferredRemovalSupport { + return -1 + } + + Cinfo := C.struct_backport_dm_info{} + defer func() { + info.Exists = int(Cinfo.exists) + info.Suspended = int(Cinfo.suspended) + info.LiveTable = int(Cinfo.live_table) + info.InactiveTable = int(Cinfo.inactive_table) + info.OpenCount = int32(Cinfo.open_count) + info.EventNr = uint32(Cinfo.event_nr) + info.Major = uint32(Cinfo.major) + info.Minor = uint32(Cinfo.minor) + info.ReadOnly = int(Cinfo.read_only) + info.TargetCount = int32(Cinfo.target_count) + info.DeferredRemove = int(Cinfo.deferred_remove) + }() + return int(C.dm_task_get_info((*C.struct_dm_task)(task), (*C.struct_dm_info)(unsafe.Pointer(&Cinfo)))) +} diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go b/components/engine/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go index 33eabe593b..8889f0f46f 100644 --- a/components/engine/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go +++ b/components/engine/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go @@ -1,8 +1,10 @@ -// +build linux,cgo,libdm_no_deferred_remove +// +build linux,cgo +// +build !libdm_dlsym_deferred_remove,libdm_no_deferred_remove package devicemapper // import "github.com/docker/docker/pkg/devicemapper" -// LibraryDeferredRemovalSupport tells if the feature is enabled in the build +// LibraryDeferredRemovalSupport tells if the feature is supported by the +// current Docker invocation. const LibraryDeferredRemovalSupport = false func dmTaskDeferredRemoveFct(task *cdmTask) int {