From 6dce6bba7710cffc9ab15626dedf767fa3da59ea Mon Sep 17 00:00:00 2001 From: Lei Jitang Date: Thu, 27 Apr 2017 11:26:51 -0400 Subject: [PATCH 01/56] Enable inspect exec if container is pause/restarting/non-running if docker exec exit and at the same the the container is pause, there could be a chance the `docker exec exit` will fail ``` $ docker exec -ti 388c7f47a06c sh / # exit Error response from daemon: Container 388c7f47a06cce0856266ffd56a2ce2901689ca7a6b9cd741b37652418448f2b is paused, unpause the container before exec ``` To reproduce this error easilly, we can add a sleep in `containerPause` ``` --- a/daemon/pause.go +++ b/daemon/pause.go @@ -2,6 +2,7 @@ package daemon import ( "fmt" + "time" "github.com/docker/docker/container" ) @@ -25,7 +26,7 @@ func (daemon *Daemon) ContainerPause(name string) error { func (daemon *Daemon) containerPause(container *container.Container) error { container.Lock() defer container.Unlock() - + time.Sleep(time.Second * 5) // We cannot Pause the container which is not running if !container.Running { return errNotRunning{container.ID} ``` Signed-off-by: Lei Jitang Upstream-commit: 0e87588bbb2482f9635a19ac43be1c0ee9652ac2 Component: engine --- components/engine/daemon/inspect.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/engine/daemon/inspect.go b/components/engine/daemon/inspect.go index 06858223f5..908a757340 100644 --- a/components/engine/daemon/inspect.go +++ b/components/engine/daemon/inspect.go @@ -196,9 +196,13 @@ func (daemon *Daemon) getInspectData(container *container.Container) (*types.Con // ContainerExecInspect returns low-level information about the exec // command. An error is returned if the exec cannot be found. func (daemon *Daemon) ContainerExecInspect(id string) (*backend.ExecInspect, error) { - e, err := daemon.getExecConfig(id) - if err != nil { - return nil, err + e := daemon.execCommands.Get(id) + if e == nil { + return nil, errExecNotFound(id) + } + + if container := daemon.containers.Get(e.ContainerID); container == nil { + return nil, errExecNotFound(id) } pc := inspectExecProcessConfig(e) From bbf5e9962e6dcae959afda866bf9892208f5bf22 Mon Sep 17 00:00:00 2001 From: gkze Date: Thu, 11 May 2017 16:40:13 -0700 Subject: [PATCH 02/56] GCP Logs: send log line as jsonPayload.message instead of jsonPayload.data Signed-off-by: gkze Upstream-commit: d42c268fc3983d19e9dbd800948e7c4445c425f0 Component: engine --- components/engine/daemon/logger/gcplogs/gcplogging.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/daemon/logger/gcplogs/gcplogging.go b/components/engine/daemon/logger/gcplogs/gcplogging.go index 3a48067f9d..a33566ae1f 100644 --- a/components/engine/daemon/logger/gcplogs/gcplogging.go +++ b/components/engine/daemon/logger/gcplogs/gcplogging.go @@ -61,7 +61,7 @@ type gcplogs struct { type dockerLogEntry struct { Instance *instanceInfo `json:"instance,omitempty"` Container *containerInfo `json:"container,omitempty"` - Data string `json:"data,omitempty"` + Message string `json:"message,omitempty"` } type instanceInfo struct { @@ -219,7 +219,7 @@ func ValidateLogOpts(cfg map[string]string) error { } func (l *gcplogs) Log(m *logger.Message) error { - data := string(m.Line) + message := string(m.Line) ts := m.Timestamp logger.PutMessage(m) @@ -228,7 +228,7 @@ func (l *gcplogs) Log(m *logger.Message) error { Payload: &dockerLogEntry{ Instance: l.instance, Container: l.container, - Data: data, + Message: message, }, }) return nil From 9968cd1aa2e12055726840cafa5b86aab4230fc9 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 2 Jun 2017 12:32:25 -0400 Subject: [PATCH 03/56] Add stream format details for attach/logs endpoint Signed-off-by: Brian Goff Upstream-commit: 48829ddf885eb656bff6dcf053bf96471b106bf0 Component: engine --- components/engine/client/container_attach.go | 20 ++++++++++++++++++++ components/engine/client/container_logs.go | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/components/engine/client/container_attach.go b/components/engine/client/container_attach.go index eea4682158..0fdf3ed0ce 100644 --- a/components/engine/client/container_attach.go +++ b/components/engine/client/container_attach.go @@ -11,6 +11,26 @@ import ( // It returns a types.HijackedConnection with the hijacked connection // and the a reader to get output. It's up to the called to close // the hijacked connection by calling types.HijackedResponse.Close. +// +// The stream format on the response will be in one of two formats: +// +// If the container is using a TTY, there is only a single stream (stdout), and +// data is copied directly from the container output stream, no extra +// multiplexing or headers. +// +// If the container is *not* using a TTY, streams for stdout and stderr are +// multiplexed. +// The format of the multiplexed stream is as follows: +// +// [8]byte{STREAM_TYPE, 0, 0, 0, SIZE1, SIZE2, SIZE3, SIZE4}[]byte{OUTPUT} +// +// STREAM_TYPE can be 1 for stdout and 2 for stderr +// +// SIZE1, SIZE2, SIZE3, and SIZE4 are four bytes of uint32 encoded as big endian. +// This is the size of OUTPUT. +// +// You can use github.com/docker/docker/pkg/stdcopy.StdCopy to demultiplex this +// stream. func (cli *Client) ContainerAttach(ctx context.Context, container string, options types.ContainerAttachOptions) (types.HijackedResponse, error) { query := url.Values{} if options.Stream { diff --git a/components/engine/client/container_logs.go b/components/engine/client/container_logs.go index 69056b6321..0f32e9f12b 100644 --- a/components/engine/client/container_logs.go +++ b/components/engine/client/container_logs.go @@ -13,6 +13,26 @@ import ( // ContainerLogs returns the logs generated by a container in an io.ReadCloser. // It's up to the caller to close the stream. +// +// The stream format on the response will be in one of two formats: +// +// If the container is using a TTY, there is only a single stream (stdout), and +// data is copied directly from the container output stream, no extra +// multiplexing or headers. +// +// If the container is *not* using a TTY, streams for stdout and stderr are +// multiplexed. +// The format of the multiplexed stream is as follows: +// +// [8]byte{STREAM_TYPE, 0, 0, 0, SIZE1, SIZE2, SIZE3, SIZE4}[]byte{OUTPUT} +// +// STREAM_TYPE can be 1 for stdout and 2 for stderr +// +// SIZE1, SIZE2, SIZE3, and SIZE4 are four bytes of uint32 encoded as big endian. +// This is the size of OUTPUT. +// +// You can use github.com/docker/docker/pkg/stdcopy.StdCopy to demultiplex this +// stream. func (cli *Client) ContainerLogs(ctx context.Context, container string, options types.ContainerLogsOptions) (io.ReadCloser, error) { query := url.Values{} if options.ShowStdout { From 5f0ae0b377aa8c443e4688d3f42015aac6a15bfd Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Tue, 6 Jun 2017 16:28:40 +0300 Subject: [PATCH 04/56] integration-cli: Replace sleeps with polling in swarm lock/unlock tests This will hopefully make the tests more robust by replacing a fixed 3s sleep with a polling loop that looks at whether the key PEM file is encrypted or not. Signed-off-by: Aaron Lehmann Upstream-commit: 17173efbe00d422392f91c672350266314808b28 Component: engine --- .../integration-cli/docker_cli_swarm_test.go | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/components/engine/integration-cli/docker_cli_swarm_test.go b/components/engine/integration-cli/docker_cli_swarm_test.go index 086a6632d2..a7d4a60aa8 100644 --- a/components/engine/integration-cli/docker_cli_swarm_test.go +++ b/components/engine/integration-cli/docker_cli_swarm_test.go @@ -4,7 +4,9 @@ package main import ( "bytes" + "crypto/x509" "encoding/json" + "encoding/pem" "fmt" "io/ioutil" "net/http" @@ -992,32 +994,35 @@ func getNodeStatus(c *check.C, d *daemon.Swarm) swarm.LocalNodeState { return info.LocalNodeState } -func checkSwarmLockedToUnlocked(c *check.C, d *daemon.Swarm, unlockKey string) { - d.Restart(c) - status := getNodeStatus(c, d) - if status == swarm.LocalNodeStateLocked { - // it must not have updated to be unlocked in time - unlock, wait 3 seconds, and try again - cmd := d.Command("swarm", "unlock") - cmd.Stdin = bytes.NewBufferString(unlockKey) - icmd.RunCmd(cmd).Assert(c, icmd.Success) +func checkKeyIsEncrypted(d *daemon.Swarm) func(*check.C) (interface{}, check.CommentInterface) { + return func(c *check.C) (interface{}, check.CommentInterface) { + keyBytes, err := ioutil.ReadFile(filepath.Join(d.Folder, "root", "swarm", "certificates", "swarm-node.key")) + if err != nil { + return fmt.Errorf("error reading key: %v", err), nil + } - c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive) + keyBlock, _ := pem.Decode(keyBytes) + if keyBlock == nil { + return fmt.Errorf("invalid PEM-encoded private key"), nil + } - time.Sleep(3 * time.Second) - d.Restart(c) + return x509.IsEncryptedPEMBlock(keyBlock), nil } +} +func checkSwarmLockedToUnlocked(c *check.C, d *daemon.Swarm, unlockKey string) { + // Wait for the PEM file to become unencrypted + waitAndAssert(c, defaultReconciliationTimeout, checkKeyIsEncrypted(d), checker.Equals, false) + + d.Restart(c) c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive) } func checkSwarmUnlockedToLocked(c *check.C, d *daemon.Swarm) { + // Wait for the PEM file to become encrypted + waitAndAssert(c, defaultReconciliationTimeout, checkKeyIsEncrypted(d), checker.Equals, true) + d.Restart(c) - status := getNodeStatus(c, d) - if status == swarm.LocalNodeStateActive { - // it must not have updated to be unlocked in time - wait 3 seconds, and try again - time.Sleep(3 * time.Second) - d.Restart(c) - } c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked) } From c41db1dd55cd187d88e9c281a1e3be354cc4f93c Mon Sep 17 00:00:00 2001 From: Deng Guangxing Date: Tue, 20 Jun 2017 11:56:25 +0800 Subject: [PATCH 05/56] fd leak and error handling Signed-off-by: Deng Guangxing Upstream-commit: 0042f992d80b59b97380bbbbeb271fd3030edd90 Component: engine --- .../daemon/logger/jsonfilelog/multireader/multireader.go | 8 ++++++-- components/engine/layer/filestore.go | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/components/engine/daemon/logger/jsonfilelog/multireader/multireader.go b/components/engine/daemon/logger/jsonfilelog/multireader/multireader.go index 06a7307ce4..1993f1d76c 100644 --- a/components/engine/daemon/logger/jsonfilelog/multireader/multireader.go +++ b/components/engine/daemon/logger/jsonfilelog/multireader/multireader.go @@ -46,7 +46,9 @@ func (r *multiReadSeeker) Seek(offset int64, whence int) (int64, error) { rdrOffset := offset - tmpOffset idx := i - rdr.Seek(rdrOffset, os.SEEK_SET) + if _, err := rdr.Seek(rdrOffset, os.SEEK_SET); err != nil { + return -1, err + } // make sure all following readers are at 0 for _, rdr := range r.readers[i+1:] { rdr.Seek(0, os.SEEK_SET) @@ -67,7 +69,9 @@ func (r *multiReadSeeker) Seek(offset int64, whence int) (int64, error) { } tmpOffset += s } - r.Seek(tmpOffset+offset, os.SEEK_SET) + if _, err := r.Seek(tmpOffset+offset, os.SEEK_SET); err != nil { + return -1, err + } return tmpOffset + offset, nil case os.SEEK_CUR: if r.pos == nil { diff --git a/components/engine/layer/filestore.go b/components/engine/layer/filestore.go index 7ea418cd5a..533f45481a 100644 --- a/components/engine/layer/filestore.go +++ b/components/engine/layer/filestore.go @@ -226,6 +226,7 @@ func (fms *fileMetadataStore) TarSplitReader(layer ChainID) (io.ReadCloser, erro } f, err := gzip.NewReader(fz) if err != nil { + fz.Close() return nil, err } From 9b81483a890e6cbf6a6c694d7263dc2f506872fb Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Sat, 24 Jun 2017 23:51:06 +0200 Subject: [PATCH 06/56] Enable bash completion in development container Signed-off-by: Harald Albers Upstream-commit: cec5ca75e1ba2e05095a5ff072c3738684575d86 Component: engine --- components/engine/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index afc7f5f639..a5b35e0ec7 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -217,7 +217,8 @@ COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy dockercli ENV PATH=/usr/local/cli:$PATH -# Activate bash completion if mounted with DOCKER_BASH_COMPLETION_PATH +# Activate bash completion and include Docker's completion if mounted with DOCKER_BASH_COMPLETION_PATH +RUN echo "source /usr/share/bash-completion/bash_completion" >> /etc/bash.bashrc RUN ln -s /usr/local/completion/bash/docker /etc/bash_completion.d/docker # Wrap all commands in the "docker-in-docker" script to allow nested containers From 1c25f0d80ab28d5f10e012623073a48a182f09ab Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 27 Jun 2017 22:41:00 -0700 Subject: [PATCH 07/56] builder: enable path filtering for filesync session Signed-off-by: Tonis Tiigi Upstream-commit: ad46348d7c9f8a84d61e1ba5f08c8fd0727e4ebc Component: engine --- .../engine/builder/dockerfile/builder.go | 3 +-- .../builder/dockerfile/clientsession.go | 19 +++++++++--------- .../client/session/filesync/diffcopy.go | 3 ++- .../client/session/filesync/filesync.go | 20 ++++++++++++++----- .../client/session/filesync/tarstream.go | 2 +- 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/components/engine/builder/dockerfile/builder.go b/components/engine/builder/dockerfile/builder.go index cee1436f92..fb1786225a 100644 --- a/components/engine/builder/dockerfile/builder.go +++ b/components/engine/builder/dockerfile/builder.go @@ -140,8 +140,7 @@ func (bm *BuildManager) initializeClientSession(ctx context.Context, cancel func }() if options.RemoteContext == remotecontext.ClientSessionRemote { st := time.Now() - csi, err := NewClientSessionSourceIdentifier(ctx, bm.sg, - options.SessionID, []string{"/"}) + csi, err := NewClientSessionSourceIdentifier(ctx, bm.sg, options.SessionID) if err != nil { return nil, err } diff --git a/components/engine/builder/dockerfile/clientsession.go b/components/engine/builder/dockerfile/clientsession.go index 647e4537c3..a7709ce517 100644 --- a/components/engine/builder/dockerfile/clientsession.go +++ b/components/engine/builder/dockerfile/clientsession.go @@ -30,26 +30,25 @@ func (cst *ClientSessionTransport) Copy(ctx context.Context, id fscache.RemoteId } return filesync.FSSync(ctx, csi.caller, filesync.FSSendRequestOpt{ - SrcPaths: csi.srcPaths, - DestDir: dest, - CacheUpdater: cu, + IncludePatterns: csi.includePatterns, + DestDir: dest, + CacheUpdater: cu, }) } // ClientSessionSourceIdentifier is an identifier that can be used for requesting // files from remote client type ClientSessionSourceIdentifier struct { - srcPaths []string - caller session.Caller - sharedKey string - uuid string + includePatterns []string + caller session.Caller + sharedKey string + uuid string } // NewClientSessionSourceIdentifier returns new ClientSessionSourceIdentifier instance -func NewClientSessionSourceIdentifier(ctx context.Context, sg SessionGetter, uuid string, sources []string) (*ClientSessionSourceIdentifier, error) { +func NewClientSessionSourceIdentifier(ctx context.Context, sg SessionGetter, uuid string) (*ClientSessionSourceIdentifier, error) { csi := &ClientSessionSourceIdentifier{ - uuid: uuid, - srcPaths: sources, + uuid: uuid, } caller, err := sg.Get(ctx, uuid) if err != nil { diff --git a/components/engine/client/session/filesync/diffcopy.go b/components/engine/client/session/filesync/diffcopy.go index b15e4ee4bf..533847acdf 100644 --- a/components/engine/client/session/filesync/diffcopy.go +++ b/components/engine/client/session/filesync/diffcopy.go @@ -9,9 +9,10 @@ import ( "github.com/tonistiigi/fsutil" ) -func sendDiffCopy(stream grpc.Stream, dir string, excludes []string, progress progressCb) error { +func sendDiffCopy(stream grpc.Stream, dir string, includes, excludes []string, progress progressCb) error { return fsutil.Send(stream.Context(), stream, dir, &fsutil.WalkOpt{ ExcludePatterns: excludes, + IncludePaths: includes, // TODO: rename IncludePatterns }, progress) } diff --git a/components/engine/client/session/filesync/filesync.go b/components/engine/client/session/filesync/filesync.go index fa6dafb6b0..9a2ffc8578 100644 --- a/components/engine/client/session/filesync/filesync.go +++ b/components/engine/client/session/filesync/filesync.go @@ -12,6 +12,11 @@ import ( "google.golang.org/grpc/metadata" ) +const ( + keyOverrideExcludes = "override-excludes" + keyIncludePatterns = "include-patterns" +) + type fsSyncProvider struct { root string excludes []string @@ -54,9 +59,10 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error opts, _ := metadata.FromContext(stream.Context()) // if no metadata continue with empty object var excludes []string - if len(opts["Override-Excludes"]) == 0 || opts["Override-Excludes"][0] != "true" { + if len(opts[keyOverrideExcludes]) == 0 || opts[keyOverrideExcludes][0] != "true" { excludes = sp.excludes } + includes := opts[keyIncludePatterns] var progress progressCb if sp.p != nil { @@ -69,7 +75,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) error doneCh = sp.doneCh sp.doneCh = nil } - err := pr.sendFn(stream, sp.root, excludes, progress) + err := pr.sendFn(stream, sp.root, includes, excludes, progress) if doneCh != nil { if err != nil { doneCh <- err @@ -88,7 +94,7 @@ type progressCb func(int, bool) type protocol struct { name string - sendFn func(stream grpc.Stream, srcDir string, excludes []string, progress progressCb) error + sendFn func(stream grpc.Stream, srcDir string, includes, excludes []string, progress progressCb) error recvFn func(stream grpc.Stream, destDir string, cu CacheUpdater) error } @@ -115,7 +121,7 @@ var supportedProtocols = []protocol{ // FSSendRequestOpt defines options for FSSend request type FSSendRequestOpt struct { - SrcPaths []string + IncludePatterns []string OverrideExcludes bool DestDir string CacheUpdater CacheUpdater @@ -142,7 +148,11 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error { opts := make(map[string][]string) if opt.OverrideExcludes { - opts["Override-Excludes"] = []string{"true"} + opts[keyOverrideExcludes] = []string{"true"} + } + + if opt.IncludePatterns != nil { + opts[keyIncludePatterns] = opt.IncludePatterns } ctx, cancel := context.WithCancel(ctx) diff --git a/components/engine/client/session/filesync/tarstream.go b/components/engine/client/session/filesync/tarstream.go index ee01e30a75..da139ebf5d 100644 --- a/components/engine/client/session/filesync/tarstream.go +++ b/components/engine/client/session/filesync/tarstream.go @@ -10,7 +10,7 @@ import ( "google.golang.org/grpc" ) -func sendTarStream(stream grpc.Stream, dir string, excludes []string, progress progressCb) error { +func sendTarStream(stream grpc.Stream, dir string, includes, excludes []string, progress progressCb) error { a, err := archive.TarWithOptions(dir, &archive.TarOptions{ ExcludePatterns: excludes, }) From 77cf31b37eaec07e56cf67b51932c395e13b7203 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 27 Jun 2017 22:42:28 -0700 Subject: [PATCH 08/56] add test for filesync path filtering and testutil helper Signed-off-by: Tonis Tiigi Upstream-commit: 4141d8fe5da596e7ee1eec217378aeb684d0a99e Component: engine --- .../client/session/filesync/filesync_test.go | 71 +++++++++++++++++++ components/engine/client/session/manager.go | 21 +++++- .../client/session/testutil/testutil.go | 70 ++++++++++++++++++ 3 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 components/engine/client/session/filesync/filesync_test.go create mode 100644 components/engine/client/session/testutil/testutil.go diff --git a/components/engine/client/session/filesync/filesync_test.go b/components/engine/client/session/filesync/filesync_test.go new file mode 100644 index 0000000000..b48c08b826 --- /dev/null +++ b/components/engine/client/session/filesync/filesync_test.go @@ -0,0 +1,71 @@ +package filesync + +import ( + "context" + "io/ioutil" + "path/filepath" + "testing" + + "github.com/docker/docker/client/session" + "github.com/docker/docker/client/session/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +func TestFileSyncIncludePatterns(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "fsynctest") + require.NoError(t, err) + + destDir, err := ioutil.TempDir("", "fsynctest") + require.NoError(t, err) + + err = ioutil.WriteFile(filepath.Join(tmpDir, "foo"), []byte("content1"), 0600) + require.NoError(t, err) + + err = ioutil.WriteFile(filepath.Join(tmpDir, "bar"), []byte("content2"), 0600) + require.NoError(t, err) + + s, err := session.NewSession("foo", "bar") + require.NoError(t, err) + + m, err := session.NewManager() + require.NoError(t, err) + + fs := NewFSSyncProvider(tmpDir, nil) + s.Allow(fs) + + dialer := session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn))) + + g, ctx := errgroup.WithContext(context.Background()) + + g.Go(func() error { + return s.Run(ctx, dialer) + }) + + g.Go(func() (reterr error) { + c, err := m.Get(ctx, s.UUID()) + if err != nil { + return err + } + if err := FSSync(ctx, c, FSSendRequestOpt{ + DestDir: destDir, + IncludePatterns: []string{"ba*"}, + }); err != nil { + return err + } + + _, err = ioutil.ReadFile(filepath.Join(destDir, "foo")) + assert.Error(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "bar")) + if err != nil { + return err + } + assert.Equal(t, "content2", string(dt)) + return s.Close() + }) + + err = g.Wait() + require.NoError(t, err) +} diff --git a/components/engine/client/session/manager.go b/components/engine/client/session/manager.go index 023e850301..9523e6f317 100644 --- a/components/engine/client/session/manager.go +++ b/components/engine/client/session/manager.go @@ -1,6 +1,7 @@ package session import ( + "net" "net/http" "strings" "sync" @@ -49,8 +50,6 @@ func (sm *Manager) HandleHTTPRequest(ctx context.Context, w http.ResponseWriter, } uuid := r.Header.Get(headerSessionUUID) - name := r.Header.Get(headerSessionName) - sharedKey := r.Header.Get(headerSessionSharedKey) proto := r.Header.Get("Upgrade") @@ -89,9 +88,25 @@ func (sm *Manager) HandleHTTPRequest(ctx context.Context, w http.ResponseWriter, conn.Write([]byte{}) resp.Write(conn) + return sm.handleConn(ctx, conn, r.Header) +} + +// HandleConn handles an incoming raw connection +func (sm *Manager) HandleConn(ctx context.Context, conn net.Conn, opts map[string][]string) error { + sm.mu.Lock() + return sm.handleConn(ctx, conn, opts) +} + +// caller needs to take lock, this function will release it +func (sm *Manager) handleConn(ctx context.Context, conn net.Conn, opts map[string][]string) error { ctx, cancel := context.WithCancel(ctx) defer cancel() + h := http.Header(opts) + uuid := h.Get(headerSessionUUID) + name := h.Get(headerSessionName) + sharedKey := h.Get(headerSessionSharedKey) + ctx, cc, err := grpcClientConn(ctx, conn) if err != nil { sm.mu.Unlock() @@ -111,7 +126,7 @@ func (sm *Manager) HandleHTTPRequest(ctx context.Context, w http.ResponseWriter, supported: make(map[string]struct{}), } - for _, m := range r.Header[headerSessionMethod] { + for _, m := range opts[headerSessionMethod] { c.supported[strings.ToLower(m)] = struct{}{} } sm.sessions[uuid] = c diff --git a/components/engine/client/session/testutil/testutil.go b/components/engine/client/session/testutil/testutil.go new file mode 100644 index 0000000000..2e145d9006 --- /dev/null +++ b/components/engine/client/session/testutil/testutil.go @@ -0,0 +1,70 @@ +package testutil + +import ( + "io" + "net" + "time" + + "github.com/Sirupsen/logrus" + "golang.org/x/net/context" +) + +// Handler is function called to handle incoming connection +type Handler func(ctx context.Context, conn net.Conn, meta map[string][]string) error + +// Dialer is a function for dialing an outgoing connection +type Dialer func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) + +// TestStream creates an in memory session dialer for a handler function +func TestStream(handler Handler) Dialer { + s1, s2 := sockPair() + return func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) { + go func() { + err := handler(context.TODO(), s1, meta) + if err != nil { + logrus.Error(err) + } + s1.Close() + }() + return s2, nil + } +} + +func sockPair() (*sock, *sock) { + pr1, pw1 := io.Pipe() + pr2, pw2 := io.Pipe() + return &sock{pw1, pr2, pw1}, &sock{pw2, pr1, pw2} +} + +type sock struct { + io.Writer + io.Reader + io.Closer +} + +func (s *sock) LocalAddr() net.Addr { + return dummyAddr{} +} +func (s *sock) RemoteAddr() net.Addr { + return dummyAddr{} +} +func (s *sock) SetDeadline(t time.Time) error { + return nil +} +func (s *sock) SetReadDeadline(t time.Time) error { + return nil +} +func (s *sock) SetWriteDeadline(t time.Time) error { + return nil +} + +type dummyAddr struct { +} + +func (d dummyAddr) Network() string { + return "tcp" +} + +func (d dummyAddr) String() string { + return "localhost" +} From c71c867f512abe859cf9fc55364e23362ab1a420 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Thu, 29 Jun 2017 16:04:47 -0700 Subject: [PATCH 09/56] middleware: Redact secret data on "secret create" With debug logging turned on, we currently log the base64-encoded secret payload. Change the middleware code to redact this. Since the field is called "Data", it requires some context-sensitivity. The URI path is examined to see which route is being invoked. Signed-off-by: Aaron Lehmann Upstream-commit: 3fbc352cbbce06cd3001d6b14b2b1ebcb4d42cd5 Component: engine --- .../engine/api/server/middleware/debug.go | 26 +++++++-- .../api/server/middleware/debug_test.go | 58 +++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 components/engine/api/server/middleware/debug_test.go diff --git a/components/engine/api/server/middleware/debug.go b/components/engine/api/server/middleware/debug.go index b931f1906b..a9a94e7f33 100644 --- a/components/engine/api/server/middleware/debug.go +++ b/components/engine/api/server/middleware/debug.go @@ -41,7 +41,7 @@ func DebugRequestMiddleware(handler func(ctx context.Context, w http.ResponseWri var postForm map[string]interface{} if err := json.Unmarshal(b, &postForm); err == nil { - maskSecretKeys(postForm) + maskSecretKeys(postForm, r.RequestURI) formStr, errMarshal := json.Marshal(postForm) if errMarshal == nil { logrus.Debugf("form data: %s", string(formStr)) @@ -54,13 +54,22 @@ func DebugRequestMiddleware(handler func(ctx context.Context, w http.ResponseWri } } -func maskSecretKeys(inp interface{}) { +func maskSecretKeys(inp interface{}, path string) { + // Remove any query string from the path + idx := strings.Index(path, "?") + if idx != -1 { + path = path[:idx] + } + // Remove trailing / characters + path = strings.TrimRight(path, "/") + if arr, ok := inp.([]interface{}); ok { for _, f := range arr { - maskSecretKeys(f) + maskSecretKeys(f, path) } return } + if form, ok := inp.(map[string]interface{}); ok { loop0: for k, v := range form { @@ -70,7 +79,16 @@ func maskSecretKeys(inp interface{}) { continue loop0 } } - maskSecretKeys(v) + maskSecretKeys(v, path) + } + + // Route-specific redactions + if strings.HasSuffix(path, "/secrets/create") { + for k := range form { + if k == "Data" { + form[k] = "*****" + } + } } } } diff --git a/components/engine/api/server/middleware/debug_test.go b/components/engine/api/server/middleware/debug_test.go new file mode 100644 index 0000000000..87ecafd145 --- /dev/null +++ b/components/engine/api/server/middleware/debug_test.go @@ -0,0 +1,58 @@ +package middleware + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMaskSecretKeys(t *testing.T) { + tests := []struct { + path string + input map[string]interface{} + expected map[string]interface{} + }{ + { + path: "/v1.30/secrets/create", + input: map[string]interface{}{"Data": "foo", "Name": "name", "Labels": map[string]interface{}{}}, + expected: map[string]interface{}{"Data": "*****", "Name": "name", "Labels": map[string]interface{}{}}, + }, + { + path: "/v1.30/secrets/create//", + input: map[string]interface{}{"Data": "foo", "Name": "name", "Labels": map[string]interface{}{}}, + expected: map[string]interface{}{"Data": "*****", "Name": "name", "Labels": map[string]interface{}{}}, + }, + + { + path: "/secrets/create?key=val", + input: map[string]interface{}{"Data": "foo", "Name": "name", "Labels": map[string]interface{}{}}, + expected: map[string]interface{}{"Data": "*****", "Name": "name", "Labels": map[string]interface{}{}}, + }, + { + path: "/v1.30/some/other/path", + input: map[string]interface{}{ + "password": "pass", + "other": map[string]interface{}{ + "secret": "secret", + "jointoken": "jointoken", + "unlockkey": "unlockkey", + "signingcakey": "signingcakey", + }, + }, + expected: map[string]interface{}{ + "password": "*****", + "other": map[string]interface{}{ + "secret": "*****", + "jointoken": "*****", + "unlockkey": "*****", + "signingcakey": "*****", + }, + }, + }, + } + + for _, testcase := range tests { + maskSecretKeys(testcase.input, testcase.path) + assert.Equal(t, testcase.expected, testcase.input) + } +} From 5852d59e6d8e5a702123a845ea090a666a8f9136 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 29 Jun 2017 22:08:42 -0700 Subject: [PATCH 10/56] Fix NewVersionError() for clients using default version The NewVersionError checks if the client is using the API version required for using a specific feature. If the client is initialized without setting a specific version, an error would be generated because it was not possible to compare versions. However, a client without explicit version set is running the latest supported version. This patch changes the behavior to only generate an error if a version was set. Signed-off-by: Sebastiaan van Stijn Upstream-commit: ff2ed1853099a210245814b4263ce1c92b14c153 Component: engine --- components/engine/client/errors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/client/errors.go b/components/engine/client/errors.go index e0effafc0a..fc7df9f1eb 100644 --- a/components/engine/client/errors.go +++ b/components/engine/client/errors.go @@ -228,7 +228,7 @@ func IsErrPluginPermissionDenied(err error) bool { // NewVersionError returns an error if the APIVersion required // if less than the current supported version func (cli *Client) NewVersionError(APIrequired, feature string) error { - if versions.LessThan(cli.version, APIrequired) { + if cli.version != "" && versions.LessThan(cli.version, APIrequired) { return fmt.Errorf("%q requires API version %s, but the Docker daemon API version is %s", feature, APIrequired, cli.version) } return nil From 7e46ec70677ecfbbb171a36592ae17b877c0279f Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Mon, 26 Jun 2017 14:54:14 -0400 Subject: [PATCH 11/56] Make plugin removes more resilient to failure Before this patch, if the plugin's `config.json` is successfully removed but the main plugin state dir could not be removed for some reason (e.g. leaked mount), it will prevent the daemon from being able to be restarted. This patches changes this to atomically remove the plugin such that on daemon restart we can detect that there was an error and re-try. It also changes the logic so that it only logs errors on restore rather than erroring out the daemon. This also removes some code which is now duplicated elsewhere. Signed-off-by: Brian Goff Upstream-commit: 11cf394e5ea964636294a219872b188fe5bdf4dd Component: engine --- .../integration-cli/docker_cli_daemon_test.go | 46 +++++++++++++++++++ components/engine/plugin/backend_linux.go | 39 +++++----------- components/engine/plugin/manager.go | 23 +++++++++- components/engine/plugin/manager_linux.go | 2 +- 4 files changed, 81 insertions(+), 29 deletions(-) diff --git a/components/engine/integration-cli/docker_cli_daemon_test.go b/components/engine/integration-cli/docker_cli_daemon_test.go index 9046fe6ea6..2a295c946e 100644 --- a/components/engine/integration-cli/docker_cli_daemon_test.go +++ b/components/engine/integration-cli/docker_cli_daemon_test.go @@ -5,6 +5,7 @@ package main import ( "bufio" "bytes" + "context" "encoding/json" "fmt" "io" @@ -25,6 +26,9 @@ import ( "crypto/x509" "github.com/cloudflare/cfssl/helpers" + "github.com/docker/docker/api" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" "github.com/docker/docker/integration-cli/checker" "github.com/docker/docker/integration-cli/cli" "github.com/docker/docker/integration-cli/daemon" @@ -2980,3 +2984,45 @@ func (s *DockerDaemonSuite) TestShmSizeReload(c *check.C) { c.Assert(err, check.IsNil, check.Commentf("Output: %s", out)) c.Assert(strings.TrimSpace(out), check.Equals, fmt.Sprintf("%v", size)) } + +// TestFailedPluginRemove makes sure that a failed plugin remove does not block +// the daemon from starting +func (s *DockerDaemonSuite) TestFailedPluginRemove(c *check.C) { + testRequires(c, DaemonIsLinux, IsAmd64, SameHostDaemon) + d := daemon.New(c, dockerBinary, dockerdBinary, daemon.Config{}) + d.Start(c) + cli, err := client.NewClient(d.Sock(), api.DefaultVersion, nil, nil) + c.Assert(err, checker.IsNil) + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + + name := "test-plugin-rm-fail" + out, err := cli.PluginInstall(ctx, name, types.PluginInstallOptions{ + Disabled: true, + AcceptAllPermissions: true, + RemoteRef: "cpuguy83/docker-logdriver-test", + }) + c.Assert(err, checker.IsNil) + defer out.Close() + io.Copy(ioutil.Discard, out) + + ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + p, _, err := cli.PluginInspectWithRaw(ctx, name) + c.Assert(err, checker.IsNil) + + // simulate a bad/partial removal by removing the plugin config. + configPath := filepath.Join(d.Root, "plugins", p.ID, "config.json") + c.Assert(os.Remove(configPath), checker.IsNil) + + d.Restart(c) + ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, err = cli.Ping(ctx) + c.Assert(err, checker.IsNil) + + _, _, err = cli.PluginInspectWithRaw(ctx, name) + // plugin should be gone since the config.json is gone + c.Assert(err, checker.NotNil) +} diff --git a/components/engine/plugin/backend_linux.go b/components/engine/plugin/backend_linux.go index c2baab5c94..b22cc155d2 100644 --- a/components/engine/plugin/backend_linux.go +++ b/components/engine/plugin/backend_linux.go @@ -13,7 +13,6 @@ import ( "os" "path" "path/filepath" - "sort" "strings" "github.com/Sirupsen/logrus" @@ -32,6 +31,7 @@ import ( "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/progress" + "github.com/docker/docker/pkg/system" "github.com/docker/docker/plugin/v2" refstore "github.com/docker/docker/reference" "github.com/opencontainers/go-digest" @@ -624,14 +624,20 @@ func (pm *Manager) Remove(name string, config *types.PluginRmConfig) error { }() id := p.GetID() - pm.config.Store.Remove(p) pluginDir := filepath.Join(pm.config.Root, id) - if err := recursiveUnmount(pluginDir); err != nil { - logrus.WithField("dir", pluginDir).WithField("id", id).Warn(err) + + if err := mount.RecursiveUnmount(pluginDir); err != nil { + return errors.Wrap(err, "error unmounting plugin data") } - if err := os.RemoveAll(pluginDir); err != nil { - logrus.Warnf("unable to remove %q from plugin remove: %v", pluginDir, err) + + if err := os.Rename(pluginDir, pluginDir+"-removing"); err != nil { + return errors.Wrap(err, "error performing atomic remove of plugin dir") } + + if err := system.EnsureRemoveAll(pluginDir); err != nil { + return errors.Wrap(err, "error removing plugin dir") + } + pm.config.Store.Remove(p) pm.config.LogPluginEvent(id, name, "remove") return nil } @@ -652,27 +658,6 @@ func getMounts(root string) ([]string, error) { return mounts, nil } -func recursiveUnmount(root string) error { - mounts, err := getMounts(root) - if err != nil { - return err - } - - // sort in reverse-lexicographic order so the root mount will always be last - sort.Sort(sort.Reverse(sort.StringSlice(mounts))) - - for i, m := range mounts { - if err := mount.Unmount(m); err != nil { - if i == len(mounts)-1 { - return errors.Wrapf(err, "error performing recursive unmount on %s", root) - } - logrus.WithError(err).WithField("mountpoint", m).Warn("could not unmount") - } - } - - return nil -} - // Set sets plugin args func (pm *Manager) Set(name string, args []string) error { p, err := pm.config.Store.GetV2Plugin(name) diff --git a/components/engine/plugin/manager.go b/components/engine/plugin/manager.go index 02d372741f..580152a930 100644 --- a/components/engine/plugin/manager.go +++ b/components/engine/plugin/manager.go @@ -163,6 +163,19 @@ func (pm *Manager) StateChanged(id string, e libcontainerd.StateInfo) error { return nil } +func handleLoadError(err error, id string) { + if err == nil { + return + } + logger := logrus.WithError(err).WithField("id", id) + if os.IsNotExist(errors.Cause(err)) { + // Likely some error while removing on an older version of docker + logger.Warn("missing plugin config, skipping: this may be caused due to a failed remove and requires manual cleanup.") + return + } + logger.Error("error loading plugin, skipping") +} + func (pm *Manager) reload() error { // todo: restore dir, err := ioutil.ReadDir(pm.config.Root) if err != nil { @@ -173,9 +186,17 @@ func (pm *Manager) reload() error { // todo: restore if validFullID.MatchString(v.Name()) { p, err := pm.loadPlugin(v.Name()) if err != nil { - return err + handleLoadError(err, v.Name()) + continue } plugins[p.GetID()] = p + } else { + if validFullID.MatchString(strings.TrimSuffix(v.Name(), "-removing")) { + // There was likely some error while removing this plugin, let's try to remove again here + if err := system.EnsureRemoveAll(v.Name()); err != nil { + logrus.WithError(err).WithField("id", v.Name()).Warn("error while attempting to clean up previously removed plugin") + } + } } } diff --git a/components/engine/plugin/manager_linux.go b/components/engine/plugin/manager_linux.go index 1e8994043f..678be84b3f 100644 --- a/components/engine/plugin/manager_linux.go +++ b/components/engine/plugin/manager_linux.go @@ -204,7 +204,7 @@ func (pm *Manager) upgradePlugin(p *v2.Plugin, configDigest digest.Digest, blobs // Make sure nothing is mounted // This could happen if the plugin was disabled with `-f` with active mounts. // If there is anything in `orig` is still mounted, this should error out. - if err := recursiveUnmount(orig); err != nil { + if err := mount.RecursiveUnmount(orig); err != nil { return err } From 56378645d09be13db1ca6bf754157d05432d3155 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Fri, 30 Jun 2017 14:27:26 -0400 Subject: [PATCH 12/56] Wait for device removal if deferredRemoval=true and deferredDeletion=false There have been some cases where umount, a device can be busy for a very short duration. Maybe its udev rules, or maybe it is runc related races or probably it is something else. We don't know yet. If deferred removal is enabled but deferred deletion is not, then for the case of "docker run -ti --rm fedora bash", a container will exit, device will be deferred removed and then immediately a call will come to delete the device. It is possible that deletion will fail if device was busy at that time. A device can't be deleted if it can't be removed/deactivated first. There is only one exception and that is when deferred deletion is on. In that case graph driver will keep track of deleted device and try to delete it later and return success to caller. Always make sure that device deactivation is synchronous when device is being deleted (except the case when deferred deletion is enabled). This should also take care of small races when device is busy for a short duration and it is being deleted. Signed-off-by: Vivek Goyal Upstream-commit: 36cb6efebc599900b691e206fb9e99d3aa2fb9a3 Component: engine --- .../daemon/graphdriver/devmapper/deviceset.go | 33 ++++++++++++++----- .../engine/pkg/devicemapper/devmapper.go | 8 +++++ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/components/engine/daemon/graphdriver/devmapper/deviceset.go b/components/engine/daemon/graphdriver/devmapper/deviceset.go index c0116c7993..3218e8a8cc 100644 --- a/components/engine/daemon/graphdriver/devmapper/deviceset.go +++ b/components/engine/daemon/graphdriver/devmapper/deviceset.go @@ -2088,7 +2088,16 @@ func (devices *DeviceSet) deleteDevice(info *devInfo, syncDelete bool) error { } // Try to deactivate device in case it is active. - if err := devices.deactivateDevice(info); err != nil { + // If deferred removal is enabled and deferred deletion is disabled + // then make sure device is removed synchronously. There have been + // some cases of device being busy for short duration and we would + // rather busy wait for device removal to take care of these cases. + deferredRemove := devices.deferredRemove + if !devices.deferredDelete { + deferredRemove = false + } + + if err := devices.deactivateDeviceMode(info, deferredRemove); err != nil { logrus.Debugf("devmapper: Error deactivating device: %s", err) return err } @@ -2145,6 +2154,11 @@ func (devices *DeviceSet) deactivatePool() error { } func (devices *DeviceSet) deactivateDevice(info *devInfo) error { + return devices.deactivateDeviceMode(info, devices.deferredRemove) +} + +func (devices *DeviceSet) deactivateDeviceMode(info *devInfo, deferredRemove bool) error { + var err error logrus.Debugf("devmapper: deactivateDevice START(%s)", info.Hash) defer logrus.Debugf("devmapper: deactivateDevice END(%s)", info.Hash) @@ -2157,14 +2171,17 @@ func (devices *DeviceSet) deactivateDevice(info *devInfo) error { return nil } - if devices.deferredRemove { - if err := devicemapper.RemoveDeviceDeferred(info.Name()); err != nil { - return err - } + if deferredRemove { + err = devicemapper.RemoveDeviceDeferred(info.Name()) } else { - if err := devices.removeDevice(info.Name()); err != nil { - return err - } + err = devices.removeDevice(info.Name()) + } + + // This function's semantics is such that it does not return an + // error if device does not exist. So if device went away by + // the time we actually tried to remove it, do not return error. + if err != devicemapper.ErrEnxio { + return err } return nil } diff --git a/components/engine/pkg/devicemapper/devmapper.go b/components/engine/pkg/devicemapper/devmapper.go index 73433c0e5a..fa77fc6cb1 100644 --- a/components/engine/pkg/devicemapper/devmapper.go +++ b/components/engine/pkg/devicemapper/devmapper.go @@ -336,10 +336,14 @@ func RemoveDevice(name string) error { defer UdevWait(cookie) dmSawBusy = false // reset before the task is run + dmSawEnxio = false if err = task.run(); err != nil { if dmSawBusy { return ErrBusy } + if dmSawEnxio { + return ErrEnxio + } return fmt.Errorf("devicemapper: Error running RemoveDevice %s", err) } @@ -380,7 +384,11 @@ func RemoveDeviceDeferred(name string) error { // by udev, what UdevWait is just cleaning up the semaphore. defer UdevWait(cookie) + dmSawEnxio = false if err = task.run(); err != nil { + if dmSawEnxio { + return ErrEnxio + } return fmt.Errorf("devicemapper: Error running RemoveDeviceDeferred %s", err) } From f8f582f3037b0ea702e918b36a332ca32ec2243d Mon Sep 17 00:00:00 2001 From: Wentao Zhang Date: Wed, 28 Jun 2017 04:14:58 +0800 Subject: [PATCH 13/56] Set unpasued state when receiving 'stateExit' event Description: 1. start a container with restart=always. `docker run -d --restart=always ubuntu sleep 3` 2. container init process exits. 3. use `docker pause ` to pause this container. if the pause action is before cgroup data is removed and after the init process died. `Pause` operation will success to write cgroup data, but actually do not freeze any process. And then docker received pause event and stateExit event from containerd, the docker state will be Running(paused), but the container is free running. Then we can not remove it, stop it , pause it and unpause it. Signed-off-by: Wentao Zhang Upstream-commit: fe1b4cfba6320793373c5397641d743d9fe94cf8 Component: engine --- components/engine/container/state.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/container/state.go b/components/engine/container/state.go index b87577b1c0..4ded406411 100644 --- a/components/engine/container/state.go +++ b/components/engine/container/state.go @@ -278,6 +278,7 @@ func (s *State) SetRunning(pid int, initial bool) { s.ErrorMsg = "" s.Running = true s.Restarting = false + s.Paused = false s.ExitCodeValue = 0 s.Pid = pid if initial { @@ -304,6 +305,7 @@ func (s *State) SetRestarting(exitStatus *ExitStatus) { // all the checks in docker around rm/stop/etc s.Running = true s.Restarting = true + s.Paused = false s.Pid = 0 s.FinishedAt = time.Now().UTC() s.setFromExitStatus(exitStatus) From 9421df98fb3cdf9564023f887f8aeb7d05514623 Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Wed, 28 Jun 2017 01:44:31 +1000 Subject: [PATCH 14/56] devicemapper: remove 256 character limit of libdm logs This limit is unecessary and can lead to the truncation of long libdm logs (which is quite annoying). Fixes: b440ec013 ("device-mapper: Move all devicemapper spew to log through utils.Debugf().") Signed-off-by: Aleksa Sarai Upstream-commit: 63328c6882c3d1f54c66499ef9963adfbf1883f0 Component: engine --- .../engine/pkg/devicemapper/devmapper_wrapper.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper.go b/components/engine/pkg/devicemapper/devmapper_wrapper.go index 4f270fba0b..cdd9db95ff 100644 --- a/components/engine/pkg/devicemapper/devmapper_wrapper.go +++ b/components/engine/pkg/devicemapper/devmapper_wrapper.go @@ -4,6 +4,7 @@ package devicemapper /* #cgo LDFLAGS: -L. -ldevmapper +#define _GNU_SOURCE #include #include // FIXME: present only for BLKGETSIZE64, maybe we can remove it? @@ -12,19 +13,20 @@ extern void DevmapperLogCallback(int level, char *file, int line, int dm_errno_o static void log_cb(int level, const char *file, int line, int dm_errno_or_class, const char *f, ...) { - char buffer[256]; - va_list ap; + char *buffer = NULL; + va_list ap; - va_start(ap, f); - vsnprintf(buffer, 256, f, ap); - va_end(ap); + va_start(ap, f); + vasprintf(&buffer, f, ap); + va_end(ap); - DevmapperLogCallback(level, (char *)file, line, dm_errno_or_class, buffer); + DevmapperLogCallback(level, (char *)file, line, dm_errno_or_class, buffer); + free(buffer); } static void log_with_errno_init() { - dm_log_with_errno_init(log_cb); + dm_log_with_errno_init(log_cb); } */ import "C" From 01199b4ca46299b10d95aa87ac83dc41da9d7451 Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Wed, 28 Jun 2017 01:45:24 +1000 Subject: [PATCH 15/56] devicemapper: actually remove DmLogInitVerbose e07d3cd9a ("devmapper: Fix libdm logging") removed all of the callers of DmLogInitVerbose, but we still kept around the wrapper. However, the libdm dm_log_init_verbose API changes the verbosity of the *default* libdm logger. Because pkg/devicemapper internally *relies* on using logging callbacks to understand what errors were encountered by libdm, this wrapper is useless (it only makes sense for the default logger which we do not user). Any user not inside Docker of this function almost certainly was not using this API correctly, because pkg/devicemapper will misbehave if our logging callbacks were not registered. Signed-off-by: Aleksa Sarai Upstream-commit: c654147f0635736fff0035ecdfa53a41c7fad0d0 Component: engine --- components/engine/pkg/devicemapper/devmapper.go | 5 ----- components/engine/pkg/devicemapper/devmapper_wrapper.go | 5 ----- 2 files changed, 10 deletions(-) diff --git a/components/engine/pkg/devicemapper/devmapper.go b/components/engine/pkg/devicemapper/devmapper.go index 73433c0e5a..8f35914366 100644 --- a/components/engine/pkg/devicemapper/devmapper.go +++ b/components/engine/pkg/devicemapper/devmapper.go @@ -264,11 +264,6 @@ func UdevWait(cookie *uint) error { return nil } -// LogInitVerbose is an interface to initialize the verbose logger for the device mapper library. -func LogInitVerbose(level int) { - DmLogInitVerbose(level) -} - var dmLogger DevmapperLogger // LogInit initializes the logger for the device mapper library. diff --git a/components/engine/pkg/devicemapper/devmapper_wrapper.go b/components/engine/pkg/devicemapper/devmapper_wrapper.go index cdd9db95ff..0fb70fe5b3 100644 --- a/components/engine/pkg/devicemapper/devmapper_wrapper.go +++ b/components/engine/pkg/devicemapper/devmapper_wrapper.go @@ -58,7 +58,6 @@ const ( var ( DmGetLibraryVersion = dmGetLibraryVersionFct DmGetNextTarget = dmGetNextTargetFct - DmLogInitVerbose = dmLogInitVerboseFct DmSetDevDir = dmSetDevDirFct DmTaskAddTarget = dmTaskAddTargetFct DmTaskCreate = dmTaskCreateFct @@ -228,10 +227,6 @@ func dmCookieSupportedFct() int { return int(C.dm_cookie_supported()) } -func dmLogInitVerboseFct(level int) { - C.dm_log_init_verbose(C.int(level)) -} - func logWithErrnoInitFct() { C.log_with_errno_init() } From 00264ebf6bfddc31d3313dc5fb4519a28e63bbfc Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Wed, 28 Jun 2017 01:46:47 +1000 Subject: [PATCH 16/56] devicemapper: change LogInit and move all callbacks to pkg LogInit used to act as a manual way of registering the *necessary* pkg/devicemapper logging callbacks. In addition, it was used to split up the logic of pkg/devicemapper into daemon/graphdriver/devmapper (such that some things were logged from libdm). The manual aspect of this API was completely non-sensical and was just begging for incorrect usage of pkg/devicemapper, so remove that semantic and always register our own libdm callbacks. In addition, recombine the split out logging callbacks into pkg/devicemapper so that the default logger is local to the library and also shown to be the recommended logger. This makes the code substantially easier to read. Also the new DefaultLogger now has configurable upper-bound for the log level, which allows for dynamically changing the logging level. Signed-off-by: Aleksa Sarai Upstream-commit: cfd39e8d6d79eee8a71fbe6820dd67babf05d951 Component: engine --- .../daemon/graphdriver/devmapper/deviceset.go | 41 ++------- .../engine/pkg/devicemapper/devmapper.go | 13 --- .../engine/pkg/devicemapper/devmapper_log.go | 92 ++++++++++++++++++- 3 files changed, 96 insertions(+), 50 deletions(-) diff --git a/components/engine/daemon/graphdriver/devmapper/deviceset.go b/components/engine/daemon/graphdriver/devmapper/deviceset.go index c0116c7993..c785962ceb 100644 --- a/components/engine/daemon/graphdriver/devmapper/deviceset.go +++ b/components/engine/daemon/graphdriver/devmapper/deviceset.go @@ -35,17 +35,13 @@ import ( ) var ( - defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 - defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 - defaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 - defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors - defaultUdevSyncOverride = false - maxDeviceID = 0xffffff // 24 bit, pool limit - deviceIDMapSz = (maxDeviceID + 1) / 8 - // We retry device removal so many a times that even error messages - // will fill up console during normal operation. So only log Fatal - // messages by default. - logLevel = devicemapper.LogLevelFatal + defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 + defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 + defaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 + defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors + defaultUdevSyncOverride = false + maxDeviceID = 0xffffff // 24 bit, pool limit + deviceIDMapSz = (maxDeviceID + 1) / 8 driverDeferredRemovalSupport = false enableDeferredRemoval = false enableDeferredDeletion = false @@ -1273,26 +1269,6 @@ func setCloseOnExec(name string) { } } -// DMLog implements logging using DevMapperLogger interface. -func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) { - // By default libdm sends us all the messages including debug ones. - // We need to filter out messages here and figure out which one - // should be printed. - if level > logLevel { - return - } - - // FIXME(vbatts) push this back into ./pkg/devicemapper/ - if level <= devicemapper.LogLevelErr { - logrus.Errorf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) - } else if level <= devicemapper.LogLevelInfo { - logrus.Infof("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) - } else { - // FIXME(vbatts) push this back into ./pkg/devicemapper/ - logrus.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) - } -} - func major(device uint64) uint64 { return (device >> 8) & 0xfff } @@ -1690,9 +1666,6 @@ func (devices *DeviceSet) enableDeferredRemovalDeletion() error { } func (devices *DeviceSet) initDevmapper(doInit bool) (retErr error) { - // give ourselves to libdm as a log handler - devicemapper.LogInit(devices) - if err := devices.enableDeferredRemovalDeletion(); err != nil { return err } diff --git a/components/engine/pkg/devicemapper/devmapper.go b/components/engine/pkg/devicemapper/devmapper.go index 8f35914366..ebd42bce2f 100644 --- a/components/engine/pkg/devicemapper/devmapper.go +++ b/components/engine/pkg/devicemapper/devmapper.go @@ -13,11 +13,6 @@ import ( "github.com/Sirupsen/logrus" ) -// DevmapperLogger defines methods for logging with devicemapper. -type DevmapperLogger interface { - DMLog(level int, file string, line int, dmError int, message string) -} - const ( deviceCreate TaskType = iota deviceReload @@ -264,14 +259,6 @@ func UdevWait(cookie *uint) error { return nil } -var dmLogger DevmapperLogger - -// LogInit initializes the logger for the device mapper library. -func LogInit(logger DevmapperLogger) { - dmLogger = logger - LogWithErrnoInit() -} - // SetDevDir sets the dev folder for the device mapper library (usually /dev). func SetDevDir(dir string) error { if res := DmSetDevDir(dir); res != 1 { diff --git a/components/engine/pkg/devicemapper/devmapper_log.go b/components/engine/pkg/devicemapper/devmapper_log.go index 7dd0c60891..098d2405ed 100644 --- a/components/engine/pkg/devicemapper/devmapper_log.go +++ b/components/engine/pkg/devicemapper/devmapper_log.go @@ -5,17 +5,45 @@ package devicemapper import "C" import ( + "fmt" "strings" + + "github.com/Sirupsen/logrus" ) +// DevmapperLogger defines methods required to register as a callback for +// logging events recieved from devicemapper. Note that devicemapper will send +// *all* logs regardless to callbacks (including debug logs) so it's +// recommended to not spam the console with the outputs. +type DevmapperLogger interface { + // DMLog is the logging callback containing all of the information from + // devicemapper. The interface is identical to the C libdm counterpart. + DMLog(level int, file string, line int, dmError int, message string) +} + +// dmLogger is the current logger in use that is being forwarded our messages. +var dmLogger DevmapperLogger + +// LogInit changes the logging callback called after processing libdm logs for +// error message information. The default logger simply forwards all logs to +// logrus. Calling LogInit(nil) disables the calling of callbacks. +func LogInit(logger DevmapperLogger) { + dmLogger = logger +} + // Due to the way cgo works this has to be in a separate file, as devmapper.go has // definitions in the cgo block, which is incompatible with using "//export" -// DevmapperLogCallback exports the devmapper log callback for cgo. +// DevmapperLogCallback exports the devmapper log callback for cgo. Note that +// because we are using callbacks, this function will be called for *every* log +// in libdm (even debug ones because there's no way of setting the verbosity +// level for an external logging callback). //export DevmapperLogCallback -func DevmapperLogCallback(level C.int, file *C.char, line C.int, dmErrnoOrClass C.int, message *C.char) { +func DevmapperLogCallback(level C.int, file *C.char, line, dmErrnoOrClass C.int, message *C.char) { msg := C.GoString(message) - if level < 7 { + + // Track what errno libdm saw, because the library only gives us 0 or 1. + if level < LogLevelDebug { if strings.Contains(msg, "busy") { dmSawBusy = true } @@ -33,3 +61,61 @@ func DevmapperLogCallback(level C.int, file *C.char, line C.int, dmErrnoOrClass dmLogger.DMLog(int(level), C.GoString(file), int(line), int(dmErrnoOrClass), msg) } } + +// DefaultLogger is the default logger used by pkg/devicemapper. It forwards +// all logs that are of higher or equal priority to the given level to the +// corresponding logrus level. +type DefaultLogger struct { + // Level corresponds to the highest libdm level that will be forwarded to + // logrus. In order to change this, register a new DefaultLogger. + Level int +} + +// DMLog is the logging callback containing all of the information from +// devicemapper. The interface is identical to the C libdm counterpart. +func (l DefaultLogger) DMLog(level int, file string, line, dmError int, message string) { + if int(level) <= l.Level { + // Forward the log to the correct logrus level, if allowed by dmLogLevel. + logMsg := fmt.Sprintf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) + switch level { + case LogLevelFatal, LogLevelErr: + logrus.Error(logMsg) + case LogLevelWarn: + logrus.Warn(logMsg) + case LogLevelNotice, LogLevelInfo: + logrus.Info(logMsg) + case LogLevelDebug: + logrus.Debug(logMsg) + default: + // Don't drop any "unknown" levels. + logrus.Info(logMsg) + } + } +} + +// registerLogCallback registers our own logging callback function for libdm +// (which is DevmapperLogCallback). +// +// Because libdm only gives us {0,1} error codes we need to parse the logs +// produced by libdm (to set dmSawBusy and so on). Note that by registering a +// callback using DevmapperLogCallback, libdm will no longer output logs to +// stderr so we have to log everything ourselves. None of this handling is +// optional because we depend on log callbacks to parse the logs, and if we +// don't forward the log information we'll be in a lot of trouble when +// debugging things. +func registerLogCallback() { + LogWithErrnoInit() +} + +func init() { + // Use the default logger by default. We only allow LogLevelFatal by + // default, because internally we mask a lot of libdm errors by retrying + // and similar tricks. Also, libdm is very chatty and we don't want to + // worry users for no reason. + dmLogger = DefaultLogger{ + Level: LogLevelFatal, + } + + // Register as early as possible so we don't miss anything. + registerLogCallback() +} From 666b2cc7345366dd91f7a4cc5fb7511539cbc668 Mon Sep 17 00:00:00 2001 From: Aleksa Sarai Date: Wed, 28 Jun 2017 01:47:05 +1000 Subject: [PATCH 17/56] devicemapper: add --storage-opt dm.libdm_log_level=X option Because we use our own logging callbacks in order to use libdm effectively, it is quite difficult to debug complicated devicemapper issues (because any warnings or notices from libdm are muted by our own callback function). e07d3cd9a ("devmapper: Fix libdm logging") further reduced the ability of this debugging by only allowing _LOG_FATAL errors to be passed to the output. Unfortunately libdm is very chatty, so in order to avoid making the logs even more crowded, add a dm.libdm_log_level storage option that allows people who are debugging the lovely world of libdm to be able to dive in without recompiling binaries. The valid values of dm.libdm_log_level map directly to the libdm logging levels, and are in the range [2,7] as of the time of writing with 7 being _LOG_DEBUG and 2 being _LOG_FATAL. The default is _LOG_FATAL. Signed-off-by: Aleksa Sarai Upstream-commit: 198f83bba120c6c9bda679d33a55acab6cc9f33d Component: engine --- .../engine/daemon/graphdriver/devmapper/deviceset.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/components/engine/daemon/graphdriver/devmapper/deviceset.go b/components/engine/daemon/graphdriver/devmapper/deviceset.go index c785962ceb..5c1856a79f 100644 --- a/components/engine/daemon/graphdriver/devmapper/deviceset.go +++ b/components/engine/daemon/graphdriver/devmapper/deviceset.go @@ -2761,6 +2761,18 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [ return nil, errors.New("dm.thinp_autoextend_threshold must be greater than 0 and less than 100") } lvmSetupConfig.AutoExtendThreshold = per + case "dm.libdm_log_level": + level, err := strconv.ParseInt(val, 10, 32) + if err != nil { + return nil, errors.Wrapf(err, "could not parse `dm.libdm_log_level=%s`", val) + } + if level < devicemapper.LogLevelFatal || level > devicemapper.LogLevelDebug { + return nil, errors.Errorf("dm.libdm_log_level must be in range [%d,%d]", devicemapper.LogLevelFatal, devicemapper.LogLevelDebug) + } + // Register a new logging callback with the specified level. + devicemapper.LogInit(devicemapper.DefaultLogger{ + Level: int(level), + }) default: return nil, fmt.Errorf("devmapper: Unknown option %s\n", key) } From c2eb21415c5c0c3d123c43f447217a62be0b0a9b Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Fri, 26 May 2017 06:25:58 +0000 Subject: [PATCH 18/56] gofmt: 5 files gofmt from go1.8.3 hg locate '*.go' |xargs ~/go/bin/gofmt -s -w Signed-off-by: Josh Soref Signed-off-by: Sebastiaan van Stijn Upstream-commit: 6284cf5e9a4bc121b73422c454cdbf8dc18f42f3 Component: engine --- .../engine/api/types/filters/parse_test.go | 80 +++++++++---------- components/engine/image/cache/compare_test.go | 46 +++++------ .../pkg/jsonlog/jsonlog_marshalling_test.go | 20 ++--- .../engine/pkg/jsonlog/jsonlogbytes_test.go | 26 +++--- .../engine/runconfig/hostconfig_test.go | 10 +-- 5 files changed, 91 insertions(+), 91 deletions(-) diff --git a/components/engine/api/types/filters/parse_test.go b/components/engine/api/types/filters/parse_test.go index 8198f89d7e..fb8ebec314 100644 --- a/components/engine/api/types/filters/parse_test.go +++ b/components/engine/api/types/filters/parse_test.go @@ -90,15 +90,15 @@ func TestFromParam(t *testing.T) { `{"key": "value"}`, } valid := map[*Args][]string{ - &Args{fields: map[string]map[string]bool{"key": {"value": true}}}: { + {fields: map[string]map[string]bool{"key": {"value": true}}}: { `{"key": ["value"]}`, `{"key": {"value": true}}`, }, - &Args{fields: map[string]map[string]bool{"key": {"value1": true, "value2": true}}}: { + {fields: map[string]map[string]bool{"key": {"value1": true, "value2": true}}}: { `{"key": ["value1", "value2"]}`, `{"key": {"value1": true, "value2": true}}`, }, - &Args{fields: map[string]map[string]bool{"key1": {"value1": true}, "key2": {"value2": true}}}: { + {fields: map[string]map[string]bool{"key1": {"value1": true}, "key2": {"value2": true}}}: { `{"key1": ["value1"], "key2": ["value2"]}`, `{"key1": {"value1": true}, "key2": {"value2": true}}`, }, @@ -172,14 +172,14 @@ func TestArgsMatchKVList(t *testing.T) { } matches := map[*Args]string{ - &Args{}: "field", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"today": true}, - "labels": map[string]bool{"key1": true}}, + {}: "field", + {map[string]map[string]bool{ + "created": {"today": true}, + "labels": {"key1": true}}, }: "labels", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"today": true}, - "labels": map[string]bool{"key1=value1": true}}, + {map[string]map[string]bool{ + "created": {"today": true}, + "labels": {"key1=value1": true}}, }: "labels", } @@ -190,16 +190,16 @@ func TestArgsMatchKVList(t *testing.T) { } differs := map[*Args]string{ - &Args{map[string]map[string]bool{ - "created": map[string]bool{"today": true}}, + {map[string]map[string]bool{ + "created": {"today": true}}, }: "created", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"today": true}, - "labels": map[string]bool{"key4": true}}, + {map[string]map[string]bool{ + "created": {"today": true}, + "labels": {"key4": true}}, }: "labels", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"today": true}, - "labels": map[string]bool{"key1=value3": true}}, + {map[string]map[string]bool{ + "created": {"today": true}, + "labels": {"key1=value3": true}}, }: "labels", } @@ -214,21 +214,21 @@ func TestArgsMatch(t *testing.T) { source := "today" matches := map[*Args]string{ - &Args{}: "field", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"today": true}}, + {}: "field", + {map[string]map[string]bool{ + "created": {"today": true}}, }: "today", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"to*": true}}, + {map[string]map[string]bool{ + "created": {"to*": true}}, }: "created", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"to(.*)": true}}, + {map[string]map[string]bool{ + "created": {"to(.*)": true}}, }: "created", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"tod": true}}, + {map[string]map[string]bool{ + "created": {"tod": true}}, }: "created", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"anyting": true, "to*": true}}, + {map[string]map[string]bool{ + "created": {"anyting": true, "to*": true}}, }: "created", } @@ -239,21 +239,21 @@ func TestArgsMatch(t *testing.T) { } differs := map[*Args]string{ - &Args{map[string]map[string]bool{ - "created": map[string]bool{"tomorrow": true}}, + {map[string]map[string]bool{ + "created": {"tomorrow": true}}, }: "created", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"to(day": true}}, + {map[string]map[string]bool{ + "created": {"to(day": true}}, }: "created", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"tom(.*)": true}}, + {map[string]map[string]bool{ + "created": {"tom(.*)": true}}, }: "created", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"tom": true}}, + {map[string]map[string]bool{ + "created": {"tom": true}}, }: "created", - &Args{map[string]map[string]bool{ - "created": map[string]bool{"today1": true}, - "labels": map[string]bool{"today": true}}, + {map[string]map[string]bool{ + "created": {"today1": true}, + "labels": {"today": true}}, }: "created", } diff --git a/components/engine/image/cache/compare_test.go b/components/engine/image/cache/compare_test.go index 7cc058933e..10e464b438 100644 --- a/components/engine/image/cache/compare_test.go +++ b/components/engine/image/cache/compare_test.go @@ -46,9 +46,9 @@ func TestCompare(t *testing.T) { sameConfigs := map[*container.Config]*container.Config{ // Empty config - &container.Config{}: {}, + {}: {}, // Does not compare hostname, domainname & image - &container.Config{ + { Hostname: "host1", Domainname: "domain1", Image: "image1", @@ -60,23 +60,23 @@ func TestCompare(t *testing.T) { User: "user", }, // only OpenStdin - &container.Config{OpenStdin: false}: {OpenStdin: false}, + {OpenStdin: false}: {OpenStdin: false}, // only env - &container.Config{Env: envs1}: {Env: envs1}, + {Env: envs1}: {Env: envs1}, // only cmd - &container.Config{Cmd: cmd1}: {Cmd: cmd1}, + {Cmd: cmd1}: {Cmd: cmd1}, // only labels - &container.Config{Labels: labels1}: {Labels: labels1}, + {Labels: labels1}: {Labels: labels1}, // only exposedPorts - &container.Config{ExposedPorts: ports1}: {ExposedPorts: ports1}, + {ExposedPorts: ports1}: {ExposedPorts: ports1}, // only entrypoints - &container.Config{Entrypoint: entrypoint1}: {Entrypoint: entrypoint1}, + {Entrypoint: entrypoint1}: {Entrypoint: entrypoint1}, // only volumes - &container.Config{Volumes: volumes1}: {Volumes: volumes1}, + {Volumes: volumes1}: {Volumes: volumes1}, } differentConfigs := map[*container.Config]*container.Config{ nil: nil, - &container.Config{ + { Hostname: "host1", Domainname: "domain1", Image: "image1", @@ -88,30 +88,30 @@ func TestCompare(t *testing.T) { User: "user2", }, // only OpenStdin - &container.Config{OpenStdin: false}: {OpenStdin: true}, - &container.Config{OpenStdin: true}: {OpenStdin: false}, + {OpenStdin: false}: {OpenStdin: true}, + {OpenStdin: true}: {OpenStdin: false}, // only env - &container.Config{Env: envs1}: {Env: envs2}, + {Env: envs1}: {Env: envs2}, // only cmd - &container.Config{Cmd: cmd1}: {Cmd: cmd2}, + {Cmd: cmd1}: {Cmd: cmd2}, // not the same number of parts - &container.Config{Cmd: cmd1}: {Cmd: cmd3}, + {Cmd: cmd1}: {Cmd: cmd3}, // only labels - &container.Config{Labels: labels1}: {Labels: labels2}, + {Labels: labels1}: {Labels: labels2}, // not the same number of labels - &container.Config{Labels: labels1}: {Labels: labels3}, + {Labels: labels1}: {Labels: labels3}, // only exposedPorts - &container.Config{ExposedPorts: ports1}: {ExposedPorts: ports2}, + {ExposedPorts: ports1}: {ExposedPorts: ports2}, // not the same number of ports - &container.Config{ExposedPorts: ports1}: {ExposedPorts: ports3}, + {ExposedPorts: ports1}: {ExposedPorts: ports3}, // only entrypoints - &container.Config{Entrypoint: entrypoint1}: {Entrypoint: entrypoint2}, + {Entrypoint: entrypoint1}: {Entrypoint: entrypoint2}, // not the same number of parts - &container.Config{Entrypoint: entrypoint1}: {Entrypoint: entrypoint3}, + {Entrypoint: entrypoint1}: {Entrypoint: entrypoint3}, // only volumes - &container.Config{Volumes: volumes1}: {Volumes: volumes2}, + {Volumes: volumes1}: {Volumes: volumes2}, // not the same number of labels - &container.Config{Volumes: volumes1}: {Volumes: volumes3}, + {Volumes: volumes1}: {Volumes: volumes3}, } for config1, config2 := range sameConfigs { if !compare(config1, config2) { diff --git a/components/engine/pkg/jsonlog/jsonlog_marshalling_test.go b/components/engine/pkg/jsonlog/jsonlog_marshalling_test.go index 3edb271410..8b0d072cd3 100644 --- a/components/engine/pkg/jsonlog/jsonlog_marshalling_test.go +++ b/components/engine/pkg/jsonlog/jsonlog_marshalling_test.go @@ -7,17 +7,17 @@ import ( func TestJSONLogMarshalJSON(t *testing.T) { logs := map[*JSONLog]string{ - &JSONLog{Log: `"A log line with \\"`}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":\".{20,}\"}$`, - &JSONLog{Log: "A log line"}: `^{\"log\":\"A log line\",\"time\":\".{20,}\"}$`, - &JSONLog{Log: "A log line with \r"}: `^{\"log\":\"A log line with \\r\",\"time\":\".{20,}\"}$`, - &JSONLog{Log: "A log line with & < >"}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":\".{20,}\"}$`, - &JSONLog{Log: "A log line with utf8 : 🚀 ψ ω β"}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":\".{20,}\"}$`, - &JSONLog{Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":\".{20,}\"}$`, - &JSONLog{}: `^{\"time\":\".{20,}\"}$`, + {Log: `"A log line with \\"`}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":\".{20,}\"}$`, + {Log: "A log line"}: `^{\"log\":\"A log line\",\"time\":\".{20,}\"}$`, + {Log: "A log line with \r"}: `^{\"log\":\"A log line with \\r\",\"time\":\".{20,}\"}$`, + {Log: "A log line with & < >"}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":\".{20,}\"}$`, + {Log: "A log line with utf8 : 🚀 ψ ω β"}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":\".{20,}\"}$`, + {Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":\".{20,}\"}$`, + {}: `^{\"time\":\".{20,}\"}$`, // These ones are a little weird - &JSONLog{Log: "\u2028 \u2029"}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":\".{20,}\"}$`, - &JSONLog{Log: string([]byte{0xaF})}: `^{\"log\":\"\\ufffd\",\"time\":\".{20,}\"}$`, - &JSONLog{Log: string([]byte{0x7F})}: `^{\"log\":\"\x7f\",\"time\":\".{20,}\"}$`, + {Log: "\u2028 \u2029"}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":\".{20,}\"}$`, + {Log: string([]byte{0xaF})}: `^{\"log\":\"\\ufffd\",\"time\":\".{20,}\"}$`, + {Log: string([]byte{0x7F})}: `^{\"log\":\"\x7f\",\"time\":\".{20,}\"}$`, } for jsonLog, expression := range logs { data, err := jsonLog.MarshalJSON() diff --git a/components/engine/pkg/jsonlog/jsonlogbytes_test.go b/components/engine/pkg/jsonlog/jsonlogbytes_test.go index 6d6ad21583..41049aaea8 100644 --- a/components/engine/pkg/jsonlog/jsonlogbytes_test.go +++ b/components/engine/pkg/jsonlog/jsonlogbytes_test.go @@ -8,21 +8,21 @@ import ( func TestJSONLogsMarshalJSONBuf(t *testing.T) { logs := map[*JSONLogs]string{ - &JSONLogs{Log: []byte(`"A log line with \\"`)}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":}$`, - &JSONLogs{Log: []byte("A log line")}: `^{\"log\":\"A log line\",\"time\":}$`, - &JSONLogs{Log: []byte("A log line with \r")}: `^{\"log\":\"A log line with \\r\",\"time\":}$`, - &JSONLogs{Log: []byte("A log line with & < >")}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":}$`, - &JSONLogs{Log: []byte("A log line with utf8 : 🚀 ψ ω β")}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":}$`, - &JSONLogs{Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":}$`, - &JSONLogs{Stream: "stdout", Log: []byte("A log line")}: `^{\"log\":\"A log line\",\"stream\":\"stdout\",\"time\":}$`, - &JSONLogs{Created: "time"}: `^{\"time\":time}$`, - &JSONLogs{}: `^{\"time\":}$`, + {Log: []byte(`"A log line with \\"`)}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":}$`, + {Log: []byte("A log line")}: `^{\"log\":\"A log line\",\"time\":}$`, + {Log: []byte("A log line with \r")}: `^{\"log\":\"A log line with \\r\",\"time\":}$`, + {Log: []byte("A log line with & < >")}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":}$`, + {Log: []byte("A log line with utf8 : 🚀 ψ ω β")}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":}$`, + {Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":}$`, + {Stream: "stdout", Log: []byte("A log line")}: `^{\"log\":\"A log line\",\"stream\":\"stdout\",\"time\":}$`, + {Created: "time"}: `^{\"time\":time}$`, + {}: `^{\"time\":}$`, // These ones are a little weird - &JSONLogs{Log: []byte("\u2028 \u2029")}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":}$`, - &JSONLogs{Log: []byte{0xaF}}: `^{\"log\":\"\\ufffd\",\"time\":}$`, - &JSONLogs{Log: []byte{0x7F}}: `^{\"log\":\"\x7f\",\"time\":}$`, + {Log: []byte("\u2028 \u2029")}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":}$`, + {Log: []byte{0xaF}}: `^{\"log\":\"\\ufffd\",\"time\":}$`, + {Log: []byte{0x7F}}: `^{\"log\":\"\x7f\",\"time\":}$`, // with raw attributes - &JSONLogs{Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":}$`, + {Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":}$`, } for jsonLog, expression := range logs { var buf bytes.Buffer diff --git a/components/engine/runconfig/hostconfig_test.go b/components/engine/runconfig/hostconfig_test.go index 7f39cf5216..a6a3eef7cd 100644 --- a/components/engine/runconfig/hostconfig_test.go +++ b/components/engine/runconfig/hostconfig_test.go @@ -167,11 +167,11 @@ func TestPidModeTest(t *testing.T) { func TestRestartPolicy(t *testing.T) { restartPolicies := map[container.RestartPolicy][]bool{ // none, always, failure - container.RestartPolicy{}: {true, false, false}, - container.RestartPolicy{Name: "something", MaximumRetryCount: 0}: {false, false, false}, - container.RestartPolicy{Name: "no", MaximumRetryCount: 0}: {true, false, false}, - container.RestartPolicy{Name: "always", MaximumRetryCount: 0}: {false, true, false}, - container.RestartPolicy{Name: "on-failure", MaximumRetryCount: 0}: {false, false, true}, + {}: {true, false, false}, + {Name: "something", MaximumRetryCount: 0}: {false, false, false}, + {Name: "no", MaximumRetryCount: 0}: {true, false, false}, + {Name: "always", MaximumRetryCount: 0}: {false, true, false}, + {Name: "on-failure", MaximumRetryCount: 0}: {false, false, true}, } for restartPolicy, state := range restartPolicies { if restartPolicy.IsNone() != state[0] { From 654d76c922d232e9c96c9138266812cf98fe4392 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 30 Jun 2017 10:34:40 -0700 Subject: [PATCH 19/56] Do not set -1 for swappiness Do not set a default value for swappiness as the default value should be `nil` Signed-off-by: Michael Crosby Upstream-commit: 9d87e6e0fb799d6ef3bb9a97bc523f8d343b5fb3 Component: engine --- components/engine/daemon/daemon.go | 8 ++++++++ components/engine/daemon/daemon_solaris.go | 3 ++- components/engine/daemon/daemon_unix.go | 9 +++------ components/engine/daemon/daemon_windows.go | 4 ++-- .../engine/integration-cli/docker_api_containers_test.go | 2 +- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go index ac03b75c2c..cdaa5c9a41 100644 --- a/components/engine/daemon/daemon.go +++ b/components/engine/daemon/daemon.go @@ -1243,3 +1243,11 @@ func (daemon *Daemon) checkpointAndSave(container *container.Container) error { } return nil } + +// because the CLI sends a -1 when it wants to unset the swappiness value +// we need to clear it on the server side +func fixMemorySwappiness(resources *containertypes.Resources) { + if resources.MemorySwappiness != nil && *resources.MemorySwappiness == -1 { + resources.MemorySwappiness = nil + } +} diff --git a/components/engine/daemon/daemon_solaris.go b/components/engine/daemon/daemon_solaris.go index 7f2004e65a..f464ee34b8 100644 --- a/components/engine/daemon/daemon_solaris.go +++ b/components/engine/daemon/daemon_solaris.go @@ -143,6 +143,7 @@ func UsingSystemd(config *Config) bool { // verifyPlatformContainerSettings performs platform-specific validation of the // hostconfig and config structures. func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes.HostConfig, config *containertypes.Config, update bool) ([]string, error) { + fixMemorySwappiness(resources) warnings := []string{} sysInfo := sysinfo.New(true) // NOTE: We do not enforce a minimum value for swap limits for zones on Solaris and @@ -163,7 +164,7 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes. } // Solaris NOTE: We allow and encourage setting the swap without setting the memory limit. - if hostConfig.MemorySwappiness != nil && *hostConfig.MemorySwappiness != -1 && !sysInfo.MemorySwappiness { + if hostConfig.MemorySwappiness != nil && !sysInfo.MemorySwappiness { warnings = append(warnings, "Your kernel does not support memory swappiness capabilities, memory swappiness discarded.") logrus.Warnf("Your kernel does not support memory swappiness capabilities, memory swappiness discarded.") hostConfig.MemorySwappiness = nil diff --git a/components/engine/daemon/daemon_unix.go b/components/engine/daemon/daemon_unix.go index 0778dde4f7..a7ba9c37b2 100644 --- a/components/engine/daemon/daemon_unix.go +++ b/components/engine/daemon/daemon_unix.go @@ -282,10 +282,6 @@ func (daemon *Daemon) adaptContainerSettings(hostConfig *containertypes.HostConf return err } hostConfig.SecurityOpt = append(hostConfig.SecurityOpt, opts...) - if hostConfig.MemorySwappiness == nil { - defaultSwappiness := int64(-1) - hostConfig.MemorySwappiness = &defaultSwappiness - } if hostConfig.OomKillDisable == nil { defaultOomKillDisable := false hostConfig.OomKillDisable = &defaultOomKillDisable @@ -296,6 +292,7 @@ func (daemon *Daemon) adaptContainerSettings(hostConfig *containertypes.HostConf func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysinfo.SysInfo, update bool) ([]string, error) { warnings := []string{} + fixMemorySwappiness(resources) // memory subsystem checks and adjustments if resources.Memory != 0 && resources.Memory < linuxMinMemory { @@ -318,14 +315,14 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi if resources.Memory == 0 && resources.MemorySwap > 0 && !update { return warnings, fmt.Errorf("You should always set the Memory limit when using Memoryswap limit, see usage") } - if resources.MemorySwappiness != nil && *resources.MemorySwappiness != -1 && !sysInfo.MemorySwappiness { + if resources.MemorySwappiness != nil && !sysInfo.MemorySwappiness { warnings = append(warnings, "Your kernel does not support memory swappiness capabilities or the cgroup is not mounted. Memory swappiness discarded.") logrus.Warn("Your kernel does not support memory swappiness capabilities, or the cgroup is not mounted. Memory swappiness discarded.") resources.MemorySwappiness = nil } if resources.MemorySwappiness != nil { swappiness := *resources.MemorySwappiness - if swappiness < -1 || swappiness > 100 { + if swappiness < 0 || swappiness > 100 { return warnings, fmt.Errorf("Invalid value: %v, valid memory swappiness range is 0-100", swappiness) } } diff --git a/components/engine/daemon/daemon_windows.go b/components/engine/daemon/daemon_windows.go index c77f32a5ef..bb9c85b80e 100644 --- a/components/engine/daemon/daemon_windows.go +++ b/components/engine/daemon/daemon_windows.go @@ -100,7 +100,7 @@ func (daemon *Daemon) adaptContainerSettings(hostConfig *containertypes.HostConf func verifyContainerResources(resources *containertypes.Resources, isHyperv bool) ([]string, error) { warnings := []string{} - + fixMemorySwappiness(resources) if !isHyperv { // The processor resource controls are mutually exclusive on // Windows Server Containers, the order of precedence is @@ -197,7 +197,7 @@ func verifyContainerResources(resources *containertypes.Resources, isHyperv bool if resources.MemorySwap != 0 { return warnings, fmt.Errorf("invalid option: Windows does not support MemorySwap") } - if resources.MemorySwappiness != nil && *resources.MemorySwappiness != -1 { + if resources.MemorySwappiness != nil { return warnings, fmt.Errorf("invalid option: Windows does not support MemorySwappiness") } if resources.OomKillDisable != nil && *resources.OomKillDisable { diff --git a/components/engine/integration-cli/docker_api_containers_test.go b/components/engine/integration-cli/docker_api_containers_test.go index 84e009207a..25c724425e 100644 --- a/components/engine/integration-cli/docker_api_containers_test.go +++ b/components/engine/integration-cli/docker_api_containers_test.go @@ -1448,7 +1448,7 @@ func (s *DockerSuite) TestPostContainersCreateMemorySwappinessHostConfigOmitted( var containerJSON types.ContainerJSON c.Assert(json.Unmarshal(body, &containerJSON), check.IsNil) - c.Assert(*containerJSON.HostConfig.MemorySwappiness, check.Equals, int64(-1)) + c.Assert(containerJSON.HostConfig.MemorySwappiness, check.IsNil) } // check validation is done daemon side and not only in cli From 643654c2f0a59f66e643d7372af9367d90dbb354 Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Sun, 21 May 2017 23:24:07 +0000 Subject: [PATCH 20/56] Spelling fixes * additional * ambiguous * anonymous * anything * application * because * before * building * capabilities * circumstances * commit * committer * compresses * concatenated * config * container * container's * current * definition * delimiter * disassociates * discovery * distributed * doesnotexist * downloads * duplicates * either * enhancing * enumerate * escapable * exactly * expect * expectations * expected * explicitly * false * filesystem * following * forbidden * git with * healthcheck * ignore * independent * inheritance * investigating * irrelevant * it * logging * looking * membership * mimic * minimum * modify * mountpoint * multiline * notifier * outputting * outside * overridden * override * parsable * plugins * precedence * propagation * provided * provides * registries * repositories * returning * settings * should * signals * someone * something * specifically * successfully * synchronize * they've * thinking * uninitialized * unintentionally * unmarshaling * unnamed * unreferenced * verify Signed-off-by: Josh Soref Signed-off-by: Sebastiaan van Stijn Upstream-commit: 39bcaee47b8a284a46b761afe218ba7deda0d482 Component: engine --- components/engine/CHANGELOG.md | 10 ++-- components/engine/Dockerfile.aarch64 | 2 +- .../router/container/container_routes.go | 2 +- components/engine/api/swagger.yaml | 2 +- .../engine/api/types/filters/parse_test.go | 2 +- components/engine/api/types/swarm/swarm.go | 2 +- .../builder/dockerfile/buildargs_test.go | 6 +- components/engine/client/client.go | 2 +- components/engine/client/container_wait.go | 4 +- components/engine/container/container_unix.go | 2 +- components/engine/container/state.go | 2 +- .../cluster/executor/container/controller.go | 2 +- .../engine/daemon/discovery/discovery_test.go | 4 +- .../engine/daemon/events/events_test.go | 2 +- .../graphdriver/devmapper/device_setup.go | 10 ++-- .../engine/daemon/graphdriver/driver_linux.go | 2 +- .../daemon/graphdriver/driver_solaris.go | 2 +- .../daemon/graphdriver/quota/projectquota.go | 2 +- .../daemon/graphdriver/windows/windows.go | 2 +- .../daemon/logger/awslogs/cloudwatchlogs.go | 2 +- components/engine/daemon/logger/ring.go | 2 +- components/engine/daemon/oci_windows.go | 2 +- components/engine/daemon/reload.go | 6 +- components/engine/daemon/volumes_unix_test.go | 2 +- .../metadata/v2_metadata_service.go | 2 +- .../engine/distribution/push_v2_test.go | 2 +- components/engine/dockerversion/useragent.go | 4 +- components/engine/docs/api/v1.24.md | 2 +- components/engine/hack/Jenkins/W2L/setup.sh | 4 +- .../host/enumerate.go | 2 +- .../integration-cli-on-swarm/host/host.go | 4 +- components/engine/hack/make.ps1 | 2 +- .../engine/hack/make/generate-index-listing | 2 +- .../integration-cli/docker_api_swarm_test.go | 2 +- .../integration-cli/docker_cli_build_test.go | 60 +++++++++---------- .../integration-cli/docker_cli_commit_test.go | 4 +- .../integration-cli/docker_cli_daemon_test.go | 6 +- ...cker_cli_external_graphdriver_unix_test.go | 2 +- .../integration-cli/docker_cli_login_test.go | 2 +- .../docker_cli_network_unix_test.go | 2 +- .../docker_cli_plugins_test.go | 2 +- .../integration-cli/docker_cli_ps_test.go | 4 +- .../docker_cli_service_logs_test.go | 2 +- .../integration-cli/docker_cli_swarm_test.go | 2 +- .../integration-cli/docker_cli_volume_test.go | 10 ++-- .../docker_experimental_network_test.go | 2 +- components/engine/opts/ip.go | 2 +- components/engine/opts/opts_test.go | 2 +- components/engine/pkg/archive/archive.go | 2 +- components/engine/pkg/archive/changes_test.go | 6 +- .../pkg/authorization/authz_unix_test.go | 2 +- .../engine/pkg/devicemapper/devmapper.go | 2 +- components/engine/pkg/filenotify/fsnotify.go | 2 +- .../pkg/plugins/pluginrpc-gen/parser_test.go | 2 +- .../engine/pkg/registrar/registrar_test.go | 2 +- components/engine/pkg/stdcopy/stdcopy_test.go | 2 +- components/engine/pkg/symlink/fs.go | 2 +- components/engine/pkg/system/path_windows.go | 2 +- components/engine/pkg/system/rm.go | 2 +- components/engine/pkg/templates/templates.go | 2 +- components/engine/pkg/testutil/cmd/command.go | 2 +- components/engine/plugin/manager.go | 2 +- components/engine/plugin/v2/settable_test.go | 2 +- components/engine/reference/store.go | 2 +- components/engine/registry/config.go | 2 +- components/engine/registry/endpoint_v1.go | 2 +- components/engine/reports/2017-05-08.md | 2 +- components/engine/reports/2017-05-15.md | 2 +- components/engine/volume/drivers/adapter.go | 2 +- .../engine/volume/testutils/testutils.go | 4 +- components/engine/volume/volume.go | 2 +- components/engine/volume/volume_linux.go | 2 +- 72 files changed, 129 insertions(+), 129 deletions(-) diff --git a/components/engine/CHANGELOG.md b/components/engine/CHANGELOG.md index 18668ce549..bbedc491e9 100644 --- a/components/engine/CHANGELOG.md +++ b/components/engine/CHANGELOG.md @@ -190,7 +190,7 @@ be found. * Update runc to 54296cf40ad8143b62dbcaa1d90e520a2136ddfe [#31666](https://github.com/docker/docker/pull/31666) * Ignore cgroup2 mountpoints [opencontainers/runc#1266](https://github.com/opencontainers/runc/pull/1266) * Update containerd to 4ab9917febca54791c5f071a9d1f404867857fcc [#31662](https://github.com/docker/docker/pull/31662) [#31852](https://github.com/docker/docker/pull/31852) - * Register healtcheck service before calling restore() [docker/containerd#609](https://github.com/docker/containerd/pull/609) + * Register healthcheck service before calling restore() [docker/containerd#609](https://github.com/docker/containerd/pull/609) * Fix `docker exec` not working after unattended upgrades that reload apparmor profiles [#31773](https://github.com/docker/docker/pull/31773) * Fix unmounting layer without merge dir with Overlay2 [#31069](https://github.com/docker/docker/pull/31069) * Do not ignore "volume in use" errors when force-delete [#31450](https://github.com/docker/docker/pull/31450) @@ -1087,12 +1087,12 @@ installing docker, please make sure to update them accordingly. + Add security options to `docker info` output [#21172](https://github.com/docker/docker/pull/21172) [#23520](https://github.com/docker/docker/pull/23520) + Add insecure registries to `docker info` output [#20410](https://github.com/docker/docker/pull/20410) + Extend Docker authorization with TLS user information [#21556](https://github.com/docker/docker/pull/21556) -+ devicemapper: expose Mininum Thin Pool Free Space through `docker info` [#21945](https://github.com/docker/docker/pull/21945) ++ devicemapper: expose Minimum Thin Pool Free Space through `docker info` [#21945](https://github.com/docker/docker/pull/21945) * API now returns a JSON object when an error occurs making it more consistent [#22880](https://github.com/docker/docker/pull/22880) - Prevent `docker run -i --restart` from hanging on exit [#22777](https://github.com/docker/docker/pull/22777) - Fix API/CLI discrepancy on hostname validation [#21641](https://github.com/docker/docker/pull/21641) - Fix discrepancy in the format of sizes in `stats` from HumanSize to BytesSize [#21773](https://github.com/docker/docker/pull/21773) -- authz: when request is denied return forbbiden exit code (403) [#22448](https://github.com/docker/docker/pull/22448) +- authz: when request is denied return forbidden exit code (403) [#22448](https://github.com/docker/docker/pull/22448) - Windows: fix tty-related displaying issues [#23878](https://github.com/docker/docker/pull/23878) ### Runtime @@ -1887,7 +1887,7 @@ by another client (#15489) #### Remote API -- Fix unmarshalling of Command and Entrypoint +- Fix unmarshaling of Command and Entrypoint - Set limit for minimum client version supported - Validate port specification - Return proper errors when attach/reattach fail @@ -2572,7 +2572,7 @@ With the ongoing changes to the networking and execution subsystems of docker te - Fix ADD caching issue with . prefixed path - Fix docker build on devicemapper by reverting sparse file tar option - Fix issue with file caching and prevent wrong cache hit -* Use same error handling while unmarshalling CMD and ENTRYPOINT +* Use same error handling while unmarshaling CMD and ENTRYPOINT #### Documentation diff --git a/components/engine/Dockerfile.aarch64 b/components/engine/Dockerfile.aarch64 index 209d92e0ac..7a8f5f793c 100644 --- a/components/engine/Dockerfile.aarch64 +++ b/components/engine/Dockerfile.aarch64 @@ -93,7 +93,7 @@ RUN set -x \ && rm -rf "$SECCOMP_PATH" # Install Go -# We don't have official binary golang 1.7.5 tarballs for ARM64, eigher for Go or +# We don't have official binary golang 1.7.5 tarballs for ARM64, either for Go or # bootstrap, so we use golang-go (1.6) as bootstrap to build Go from source code. # We don't use the official ARMv6 released binaries as a GOROOT_BOOTSTRAP, because # not all ARM64 platforms support 32-bit mode. 32-bit mode is optional for ARMv8. diff --git a/components/engine/api/server/router/container/container_routes.go b/components/engine/api/server/router/container/container_routes.go index 0032fea7aa..96b1010e19 100644 --- a/components/engine/api/server/router/container/container_routes.go +++ b/components/engine/api/server/router/container/container_routes.go @@ -102,7 +102,7 @@ func (s *containerRouter) getContainersLogs(ctx context.Context, w http.Response } // doesn't matter what version the client is on, we're using this internally only - // also do we need size? i'm thinkin no we don't + // also do we need size? i'm thinking no we don't raw, err := s.backend.ContainerInspect(containerName, false, api.DefaultVersion) if err != nil { return err diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index fd0c4c1715..f899b6d731 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -1637,7 +1637,7 @@ definitions: may not be applied if the version number has changed from the last read. In other words, if two update requests specify the same base version, only one of the requests can succeed. As a result, two separate update requests that happen at the same time will not - unintentially overwrite each other. + unintentionally overwrite each other. type: "object" properties: Index: diff --git a/components/engine/api/types/filters/parse_test.go b/components/engine/api/types/filters/parse_test.go index fb8ebec314..ccd1684a07 100644 --- a/components/engine/api/types/filters/parse_test.go +++ b/components/engine/api/types/filters/parse_test.go @@ -228,7 +228,7 @@ func TestArgsMatch(t *testing.T) { "created": {"tod": true}}, }: "created", {map[string]map[string]bool{ - "created": {"anyting": true, "to*": true}}, + "created": {"anything": true, "to*": true}}, }: "created", } diff --git a/components/engine/api/types/swarm/swarm.go b/components/engine/api/types/swarm/swarm.go index 5b74f14b11..b65fa86dac 100644 --- a/components/engine/api/types/swarm/swarm.go +++ b/components/engine/api/types/swarm/swarm.go @@ -2,7 +2,7 @@ package swarm import "time" -// ClusterInfo represents info about the cluster for outputing in "info" +// ClusterInfo represents info about the cluster for outputting in "info" // it contains the same information as "Swarm", but without the JoinTokens type ClusterInfo struct { ID string diff --git a/components/engine/builder/dockerfile/buildargs_test.go b/components/engine/builder/dockerfile/buildargs_test.go index 77113ea21b..241bc84470 100644 --- a/components/engine/builder/dockerfile/buildargs_test.go +++ b/components/engine/builder/dockerfile/buildargs_test.go @@ -20,7 +20,7 @@ func TestGetAllAllowed(t *testing.T) { }) buildArgs.AddMetaArg("ArgFromMeta", strPtr("frommeta1")) - buildArgs.AddMetaArg("ArgFromMetaOverriden", strPtr("frommeta2")) + buildArgs.AddMetaArg("ArgFromMetaOverridden", strPtr("frommeta2")) buildArgs.AddMetaArg("ArgFromMetaNotUsed", strPtr("frommeta3")) buildArgs.AddArg("ArgOverriddenByOptions", strPtr("fromdockerfile2")) @@ -28,7 +28,7 @@ func TestGetAllAllowed(t *testing.T) { buildArgs.AddArg("ArgNoDefaultInDockerfile", nil) buildArgs.AddArg("ArgNoDefaultInDockerfileFromOptions", nil) buildArgs.AddArg("ArgFromMeta", nil) - buildArgs.AddArg("ArgFromMetaOverriden", strPtr("fromdockerfile3")) + buildArgs.AddArg("ArgFromMetaOverridden", strPtr("fromdockerfile3")) all := buildArgs.GetAllAllowed() expected := map[string]string{ @@ -37,7 +37,7 @@ func TestGetAllAllowed(t *testing.T) { "ArgWithDefaultInDockerfile": "fromdockerfile1", "ArgNoDefaultInDockerfileFromOptions": "fromopt3", "ArgFromMeta": "frommeta1", - "ArgFromMetaOverriden": "fromdockerfile3", + "ArgFromMetaOverridden": "fromdockerfile3", } assert.Equal(t, expected, all) } diff --git a/components/engine/client/client.go b/components/engine/client/client.go index 62a3db360a..7e14531723 100644 --- a/components/engine/client/client.go +++ b/components/engine/client/client.go @@ -91,7 +91,7 @@ type Client struct { // CheckRedirect specifies the policy for dealing with redirect responses: // If the request is non-GET return `ErrRedirect`. Otherwise use the last response. // -// Go 1.8 changes behavior for HTTP redirects (specificlaly 301, 307, and 308) in the client . +// Go 1.8 changes behavior for HTTP redirects (specifically 301, 307, and 308) in the client . // The Docker client (and by extension docker API client) can be made to to send a request // like POST /containers//start where what would normally be in the name section of the URL is empty. // This triggers an HTTP 301 from the daemon. diff --git a/components/engine/client/container_wait.go b/components/engine/client/container_wait.go index 13b5eea6a9..854c6c053b 100644 --- a/components/engine/client/container_wait.go +++ b/components/engine/client/container_wait.go @@ -14,7 +14,7 @@ import ( // indicated by the given condition, either "not-running" (default), // "next-exit", or "removed". // -// If this client's API version is beforer 1.30, condition is ignored and +// If this client's API version is before 1.30, condition is ignored and // ContainerWait will return immediately with the two channels, as the server // will wait as if the condition were "not-running". // @@ -23,7 +23,7 @@ import ( // then returns two channels on which the caller can wait for the exit status // of the container or an error if there was a problem either beginning the // wait request or in getting the response. This allows the caller to -// sychronize ContainerWait with other calls, such as specifying a +// synchronize ContainerWait with other calls, such as specifying a // "next-exit" condition before issuing a ContainerStart request. func (cli *Client) ContainerWait(ctx context.Context, containerID string, condition container.WaitCondition) (<-chan container.ContainerWaitOKBody, <-chan error) { if versions.LessThan(cli.ClientVersion(), "1.30") { diff --git a/components/engine/container/container_unix.go b/components/engine/container/container_unix.go index 0cd9312be3..327f950ff8 100644 --- a/components/engine/container/container_unix.go +++ b/components/engine/container/container_unix.go @@ -269,7 +269,7 @@ func (container *Container) UpdateContainer(hostConfig *containertypes.HostConfi cResources := &container.HostConfig.Resources // validate NanoCPUs, CPUPeriod, and CPUQuota - // Becuase NanoCPU effectively updates CPUPeriod/CPUQuota, + // Because NanoCPU effectively updates CPUPeriod/CPUQuota, // once NanoCPU is already set, updating CPUPeriod/CPUQuota will be blocked, and vice versa. // In the following we make sure the intended update (resources) does not conflict with the existing (cResource). if resources.NanoCPUs > 0 && cResources.CPUPeriod > 0 { diff --git a/components/engine/container/state.go b/components/engine/container/state.go index 4ded406411..01f7ab4584 100644 --- a/components/engine/container/state.go +++ b/components/engine/container/state.go @@ -185,7 +185,7 @@ const ( // timeouts, and avoiding goroutine leaks. Wait must be called without holding // the state lock. Returns a channel from which the caller will receive the // result. If the container exited on its own, the result's Err() method will -// be nil and its ExitCode() method will return the conatiners exit code, +// be nil and its ExitCode() method will return the container's exit code, // otherwise, the results Err() method will return an error indicating why the // wait operation failed. func (s *State) Wait(ctx context.Context, condition WaitCondition) <-chan StateStatus { diff --git a/components/engine/daemon/cluster/executor/container/controller.go b/components/engine/daemon/cluster/executor/container/controller.go index 8e95816138..163643e398 100644 --- a/components/engine/daemon/cluster/executor/container/controller.go +++ b/components/engine/daemon/cluster/executor/container/controller.go @@ -343,7 +343,7 @@ func (r *controller) Shutdown(ctx context.Context) error { } // add a delay for gossip converge - // TODO(dongluochen): this delay shoud be configurable to fit different cluster size and network delay. + // TODO(dongluochen): this delay should be configurable to fit different cluster size and network delay. time.Sleep(defaultGossipConvergeDelay) } diff --git a/components/engine/daemon/discovery/discovery_test.go b/components/engine/daemon/discovery/discovery_test.go index 781442939b..f084a649a7 100644 --- a/components/engine/daemon/discovery/discovery_test.go +++ b/components/engine/daemon/discovery/discovery_test.go @@ -87,8 +87,8 @@ func TestDiscoveryOpts(t *testing.T) { t.Fatalf("Heartbeat - Expected : %v, Actual : %v", expected, heartbeat) } - discaveryTTL := fmt.Sprintf("%d", defaultDiscoveryTTLFactor-1) - clusterOpts = map[string]string{"discovery.ttl": discaveryTTL} + discoveryTTL := fmt.Sprintf("%d", defaultDiscoveryTTLFactor-1) + clusterOpts = map[string]string{"discovery.ttl": discoveryTTL} heartbeat, ttl, err = discoveryOpts(clusterOpts) if err == nil && heartbeat == 0 { t.Fatal("discovery.heartbeat must be positive") diff --git a/components/engine/daemon/events/events_test.go b/components/engine/daemon/events/events_test.go index ffb4e50bf9..ebb222cfbd 100644 --- a/components/engine/daemon/events/events_test.go +++ b/components/engine/daemon/events/events_test.go @@ -247,7 +247,7 @@ func TestLoadBufferedEventsOnlyFromPast(t *testing.T) { } // #13753 -func TestIngoreBufferedWhenNoTimes(t *testing.T) { +func TestIgnoreBufferedWhenNoTimes(t *testing.T) { m1, err := eventstestutils.Scan("2016-03-07T17:28:03.022433271+02:00 container die 0b863f2a26c18557fc6cdadda007c459f9ec81b874780808138aea78a3595079 (image=ubuntu, name=small_hoover)") if err != nil { t.Fatal(err) diff --git a/components/engine/daemon/graphdriver/devmapper/device_setup.go b/components/engine/daemon/graphdriver/devmapper/device_setup.go index 31eeae79f1..ef6cffbf20 100644 --- a/components/engine/daemon/graphdriver/devmapper/device_setup.go +++ b/components/engine/daemon/graphdriver/devmapper/device_setup.go @@ -174,27 +174,27 @@ func writeLVMConfig(root string, cfg directLVMConfig) error { func setupDirectLVM(cfg directLVMConfig) error { pvCreate, err := exec.LookPath("pvcreate") if err != nil { - return errors.Wrap(err, "error lookuping up command `pvcreate` while setting up direct lvm") + return errors.Wrap(err, "error looking up command `pvcreate` while setting up direct lvm") } vgCreate, err := exec.LookPath("vgcreate") if err != nil { - return errors.Wrap(err, "error lookuping up command `vgcreate` while setting up direct lvm") + return errors.Wrap(err, "error looking up command `vgcreate` while setting up direct lvm") } lvCreate, err := exec.LookPath("lvcreate") if err != nil { - return errors.Wrap(err, "error lookuping up command `lvcreate` while setting up direct lvm") + return errors.Wrap(err, "error looking up command `lvcreate` while setting up direct lvm") } lvConvert, err := exec.LookPath("lvconvert") if err != nil { - return errors.Wrap(err, "error lookuping up command `lvconvert` while setting up direct lvm") + return errors.Wrap(err, "error looking up command `lvconvert` while setting up direct lvm") } lvChange, err := exec.LookPath("lvchange") if err != nil { - return errors.Wrap(err, "error lookuping up command `lvchange` while setting up direct lvm") + return errors.Wrap(err, "error looking up command `lvchange` while setting up direct lvm") } if cfg.AutoExtendPercent == 0 { diff --git a/components/engine/daemon/graphdriver/driver_linux.go b/components/engine/daemon/graphdriver/driver_linux.go index 5c8d0e2301..46b6eec09b 100644 --- a/components/engine/daemon/graphdriver/driver_linux.go +++ b/components/engine/daemon/graphdriver/driver_linux.go @@ -95,7 +95,7 @@ func GetFSMagic(rootpath string) (FsMagic, error) { return FsMagic(buf.Type), nil } -// NewFsChecker returns a checker configured for the provied FsMagic +// NewFsChecker returns a checker configured for the provided FsMagic func NewFsChecker(t FsMagic) Checker { return &fsChecker{ t: t, diff --git a/components/engine/daemon/graphdriver/driver_solaris.go b/components/engine/daemon/graphdriver/driver_solaris.go index 7daf01c32d..06dc360cf5 100644 --- a/components/engine/daemon/graphdriver/driver_solaris.go +++ b/components/engine/daemon/graphdriver/driver_solaris.go @@ -54,7 +54,7 @@ func (c *fsChecker) IsMounted(path string) bool { return m } -// NewFsChecker returns a checker configured for the provied FsMagic +// NewFsChecker returns a checker configured for the provided FsMagic func NewFsChecker(t FsMagic) Checker { return &fsChecker{ t: t, diff --git a/components/engine/daemon/graphdriver/quota/projectquota.go b/components/engine/daemon/graphdriver/quota/projectquota.go index e408d5f906..48ade10348 100644 --- a/components/engine/daemon/graphdriver/quota/projectquota.go +++ b/components/engine/daemon/graphdriver/quota/projectquota.go @@ -328,7 +328,7 @@ func makeBackingFsDev(home string) (string, error) { } backingFsBlockDev := path.Join(home, "backingFsBlockDev") - // Re-create just in case comeone copied the home directory over to a new device + // Re-create just in case someone copied the home directory over to a new device syscall.Unlink(backingFsBlockDev) stat := fileinfo.Sys().(*syscall.Stat_t) if err := syscall.Mknod(backingFsBlockDev, syscall.S_IFBLK|0600, int(stat.Dev)); err != nil { diff --git a/components/engine/daemon/graphdriver/windows/windows.go b/components/engine/daemon/graphdriver/windows/windows.go index 514650aa7c..6b18c8c242 100644 --- a/components/engine/daemon/graphdriver/windows/windows.go +++ b/components/engine/daemon/graphdriver/windows/windows.go @@ -300,7 +300,7 @@ func (d *Driver) Remove(id string) error { // // TODO @jhowardmsft - For RS3, we can remove the retries. Also consider // using platform APIs (if available) to get this more succinctly. Also - // consider enlighting the Remove() interface to have context of why + // consider enhancing the Remove() interface to have context of why // the remove is being called - that could improve efficiency by not // enumerating compute systems during a remove of a container as it's // not required. diff --git a/components/engine/daemon/logger/awslogs/cloudwatchlogs.go b/components/engine/daemon/logger/awslogs/cloudwatchlogs.go index d44f09d6a0..4d98468a79 100644 --- a/components/engine/daemon/logger/awslogs/cloudwatchlogs.go +++ b/components/engine/daemon/logger/awslogs/cloudwatchlogs.go @@ -363,7 +363,7 @@ var newTicker = func(freq time.Duration) *time.Ticker { // awslogs-datetime-format options have been configured, multiline processing // is enabled, where log messages are stored in an event buffer until a multiline // pattern match is found, at which point the messages in the event buffer are -// pushed to CloudWatch logs as a single log event. Multline messages are processed +// pushed to CloudWatch logs as a single log event. Multiline messages are processed // according to the maximumBytesPerPut constraint, and the implementation only // allows for messages to be buffered for a maximum of 2*batchPublishFrequency // seconds. When events are ready to be processed for submission to CloudWatch diff --git a/components/engine/daemon/logger/ring.go b/components/engine/daemon/logger/ring.go index 90769d71e1..5c55955474 100644 --- a/components/engine/daemon/logger/ring.go +++ b/components/engine/daemon/logger/ring.go @@ -121,7 +121,7 @@ func (r *RingLogger) run() { type messageRing struct { mu sync.Mutex - // singals callers of `Dequeue` to wake up either on `Close` or when a new `Message` is added + // signals callers of `Dequeue` to wake up either on `Close` or when a new `Message` is added wait *sync.Cond sizeBytes int64 // current buffer size diff --git a/components/engine/daemon/oci_windows.go b/components/engine/daemon/oci_windows.go index 9f44c66523..f114230efa 100644 --- a/components/engine/daemon/oci_windows.go +++ b/components/engine/daemon/oci_windows.go @@ -55,7 +55,7 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { } // If the container has not been started, and has configs or secrets - // secrets, create symlinks to each confing and secret. If it has been + // secrets, create symlinks to each config and secret. If it has been // started before, the symlinks should have already been created. Also, it // is important to not mount a Hyper-V container that has been started // before, to protect the host from the container; for example, from diff --git a/components/engine/daemon/reload.go b/components/engine/daemon/reload.go index e47455946e..0200bcf06e 100644 --- a/components/engine/daemon/reload.go +++ b/components/engine/daemon/reload.go @@ -39,7 +39,7 @@ func (daemon *Daemon) Reload(conf *config.Config) (err error) { daemon.reloadPlatform(conf, attributes) daemon.reloadDebug(conf, attributes) - daemon.reloadMaxConcurrentDowloadsAndUploads(conf, attributes) + daemon.reloadMaxConcurrentDownloadsAndUploads(conf, attributes) daemon.reloadShutdownTimeout(conf, attributes) if err := daemon.reloadClusterDiscovery(conf, attributes); err != nil { @@ -74,9 +74,9 @@ func (daemon *Daemon) reloadDebug(conf *config.Config, attributes map[string]str attributes["debug"] = fmt.Sprintf("%t", daemon.configStore.Debug) } -// reloadMaxConcurrentDowloadsAndUploads updates configuration with max concurrent +// reloadMaxConcurrentDownloadsAndUploads updates configuration with max concurrent // download and upload options and updates the passed attributes -func (daemon *Daemon) reloadMaxConcurrentDowloadsAndUploads(conf *config.Config, attributes map[string]string) { +func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(conf *config.Config, attributes map[string]string) { // If no value is set for max-concurrent-downloads we assume it is the default value // We always "reset" as the cost is lightweight and easy to maintain. if conf.IsValueSet("max-concurrent-downloads") && conf.MaxConcurrentDownloads != nil { diff --git a/components/engine/daemon/volumes_unix_test.go b/components/engine/daemon/volumes_unix_test.go index dcd6977eaa..3a81eeeb7c 100644 --- a/components/engine/daemon/volumes_unix_test.go +++ b/components/engine/daemon/volumes_unix_test.go @@ -206,7 +206,7 @@ func TestBackportMountSpec(t *testing.T) { BindOptions: &mounttypes.BindOptions{Propagation: "shared"}, }, }, - comment: "bind mount with read/write + shared propgation", + comment: "bind mount with read/write + shared propagation", }, { mp: &volume.MountPoint{ diff --git a/components/engine/distribution/metadata/v2_metadata_service.go b/components/engine/distribution/metadata/v2_metadata_service.go index c31c7214e7..7524f63ce7 100644 --- a/components/engine/distribution/metadata/v2_metadata_service.go +++ b/components/engine/distribution/metadata/v2_metadata_service.go @@ -203,7 +203,7 @@ func (serv *v2MetadataService) TagAndAdd(diffID layer.DiffID, hmacKey []byte, me return serv.Add(diffID, meta) } -// Remove unassociates a metadata entry from a layer DiffID. +// Remove disassociates a metadata entry from a layer DiffID. func (serv *v2MetadataService) Remove(metadata V2Metadata) error { if serv.store == nil { // Support a service which has no backend storage, in this case diff --git a/components/engine/distribution/push_v2_test.go b/components/engine/distribution/push_v2_test.go index e8e2b1af25..99f5acb0c2 100644 --- a/components/engine/distribution/push_v2_test.go +++ b/components/engine/distribution/push_v2_test.go @@ -185,7 +185,7 @@ func TestLayerAlreadyExists(t *testing.T) { expectedRequests: []string{"apple"}, }, { - name: "not matching reposies", + name: "not matching repositories", targetRepo: "busybox", maxExistenceChecks: 3, metadata: []metadata.V2Metadata{ diff --git a/components/engine/dockerversion/useragent.go b/components/engine/dockerversion/useragent.go index 53632cbc35..c02d0fda1a 100644 --- a/components/engine/dockerversion/useragent.go +++ b/components/engine/dockerversion/useragent.go @@ -52,8 +52,8 @@ func escapeStr(s string, charsToEscape string) string { var ret string for _, currRune := range s { appended := false - for _, escapeableRune := range charsToEscape { - if currRune == escapeableRune { + for _, escapableRune := range charsToEscape { + if currRune == escapableRune { ret += `\` + string(currRune) appended = true break diff --git a/components/engine/docs/api/v1.24.md b/components/engine/docs/api/v1.24.md index fde0144212..d07ea84b34 100644 --- a/components/engine/docs/api/v1.24.md +++ b/components/engine/docs/api/v1.24.md @@ -826,7 +826,7 @@ Get `stdout` and `stderr` logs from the container ``id`` **Query parameters**: -- **details** - 1/True/true or 0/False/flase, Show extra details provided to logs. Default `false`. +- **details** - 1/True/true or 0/False/false, Show extra details provided to logs. Default `false`. - **follow** – 1/True/true or 0/False/false, return stream. Default `false`. - **stdout** – 1/True/true or 0/False/false, show `stdout` log. Default `false`. - **stderr** – 1/True/true or 0/False/false, show `stderr` log. Default `false`. diff --git a/components/engine/hack/Jenkins/W2L/setup.sh b/components/engine/hack/Jenkins/W2L/setup.sh index 3d211be6f7..a3d86b857a 100644 --- a/components/engine/hack/Jenkins/W2L/setup.sh +++ b/components/engine/hack/Jenkins/W2L/setup.sh @@ -13,7 +13,7 @@ SCRIPT_VER="Wed Apr 20 18:30:19 UTC 2016" # - Error if running 32-bit posix tools. Probably can take from bash --version and check contains "x86_64" # - Warn if the CI directory cannot be deleted afterwards. Otherwise turdlets are left behind # - Use %systemdrive% ($SYSTEMDRIVE) rather than hard code to c: for TEMP -# - Consider cross builing the Windows binary and copy across. That's a bit of a heavy lift. Only reason +# - Consider cross building the Windows binary and copy across. That's a bit of a heavy lift. Only reason # for doing that is that it mirrors the actual release process for docker.exe which is cross-built. # However, should absolutely not be a problem if built natively, so nit-picking. # - Tidy up of images and containers. Either here, or in the teardown script. @@ -116,7 +116,7 @@ fi # Get the commit has and verify we have something if [ $ec -eq 0 ]; then export COMMITHASH=$(git rev-parse --short HEAD) - echo INFO: Commmit hash is $COMMITHASH + echo INFO: Commit hash is $COMMITHASH if [ -z $COMMITHASH ]; then echo "ERROR: Failed to get commit hash. Are you sure this is a docker repository?" ec=1 diff --git a/components/engine/hack/integration-cli-on-swarm/host/enumerate.go b/components/engine/hack/integration-cli-on-swarm/host/enumerate.go index 08e5ac7f19..56c03e38d5 100644 --- a/components/engine/hack/integration-cli-on-swarm/host/enumerate.go +++ b/components/engine/hack/integration-cli-on-swarm/host/enumerate.go @@ -24,7 +24,7 @@ func enumerateTestsForBytes(b []byte) ([]string, error) { return tests, nil } -// enumareteTests enumerates valid `-check.f` strings for all the test functions. +// enumerateTests enumerates valid `-check.f` strings for all the test functions. // Note that we use regexp rather than parsing Go files for performance reason. // (Try `TESTFLAGS=-check.list make test-integration-cli` to see the slowness of parsing) // The files needs to be `gofmt`-ed diff --git a/components/engine/hack/integration-cli-on-swarm/host/host.go b/components/engine/hack/integration-cli-on-swarm/host/host.go index d881b7a262..6823a76686 100644 --- a/components/engine/hack/integration-cli-on-swarm/host/host.go +++ b/components/engine/hack/integration-cli-on-swarm/host/host.go @@ -36,10 +36,10 @@ func xmain() (int, error) { // Should we use cobra maybe? replicas := flag.Int("replicas", 1, "Number of worker service replica") chunks := flag.Int("chunks", 0, "Number of test chunks executed in batch (0 == replicas)") - pushWorkerImage := flag.String("push-worker-image", "", "Push the worker image to the registry. Required for distribuetd execution. (empty == not to push)") + pushWorkerImage := flag.String("push-worker-image", "", "Push the worker image to the registry. Required for distributed execution. (empty == not to push)") shuffle := flag.Bool("shuffle", false, "Shuffle the input so as to mitigate makespan nonuniformity") // flags below are rarely used - randSeed := flag.Int64("rand-seed", int64(0), "Random seed used for shuffling (0 == curent time)") + randSeed := flag.Int64("rand-seed", int64(0), "Random seed used for shuffling (0 == current time)") filtersFile := flag.String("filters-file", "", "Path to optional file composed of `-check.f` filter strings") dryRun := flag.Bool("dry-run", false, "Dry run") keepExecutor := flag.Bool("keep-executor", false, "Do not auto-remove executor containers, which is used for running privileged programs on Swarm") diff --git a/components/engine/hack/make.ps1 b/components/engine/hack/make.ps1 index c1cdad37ee..ac3e369048 100644 --- a/components/engine/hack/make.ps1 +++ b/components/engine/hack/make.ps1 @@ -175,7 +175,7 @@ Function Execute-Build($type, $additionalBuildTags, $directory) { if ($Race) { Write-Warning "Using race detector"; $raceParm=" -race"} if ($ForceBuildAll) { $allParm=" -a" } if ($NoOpt) { $optParm=" -gcflags "+""""+"-N -l"+"""" } - if ($addtionalBuildTags -ne "") { $buildTags += $(" " + $additionalBuildTags) } + if ($additionalBuildTags -ne "") { $buildTags += $(" " + $additionalBuildTags) } # Do the go build in the appropriate directory # Note -linkmode=internal is required to be able to debug on Windows. diff --git a/components/engine/hack/make/generate-index-listing b/components/engine/hack/make/generate-index-listing index 7094ad7dbd..9f1208403f 100755 --- a/components/engine/hack/make/generate-index-listing +++ b/components/engine/hack/make/generate-index-listing @@ -40,7 +40,7 @@ create_index() { # change IFS locally within subshell so the for loop saves line correctly to L var IFS=$'\n'; - # pretty sweet, will mimick the normal apache output. skipping "index" and hidden files + # pretty sweet, will mimic the normal apache output. skipping "index" and hidden files for L in $(find -L . -mount -depth -maxdepth 1 -type f ! -name 'index' ! -name '.*' -prune -printf "%f|@_@%Td-%Tb-%TY %Tk:%TM @%f@\n"|sort|column -t -s '|' | sed 's,\([\ ]\+\)@_@,\1,g'); do # file diff --git a/components/engine/integration-cli/docker_api_swarm_test.go b/components/engine/integration-cli/docker_api_swarm_test.go index 03cf899d71..a146bcc519 100644 --- a/components/engine/integration-cli/docker_api_swarm_test.go +++ b/components/engine/integration-cli/docker_api_swarm_test.go @@ -985,7 +985,7 @@ func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) { if cert != nil { c.Assert(clusterTLSInfo.TrustRoot, checker.Equals, expectedCert) } - // could take another second or two for the nodes to trust the new roots after the've all gotten + // could take another second or two for the nodes to trust the new roots after they've all gotten // new TLS certificates for j := 0; j < 18; j++ { mInfo := m.GetNode(c, m.NodeID).Description.TLSInfo diff --git a/components/engine/integration-cli/docker_cli_build_test.go b/components/engine/integration-cli/docker_cli_build_test.go index 77b151e89f..5a3d3efc65 100644 --- a/components/engine/integration-cli/docker_cli_build_test.go +++ b/components/engine/integration-cli/docker_cli_build_test.go @@ -1712,7 +1712,7 @@ func (s *DockerSuite) TestBuildEntrypoint(c *check.C) { } // #6445 ensure ONBUILD triggers aren't committed to grandchildren -func (s *DockerSuite) TestBuildOnBuildLimitedInheritence(c *check.C) { +func (s *DockerSuite) TestBuildOnBuildLimitedInheritance(c *check.C) { buildImageSuccessfully(c, "testonbuildtrigger1", build.WithDockerfile(` FROM busybox RUN echo "GRANDPARENT" @@ -3063,7 +3063,7 @@ func (s *DockerSuite) TestBuildFromGitWithContext(c *check.C) { } } -func (s *DockerSuite) TestBuildFromGitwithF(c *check.C) { +func (s *DockerSuite) TestBuildFromGitWithF(c *check.C) { name := "testbuildfromgitwithf" git := fakegit.New(c, "repo", map[string]string{ "myApp/myDockerfile": `FROM busybox @@ -3225,7 +3225,7 @@ func (s *DockerSuite) TestBuildCmdJSONNoShDashC(c *check.C) { } } -func (s *DockerSuite) TestBuildEntrypointCanBeOverridenByChild(c *check.C) { +func (s *DockerSuite) TestBuildEntrypointCanBeOverriddenByChild(c *check.C) { buildImageSuccessfully(c, "parent", build.WithDockerfile(` FROM busybox ENTRYPOINT exit 130 @@ -3245,7 +3245,7 @@ func (s *DockerSuite) TestBuildEntrypointCanBeOverridenByChild(c *check.C) { }) } -func (s *DockerSuite) TestBuildEntrypointCanBeOverridenByChildInspect(c *check.C) { +func (s *DockerSuite) TestBuildEntrypointCanBeOverriddenByChildInspect(c *check.C) { var ( name = "testbuildepinherit" name2 = "testbuildepinherit2" @@ -4472,26 +4472,26 @@ func (s *DockerSuite) TestBuildBuildTimeArgOverrideArgDefinedBeforeEnv(c *check. imgName := "bldargtest" envKey := "foo" envVal := "bar" - envValOveride := "barOverride" + envValOverride := "barOverride" dockerfile := fmt.Sprintf(`FROM busybox ARG %s ENV %s %s RUN echo $%s CMD echo $%s - `, envKey, envKey, envValOveride, envKey, envKey) + `, envKey, envKey, envValOverride, envKey, envKey) result := buildImage(imgName, cli.WithFlags("--build-arg", fmt.Sprintf("%s=%s", envKey, envVal)), build.WithDockerfile(dockerfile), ) result.Assert(c, icmd.Success) - if strings.Count(result.Combined(), envValOveride) != 2 { - c.Fatalf("failed to access environment variable in output: %q expected: %q", result.Combined(), envValOveride) + if strings.Count(result.Combined(), envValOverride) != 2 { + c.Fatalf("failed to access environment variable in output: %q expected: %q", result.Combined(), envValOverride) } containerName := "bldargCont" - if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); !strings.Contains(out, envValOveride) { - c.Fatalf("run produced invalid output: %q, expected %q", out, envValOveride) + if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); !strings.Contains(out, envValOverride) { + c.Fatalf("run produced invalid output: %q, expected %q", out, envValOverride) } } @@ -4501,25 +4501,25 @@ func (s *DockerSuite) TestBuildBuildTimeArgOverrideEnvDefinedBeforeArg(c *check. imgName := "bldargtest" envKey := "foo" envVal := "bar" - envValOveride := "barOverride" + envValOverride := "barOverride" dockerfile := fmt.Sprintf(`FROM busybox ENV %s %s ARG %s RUN echo $%s CMD echo $%s - `, envKey, envValOveride, envKey, envKey, envKey) + `, envKey, envValOverride, envKey, envKey, envKey) result := buildImage(imgName, cli.WithFlags("--build-arg", fmt.Sprintf("%s=%s", envKey, envVal)), build.WithDockerfile(dockerfile), ) result.Assert(c, icmd.Success) - if strings.Count(result.Combined(), envValOveride) != 2 { - c.Fatalf("failed to access environment variable in output: %q expected: %q", result.Combined(), envValOveride) + if strings.Count(result.Combined(), envValOverride) != 2 { + c.Fatalf("failed to access environment variable in output: %q expected: %q", result.Combined(), envValOverride) } containerName := "bldargCont" - if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); !strings.Contains(out, envValOveride) { - c.Fatalf("run produced invalid output: %q, expected %q", out, envValOveride) + if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); !strings.Contains(out, envValOverride) { + c.Fatalf("run produced invalid output: %q, expected %q", out, envValOverride) } } @@ -4616,25 +4616,25 @@ func (s *DockerSuite) TestBuildBuildTimeArgExpansionOverride(c *check.C) { envKey := "foo" envVal := "bar" envKey1 := "foo1" - envValOveride := "barOverride" + envValOverride := "barOverride" dockerfile := fmt.Sprintf(`FROM busybox ARG %s ENV %s %s ENV %s ${%s} RUN echo $%s - CMD echo $%s`, envKey, envKey, envValOveride, envKey1, envKey, envKey1, envKey1) + CMD echo $%s`, envKey, envKey, envValOverride, envKey1, envKey, envKey1, envKey1) result := buildImage(imgName, cli.WithFlags("--build-arg", fmt.Sprintf("%s=%s", envKey, envVal)), build.WithDockerfile(dockerfile), ) result.Assert(c, icmd.Success) - if strings.Count(result.Combined(), envValOveride) != 2 { - c.Fatalf("failed to access environment variable in output: %q expected: %q", result.Combined(), envValOveride) + if strings.Count(result.Combined(), envValOverride) != 2 { + c.Fatalf("failed to access environment variable in output: %q expected: %q", result.Combined(), envValOverride) } containerName := "bldargCont" - if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); !strings.Contains(out, envValOveride) { - c.Fatalf("run produced invalid output: %q, expected %q", out, envValOveride) + if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); !strings.Contains(out, envValOverride) { + c.Fatalf("run produced invalid output: %q, expected %q", out, envValOverride) } } @@ -4690,24 +4690,24 @@ func (s *DockerSuite) TestBuildBuildTimeArgDefaultOverride(c *check.C) { imgName := "bldargtest" envKey := "foo" envVal := "bar" - envValOveride := "barOverride" + envValOverride := "barOverride" dockerfile := fmt.Sprintf(`FROM busybox ARG %s=%s ENV %s $%s RUN echo $%s CMD echo $%s`, envKey, envVal, envKey, envKey, envKey, envKey) result := buildImage(imgName, - cli.WithFlags("--build-arg", fmt.Sprintf("%s=%s", envKey, envValOveride)), + cli.WithFlags("--build-arg", fmt.Sprintf("%s=%s", envKey, envValOverride)), build.WithDockerfile(dockerfile), ) result.Assert(c, icmd.Success) - if strings.Count(result.Combined(), envValOveride) != 1 { - c.Fatalf("failed to access environment variable in output: %q expected: %q", result.Combined(), envValOveride) + if strings.Count(result.Combined(), envValOverride) != 1 { + c.Fatalf("failed to access environment variable in output: %q expected: %q", result.Combined(), envValOverride) } containerName := "bldargCont" - if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); !strings.Contains(out, envValOveride) { - c.Fatalf("run produced invalid output: %q, expected %q", out, envValOveride) + if out, _ := dockerCmd(c, "run", "--name", containerName, imgName); !strings.Contains(out, envValOverride) { + c.Fatalf("run produced invalid output: %q, expected %q", out, envValOverride) } } @@ -4824,7 +4824,7 @@ func (s *DockerSuite) TestBuildBuildTimeArgEmptyValVariants(c *check.C) { buildImageSuccessfully(c, imgName, build.WithDockerfile(dockerfile)) } -func (s *DockerSuite) TestBuildBuildTimeArgDefintionWithNoEnvInjection(c *check.C) { +func (s *DockerSuite) TestBuildBuildTimeArgDefinitionWithNoEnvInjection(c *check.C) { imgName := "bldargtest" envKey := "foo" dockerfile := fmt.Sprintf(`FROM busybox @@ -5785,7 +5785,7 @@ func (s *DockerSuite) TestBuildWithExtraHostInvalidFormat(c *check.C) { buildFlag string }{ {"extra_host_missing_ip", dockerfile, "--add-host=foo"}, - {"extra_host_missing_ip_with_delimeter", dockerfile, "--add-host=foo:"}, + {"extra_host_missing_ip_with_delimiter", dockerfile, "--add-host=foo:"}, {"extra_host_missing_hostname", dockerfile, "--add-host=:127.0.0.1"}, {"extra_host_invalid_ipv4", dockerfile, "--add-host=foo:101.10.2"}, {"extra_host_invalid_ipv6", dockerfile, "--add-host=foo:2001::1::3F"}, diff --git a/components/engine/integration-cli/docker_cli_commit_test.go b/components/engine/integration-cli/docker_cli_commit_test.go index 549b35a935..b054c79c30 100644 --- a/components/engine/integration-cli/docker_cli_commit_test.go +++ b/components/engine/integration-cli/docker_cli_commit_test.go @@ -54,9 +54,9 @@ func (s *DockerSuite) TestCommitPausedContainer(c *check.C) { } func (s *DockerSuite) TestCommitNewFile(c *check.C) { - dockerCmd(c, "run", "--name", "commiter", "busybox", "/bin/sh", "-c", "echo koye > /foo") + dockerCmd(c, "run", "--name", "committer", "busybox", "/bin/sh", "-c", "echo koye > /foo") - imageID, _ := dockerCmd(c, "commit", "commiter") + imageID, _ := dockerCmd(c, "commit", "committer") imageID = strings.TrimSpace(imageID) out, _ := dockerCmd(c, "run", imageID, "cat", "/foo") diff --git a/components/engine/integration-cli/docker_cli_daemon_test.go b/components/engine/integration-cli/docker_cli_daemon_test.go index b551ad4509..b35e0c5272 100644 --- a/components/engine/integration-cli/docker_cli_daemon_test.go +++ b/components/engine/integration-cli/docker_cli_daemon_test.go @@ -965,7 +965,7 @@ func (s *DockerDaemonSuite) TestDaemonUlimitDefaults(c *check.C) { c.Fatalf("expected `ulimit -n` to be `42`, got: %s", nofile) } if nproc != "2048" { - c.Fatalf("exepcted `ulimit -p` to be 2048, got: %s", nproc) + c.Fatalf("expected `ulimit -p` to be 2048, got: %s", nproc) } // Now restart daemon with a new default @@ -987,7 +987,7 @@ func (s *DockerDaemonSuite) TestDaemonUlimitDefaults(c *check.C) { c.Fatalf("expected `ulimit -n` to be `43`, got: %s", nofile) } if nproc != "2048" { - c.Fatalf("exepcted `ulimit -p` to be 2048, got: %s", nproc) + c.Fatalf("expected `ulimit -p` to be 2048, got: %s", nproc) } } @@ -1408,7 +1408,7 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithSocketAsVolume(c *check.C) { } // os.Kill should kill daemon ungracefully, leaving behind container mounts. -// A subsequent daemon restart shoud clean up said mounts. +// A subsequent daemon restart should clean up said mounts. func (s *DockerDaemonSuite) TestCleanupMountsAfterDaemonAndContainerKill(c *check.C) { d := daemon.New(c, dockerBinary, dockerdBinary, daemon.Config{ Experimental: testEnv.ExperimentalDaemon(), diff --git a/components/engine/integration-cli/docker_cli_external_graphdriver_unix_test.go b/components/engine/integration-cli/docker_cli_external_graphdriver_unix_test.go index d736c4bdd2..16023c9a75 100644 --- a/components/engine/integration-cli/docker_cli_external_graphdriver_unix_test.go +++ b/components/engine/integration-cli/docker_cli_external_graphdriver_unix_test.go @@ -111,7 +111,7 @@ func (s *DockerExternalGraphdriverSuite) setUpPlugin(c *check.C, name string, ex } respond := func(w http.ResponseWriter, data interface{}) { - w.Header().Set("Content-Type", "appplication/vnd.docker.plugins.v1+json") + w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") switch t := data.(type) { case error: fmt.Fprintln(w, fmt.Sprintf(`{"Err": %q}`, t.Error())) diff --git a/components/engine/integration-cli/docker_cli_login_test.go b/components/engine/integration-cli/docker_cli_login_test.go index 94dc03778f..cb261bed85 100644 --- a/components/engine/integration-cli/docker_cli_login_test.go +++ b/components/engine/integration-cli/docker_cli_login_test.go @@ -16,7 +16,7 @@ func (s *DockerSuite) TestLoginWithoutTTY(c *check.C) { // run the command and block until it's done err := cmd.Run() - c.Assert(err, checker.NotNil) //"Expected non nil err when loginning in & TTY not available" + c.Assert(err, checker.NotNil) //"Expected non nil err when logging in & TTY not available" } func (s *DockerRegistryAuthHtpasswdSuite) TestLoginToPrivateRegistry(c *check.C) { diff --git a/components/engine/integration-cli/docker_cli_network_unix_test.go b/components/engine/integration-cli/docker_cli_network_unix_test.go index cdcb9ff6e3..28581bf503 100644 --- a/components/engine/integration-cli/docker_cli_network_unix_test.go +++ b/components/engine/integration-cli/docker_cli_network_unix_test.go @@ -1151,7 +1151,7 @@ func (s *DockerNetworkSuite) TestDockerNetworkHostModeUngracefulDaemonRestart(c out, err := s.d.Cmd("run", "-d", "--name", cName, "--net=host", "--restart=always", "busybox", "top") c.Assert(err, checker.IsNil, check.Commentf(out)) - // verfiy container has finished starting before killing daemon + // verify container has finished starting before killing daemon err = s.d.WaitRun(cName) c.Assert(err, checker.IsNil) } diff --git a/components/engine/integration-cli/docker_cli_plugins_test.go b/components/engine/integration-cli/docker_cli_plugins_test.go index 46b0b26468..e1fcaf2c3e 100644 --- a/components/engine/integration-cli/docker_cli_plugins_test.go +++ b/components/engine/integration-cli/docker_cli_plugins_test.go @@ -475,6 +475,6 @@ func (s *DockerSuite) TestPluginMetricsCollector(c *check.C) { b, err := ioutil.ReadAll(resp.Body) c.Assert(err, checker.IsNil) - // check that a known metric is there... don't epect this metric to change over time.. probably safe + // check that a known metric is there... don't expect this metric to change over time.. probably safe c.Assert(string(b), checker.Contains, "container_actions") } diff --git a/components/engine/integration-cli/docker_cli_ps_test.go b/components/engine/integration-cli/docker_cli_ps_test.go index b6c9350902..98a20f4267 100644 --- a/components/engine/integration-cli/docker_cli_ps_test.go +++ b/components/engine/integration-cli/docker_cli_ps_test.go @@ -746,7 +746,7 @@ func (s *DockerSuite) TestPsShowMounts(c *check.C) { fields = strings.Fields(lines[1]) c.Assert(fields, checker.HasLen, 2) - annonymounsVolumeID := fields[1] + anonymousVolumeID := fields[1] fields = strings.Fields(lines[2]) c.Assert(fields[1], checker.Equals, "ps-volume-test") @@ -771,7 +771,7 @@ func (s *DockerSuite) TestPsShowMounts(c *check.C) { c.Assert(lines, checker.HasLen, 2) fields = strings.Fields(lines[0]) - c.Assert(fields[1], checker.Equals, annonymounsVolumeID) + c.Assert(fields[1], checker.Equals, anonymousVolumeID) fields = strings.Fields(lines[1]) c.Assert(fields[1], checker.Equals, "ps-volume-test") diff --git a/components/engine/integration-cli/docker_cli_service_logs_test.go b/components/engine/integration-cli/docker_cli_service_logs_test.go index d38cdefea8..d2ce36def0 100644 --- a/components/engine/integration-cli/docker_cli_service_logs_test.go +++ b/components/engine/integration-cli/docker_cli_service_logs_test.go @@ -212,7 +212,7 @@ func (s *DockerSwarmSuite) TestServiceLogsTaskLogs(c *check.C) { fmt.Sprintf("--replicas=%v", replicas), // which has this the task id as an environment variable templated in "--env", "TASK={{.Task.ID}}", - // and runs this command to print exaclty 6 logs lines + // and runs this command to print exactly 6 logs lines "busybox", "sh", "-c", "for line in $(seq 0 5); do echo $TASK log test $line; done; sleep 100000", )) result.Assert(c, icmd.Expected{}) diff --git a/components/engine/integration-cli/docker_cli_swarm_test.go b/components/engine/integration-cli/docker_cli_swarm_test.go index 086a6632d2..8b57c5dd25 100644 --- a/components/engine/integration-cli/docker_cli_swarm_test.go +++ b/components/engine/integration-cli/docker_cli_swarm_test.go @@ -1887,7 +1887,7 @@ func (s *DockerSwarmSuite) TestNetworkInspectWithDuplicateNames(c *check.C) { out, err = d.Cmd("network", "rm", n2.ID) c.Assert(err, checker.IsNil, check.Commentf(out)) - // Dupliates with name but with different driver + // Duplicates with name but with different driver networkCreateRequest.NetworkCreate.Driver = "overlay" status, body, err = d.SockRequest("POST", "/networks/create", networkCreateRequest) diff --git a/components/engine/integration-cli/docker_cli_volume_test.go b/components/engine/integration-cli/docker_cli_volume_test.go index e8c837c9c0..e0bf7cafe3 100644 --- a/components/engine/integration-cli/docker_cli_volume_test.go +++ b/components/engine/integration-cli/docker_cli_volume_test.go @@ -34,7 +34,7 @@ func (s *DockerSuite) TestVolumeCLICreate(c *check.C) { func (s *DockerSuite) TestVolumeCLIInspect(c *check.C) { c.Assert( - exec.Command(dockerBinary, "volume", "inspect", "doesntexist").Run(), + exec.Command(dockerBinary, "volume", "inspect", "doesnotexist").Run(), check.Not(check.IsNil), check.Commentf("volume inspect should error on non-existent volume"), ) @@ -54,10 +54,10 @@ func (s *DockerSuite) TestVolumeCLIInspectMulti(c *check.C) { dockerCmd(c, "volume", "create", "test2") dockerCmd(c, "volume", "create", "test3") - result := dockerCmdWithResult("volume", "inspect", "--format={{ .Name }}", "test1", "test2", "doesntexist", "test3") + result := dockerCmdWithResult("volume", "inspect", "--format={{ .Name }}", "test1", "test2", "doesnotexist", "test3") c.Assert(result, icmd.Matches, icmd.Expected{ ExitCode: 1, - Err: "No such volume: doesntexist", + Err: "No such volume: doesnotexist", }) out := result.Stdout() @@ -185,7 +185,7 @@ func (s *DockerSuite) TestVolumeCLILsFilterDangling(c *check.C) { out, _ = dockerCmd(c, "volume", "ls", "--filter", "name=testisin") c.Assert(out, check.Not(checker.Contains), "testnotinuse1\n", check.Commentf("expected volume 'testnotinuse1' in output")) - c.Assert(out, checker.Contains, "testisinuse1\n", check.Commentf("execpeted volume 'testisinuse1' in output")) + c.Assert(out, checker.Contains, "testisinuse1\n", check.Commentf("expected volume 'testisinuse1' in output")) c.Assert(out, checker.Contains, "testisinuse2\n", check.Commentf("expected volume 'testisinuse2' in output")) } @@ -234,7 +234,7 @@ func (s *DockerSuite) TestVolumeCLIRm(c *check.C) { dockerCmd(c, "volume", "rm", volumeID) c.Assert( - exec.Command("volume", "rm", "doesntexist").Run(), + exec.Command("volume", "rm", "doesnotexist").Run(), check.Not(check.IsNil), check.Commentf("volume rm should fail with non-existent volume"), ) diff --git a/components/engine/integration-cli/docker_experimental_network_test.go b/components/engine/integration-cli/docker_experimental_network_test.go index 1240392cac..f352050d3b 100644 --- a/components/engine/integration-cli/docker_experimental_network_test.go +++ b/components/engine/integration-cli/docker_experimental_network_test.go @@ -155,7 +155,7 @@ func (s *DockerNetworkSuite) TestDockerNetworkMacvlanMultiSubnet(c *check.C) { _, _, err := dockerCmdWithError("exec", "second", "ping", "-c", "1", strings.TrimSpace(ip)) c.Assert(err, check.IsNil) // verify ipv6 connectivity to the explicit --ipv6 address second to first - c.Skip("Temporarily skipping while invesitigating sporadic v6 CI issues") + c.Skip("Temporarily skipping while investigating sporadic v6 CI issues") _, _, err = dockerCmdWithError("exec", "second", "ping6", "-c", "1", strings.TrimSpace(ip6)) c.Assert(err, check.IsNil) diff --git a/components/engine/opts/ip.go b/components/engine/opts/ip.go index fb03b50111..1095063977 100644 --- a/components/engine/opts/ip.go +++ b/components/engine/opts/ip.go @@ -22,7 +22,7 @@ func NewIPOpt(ref *net.IP, defaultVal string) *IPOpt { } // Set sets an IPv4 or IPv6 address from a given string. If the given -// string is not parseable as an IP address it returns an error. +// string is not parsable as an IP address it returns an error. func (o *IPOpt) Set(val string) error { ip := net.ParseIP(val) if ip == nil { diff --git a/components/engine/opts/opts_test.go b/components/engine/opts/opts_test.go index 1afe3c130d..269f88639b 100644 --- a/components/engine/opts/opts_test.go +++ b/components/engine/opts/opts_test.go @@ -157,7 +157,7 @@ func TestValidateDNSSearch(t *testing.T) { `foo.bar-.baz`, `foo.-bar`, `foo.-bar.baz`, - `foo.bar.baz.this.should.fail.on.long.name.beause.it.is.longer.thanisshouldbethis.should.fail.on.long.name.beause.it.is.longer.thanisshouldbethis.should.fail.on.long.name.beause.it.is.longer.thanisshouldbethis.should.fail.on.long.name.beause.it.is.longer.thanisshouldbe`, + `foo.bar.baz.this.should.fail.on.long.name.because.it.is.longer.thanitshouldbethis.should.fail.on.long.name.because.it.is.longer.thanitshouldbethis.should.fail.on.long.name.because.it.is.longer.thanitshouldbethis.should.fail.on.long.name.because.it.is.longer.thanitshouldbe`, } for _, domain := range valid { diff --git a/components/engine/pkg/archive/archive.go b/components/engine/pkg/archive/archive.go index c6ad7c58a8..79d871e20e 100644 --- a/components/engine/pkg/archive/archive.go +++ b/components/engine/pkg/archive/archive.go @@ -180,7 +180,7 @@ func DecompressStream(archive io.Reader) (io.ReadCloser, error) { } } -// CompressStream compresseses the dest with specified compression algorithm. +// CompressStream compresses the dest with specified compression algorithm. func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) { p := pools.BufioWriter32KPool buf := p.Get(dest) diff --git a/components/engine/pkg/archive/changes_test.go b/components/engine/pkg/archive/changes_test.go index eae1d022c7..c5d1629e72 100644 --- a/components/engine/pkg/archive/changes_test.go +++ b/components/engine/pkg/archive/changes_test.go @@ -102,10 +102,10 @@ func createSampleDir(t *testing.T, root string) { } func TestChangeString(t *testing.T) { - modifiyChange := Change{"change", ChangeModify} - toString := modifiyChange.String() + modifyChange := Change{"change", ChangeModify} + toString := modifyChange.String() if toString != "C change" { - t.Fatalf("String() of a change with ChangeModifiy Kind should have been %s but was %s", "C change", toString) + t.Fatalf("String() of a change with ChangeModify Kind should have been %s but was %s", "C change", toString) } addChange := Change{"change", ChangeAdd} toString = addChange.String() diff --git a/components/engine/pkg/authorization/authz_unix_test.go b/components/engine/pkg/authorization/authz_unix_test.go index a787f3cd8c..a692802d55 100644 --- a/components/engine/pkg/authorization/authz_unix_test.go +++ b/components/engine/pkg/authorization/authz_unix_test.go @@ -99,7 +99,7 @@ func TestAuthZResponsePlugin(t *testing.T) { request := Request{ User: "user", - RequestURI: "someting.com/auth", + RequestURI: "something.com/auth", RequestBody: []byte("sample body"), } server.replayResponse = Response{ diff --git a/components/engine/pkg/devicemapper/devmapper.go b/components/engine/pkg/devicemapper/devmapper.go index 73433c0e5a..08e0c06aad 100644 --- a/components/engine/pkg/devicemapper/devmapper.go +++ b/components/engine/pkg/devicemapper/devmapper.go @@ -373,7 +373,7 @@ func RemoveDeviceDeferred(name string) error { // semaphores created in `task.setCookie` will be cleaned up in `UdevWait`. // So these two function call must come in pairs, otherwise semaphores will // be leaked, and the limit of number of semaphores defined in `/proc/sys/kernel/sem` - // will be reached, which will eventually make all follwing calls to 'task.SetCookie' + // will be reached, which will eventually make all following calls to 'task.SetCookie' // fail. // this call will not wait for the deferred removal's final executing, since no // udev event will be generated, and the semaphore's value will not be incremented diff --git a/components/engine/pkg/filenotify/fsnotify.go b/components/engine/pkg/filenotify/fsnotify.go index 5d08a997a0..2614e05d31 100644 --- a/components/engine/pkg/filenotify/fsnotify.go +++ b/components/engine/pkg/filenotify/fsnotify.go @@ -2,7 +2,7 @@ package filenotify import "github.com/fsnotify/fsnotify" -// fsNotifyWatcher wraps the fsnotify package to satisfy the FileNotifer interface +// fsNotifyWatcher wraps the fsnotify package to satisfy the FileNotifier interface type fsNotifyWatcher struct { *fsnotify.Watcher } diff --git a/components/engine/pkg/plugins/pluginrpc-gen/parser_test.go b/components/engine/pkg/plugins/pluginrpc-gen/parser_test.go index a1b1ac9567..fe7fa5ade6 100644 --- a/components/engine/pkg/plugins/pluginrpc-gen/parser_test.go +++ b/components/engine/pkg/plugins/pluginrpc-gen/parser_test.go @@ -136,7 +136,7 @@ func TestParseWithMultipleFuncs(t *testing.T) { } } -func TestParseWithUnamedReturn(t *testing.T) { +func TestParseWithUnnamedReturn(t *testing.T) { _, err := Parse(testFixture, "Fooer4") if !strings.HasSuffix(err.Error(), errBadReturn.Error()) { t.Fatalf("expected ErrBadReturn, got %v", err) diff --git a/components/engine/pkg/registrar/registrar_test.go b/components/engine/pkg/registrar/registrar_test.go index 0c1ef312ae..70f8084b30 100644 --- a/components/engine/pkg/registrar/registrar_test.go +++ b/components/engine/pkg/registrar/registrar_test.go @@ -60,7 +60,7 @@ func TestGetNames(t *testing.T) { } if !reflect.DeepEqual(names, names2) { - t.Fatalf("Exepected: %v, Got: %v", names, names2) + t.Fatalf("Expected: %v, Got: %v", names, names2) } } diff --git a/components/engine/pkg/stdcopy/stdcopy_test.go b/components/engine/pkg/stdcopy/stdcopy_test.go index b3e2c4dfd8..3f992fda6a 100644 --- a/components/engine/pkg/stdcopy/stdcopy_test.go +++ b/components/engine/pkg/stdcopy/stdcopy_test.go @@ -16,7 +16,7 @@ func TestNewStdWriter(t *testing.T) { } } -func TestWriteWithUnitializedStdWriter(t *testing.T) { +func TestWriteWithUninitializedStdWriter(t *testing.T) { writer := stdWriter{ Writer: nil, prefix: byte(Stdout), diff --git a/components/engine/pkg/symlink/fs.go b/components/engine/pkg/symlink/fs.go index f6bc2231f6..52fb9a691b 100644 --- a/components/engine/pkg/symlink/fs.go +++ b/components/engine/pkg/symlink/fs.go @@ -40,7 +40,7 @@ func FollowSymlinkInScope(path, root string) (string, error) { // // Example: // If /foo/bar -> /outside, -// FollowSymlinkInScope("/foo/bar", "/foo") == "/foo/outside" instead of "/oustide" +// FollowSymlinkInScope("/foo/bar", "/foo") == "/foo/outside" instead of "/outside" // // IMPORTANT: it is the caller's responsibility to call evalSymlinksInScope *after* relevant symlinks // are created and not to create subsequently, additional symlinks that could potentially make a diff --git a/components/engine/pkg/system/path_windows.go b/components/engine/pkg/system/path_windows.go index 3fc4744948..aab891522d 100644 --- a/components/engine/pkg/system/path_windows.go +++ b/components/engine/pkg/system/path_windows.go @@ -12,7 +12,7 @@ import ( // This is used, for example, when validating a user provided path in docker cp. // If a drive letter is supplied, it must be the system drive. The drive letter // is always removed. Also, it translates it to OS semantics (IOW / to \). We -// need the path in this syntax so that it can ultimately be contatenated with +// need the path in this syntax so that it can ultimately be concatenated with // a Windows long-path which doesn't support drive-letters. Examples: // C: --> Fail // C:\ --> \ diff --git a/components/engine/pkg/system/rm.go b/components/engine/pkg/system/rm.go index ca0621f04f..101b569a56 100644 --- a/components/engine/pkg/system/rm.go +++ b/components/engine/pkg/system/rm.go @@ -20,7 +20,7 @@ import ( // These types of errors do not need to be returned since it's ok for the dir to // be gone we can just retry the remove operation. // -// This should not return a `os.ErrNotExist` kind of error under any cirucmstances +// This should not return a `os.ErrNotExist` kind of error under any circumstances func EnsureRemoveAll(dir string) error { notExistErr := make(map[string]bool) diff --git a/components/engine/pkg/templates/templates.go b/components/engine/pkg/templates/templates.go index 75a3dd974e..d2d7e0c3d7 100644 --- a/components/engine/pkg/templates/templates.go +++ b/components/engine/pkg/templates/templates.go @@ -30,7 +30,7 @@ var basicFunctions = template.FuncMap{ // HeaderFunctions are used to created headers of a table. // This is a replacement of basicFunctions for header generation // because we want the header to remain intact. -// Some functions like `split` are irrevelant so not added. +// Some functions like `split` are irrelevant so not added. var HeaderFunctions = template.FuncMap{ "json": func(v string) string { return v diff --git a/components/engine/pkg/testutil/cmd/command.go b/components/engine/pkg/testutil/cmd/command.go index 36aba593b2..6f36d67908 100644 --- a/components/engine/pkg/testutil/cmd/command.go +++ b/components/engine/pkg/testutil/cmd/command.go @@ -53,7 +53,7 @@ type Result struct { } // Assert compares the Result against the Expected struct, and fails the test if -// any of the expcetations are not met. +// any of the expectations are not met. func (r *Result) Assert(t testingT, exp Expected) *Result { err := r.Compare(exp) if err == nil { diff --git a/components/engine/plugin/manager.go b/components/engine/plugin/manager.go index 02d372741f..29d3902c31 100644 --- a/components/engine/plugin/manager.go +++ b/components/engine/plugin/manager.go @@ -271,7 +271,7 @@ func (pm *Manager) save(p *v2.Plugin) error { return nil } -// GC cleans up unrefrenced blobs. This is recommended to run in a goroutine +// GC cleans up unreferenced blobs. This is recommended to run in a goroutine func (pm *Manager) GC() { pm.muGC.Lock() defer pm.muGC.Unlock() diff --git a/components/engine/plugin/v2/settable_test.go b/components/engine/plugin/v2/settable_test.go index 7183f3a679..1094c472b0 100644 --- a/components/engine/plugin/v2/settable_test.go +++ b/components/engine/plugin/v2/settable_test.go @@ -68,7 +68,7 @@ func TestIsSettable(t *testing.T) { } } -func TestUpdateSettinsEnv(t *testing.T) { +func TestUpdateSettingsEnv(t *testing.T) { contexts := []struct { env []string set settable diff --git a/components/engine/reference/store.go b/components/engine/reference/store.go index 6599235401..5b68c437c8 100644 --- a/components/engine/reference/store.go +++ b/components/engine/reference/store.go @@ -221,7 +221,7 @@ func (store *store) Delete(ref reference.Named) (bool, error) { func (store *store) Get(ref reference.Named) (digest.Digest, error) { if canonical, ok := ref.(reference.Canonical); ok { // If reference contains both tag and digest, only - // lookup by digest as it takes precendent over + // lookup by digest as it takes precedence over // tag, until tag/digest combos are stored. if _, ok := ref.(reference.Tagged); ok { var err error diff --git a/components/engine/registry/config.go b/components/engine/registry/config.go index 651bd73097..182599e38d 100644 --- a/components/engine/registry/config.go +++ b/components/engine/registry/config.go @@ -252,7 +252,7 @@ skip: return nil } -// allowNondistributableArtifacts returns true if the provided hostname is part of the list of regsitries +// allowNondistributableArtifacts returns true if the provided hostname is part of the list of registries // that allow push of nondistributable artifacts. // // The list can contain elements with CIDR notation to specify a whole subnet. If the subnet contains an IP diff --git a/components/engine/registry/endpoint_v1.go b/components/engine/registry/endpoint_v1.go index 6bcf8c935d..c5ca961dd4 100644 --- a/components/engine/registry/endpoint_v1.go +++ b/components/engine/registry/endpoint_v1.go @@ -175,7 +175,7 @@ func (e *V1Endpoint) Ping() (PingResult, error) { Standalone: true, } if err := json.Unmarshal(jsonString, &info); err != nil { - logrus.Debugf("Error unmarshalling the _ping PingResult: %s", err) + logrus.Debugf("Error unmarshaling the _ping PingResult: %s", err) // don't stop here. Just assume sane defaults } if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" { diff --git a/components/engine/reports/2017-05-08.md b/components/engine/reports/2017-05-08.md index 86d011a00a..7f03335416 100644 --- a/components/engine/reports/2017-05-08.md +++ b/components/engine/reports/2017-05-08.md @@ -9,7 +9,7 @@ During this meeting, we are talking about the [tasks](https://github.com/moby/mo ### The CLI split -The Docker CLI was succesfully moved to [https://github.com/docker/cli](https://github.com/docker/cli) last week thanks to @tiborvass +The Docker CLI was successfully moved to [https://github.com/docker/cli](https://github.com/docker/cli) last week thanks to @tiborvass The Docker CLI is now compiled from the [Dockerfile](https://github.com/moby/moby/blob/a762ceace4e8c1c7ce4fb582789af9d8074be3e1/Dockerfile#L248) ### Mailing list diff --git a/components/engine/reports/2017-05-15.md b/components/engine/reports/2017-05-15.md index 372630c2c2..7556f9cc4a 100644 --- a/components/engine/reports/2017-05-15.md +++ b/components/engine/reports/2017-05-15.md @@ -27,7 +27,7 @@ breaking up / removing existing packages that likely are not good candidates to With the removal of the CLI from the moby repository, new pull requests will have to be tested using API tests instead of using the CLI. Discussion took place whether or not these tests should use the API `client` package, or be completely -independend, and make raw HTTP calls. +independent, and make raw HTTP calls. A topic was created on the forum to discuss options: [evolution of testing](https://forums.mobyproject.org/t/evolution-of-testing-moby/38) diff --git a/components/engine/volume/drivers/adapter.go b/components/engine/volume/drivers/adapter.go index 0ec68dad5f..304c81bc00 100644 --- a/components/engine/volume/drivers/adapter.go +++ b/components/engine/volume/drivers/adapter.go @@ -102,7 +102,7 @@ func (a *volumeDriverAdapter) getCapabilities() volume.Capability { if err != nil { // `GetCapabilities` is a not a required endpoint. // On error assume it's a local-only driver - logrus.Warnf("Volume driver %s returned an error while trying to query its capabilities, using default capabilties: %v", a.name, err) + logrus.Warnf("Volume driver %s returned an error while trying to query its capabilities, using default capabilities: %v", a.name, err) return volume.Capability{Scope: volume.LocalScope} } diff --git a/components/engine/volume/testutils/testutils.go b/components/engine/volume/testutils/testutils.go index d54ba44be6..359d923822 100644 --- a/components/engine/volume/testutils/testutils.go +++ b/components/engine/volume/testutils/testutils.go @@ -25,7 +25,7 @@ func (NoopVolume) Mount(_ string) (string, error) { return "noop", nil } // Unmount unmounts the volume from the container func (NoopVolume) Unmount(_ string) error { return nil } -// Status proivdes low-level details about the volume +// Status provides low-level details about the volume func (NoopVolume) Status() map[string]interface{} { return nil } // CreatedAt provides the time the volume (directory) was created at @@ -57,7 +57,7 @@ func (FakeVolume) Mount(_ string) (string, error) { return "fake", nil } // Unmount unmounts the volume from the container func (FakeVolume) Unmount(_ string) error { return nil } -// Status proivdes low-level details about the volume +// Status provides low-level details about the volume func (FakeVolume) Status() map[string]interface{} { return nil } // CreatedAt provides the time the volume (directory) was created at diff --git a/components/engine/volume/volume.go b/components/engine/volume/volume.go index 2de248aa5f..8598d4cb8f 100644 --- a/components/engine/volume/volume.go +++ b/components/engine/volume/volume.go @@ -125,7 +125,7 @@ type MountPoint struct { Spec mounttypes.Mount // Track usage of this mountpoint - // Specicially needed for containers which are running and calls to `docker cp` + // Specifically needed for containers which are running and calls to `docker cp` // because both these actions require mounting the volumes. active int } diff --git a/components/engine/volume/volume_linux.go b/components/engine/volume/volume_linux.go index d4b4d800b2..fdf7b63e4b 100644 --- a/components/engine/volume/volume_linux.go +++ b/components/engine/volume/volume_linux.go @@ -26,7 +26,7 @@ func ConvertTmpfsOptions(opt *mounttypes.TmpfsOptions, readOnly bool) (string, e // okay, since API is that way anyways. // we do this by finding the suffix that divides evenly into the - // value, returing the value itself, with no suffix, if it fails. + // value, returning the value itself, with no suffix, if it fails. // // For the most part, we don't enforce any semantic to this values. // The operating system will usually align this and enforce minimum From e720e150d412e1882736e81f0320c75901d0f431 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 4 Jul 2017 03:09:56 -0700 Subject: [PATCH 21/56] Add missing API documentatoin for DataPathAddr COmmit 0307fe1a0bcdc02583a24add41eb783c117bad8c added a new `DataPathAddr` property to the swarm/init and swarm/join endpoints. This property was not yet added to the documentation. Signed-off-by: Sebastiaan van Stijn Upstream-commit: c79c16910c0f3d6e88f2dc6ef609ecc3b02ccef9 Component: engine --- components/engine/api/swagger.yaml | 21 +++++++++++++++++++ components/engine/docs/api/version-history.md | 2 ++ 2 files changed, 23 insertions(+) diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index fd0c4c1715..b9e6932994 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -7394,6 +7394,16 @@ paths: AdvertiseAddr: description: "Externally reachable address advertised to other nodes. This can either be an address/port combination in the form `192.168.1.1:4567`, or an interface followed by a port number, like `eth0:4567`. If the port number is omitted, the port number from the listen address is used. If `AdvertiseAddr` is not specified, it will be automatically detected when possible." type: "string" + DataPathAddr: + description: | + Address or interface to use for data path traffic (format: ``), for example, `192.168.1.1`, + or an interface, like `eth0`. If `DataPathAddr` is unspecified, the same address as `AdvertiseAddr` + is used. + + The `DataPathAddr` specifies the address that global scope network drivers will publish towards other + nodes in order to reach the containers running on this node. Using this parameter it is possible to + separate the container data traffic from the management traffic of the cluster. + type: "string" ForceNewCluster: description: "Force creation of a new swarm." type: "boolean" @@ -7442,6 +7452,17 @@ paths: type: "string" AdvertiseAddr: description: "Externally reachable address advertised to other nodes. This can either be an address/port combination in the form `192.168.1.1:4567`, or an interface followed by a port number, like `eth0:4567`. If the port number is omitted, the port number from the listen address is used. If `AdvertiseAddr` is not specified, it will be automatically detected when possible." + type: "string" + DataPathAddr: + description: | + Address or interface to use for data path traffic (format: ``), for example, `192.168.1.1`, + or an interface, like `eth0`. If `DataPathAddr` is unspecified, the same address as `AdvertiseAddr` + is used. + + The `DataPathAddr` specifies the address that global scope network drivers will publish towards other + nodes in order to reach the containers running on this node. Using this parameter it is possible to + separate the container data traffic from the management traffic of the cluster. + type: "string" RemoteAddrs: description: "Addresses of manager nodes already participating in the swarm." diff --git a/components/engine/docs/api/version-history.md b/components/engine/docs/api/version-history.md index aa089ed0a1..26f7585860 100644 --- a/components/engine/docs/api/version-history.md +++ b/components/engine/docs/api/version-history.md @@ -26,6 +26,8 @@ keywords: "API, Docker, rcli, REST, documentation" the daemon. This endpoint is experimental and only available if the daemon is started with experimental features enabled. * `GET /images/(name)/get` now includes an `ImageMetadata` field which contains image metadata that is local to the engine and not part of the image config. +* `POST /swarm/init` now accepts a `DataPathAddr` property to set the IP-address or network interface to use for data traffic +* `POST /swarm/join` now accepts a `DataPathAddr` property to set the IP-address or network interface to use for data traffic ## v1.30 API changes From b4c3d928eb49824534cdb9807c3f58851a538e1a Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 4 Jul 2017 07:37:26 -0400 Subject: [PATCH 22/56] Fix plugin remove dir name after rename. Signed-off-by: Brian Goff Upstream-commit: 4bf263c19873718394f8161dbc020bf4be30f9d6 Component: engine --- components/engine/plugin/backend_linux.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/engine/plugin/backend_linux.go b/components/engine/plugin/backend_linux.go index b22cc155d2..d65b90a2b7 100644 --- a/components/engine/plugin/backend_linux.go +++ b/components/engine/plugin/backend_linux.go @@ -630,11 +630,12 @@ func (pm *Manager) Remove(name string, config *types.PluginRmConfig) error { return errors.Wrap(err, "error unmounting plugin data") } - if err := os.Rename(pluginDir, pluginDir+"-removing"); err != nil { + removeDir := pluginDir + "-removing" + if err := os.Rename(pluginDir, removeDir); err != nil { return errors.Wrap(err, "error performing atomic remove of plugin dir") } - if err := system.EnsureRemoveAll(pluginDir); err != nil { + if err := system.EnsureRemoveAll(removeDir); err != nil { return errors.Wrap(err, "error removing plugin dir") } pm.config.Store.Remove(p) From 64645433a64c8e0e79bf392a6eec0a1679faa8f7 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Tue, 4 Jul 2017 06:34:15 +0000 Subject: [PATCH 23/56] pkg/archive.FileInfoHeader: fill file type bits Go 1.9 (golang/go@66b5a2f) removed file type bits from archive/tar.FileInfoHeader(). This commit ensures file type bits are filled even on Go 1.9 for compatibility. Signed-off-by: Akihiro Suda Upstream-commit: 1a451d9a7bb9cd7d437b42d4e73b0560fbf84348 Component: engine --- components/engine/pkg/archive/archive.go | 39 +++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/components/engine/pkg/archive/archive.go b/components/engine/pkg/archive/archive.go index c6ad7c58a8..2bed910fb4 100644 --- a/components/engine/pkg/archive/archive.go +++ b/components/engine/pkg/archive/archive.go @@ -93,6 +93,16 @@ const ( OverlayWhiteoutFormat ) +const ( + modeISDIR = 040000 // Directory + modeISFIFO = 010000 // FIFO + modeISREG = 0100000 // Regular file + modeISLNK = 0120000 // Symbolic link + modeISBLK = 060000 // Block special file + modeISCHR = 020000 // Character special file + modeISSOCK = 0140000 // Socket +) + // IsArchivePath checks if the (possibly compressed) file at the given path // starts with a tar file header. func IsArchivePath(path string) bool { @@ -305,12 +315,14 @@ func (compression *Compression) Extension() string { // FileInfoHeader creates a populated Header from fi. // Compared to archive pkg this function fills in more information. +// Also, regardless of Go version, this function fills file type bits (e.g. hdr.Mode |= modeISDIR), +// which have been deleted since Go 1.9 archive/tar. func FileInfoHeader(name string, fi os.FileInfo, link string) (*tar.Header, error) { hdr, err := tar.FileInfoHeader(fi, link) if err != nil { return nil, err } - hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) + hdr.Mode = fillGo18FileTypeBits(int64(chmodTarEntry(os.FileMode(hdr.Mode))), fi) name, err = canonicalTarName(name, fi.IsDir()) if err != nil { return nil, fmt.Errorf("tar: cannot canonicalize path: %v", err) @@ -322,6 +334,31 @@ func FileInfoHeader(name string, fi os.FileInfo, link string) (*tar.Header, erro return hdr, nil } +// fillGo18FileTypeBits fills type bits which have been removed on Go 1.9 archive/tar +// https://github.com/golang/go/commit/66b5a2f +func fillGo18FileTypeBits(mode int64, fi os.FileInfo) int64 { + fm := fi.Mode() + switch { + case fm.IsRegular(): + mode |= modeISREG + case fi.IsDir(): + mode |= modeISDIR + case fm&os.ModeSymlink != 0: + mode |= modeISLNK + case fm&os.ModeDevice != 0: + if fm&os.ModeCharDevice != 0 { + mode |= modeISCHR + } else { + mode |= modeISBLK + } + case fm&os.ModeNamedPipe != 0: + mode |= modeISFIFO + case fm&os.ModeSocket != 0: + mode |= modeISSOCK + } + return mode +} + // ReadSecurityXattrToTarHeader reads security.capability xattr from filesystem // to a tar header func ReadSecurityXattrToTarHeader(path string, hdr *tar.Header) error { From 14554f495ca39899e355354fb369e20edaf6d778 Mon Sep 17 00:00:00 2001 From: Yang Pengfei Date: Tue, 4 Jul 2017 23:35:58 +0800 Subject: [PATCH 24/56] Fix run `docker rename new_name` concurrently, the container will have multi names When run `docker rename new_name` concurrently, every operation will release container's old name. So container will have multi new names reserve in nameIndex. Signed-off-by: Yang Pengfei Upstream-commit: cc2340689c431504689d01c9516b3a340b2e8dbc Component: engine --- components/engine/daemon/rename.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/daemon/rename.go b/components/engine/daemon/rename.go index ad21593c19..2a8d0b22c7 100644 --- a/components/engine/daemon/rename.go +++ b/components/engine/daemon/rename.go @@ -32,6 +32,9 @@ func (daemon *Daemon) ContainerRename(oldName, newName string) error { return err } + container.Lock() + defer container.Unlock() + oldName = container.Name oldIsAnonymousEndpoint := container.NetworkSettings.IsAnonymousEndpoint @@ -39,9 +42,6 @@ func (daemon *Daemon) ContainerRename(oldName, newName string) error { return errors.New("Renaming a container with the same name as its current name") } - container.Lock() - defer container.Unlock() - links := map[string]*dockercontainer.Container{} for k, v := range daemon.linkIndex.children(container) { if !strings.HasPrefix(k, oldName) { From 2aedd8b1602866417717c4ee9c5202537773bfa2 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 30 Jun 2017 13:13:32 -0400 Subject: [PATCH 25/56] Fix log readers can block writes indefinitely Before this patch, a log reader is able to block all log writes indefinitely (and other operations) by simply opening the log stream and not consuming all the messages. The reason for this is we protect the read stream from corruption by ensuring there are no new writes while the log stream is consumed (and caught up with the live entries). We can get around this issue because log files are append only, so we can limit reads to only the section of the file that was written to when the log stream was first requested. Now logs are only blocked until all files are opened, rather than streamed to the client. Signed-off-by: Brian Goff Upstream-commit: e2209185ed1c959131d4068ec7fc93e194dc0802 Component: engine --- .../daemon/logger/jsonfilelog/jsonfilelog.go | 57 ++++++++++++------- .../engine/daemon/logger/jsonfilelog/read.go | 45 ++++++++++----- 2 files changed, 67 insertions(+), 35 deletions(-) diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go index 797644e669..e8df0ecbd0 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go @@ -7,6 +7,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "strconv" "sync" @@ -15,6 +16,7 @@ import ( "github.com/docker/docker/daemon/logger/loggerutils" "github.com/docker/docker/pkg/jsonlog" "github.com/docker/go-units" + "github.com/pkg/errors" ) // Name is the name of the file that the jsonlogger logs to. @@ -22,12 +24,13 @@ const Name = "json-file" // JSONFileLogger is Logger implementation for default Docker logging. type JSONFileLogger struct { - buf *bytes.Buffer - writer *loggerutils.RotateFileWriter - mu sync.Mutex - readers map[*logger.LogWatcher]struct{} // stores the active log followers - extra []byte // json-encoded extra attributes + extra []byte // json-encoded extra attributes + + mu sync.RWMutex + buf *bytes.Buffer // avoids allocating a new buffer on each call to `Log()` closed bool + writer *loggerutils.RotateFileWriter + readers map[*logger.LogWatcher]struct{} // stores the active log followers } func init() { @@ -90,33 +93,45 @@ func New(info logger.Info) (logger.Logger, error) { // Log converts logger.Message to jsonlog.JSONLog and serializes it to file. func (l *JSONFileLogger) Log(msg *logger.Message) error { + l.mu.Lock() + err := writeMessageBuf(l.writer, msg, l.extra, l.buf) + l.buf.Reset() + l.mu.Unlock() + return err +} + +func writeMessageBuf(w io.Writer, m *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error { + if err := marshalMessage(m, extra, buf); err != nil { + logger.PutMessage(m) + return err + } + logger.PutMessage(m) + if _, err := w.Write(buf.Bytes()); err != nil { + return errors.Wrap(err, "error writing log entry") + } + return nil +} + +func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error { timestamp, err := jsonlog.FastTimeMarshalJSON(msg.Timestamp) if err != nil { return err } - l.mu.Lock() - logline := msg.Line + logLine := msg.Line if !msg.Partial { - logline = append(msg.Line, '\n') + logLine = append(msg.Line, '\n') } err = (&jsonlog.JSONLogs{ - Log: logline, + Log: logLine, Stream: msg.Source, Created: timestamp, - RawAttrs: l.extra, - }).MarshalJSONBuf(l.buf) - logger.PutMessage(msg) + RawAttrs: extra, + }).MarshalJSONBuf(buf) if err != nil { - l.mu.Unlock() - return err + return errors.Wrap(err, "error writing log message to buffer") } - - l.buf.WriteByte('\n') - _, err = l.writer.Write(l.buf.Bytes()) - l.buf.Reset() - l.mu.Unlock() - - return err + err = buf.WriteByte('\n') + return errors.Wrap(err, "error finalizing log buffer") } // ValidateLogOpt looks for json specific log options max-file & max-size. diff --git a/components/engine/daemon/logger/jsonfilelog/read.go b/components/engine/daemon/logger/jsonfilelog/read.go index 312aa551d0..3fe5967241 100644 --- a/components/engine/daemon/logger/jsonfilelog/read.go +++ b/components/engine/daemon/logger/jsonfilelog/read.go @@ -3,7 +3,6 @@ package jsonfilelog import ( "bytes" "encoding/json" - "errors" "fmt" "io" "os" @@ -18,6 +17,7 @@ import ( "github.com/docker/docker/pkg/filenotify" "github.com/docker/docker/pkg/jsonlog" "github.com/docker/docker/pkg/tailfile" + "github.com/pkg/errors" ) const maxJSONDecodeRetry = 20000 @@ -48,10 +48,11 @@ func (l *JSONFileLogger) ReadLogs(config logger.ReadConfig) *logger.LogWatcher { func (l *JSONFileLogger) readLogs(logWatcher *logger.LogWatcher, config logger.ReadConfig) { defer close(logWatcher.Msg) - // lock so the read stream doesn't get corrupted due to rotations or other log data written while we read + // lock so the read stream doesn't get corrupted due to rotations or other log data written while we open these files // This will block writes!!! - l.mu.Lock() + l.mu.RLock() + // TODO it would be nice to move a lot of this reader implementation to the rotate logger object pth := l.writer.LogPath() var files []io.ReadSeeker for i := l.writer.MaxFiles(); i > 1; i-- { @@ -59,25 +60,36 @@ func (l *JSONFileLogger) readLogs(logWatcher *logger.LogWatcher, config logger.R if err != nil { if !os.IsNotExist(err) { logWatcher.Err <- err - break + l.mu.RUnlock() + return } continue } defer f.Close() - files = append(files, f) } latestFile, err := os.Open(pth) if err != nil { - logWatcher.Err <- err - l.mu.Unlock() + logWatcher.Err <- errors.Wrap(err, "error opening latest log file") + l.mu.RUnlock() return } defer latestFile.Close() + latestChunk, err := newSectionReader(latestFile) + + // Now we have the reader sectioned, all fd's opened, we can unlock. + // New writes/rotates will not affect seeking through these files + l.mu.RUnlock() + + if err != nil { + logWatcher.Err <- err + return + } + if config.Tail != 0 { - tailer := multireader.MultiReadSeeker(append(files, latestFile)...) + tailer := multireader.MultiReadSeeker(append(files, latestChunk)...) tailFile(tailer, logWatcher, config.Tail, config.Since) } @@ -89,19 +101,14 @@ func (l *JSONFileLogger) readLogs(logWatcher *logger.LogWatcher, config logger.R } if !config.Follow || l.closed { - l.mu.Unlock() return } - if config.Tail >= 0 { - latestFile.Seek(0, os.SEEK_END) - } - notifyRotate := l.writer.NotifyRotate() defer l.writer.NotifyRotateEvict(notifyRotate) + l.mu.Lock() l.readers[logWatcher] = struct{}{} - l.mu.Unlock() followLogs(latestFile, logWatcher, notifyRotate, config.Since) @@ -111,6 +118,16 @@ func (l *JSONFileLogger) readLogs(logWatcher *logger.LogWatcher, config logger.R l.mu.Unlock() } +func newSectionReader(f *os.File) (*io.SectionReader, error) { + // seek to the end to get the size + // we'll leave this at the end of the file since section reader does not advance the reader + size, err := f.Seek(0, os.SEEK_END) + if err != nil { + return nil, errors.Wrap(err, "error getting current file size") + } + return io.NewSectionReader(f, 0, size), nil +} + func tailFile(f io.ReadSeeker, logWatcher *logger.LogWatcher, tail int, since time.Time) { var rdr io.Reader rdr = f From 7668114cba6ade5e0dbdac6460cd864dd9ffbf87 Mon Sep 17 00:00:00 2001 From: yangshukui Date: Thu, 6 Jul 2017 17:03:38 +0800 Subject: [PATCH 26/56] Fix docker cp dir with hard link docker run --name=test ubuntu /bin/sh -c "cd /tmp && echo hi > a && ln a b" && docker cp test:/tmp tmp_ test link /root/tmp/a /root/tmp_/b: no such file or directory Signed-off-by: yangshukui Upstream-commit: d58ffa0364c04d03a8f25704d7f0489ee6cd9634 Component: engine --- components/engine/pkg/archive/copy.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/pkg/archive/copy.go b/components/engine/pkg/archive/copy.go index 0614c67cec..5281e29d18 100644 --- a/components/engine/pkg/archive/copy.go +++ b/components/engine/pkg/archive/copy.go @@ -332,6 +332,9 @@ func RebaseArchiveEntries(srcContent io.Reader, oldBase, newBase string) io.Read } hdr.Name = strings.Replace(hdr.Name, oldBase, newBase, 1) + if hdr.Typeflag == tar.TypeLink { + hdr.Linkname = strings.Replace(hdr.Linkname, oldBase, newBase, 1) + } if err = rebasedTar.WriteHeader(hdr); err != nil { w.CloseWithError(err) From 4ad1d46962ed7891914f323b23565cc3f6d2f96b Mon Sep 17 00:00:00 2001 From: Deborah Gertrude Digges Date: Thu, 6 Jul 2017 22:29:57 +0530 Subject: [PATCH 27/56] Add link to docker.com in README.md - Update README with the link to docker.com Signed off by Author: Deborah Gertrude Digges Signed-off-by: ddigges Upstream-commit: f63ec3d87467419a94f827939a1eaefa03b2f181 Component: engine --- components/engine/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/README.md b/components/engine/README.md index f768ff8449..533d7717df 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -51,7 +51,7 @@ Moby is NOT recommended for: - Application developers looking for an easy way to run their applications in containers. We recommend Docker CE instead. - Enterprise IT and development teams looking for a ready-to-use, commercially supported container platform. We recommend Docker EE instead. -- Anyone curious about containers and looking for an easy way to learn. We recommend the docker.com website instead. +- Anyone curious about containers and looking for an easy way to learn. We recommend the [docker.com](https://www.docker.com/) website instead. # Transitioning to Moby From c782cb6c440d4f07453bb5666e6a21a55f102aec Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 7 Jun 2017 13:07:01 -0400 Subject: [PATCH 28/56] Make plugin emit strongly typed, consumable events Enables other subsystems to watch actions for a plugin(s). This will be used specifically for implementing plugins on swarm where a swarm controller needs to watch the state of a plugin. Signed-off-by: Brian Goff Upstream-commit: 72c3bcf2a533a827402945e3a55872e2db4fb024 Component: engine --- .../api/server/router/plugin/backend.go | 3 +- .../engine/api/server/router/swarm/helpers.go | 2 +- components/engine/api/swagger.yaml | 30 +- .../engine/api/types/swarm/runtime/gen.go | 3 + .../api/types/swarm/runtime/plugin.pb.go | 712 ++++++++++++++++++ .../api/types/swarm/runtime/plugin.proto | 18 + components/engine/api/types/swarm/task.go | 12 +- components/engine/client/service_create.go | 106 ++- .../engine/client/service_create_test.go | 4 +- components/engine/client/service_update.go | 46 +- components/engine/cmd/dockerd/daemon.go | 1 + components/engine/daemon/cluster/cluster.go | 2 + .../cluster/controllers/plugin/controller.go | 244 +++++- .../controllers/plugin/controller_test.go | 390 ++++++++++ .../daemon/cluster/convert/container.go | 9 +- .../engine/daemon/cluster/convert/service.go | 77 +- .../daemon/cluster/convert/service_test.go | 6 +- .../engine/daemon/cluster/convert/task.go | 15 +- .../cluster/executor/container/executor.go | 14 +- components/engine/daemon/cluster/filters.go | 2 + .../engine/daemon/cluster/noderunner.go | 2 +- components/engine/daemon/cluster/services.go | 168 +++-- components/engine/daemon/cluster/tasks.go | 18 +- components/engine/docs/api/version-history.md | 1 + .../integration-cli/daemon/daemon_swarm.go | 60 +- .../docker_api_swarm_service_test.go | 78 ++ .../integration-cli/docker_api_swarm_test.go | 10 +- .../fixtures/plugin/basic/basic.go | 34 + .../integration-cli/fixtures/plugin/plugin.go | 183 +++++ components/engine/pkg/pubsub/publisher.go | 10 + components/engine/plugin/backend_linux.go | 17 +- .../engine/plugin/backend_unsupported.go | 2 +- components/engine/plugin/defs.go | 11 + components/engine/plugin/events.go | 111 +++ components/engine/plugin/manager.go | 9 + components/engine/plugin/manager_linux.go | 5 +- components/engine/plugin/v2/plugin.go | 2 + 37 files changed, 2204 insertions(+), 213 deletions(-) create mode 100644 components/engine/api/types/swarm/runtime/gen.go create mode 100644 components/engine/api/types/swarm/runtime/plugin.pb.go create mode 100644 components/engine/api/types/swarm/runtime/plugin.proto create mode 100644 components/engine/daemon/cluster/controllers/plugin/controller_test.go create mode 100644 components/engine/integration-cli/fixtures/plugin/basic/basic.go create mode 100644 components/engine/integration-cli/fixtures/plugin/plugin.go create mode 100644 components/engine/plugin/events.go diff --git a/components/engine/api/server/router/plugin/backend.go b/components/engine/api/server/router/plugin/backend.go index a5f3c9790a..1b60501fc5 100644 --- a/components/engine/api/server/router/plugin/backend.go +++ b/components/engine/api/server/router/plugin/backend.go @@ -7,6 +7,7 @@ import ( "github.com/docker/distribution/reference" enginetypes "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/plugin" "golang.org/x/net/context" ) @@ -19,7 +20,7 @@ type Backend interface { Remove(name string, config *enginetypes.PluginRmConfig) error Set(name string, args []string) error Privileges(ctx context.Context, ref reference.Named, metaHeaders http.Header, authConfig *enginetypes.AuthConfig) (enginetypes.PluginPrivileges, error) - Pull(ctx context.Context, ref reference.Named, name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig, privileges enginetypes.PluginPrivileges, outStream io.Writer) error + Pull(ctx context.Context, ref reference.Named, name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig, privileges enginetypes.PluginPrivileges, outStream io.Writer, opts ...plugin.CreateOpt) error Push(ctx context.Context, name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig, outStream io.Writer) error Upgrade(ctx context.Context, ref reference.Named, name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig, privileges enginetypes.PluginPrivileges, outStream io.Writer) error CreateFromContext(ctx context.Context, tarCtx io.ReadCloser, options *enginetypes.PluginCreateOptions) error diff --git a/components/engine/api/server/router/swarm/helpers.go b/components/engine/api/server/router/swarm/helpers.go index ea692ea368..7d2944208f 100644 --- a/components/engine/api/server/router/swarm/helpers.go +++ b/components/engine/api/server/router/swarm/helpers.go @@ -44,7 +44,7 @@ func (sr *swarmRouter) swarmLogs(ctx context.Context, w http.ResponseWriter, r * // maybe should return some context with this error? return err } - tty = s.Spec.TaskTemplate.ContainerSpec.TTY || tty + tty = (s.Spec.TaskTemplate.ContainerSpec != nil && s.Spec.TaskTemplate.ContainerSpec.TTY) || tty } for _, task := range selector.Tasks { t, err := sr.backend.GetTask(task) diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index 06e6b0bd3f..5803266b0f 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -1975,11 +1975,39 @@ definitions: description: "User modifiable task configuration." type: "object" properties: + PluginSpec: + type: "object" + description: "Invalid when specified with `ContainerSpec`." + properties: + Name: + description: "The name or 'alias' to use for the plugin." + type: "string" + Remote: + description: "The plugin image reference to use." + type: "string" + Disabled: + description: "Disable the plugin once scheduled." + type: "boolean" + PluginPrivilege: + type: "array" + items: + description: "Describes a permission accepted by the user upon installing the plugin." + type: "object" + properties: + Name: + type: "string" + Description: + type: "string" + Value: + type: "array" + items: + type: "string" ContainerSpec: type: "object" + description: "Invalid when specified with `PluginSpec`." properties: Image: - description: "The image name to use for the container." + description: "The image name to use for the container" type: "string" Labels: description: "User-defined key/value data." diff --git a/components/engine/api/types/swarm/runtime/gen.go b/components/engine/api/types/swarm/runtime/gen.go new file mode 100644 index 0000000000..47ae234ef3 --- /dev/null +++ b/components/engine/api/types/swarm/runtime/gen.go @@ -0,0 +1,3 @@ +//go:generate protoc -I . --gogofast_out=import_path=github.com/docker/docker/api/types/swarm/runtime:. plugin.proto + +package runtime diff --git a/components/engine/api/types/swarm/runtime/plugin.pb.go b/components/engine/api/types/swarm/runtime/plugin.pb.go new file mode 100644 index 0000000000..1fdc9b0436 --- /dev/null +++ b/components/engine/api/types/swarm/runtime/plugin.pb.go @@ -0,0 +1,712 @@ +// Code generated by protoc-gen-gogo. +// source: plugin.proto +// DO NOT EDIT! + +/* + Package runtime is a generated protocol buffer package. + + It is generated from these files: + plugin.proto + + It has these top-level messages: + PluginSpec + PluginPrivilege +*/ +package runtime + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +// PluginSpec defines the base payload which clients can specify for creating +// a service with the plugin runtime. +type PluginSpec struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Remote string `protobuf:"bytes,2,opt,name=remote,proto3" json:"remote,omitempty"` + Privileges []*PluginPrivilege `protobuf:"bytes,3,rep,name=privileges" json:"privileges,omitempty"` + Disabled bool `protobuf:"varint,4,opt,name=disabled,proto3" json:"disabled,omitempty"` +} + +func (m *PluginSpec) Reset() { *m = PluginSpec{} } +func (m *PluginSpec) String() string { return proto.CompactTextString(m) } +func (*PluginSpec) ProtoMessage() {} +func (*PluginSpec) Descriptor() ([]byte, []int) { return fileDescriptorPlugin, []int{0} } + +func (m *PluginSpec) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *PluginSpec) GetRemote() string { + if m != nil { + return m.Remote + } + return "" +} + +func (m *PluginSpec) GetPrivileges() []*PluginPrivilege { + if m != nil { + return m.Privileges + } + return nil +} + +func (m *PluginSpec) GetDisabled() bool { + if m != nil { + return m.Disabled + } + return false +} + +// PluginPrivilege describes a permission the user has to accept +// upon installing a plugin. +type PluginPrivilege struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Description string `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"` + Value []string `protobuf:"bytes,3,rep,name=value" json:"value,omitempty"` +} + +func (m *PluginPrivilege) Reset() { *m = PluginPrivilege{} } +func (m *PluginPrivilege) String() string { return proto.CompactTextString(m) } +func (*PluginPrivilege) ProtoMessage() {} +func (*PluginPrivilege) Descriptor() ([]byte, []int) { return fileDescriptorPlugin, []int{1} } + +func (m *PluginPrivilege) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *PluginPrivilege) GetDescription() string { + if m != nil { + return m.Description + } + return "" +} + +func (m *PluginPrivilege) GetValue() []string { + if m != nil { + return m.Value + } + return nil +} + +func init() { + proto.RegisterType((*PluginSpec)(nil), "PluginSpec") + proto.RegisterType((*PluginPrivilege)(nil), "PluginPrivilege") +} +func (m *PluginSpec) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PluginSpec) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPlugin(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Remote) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPlugin(dAtA, i, uint64(len(m.Remote))) + i += copy(dAtA[i:], m.Remote) + } + if len(m.Privileges) > 0 { + for _, msg := range m.Privileges { + dAtA[i] = 0x1a + i++ + i = encodeVarintPlugin(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.Disabled { + dAtA[i] = 0x20 + i++ + if m.Disabled { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *PluginPrivilege) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PluginPrivilege) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPlugin(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Description) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPlugin(dAtA, i, uint64(len(m.Description))) + i += copy(dAtA[i:], m.Description) + } + if len(m.Value) > 0 { + for _, s := range m.Value { + dAtA[i] = 0x1a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + +func encodeFixed64Plugin(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Plugin(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} +func encodeVarintPlugin(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *PluginSpec) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPlugin(uint64(l)) + } + l = len(m.Remote) + if l > 0 { + n += 1 + l + sovPlugin(uint64(l)) + } + if len(m.Privileges) > 0 { + for _, e := range m.Privileges { + l = e.Size() + n += 1 + l + sovPlugin(uint64(l)) + } + } + if m.Disabled { + n += 2 + } + return n +} + +func (m *PluginPrivilege) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPlugin(uint64(l)) + } + l = len(m.Description) + if l > 0 { + n += 1 + l + sovPlugin(uint64(l)) + } + if len(m.Value) > 0 { + for _, s := range m.Value { + l = len(s) + n += 1 + l + sovPlugin(uint64(l)) + } + } + return n +} + +func sovPlugin(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozPlugin(x uint64) (n int) { + return sovPlugin(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *PluginSpec) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlugin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PluginSpec: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PluginSpec: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlugin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlugin + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Remote", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlugin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlugin + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Remote = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Privileges", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlugin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPlugin + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Privileges = append(m.Privileges, &PluginPrivilege{}) + if err := m.Privileges[len(m.Privileges)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Disabled", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlugin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Disabled = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipPlugin(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPlugin + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PluginPrivilege) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlugin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PluginPrivilege: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PluginPrivilege: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlugin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlugin + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Description", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlugin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlugin + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Description = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlugin + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlugin + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Value = append(m.Value, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPlugin(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPlugin + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipPlugin(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPlugin + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPlugin + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPlugin + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthPlugin + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowPlugin + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipPlugin(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthPlugin = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowPlugin = fmt.Errorf("proto: integer overflow") +) + +func init() { proto.RegisterFile("plugin.proto", fileDescriptorPlugin) } + +var fileDescriptorPlugin = []byte{ + // 196 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0xc8, 0x29, 0x4d, + 0xcf, 0xcc, 0xd3, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x57, 0x6a, 0x63, 0xe4, 0xe2, 0x0a, 0x00, 0x0b, + 0x04, 0x17, 0xa4, 0x26, 0x0b, 0x09, 0x71, 0xb1, 0xe4, 0x25, 0xe6, 0xa6, 0x4a, 0x30, 0x2a, 0x30, + 0x6a, 0x70, 0x06, 0x81, 0xd9, 0x42, 0x62, 0x5c, 0x6c, 0x45, 0xa9, 0xb9, 0xf9, 0x25, 0xa9, 0x12, + 0x4c, 0x60, 0x51, 0x28, 0x4f, 0xc8, 0x80, 0x8b, 0xab, 0xa0, 0x28, 0xb3, 0x2c, 0x33, 0x27, 0x35, + 0x3d, 0xb5, 0x58, 0x82, 0x59, 0x81, 0x59, 0x83, 0xdb, 0x48, 0x40, 0x0f, 0x62, 0x58, 0x00, 0x4c, + 0x22, 0x08, 0x49, 0x8d, 0x90, 0x14, 0x17, 0x47, 0x4a, 0x66, 0x71, 0x62, 0x52, 0x4e, 0x6a, 0x8a, + 0x04, 0x8b, 0x02, 0xa3, 0x06, 0x47, 0x10, 0x9c, 0xaf, 0x14, 0xcb, 0xc5, 0x8f, 0xa6, 0x15, 0xab, + 0x63, 0x14, 0xb8, 0xb8, 0x53, 0x52, 0x8b, 0x93, 0x8b, 0x32, 0x0b, 0x4a, 0x32, 0xf3, 0xf3, 0xa0, + 0x2e, 0x42, 0x16, 0x12, 0x12, 0xe1, 0x62, 0x2d, 0x4b, 0xcc, 0x29, 0x4d, 0x05, 0xbb, 0x88, 0x33, + 0x08, 0xc2, 0x71, 0xe2, 0x39, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, + 0x18, 0x93, 0xd8, 0xc0, 0x9e, 0x37, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0xb8, 0x84, 0xad, 0x79, + 0x0c, 0x01, 0x00, 0x00, +} diff --git a/components/engine/api/types/swarm/runtime/plugin.proto b/components/engine/api/types/swarm/runtime/plugin.proto new file mode 100644 index 0000000000..06eb7ba650 --- /dev/null +++ b/components/engine/api/types/swarm/runtime/plugin.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +// PluginSpec defines the base payload which clients can specify for creating +// a service with the plugin runtime. +message PluginSpec { + string name = 1; + string remote = 2; + repeated PluginPrivilege privileges = 3; + bool disabled = 4; +} + +// PluginPrivilege describes a permission the user has to accept +// upon installing a plugin. +message PluginPrivilege { + string name = 1; + string description = 2; + repeated string value = 3; +} diff --git a/components/engine/api/types/swarm/task.go b/components/engine/api/types/swarm/task.go index a598a79d59..1712c06cf6 100644 --- a/components/engine/api/types/swarm/task.go +++ b/components/engine/api/types/swarm/task.go @@ -1,6 +1,10 @@ package swarm -import "time" +import ( + "time" + + "github.com/docker/docker/api/types/swarm/runtime" +) // TaskState represents the state of a task. type TaskState string @@ -51,7 +55,11 @@ type Task struct { // TaskSpec represents the spec of a task. type TaskSpec struct { - ContainerSpec ContainerSpec `json:",omitempty"` + // ContainerSpec and PluginSpec are mutually exclusive. + // PluginSpec will only be used when the `Runtime` field is set to `plugin` + ContainerSpec *ContainerSpec `json:",omitempty"` + PluginSpec *runtime.PluginSpec `json:",omitempty"` + Resources *ResourceRequirements `json:",omitempty"` RestartPolicy *RestartPolicy `json:",omitempty"` Placement *Placement `json:",omitempty"` diff --git a/components/engine/client/service_create.go b/components/engine/client/service_create.go index c2fc2776a8..a36839443c 100644 --- a/components/engine/client/service_create.go +++ b/components/engine/client/service_create.go @@ -6,9 +6,9 @@ import ( "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" - registrytypes "github.com/docker/docker/api/types/registry" "github.com/docker/docker/api/types/swarm" "github.com/opencontainers/go-digest" + "github.com/pkg/errors" "golang.org/x/net/context" ) @@ -24,24 +24,51 @@ func (cli *Client) ServiceCreate(ctx context.Context, service swarm.ServiceSpec, headers["X-Registry-Auth"] = []string{options.EncodedRegistryAuth} } - // ensure that the image is tagged - if taggedImg := imageWithTagString(service.TaskTemplate.ContainerSpec.Image); taggedImg != "" { - service.TaskTemplate.ContainerSpec.Image = taggedImg + // Make sure containerSpec is not nil when no runtime is set or the runtime is set to container + if service.TaskTemplate.ContainerSpec == nil && (service.TaskTemplate.Runtime == "" || service.TaskTemplate.Runtime == swarm.RuntimeContainer) { + service.TaskTemplate.ContainerSpec = &swarm.ContainerSpec{} } - // Contact the registry to retrieve digest and platform information - if options.QueryRegistry { - distributionInspect, err := cli.DistributionInspect(ctx, service.TaskTemplate.ContainerSpec.Image, options.EncodedRegistryAuth) - distErr = err - if err == nil { - // now pin by digest if the image doesn't already contain a digest - if img := imageWithDigestString(service.TaskTemplate.ContainerSpec.Image, distributionInspect.Descriptor.Digest); img != "" { + if err := validateServiceSpec(service); err != nil { + return types.ServiceCreateResponse{}, err + } + + // ensure that the image is tagged + var imgPlatforms []swarm.Platform + if service.TaskTemplate.ContainerSpec != nil { + if taggedImg := imageWithTagString(service.TaskTemplate.ContainerSpec.Image); taggedImg != "" { + service.TaskTemplate.ContainerSpec.Image = taggedImg + } + if options.QueryRegistry { + var img string + img, imgPlatforms, distErr = imageDigestAndPlatforms(ctx, cli, service.TaskTemplate.ContainerSpec.Image, options.EncodedRegistryAuth) + if img != "" { service.TaskTemplate.ContainerSpec.Image = img } - // add platforms that are compatible with the service - service.TaskTemplate.Placement = setServicePlatforms(service.TaskTemplate.Placement, distributionInspect) } } + + // ensure that the image is tagged + if service.TaskTemplate.PluginSpec != nil { + if taggedImg := imageWithTagString(service.TaskTemplate.PluginSpec.Remote); taggedImg != "" { + service.TaskTemplate.PluginSpec.Remote = taggedImg + } + if options.QueryRegistry { + var img string + img, imgPlatforms, distErr = imageDigestAndPlatforms(ctx, cli, service.TaskTemplate.PluginSpec.Remote, options.EncodedRegistryAuth) + if img != "" { + service.TaskTemplate.PluginSpec.Remote = img + } + } + } + + if service.TaskTemplate.Placement == nil && len(imgPlatforms) > 0 { + service.TaskTemplate.Placement = &swarm.Placement{} + } + if len(imgPlatforms) > 0 { + service.TaskTemplate.Placement.Platforms = imgPlatforms + } + var response types.ServiceCreateResponse resp, err := cli.post(ctx, "/services/create", nil, service, headers) if err != nil { @@ -58,6 +85,28 @@ func (cli *Client) ServiceCreate(ctx context.Context, service swarm.ServiceSpec, return response, err } +func imageDigestAndPlatforms(ctx context.Context, cli *Client, image, encodedAuth string) (string, []swarm.Platform, error) { + distributionInspect, err := cli.DistributionInspect(ctx, image, encodedAuth) + imageWithDigest := image + var platforms []swarm.Platform + if err != nil { + return "", nil, err + } + + imageWithDigest = imageWithDigestString(image, distributionInspect.Descriptor.Digest) + + if len(distributionInspect.Platforms) > 0 { + platforms = make([]swarm.Platform, 0, len(distributionInspect.Platforms)) + for _, p := range distributionInspect.Platforms { + platforms = append(platforms, swarm.Platform{ + Architecture: p.Architecture, + OS: p.OS, + }) + } + } + return imageWithDigest, platforms, err +} + // imageWithDigestString takes an image string and a digest, and updates // the image string if it didn't originally contain a digest. It returns // an empty string if there are no updates. @@ -86,27 +135,22 @@ func imageWithTagString(image string) string { return "" } -// setServicePlatforms sets Platforms in swarm.Placement to list all -// compatible platforms for the service, as found in distributionInspect -// and returns a pointer to the new or updated swarm.Placement struct. -func setServicePlatforms(placement *swarm.Placement, distributionInspect registrytypes.DistributionInspect) *swarm.Placement { - if placement == nil { - placement = &swarm.Placement{} - } - // reset any existing listed platforms - placement.Platforms = []swarm.Platform{} - for _, p := range distributionInspect.Platforms { - placement.Platforms = append(placement.Platforms, swarm.Platform{ - Architecture: p.Architecture, - OS: p.OS, - }) - } - return placement -} - // digestWarning constructs a formatted warning string using the // image name that could not be pinned by digest. The formatting // is hardcoded, but could me made smarter in the future func digestWarning(image string) string { return fmt.Sprintf("image %s could not be accessed on a registry to record\nits digest. Each node will access %s independently,\npossibly leading to different nodes running different\nversions of the image.\n", image, image) } + +func validateServiceSpec(s swarm.ServiceSpec) error { + if s.TaskTemplate.ContainerSpec != nil && s.TaskTemplate.PluginSpec != nil { + return errors.New("must not specify both a container spec and a plugin spec in the task template") + } + if s.TaskTemplate.PluginSpec != nil && s.TaskTemplate.Runtime != swarm.RuntimePlugin { + return errors.New("mismatched runtime with plugin spec") + } + if s.TaskTemplate.ContainerSpec != nil && (s.TaskTemplate.Runtime != "" && s.TaskTemplate.Runtime != swarm.RuntimeContainer) { + return errors.New("mismatched runtime with container spec") + } + return nil +} diff --git a/components/engine/client/service_create_test.go b/components/engine/client/service_create_test.go index eb9e1b59df..6915d636e0 100644 --- a/components/engine/client/service_create_test.go +++ b/components/engine/client/service_create_test.go @@ -112,7 +112,7 @@ func TestServiceCreateCompatiblePlatforms(t *testing.T) { }), } - spec := swarm.ServiceSpec{TaskTemplate: swarm.TaskSpec{ContainerSpec: swarm.ContainerSpec{Image: "foobar:1.0"}}} + spec := swarm.ServiceSpec{TaskTemplate: swarm.TaskSpec{ContainerSpec: &swarm.ContainerSpec{Image: "foobar:1.0"}}} r, err := client.ServiceCreate(context.Background(), spec, types.ServiceCreateOptions{QueryRegistry: true}) assert.NoError(t, err) @@ -189,7 +189,7 @@ func TestServiceCreateDigestPinning(t *testing.T) { for _, p := range pinByDigestTests { r, err := client.ServiceCreate(context.Background(), swarm.ServiceSpec{ TaskTemplate: swarm.TaskSpec{ - ContainerSpec: swarm.ContainerSpec{ + ContainerSpec: &swarm.ContainerSpec{ Image: p.img, }, }, diff --git a/components/engine/client/service_update.go b/components/engine/client/service_update.go index a72adc997c..8764f299a3 100644 --- a/components/engine/client/service_update.go +++ b/components/engine/client/service_update.go @@ -35,26 +35,46 @@ func (cli *Client) ServiceUpdate(ctx context.Context, serviceID string, version query.Set("version", strconv.FormatUint(version.Index, 10)) - // ensure that the image is tagged - if taggedImg := imageWithTagString(service.TaskTemplate.ContainerSpec.Image); taggedImg != "" { - service.TaskTemplate.ContainerSpec.Image = taggedImg + if err := validateServiceSpec(service); err != nil { + return types.ServiceUpdateResponse{}, err } - // Contact the registry to retrieve digest and platform information - // This happens only when the image has changed - if options.QueryRegistry { - distributionInspect, err := cli.DistributionInspect(ctx, service.TaskTemplate.ContainerSpec.Image, options.EncodedRegistryAuth) - distErr = err - if err == nil { - // now pin by digest if the image doesn't already contain a digest - if img := imageWithDigestString(service.TaskTemplate.ContainerSpec.Image, distributionInspect.Descriptor.Digest); img != "" { + var imgPlatforms []swarm.Platform + // ensure that the image is tagged + if service.TaskTemplate.ContainerSpec != nil { + if taggedImg := imageWithTagString(service.TaskTemplate.ContainerSpec.Image); taggedImg != "" { + service.TaskTemplate.ContainerSpec.Image = taggedImg + } + if options.QueryRegistry { + var img string + img, imgPlatforms, distErr = imageDigestAndPlatforms(ctx, cli, service.TaskTemplate.ContainerSpec.Image, options.EncodedRegistryAuth) + if img != "" { service.TaskTemplate.ContainerSpec.Image = img } - // add platforms that are compatible with the service - service.TaskTemplate.Placement = setServicePlatforms(service.TaskTemplate.Placement, distributionInspect) } } + // ensure that the image is tagged + if service.TaskTemplate.PluginSpec != nil { + if taggedImg := imageWithTagString(service.TaskTemplate.PluginSpec.Remote); taggedImg != "" { + service.TaskTemplate.PluginSpec.Remote = taggedImg + } + if options.QueryRegistry { + var img string + img, imgPlatforms, distErr = imageDigestAndPlatforms(ctx, cli, service.TaskTemplate.PluginSpec.Remote, options.EncodedRegistryAuth) + if img != "" { + service.TaskTemplate.PluginSpec.Remote = img + } + } + } + + if service.TaskTemplate.Placement == nil && len(imgPlatforms) > 0 { + service.TaskTemplate.Placement = &swarm.Placement{} + } + if len(imgPlatforms) > 0 { + service.TaskTemplate.Placement.Platforms = imgPlatforms + } + var response types.ServiceUpdateResponse resp, err := cli.post(ctx, "/services/"+serviceID+"/update", query, service, headers) if err != nil { diff --git a/components/engine/cmd/dockerd/daemon.go b/components/engine/cmd/dockerd/daemon.go index c9f7065c28..ad937ede7c 100644 --- a/components/engine/cmd/dockerd/daemon.go +++ b/components/engine/cmd/dockerd/daemon.go @@ -253,6 +253,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { Root: cli.Config.Root, Name: name, Backend: d, + PluginBackend: d.PluginManager(), NetworkSubnetsProvider: d, DefaultAdvertiseAddr: cli.Config.SwarmDefaultAdvertiseAddr, RuntimeRoot: cli.getSwarmRunRoot(), diff --git a/components/engine/daemon/cluster/cluster.go b/components/engine/daemon/cluster/cluster.go index 6874dbf0ee..57fc4d2d6f 100644 --- a/components/engine/daemon/cluster/cluster.go +++ b/components/engine/daemon/cluster/cluster.go @@ -49,6 +49,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types/network" types "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/daemon/cluster/controllers/plugin" executorpkg "github.com/docker/docker/daemon/cluster/executor" "github.com/docker/docker/pkg/signal" lncluster "github.com/docker/libnetwork/cluster" @@ -97,6 +98,7 @@ type Config struct { Root string Name string Backend executorpkg.Backend + PluginBackend plugin.Backend NetworkSubnetsProvider NetworkSubnetsProvider // DefaultAdvertiseAddr is the default host/IP or network interface to use diff --git a/components/engine/daemon/cluster/controllers/plugin/controller.go b/components/engine/daemon/cluster/controllers/plugin/controller.go index de7eb2c00f..e72edcdd75 100644 --- a/components/engine/daemon/cluster/controllers/plugin/controller.go +++ b/components/engine/daemon/cluster/controllers/plugin/controller.go @@ -1,79 +1,261 @@ package plugin import ( + "io" + "io/ioutil" + "net/http" + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/reference" + enginetypes "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/swarm/runtime" + "github.com/docker/docker/plugin" + "github.com/docker/docker/plugin/v2" "github.com/docker/swarmkit/api" + "github.com/gogo/protobuf/proto" + "github.com/pkg/errors" "golang.org/x/net/context" ) -// Controller is the controller for the plugin backend -type Controller struct{} +// Controller is the controller for the plugin backend. +// Plugins are managed as a singleton object with a desired state (different from containers). +// With the the plugin controller instead of having a strict create->start->stop->remove +// task lifecycle like containers, we manage the desired state of the plugin and let +// the plugin manager do what it already does and monitor the plugin. +// We'll also end up with many tasks all pointing to the same plugin ID. +// +// TODO(@cpuguy83): registry auth is intentionally not supported until we work out +// the right way to pass registry crednetials via secrets. +type Controller struct { + backend Backend + spec runtime.PluginSpec + logger *logrus.Entry + + pluginID string + serviceID string + taskID string + + // hook used to signal tests that `Wait()` is actually ready and waiting + signalWaitReady func() +} + +// Backend is the interface for interacting with the plugin manager +// Controller actions are passed to the configured backend to do the real work. +type Backend interface { + Disable(name string, config *enginetypes.PluginDisableConfig) error + Enable(name string, config *enginetypes.PluginEnableConfig) error + Remove(name string, config *enginetypes.PluginRmConfig) error + Pull(ctx context.Context, ref reference.Named, name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig, privileges enginetypes.PluginPrivileges, outStream io.Writer, opts ...plugin.CreateOpt) error + Upgrade(ctx context.Context, ref reference.Named, name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig, privileges enginetypes.PluginPrivileges, outStream io.Writer) error + Get(name string) (*v2.Plugin, error) + SubscribeEvents(buffer int, events ...plugin.Event) (eventCh <-chan interface{}, cancel func()) +} // NewController returns a new cluster plugin controller -func NewController() (*Controller, error) { - return &Controller{}, nil +func NewController(backend Backend, t *api.Task) (*Controller, error) { + spec, err := readSpec(t) + if err != nil { + return nil, err + } + return &Controller{ + backend: backend, + spec: spec, + serviceID: t.ServiceID, + logger: logrus.WithFields(logrus.Fields{ + "controller": "plugin", + "task": t.ID, + "plugin": spec.Name, + })}, nil +} + +func readSpec(t *api.Task) (runtime.PluginSpec, error) { + var cfg runtime.PluginSpec + + generic := t.Spec.GetGeneric() + if err := proto.Unmarshal(generic.Payload.Value, &cfg); err != nil { + return cfg, errors.Wrap(err, "error reading plugin spec") + } + return cfg, nil } // Update is the update phase from swarmkit func (p *Controller) Update(ctx context.Context, t *api.Task) error { - logrus.WithFields(logrus.Fields{ - "controller": "plugin", - }).Debug("Update") + p.logger.Debug("Update") return nil } // Prepare is the prepare phase from swarmkit -func (p *Controller) Prepare(ctx context.Context) error { - logrus.WithFields(logrus.Fields{ - "controller": "plugin", - }).Debug("Prepare") +func (p *Controller) Prepare(ctx context.Context) (err error) { + p.logger.Debug("Prepare") + + remote, err := reference.ParseNormalizedNamed(p.spec.Remote) + if err != nil { + return errors.Wrapf(err, "error parsing remote reference %q", p.spec.Remote) + } + + if p.spec.Name == "" { + p.spec.Name = remote.String() + } + + var authConfig enginetypes.AuthConfig + privs := convertPrivileges(p.spec.Privileges) + + pl, err := p.backend.Get(p.spec.Name) + + defer func() { + if pl != nil && err == nil { + pl.Acquire() + } + }() + + if err == nil && pl != nil { + if pl.SwarmServiceID != p.serviceID { + return errors.Errorf("plugin already exists: %s", p.spec.Name) + } + if pl.IsEnabled() { + if err := p.backend.Disable(pl.GetID(), &enginetypes.PluginDisableConfig{ForceDisable: true}); err != nil { + p.logger.WithError(err).Debug("could not disable plugin before running upgrade") + } + } + p.pluginID = pl.GetID() + return p.backend.Upgrade(ctx, remote, p.spec.Name, nil, &authConfig, privs, ioutil.Discard) + } + + if err := p.backend.Pull(ctx, remote, p.spec.Name, nil, &authConfig, privs, ioutil.Discard, plugin.WithSwarmService(p.serviceID)); err != nil { + return err + } + pl, err = p.backend.Get(p.spec.Name) + if err != nil { + return err + } + p.pluginID = pl.GetID() + return nil } // Start is the start phase from swarmkit func (p *Controller) Start(ctx context.Context) error { - logrus.WithFields(logrus.Fields{ - "controller": "plugin", - }).Debug("Start") + p.logger.Debug("Start") + + pl, err := p.backend.Get(p.pluginID) + if err != nil { + return err + } + + if p.spec.Disabled { + if pl.IsEnabled() { + return p.backend.Disable(p.pluginID, &enginetypes.PluginDisableConfig{ForceDisable: false}) + } + return nil + } + if !pl.IsEnabled() { + return p.backend.Enable(p.pluginID, &enginetypes.PluginEnableConfig{Timeout: 30}) + } return nil } // Wait causes the task to wait until returned func (p *Controller) Wait(ctx context.Context) error { - logrus.WithFields(logrus.Fields{ - "controller": "plugin", - }).Debug("Wait") - return nil + p.logger.Debug("Wait") + + pl, err := p.backend.Get(p.pluginID) + if err != nil { + return err + } + + events, cancel := p.backend.SubscribeEvents(1, plugin.EventDisable{Plugin: pl.PluginObj}, plugin.EventRemove{Plugin: pl.PluginObj}, plugin.EventEnable{Plugin: pl.PluginObj}) + defer cancel() + + if p.signalWaitReady != nil { + p.signalWaitReady() + } + + if !p.spec.Disabled != pl.IsEnabled() { + return errors.New("mismatched plugin state") + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case e := <-events: + p.logger.Debugf("got event %#T", e) + + switch e.(type) { + case plugin.EventEnable: + if p.spec.Disabled { + return errors.New("plugin enabled") + } + case plugin.EventRemove: + return errors.New("plugin removed") + case plugin.EventDisable: + if !p.spec.Disabled { + return errors.New("plugin disabled") + } + } + } + } +} + +func isNotFound(err error) bool { + _, ok := errors.Cause(err).(plugin.ErrNotFound) + return ok } // Shutdown is the shutdown phase from swarmkit func (p *Controller) Shutdown(ctx context.Context) error { - logrus.WithFields(logrus.Fields{ - "controller": "plugin", - }).Debug("Shutdown") + p.logger.Debug("Shutdown") return nil } // Terminate is the terminate phase from swarmkit func (p *Controller) Terminate(ctx context.Context) error { - logrus.WithFields(logrus.Fields{ - "controller": "plugin", - }).Debug("Terminate") + p.logger.Debug("Terminate") return nil } // Remove is the remove phase from swarmkit func (p *Controller) Remove(ctx context.Context) error { - logrus.WithFields(logrus.Fields{ - "controller": "plugin", - }).Debug("Remove") - return nil + p.logger.Debug("Remove") + + pl, err := p.backend.Get(p.pluginID) + if err != nil { + if isNotFound(err) { + return nil + } + return err + } + + pl.Release() + if pl.GetRefCount() > 0 { + p.logger.Debug("skipping remove due to ref count") + return nil + } + + // This may error because we have exactly 1 plugin, but potentially multiple + // tasks which are calling remove. + err = p.backend.Remove(p.pluginID, &enginetypes.PluginRmConfig{ForceRemove: true}) + if isNotFound(err) { + return nil + } + return err } // Close is the close phase from swarmkit func (p *Controller) Close() error { - logrus.WithFields(logrus.Fields{ - "controller": "plugin", - }).Debug("Close") + p.logger.Debug("Close") return nil } + +func convertPrivileges(ls []*runtime.PluginPrivilege) enginetypes.PluginPrivileges { + var out enginetypes.PluginPrivileges + for _, p := range ls { + pp := enginetypes.PluginPrivilege{ + Name: p.Name, + Description: p.Description, + Value: p.Value, + } + out = append(out, pp) + } + return out +} diff --git a/components/engine/daemon/cluster/controllers/plugin/controller_test.go b/components/engine/daemon/cluster/controllers/plugin/controller_test.go new file mode 100644 index 0000000000..17b77cc89f --- /dev/null +++ b/components/engine/daemon/cluster/controllers/plugin/controller_test.go @@ -0,0 +1,390 @@ +package plugin + +import ( + "errors" + "io" + "io/ioutil" + "net/http" + "strings" + "testing" + "time" + + "github.com/Sirupsen/logrus" + "github.com/docker/distribution/reference" + enginetypes "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/swarm/runtime" + "github.com/docker/docker/pkg/pubsub" + "github.com/docker/docker/plugin" + "github.com/docker/docker/plugin/v2" + "golang.org/x/net/context" +) + +const ( + pluginTestName = "test" + pluginTestRemote = "testremote" + pluginTestRemoteUpgrade = "testremote2" +) + +func TestPrepare(t *testing.T) { + b := newMockBackend() + c := newTestController(b, false) + ctx := context.Background() + + if err := c.Prepare(ctx); err != nil { + t.Fatal(err) + } + + if b.p == nil { + t.Fatal("pull not performed") + } + + c = newTestController(b, false) + if err := c.Prepare(ctx); err != nil { + t.Fatal(err) + } + if b.p == nil { + t.Fatal("unexpected nil") + } + if b.p.PluginObj.PluginReference != pluginTestRemoteUpgrade { + t.Fatal("upgrade not performed") + } + + c = newTestController(b, false) + c.serviceID = "1" + if err := c.Prepare(ctx); err == nil { + t.Fatal("expected error on prepare") + } +} + +func TestStart(t *testing.T) { + b := newMockBackend() + c := newTestController(b, false) + ctx := context.Background() + + if err := c.Prepare(ctx); err != nil { + t.Fatal(err) + } + + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + + if !b.p.IsEnabled() { + t.Fatal("expected plugin to be enabled") + } + + c = newTestController(b, true) + if err := c.Prepare(ctx); err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + if b.p.IsEnabled() { + t.Fatal("expected plugin to be disabled") + } + + c = newTestController(b, false) + if err := c.Prepare(ctx); err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + if !b.p.IsEnabled() { + t.Fatal("expected plugin to be enabled") + } +} + +func TestWaitCancel(t *testing.T) { + b := newMockBackend() + c := newTestController(b, true) + ctx := context.Background() + if err := c.Prepare(ctx); err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + + ctxCancel, cancel := context.WithCancel(ctx) + chErr := make(chan error) + go func() { + chErr <- c.Wait(ctxCancel) + }() + cancel() + select { + case err := <-chErr: + if err != context.Canceled { + t.Fatal(err) + } + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for cancelation") + } +} + +func TestWaitDisabled(t *testing.T) { + b := newMockBackend() + c := newTestController(b, true) + ctx := context.Background() + if err := c.Prepare(ctx); err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + + chErr := make(chan error) + go func() { + chErr <- c.Wait(ctx) + }() + + if err := b.Enable("test", nil); err != nil { + t.Fatal(err) + } + select { + case err := <-chErr: + if err == nil { + t.Fatal("expected error") + } + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for event") + } + + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + + ctxWaitReady, cancelCtxWaitReady := context.WithTimeout(ctx, 30*time.Second) + c.signalWaitReady = cancelCtxWaitReady + defer cancelCtxWaitReady() + + go func() { + chErr <- c.Wait(ctx) + }() + + chEvent, cancel := b.SubscribeEvents(1) + defer cancel() + + if err := b.Disable("test", nil); err != nil { + t.Fatal(err) + } + + select { + case <-chEvent: + <-ctxWaitReady.Done() + if err := ctxWaitReady.Err(); err == context.DeadlineExceeded { + t.Fatal(err) + } + select { + case <-chErr: + t.Fatal("wait returned unexpectedly") + default: + // all good + } + case <-chErr: + t.Fatal("wait returned unexpectedly") + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for event") + } + + if err := b.Remove("test", nil); err != nil { + t.Fatal(err) + } + select { + case err := <-chErr: + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "removed") { + t.Fatal(err) + } + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for event") + } +} + +func TestWaitEnabled(t *testing.T) { + b := newMockBackend() + c := newTestController(b, false) + ctx := context.Background() + if err := c.Prepare(ctx); err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + + chErr := make(chan error) + go func() { + chErr <- c.Wait(ctx) + }() + + if err := b.Disable("test", nil); err != nil { + t.Fatal(err) + } + select { + case err := <-chErr: + if err == nil { + t.Fatal("expected error") + } + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for event") + } + + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + + ctxWaitReady, ctxWaitCancel := context.WithCancel(ctx) + c.signalWaitReady = ctxWaitCancel + defer ctxWaitCancel() + + go func() { + chErr <- c.Wait(ctx) + }() + + chEvent, cancel := b.SubscribeEvents(1) + defer cancel() + + if err := b.Enable("test", nil); err != nil { + t.Fatal(err) + } + + select { + case <-chEvent: + <-ctxWaitReady.Done() + if err := ctxWaitReady.Err(); err == context.DeadlineExceeded { + t.Fatal(err) + } + select { + case <-chErr: + t.Fatal("wait returned unexpectedly") + default: + // all good + } + case <-chErr: + t.Fatal("wait returned unexpectedly") + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for event") + } + + if err := b.Remove("test", nil); err != nil { + t.Fatal(err) + } + select { + case err := <-chErr: + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "removed") { + t.Fatal(err) + } + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for event") + } +} + +func TestRemove(t *testing.T) { + b := newMockBackend() + c := newTestController(b, false) + ctx := context.Background() + + if err := c.Prepare(ctx); err != nil { + t.Fatal(err) + } + if err := c.Shutdown(ctx); err != nil { + t.Fatal(err) + } + + c2 := newTestController(b, false) + if err := c2.Prepare(ctx); err != nil { + t.Fatal(err) + } + + if err := c.Remove(ctx); err != nil { + t.Fatal(err) + } + if b.p == nil { + t.Fatal("plugin removed unexpectedly") + } + if err := c2.Shutdown(ctx); err != nil { + t.Fatal(err) + } + if err := c2.Remove(ctx); err != nil { + t.Fatal(err) + } + if b.p != nil { + t.Fatal("expected plugin to be removed") + } +} + +func newTestController(b Backend, disabled bool) *Controller { + return &Controller{ + logger: &logrus.Entry{Logger: &logrus.Logger{Out: ioutil.Discard}}, + backend: b, + spec: runtime.PluginSpec{ + Name: pluginTestName, + Remote: pluginTestRemote, + Disabled: disabled, + }, + } +} + +func newMockBackend() *mockBackend { + return &mockBackend{ + pub: pubsub.NewPublisher(0, 0), + } +} + +type mockBackend struct { + p *v2.Plugin + pub *pubsub.Publisher +} + +func (m *mockBackend) Disable(name string, config *enginetypes.PluginDisableConfig) error { + m.p.PluginObj.Enabled = false + m.pub.Publish(plugin.EventDisable{}) + return nil +} + +func (m *mockBackend) Enable(name string, config *enginetypes.PluginEnableConfig) error { + m.p.PluginObj.Enabled = true + m.pub.Publish(plugin.EventEnable{}) + return nil +} + +func (m *mockBackend) Remove(name string, config *enginetypes.PluginRmConfig) error { + m.p = nil + m.pub.Publish(plugin.EventRemove{}) + return nil +} + +func (m *mockBackend) Pull(ctx context.Context, ref reference.Named, name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig, privileges enginetypes.PluginPrivileges, outStream io.Writer, opts ...plugin.CreateOpt) error { + m.p = &v2.Plugin{ + PluginObj: enginetypes.Plugin{ + ID: "1234", + Name: name, + PluginReference: ref.String(), + }, + } + return nil +} + +func (m *mockBackend) Upgrade(ctx context.Context, ref reference.Named, name string, metaHeaders http.Header, authConfig *enginetypes.AuthConfig, privileges enginetypes.PluginPrivileges, outStream io.Writer) error { + m.p.PluginObj.PluginReference = pluginTestRemoteUpgrade + return nil +} + +func (m *mockBackend) Get(name string) (*v2.Plugin, error) { + if m.p == nil { + return nil, errors.New("not found") + } + return m.p, nil +} + +func (m *mockBackend) SubscribeEvents(buffer int, events ...plugin.Event) (eventCh <-chan interface{}, cancel func()) { + ch := m.pub.SubscribeTopicWithBuffer(nil, buffer) + cancel = func() { m.pub.Evict(ch) } + return ch, cancel +} diff --git a/components/engine/daemon/cluster/convert/container.go b/components/engine/daemon/cluster/convert/container.go index a468c5f846..6ac6f331f2 100644 --- a/components/engine/daemon/cluster/convert/container.go +++ b/components/engine/daemon/cluster/convert/container.go @@ -13,8 +13,11 @@ import ( gogotypes "github.com/gogo/protobuf/types" ) -func containerSpecFromGRPC(c *swarmapi.ContainerSpec) types.ContainerSpec { - containerSpec := types.ContainerSpec{ +func containerSpecFromGRPC(c *swarmapi.ContainerSpec) *types.ContainerSpec { + if c == nil { + return nil + } + containerSpec := &types.ContainerSpec{ Image: c.Image, Labels: c.Labels, Command: c.Command, @@ -211,7 +214,7 @@ func configReferencesFromGRPC(sr []*swarmapi.ConfigReference) []*types.ConfigRef return refs } -func containerToGRPC(c types.ContainerSpec) (*swarmapi.ContainerSpec, error) { +func containerToGRPC(c *types.ContainerSpec) (*swarmapi.ContainerSpec, error) { containerSpec := &swarmapi.ContainerSpec{ Image: c.Image, Labels: c.Labels, diff --git a/components/engine/daemon/cluster/convert/service.go b/components/engine/daemon/cluster/convert/service.go index 3ab212927c..947debdf52 100644 --- a/components/engine/daemon/cluster/convert/service.go +++ b/components/engine/daemon/cluster/convert/service.go @@ -1,14 +1,16 @@ package convert import ( - "errors" "fmt" "strings" types "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/api/types/swarm/runtime" "github.com/docker/docker/pkg/namesgenerator" swarmapi "github.com/docker/swarmkit/api" + "github.com/gogo/protobuf/proto" gogotypes "github.com/gogo/protobuf/types" + "github.com/pkg/errors" ) var ( @@ -85,7 +87,10 @@ func serviceSpecFromGRPC(spec *swarmapi.ServiceSpec) (*types.ServiceSpec, error) } - taskTemplate := taskSpecFromGRPC(spec.Task) + taskTemplate, err := taskSpecFromGRPC(spec.Task) + if err != nil { + return nil, err + } switch t := spec.Task.GetRuntime().(type) { case *swarmapi.TaskSpec_Container: @@ -164,19 +169,34 @@ func ServiceSpecToGRPC(s types.ServiceSpec) (swarmapi.ServiceSpec, error) { switch s.TaskTemplate.Runtime { case types.RuntimeContainer, "": // if empty runtime default to container - containerSpec, err := containerToGRPC(s.TaskTemplate.ContainerSpec) - if err != nil { - return swarmapi.ServiceSpec{}, err + if s.TaskTemplate.ContainerSpec != nil { + containerSpec, err := containerToGRPC(s.TaskTemplate.ContainerSpec) + if err != nil { + return swarmapi.ServiceSpec{}, err + } + spec.Task.Runtime = &swarmapi.TaskSpec_Container{Container: containerSpec} } - spec.Task.Runtime = &swarmapi.TaskSpec_Container{Container: containerSpec} case types.RuntimePlugin: - spec.Task.Runtime = &swarmapi.TaskSpec_Generic{ - Generic: &swarmapi.GenericRuntimeSpec{ - Kind: string(types.RuntimePlugin), - Payload: &gogotypes.Any{ - TypeUrl: string(types.RuntimeURLPlugin), + if s.Mode.Replicated != nil { + return swarmapi.ServiceSpec{}, errors.New("plugins must not use replicated mode") + } + + s.Mode.Global = &types.GlobalService{} // must always be global + + if s.TaskTemplate.PluginSpec != nil { + pluginSpec, err := proto.Marshal(s.TaskTemplate.PluginSpec) + if err != nil { + return swarmapi.ServiceSpec{}, err + } + spec.Task.Runtime = &swarmapi.TaskSpec_Generic{ + Generic: &swarmapi.GenericRuntimeSpec{ + Kind: string(types.RuntimePlugin), + Payload: &gogotypes.Any{ + TypeUrl: string(types.RuntimeURLPlugin), + Value: pluginSpec, + }, }, - }, + } } default: return swarmapi.ServiceSpec{}, ErrUnsupportedRuntime @@ -507,21 +527,14 @@ func updateConfigToGRPC(updateConfig *types.UpdateConfig) (*swarmapi.UpdateConfi return converted, nil } -func taskSpecFromGRPC(taskSpec swarmapi.TaskSpec) types.TaskSpec { +func taskSpecFromGRPC(taskSpec swarmapi.TaskSpec) (types.TaskSpec, error) { taskNetworks := make([]types.NetworkAttachmentConfig, 0, len(taskSpec.Networks)) for _, n := range taskSpec.Networks { netConfig := types.NetworkAttachmentConfig{Target: n.Target, Aliases: n.Aliases, DriverOpts: n.DriverAttachmentOpts} taskNetworks = append(taskNetworks, netConfig) } - c := taskSpec.GetContainer() - cSpec := types.ContainerSpec{} - if c != nil { - cSpec = containerSpecFromGRPC(c) - } - - return types.TaskSpec{ - ContainerSpec: cSpec, + t := types.TaskSpec{ Resources: resourcesFromGRPC(taskSpec.Resources), RestartPolicy: restartPolicyFromGRPC(taskSpec.Restart), Placement: placementFromGRPC(taskSpec.Placement), @@ -529,4 +542,26 @@ func taskSpecFromGRPC(taskSpec swarmapi.TaskSpec) types.TaskSpec { Networks: taskNetworks, ForceUpdate: taskSpec.ForceUpdate, } + + switch taskSpec.GetRuntime().(type) { + case *swarmapi.TaskSpec_Container, nil: + c := taskSpec.GetContainer() + if c != nil { + t.ContainerSpec = containerSpecFromGRPC(c) + } + case *swarmapi.TaskSpec_Generic: + g := taskSpec.GetGeneric() + if g != nil { + switch g.Kind { + case string(types.RuntimePlugin): + var p runtime.PluginSpec + if err := proto.Unmarshal(g.Payload.Value, &p); err != nil { + return t, errors.Wrap(err, "error unmarshalling plugin spec") + } + t.PluginSpec = &p + } + } + } + + return t, nil } diff --git a/components/engine/daemon/cluster/convert/service_test.go b/components/engine/daemon/cluster/convert/service_test.go index 92d80d3323..1b6598974b 100644 --- a/components/engine/daemon/cluster/convert/service_test.go +++ b/components/engine/daemon/cluster/convert/service_test.go @@ -4,6 +4,7 @@ import ( "testing" swarmtypes "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/api/types/swarm/runtime" swarmapi "github.com/docker/swarmkit/api" google_protobuf3 "github.com/gogo/protobuf/types" ) @@ -82,7 +83,8 @@ func TestServiceConvertFromGRPCGenericRuntimePlugin(t *testing.T) { func TestServiceConvertToGRPCGenericRuntimePlugin(t *testing.T) { s := swarmtypes.ServiceSpec{ TaskTemplate: swarmtypes.TaskSpec{ - Runtime: swarmtypes.RuntimePlugin, + Runtime: swarmtypes.RuntimePlugin, + PluginSpec: &runtime.PluginSpec{}, }, Mode: swarmtypes.ServiceMode{ Global: &swarmtypes.GlobalService{}, @@ -108,7 +110,7 @@ func TestServiceConvertToGRPCContainerRuntime(t *testing.T) { image := "alpine:latest" s := swarmtypes.ServiceSpec{ TaskTemplate: swarmtypes.TaskSpec{ - ContainerSpec: swarmtypes.ContainerSpec{ + ContainerSpec: &swarmtypes.ContainerSpec{ Image: image, }, }, diff --git a/components/engine/daemon/cluster/convert/task.go b/components/engine/daemon/cluster/convert/task.go index b90d24e359..e301415c6c 100644 --- a/components/engine/daemon/cluster/convert/task.go +++ b/components/engine/daemon/cluster/convert/task.go @@ -9,19 +9,22 @@ import ( ) // TaskFromGRPC converts a grpc Task to a Task. -func TaskFromGRPC(t swarmapi.Task) types.Task { +func TaskFromGRPC(t swarmapi.Task) (types.Task, error) { if t.Spec.GetAttachment() != nil { - return types.Task{} + return types.Task{}, nil } containerStatus := t.Status.GetContainer() - + taskSpec, err := taskSpecFromGRPC(t.Spec) + if err != nil { + return types.Task{}, err + } task := types.Task{ ID: t.ID, Annotations: annotationsFromGRPC(t.Annotations), ServiceID: t.ServiceID, Slot: int(t.Slot), NodeID: t.NodeID, - Spec: taskSpecFromGRPC(t.Spec), + Spec: taskSpec, Status: types.TaskStatus{ State: types.TaskState(strings.ToLower(t.Status.State.String())), Message: t.Status.Message, @@ -49,7 +52,7 @@ func TaskFromGRPC(t swarmapi.Task) types.Task { } if t.Status.PortStatus == nil { - return task + return task, nil } for _, p := range t.Status.PortStatus.Ports { @@ -62,5 +65,5 @@ func TaskFromGRPC(t swarmapi.Task) types.Task { }) } - return task + return task, nil } diff --git a/components/engine/daemon/cluster/executor/container/executor.go b/components/engine/daemon/cluster/executor/container/executor.go index 03a00cc87b..a71a9412e3 100644 --- a/components/engine/daemon/cluster/executor/container/executor.go +++ b/components/engine/daemon/cluster/executor/container/executor.go @@ -22,15 +22,17 @@ import ( ) type executor struct { - backend executorpkg.Backend - dependencies exec.DependencyManager + backend executorpkg.Backend + pluginBackend plugin.Backend + dependencies exec.DependencyManager } // NewExecutor returns an executor from the docker client. -func NewExecutor(b executorpkg.Backend) exec.Executor { +func NewExecutor(b executorpkg.Backend, p plugin.Backend) exec.Executor { return &executor{ - backend: b, - dependencies: agent.NewDependencyManager(), + backend: b, + pluginBackend: p, + dependencies: agent.NewDependencyManager(), } } @@ -181,7 +183,7 @@ func (e *executor) Controller(t *api.Task) (exec.Controller, error) { } switch runtimeKind { case string(swarmtypes.RuntimePlugin): - c, err := plugin.NewController() + c, err := plugin.NewController(e.pluginBackend, t) if err != nil { return ctlr, err } diff --git a/components/engine/daemon/cluster/filters.go b/components/engine/daemon/cluster/filters.go index 0a004af223..efda7dc8d7 100644 --- a/components/engine/daemon/cluster/filters.go +++ b/components/engine/daemon/cluster/filters.go @@ -57,6 +57,7 @@ func newListTasksFilters(filter filters.Args, transformFunc func(filters.Args) e // internal use in checking create/update progress. Therefore, // we prefix it with a '_'. "_up-to-date": true, + "runtime": true, } if err := filter.Validate(accepted); err != nil { return nil, err @@ -73,6 +74,7 @@ func newListTasksFilters(filter filters.Args, transformFunc func(filters.Args) e ServiceIDs: filter.Get("service"), NodeIDs: filter.Get("node"), UpToDate: len(filter.Get("_up-to-date")) != 0, + Runtimes: filter.Get("runtime"), } for _, s := range filter.Get("desired-state") { diff --git a/components/engine/daemon/cluster/noderunner.go b/components/engine/daemon/cluster/noderunner.go index c0c7529ed9..a1eda066b2 100644 --- a/components/engine/daemon/cluster/noderunner.go +++ b/components/engine/daemon/cluster/noderunner.go @@ -118,7 +118,7 @@ func (n *nodeRunner) start(conf nodeStartConfig) error { JoinAddr: joinAddr, StateDir: n.cluster.root, JoinToken: conf.joinToken, - Executor: container.NewExecutor(n.cluster.config.Backend), + Executor: container.NewExecutor(n.cluster.config.Backend, n.cluster.config.PluginBackend), HeartbeatTick: 1, ElectionTick: 3, UnlockKey: conf.lockKey, diff --git a/components/engine/daemon/cluster/services.go b/components/engine/daemon/cluster/services.go index f4416c24c3..42397fa00b 100644 --- a/components/engine/daemon/cluster/services.go +++ b/components/engine/daemon/cluster/services.go @@ -50,14 +50,16 @@ func (c *Cluster) GetServices(options apitypes.ServiceListOptions) ([]types.Serv return nil, err } + if len(options.Filters.Get("runtime")) == 0 { + // Default to using the container runtime filter + options.Filters.Add("runtime", string(types.RuntimeContainer)) + } + filters := &swarmapi.ListServicesRequest_Filters{ NamePrefixes: options.Filters.Get("name"), IDPrefixes: options.Filters.Get("id"), Labels: runconfigopts.ConvertKVStringsToMap(options.Filters.Get("label")), - // (ehazlett): hardcode runtime for now. eventually we will - // be able to filter for the desired runtimes once more - // are supported. - Runtimes: []string{string(types.RuntimeContainer)}, + Runtimes: options.Filters.Get("runtime"), } ctx, cancel := c.getRequestContext() @@ -134,6 +136,20 @@ func (c *Cluster) CreateService(s types.ServiceSpec, encodedAuth string, queryRe switch serviceSpec.Task.Runtime.(type) { // handle other runtimes here + case *swarmapi.TaskSpec_Generic: + switch serviceSpec.Task.GetGeneric().Kind { + case string(types.RuntimePlugin): + if s.TaskTemplate.PluginSpec == nil { + return errors.New("plugin spec must be set") + } + } + + r, err := state.controlClient.CreateService(ctx, &swarmapi.CreateServiceRequest{Spec: &serviceSpec}) + if err != nil { + return err + } + + resp.ID = r.Service.ID case *swarmapi.TaskSpec_Container: ctnr := serviceSpec.Task.GetContainer() if ctnr == nil { @@ -146,7 +162,9 @@ func (c *Cluster) CreateService(s types.ServiceSpec, encodedAuth string, queryRe // retrieve auth config from encoded auth authConfig := &apitypes.AuthConfig{} if encodedAuth != "" { - if err := json.NewDecoder(base64.NewDecoder(base64.URLEncoding, strings.NewReader(encodedAuth))).Decode(authConfig); err != nil { + authReader := strings.NewReader(encodedAuth) + dec := json.NewDecoder(base64.NewDecoder(base64.URLEncoding, authReader)) + if err := dec.Decode(authConfig); err != nil { logrus.Warnf("invalid authconfig: %v", err) } } @@ -216,75 +234,85 @@ func (c *Cluster) UpdateService(serviceIDOrName string, version uint64, spec typ return err } - newCtnr := serviceSpec.Task.GetContainer() - if newCtnr == nil { - return errors.New("service does not use container tasks") - } - - encodedAuth := flags.EncodedRegistryAuth - if encodedAuth != "" { - newCtnr.PullOptions = &swarmapi.ContainerSpec_PullOptions{RegistryAuth: encodedAuth} - } else { - // this is needed because if the encodedAuth isn't being updated then we - // shouldn't lose it, and continue to use the one that was already present - var ctnr *swarmapi.ContainerSpec - switch flags.RegistryAuthFrom { - case apitypes.RegistryAuthFromSpec, "": - ctnr = currentService.Spec.Task.GetContainer() - case apitypes.RegistryAuthFromPreviousSpec: - if currentService.PreviousSpec == nil { - return errors.New("service does not have a previous spec") - } - ctnr = currentService.PreviousSpec.Task.GetContainer() - default: - return errors.New("unsupported registryAuthFrom value") - } - if ctnr == nil { - return errors.New("service does not use container tasks") - } - newCtnr.PullOptions = ctnr.PullOptions - // update encodedAuth so it can be used to pin image by digest - if ctnr.PullOptions != nil { - encodedAuth = ctnr.PullOptions.RegistryAuth - } - } - - // retrieve auth config from encoded auth - authConfig := &apitypes.AuthConfig{} - if encodedAuth != "" { - if err := json.NewDecoder(base64.NewDecoder(base64.URLEncoding, strings.NewReader(encodedAuth))).Decode(authConfig); err != nil { - logrus.Warnf("invalid authconfig: %v", err) - } - } - resp = &apitypes.ServiceUpdateResponse{} - // pin image by digest for API versions < 1.30 - // TODO(nishanttotla): The check on "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE" - // should be removed in the future. Since integration tests only use the - // latest API version, so this is no longer required. - if os.Getenv("DOCKER_SERVICE_PREFER_OFFLINE_IMAGE") != "1" && queryRegistry { - digestImage, err := c.imageWithDigestString(ctx, newCtnr.Image, authConfig) - if err != nil { - logrus.Warnf("unable to pin image %s to digest: %s", newCtnr.Image, err.Error()) - // warning in the client response should be concise - resp.Warnings = append(resp.Warnings, digestWarning(newCtnr.Image)) - } else if newCtnr.Image != digestImage { - logrus.Debugf("pinning image %s by digest: %s", newCtnr.Image, digestImage) - newCtnr.Image = digestImage - } else { - logrus.Debugf("updating service using supplied digest reference %s", newCtnr.Image) + switch serviceSpec.Task.Runtime.(type) { + case *swarmapi.TaskSpec_Generic: + switch serviceSpec.Task.GetGeneric().Kind { + case string(types.RuntimePlugin): + if spec.TaskTemplate.PluginSpec == nil { + return errors.New("plugin spec must be set") + } + } + case *swarmapi.TaskSpec_Container: + newCtnr := serviceSpec.Task.GetContainer() + if newCtnr == nil { + return errors.New("service does not use container tasks") } - // Replace the context with a fresh one. - // If we timed out while communicating with the - // registry, then "ctx" will already be expired, which - // would cause UpdateService below to fail. Reusing - // "ctx" could make it impossible to update a service - // if the registry is slow or unresponsive. - var cancel func() - ctx, cancel = c.getRequestContext() - defer cancel() + encodedAuth := flags.EncodedRegistryAuth + if encodedAuth != "" { + newCtnr.PullOptions = &swarmapi.ContainerSpec_PullOptions{RegistryAuth: encodedAuth} + } else { + // this is needed because if the encodedAuth isn't being updated then we + // shouldn't lose it, and continue to use the one that was already present + var ctnr *swarmapi.ContainerSpec + switch flags.RegistryAuthFrom { + case apitypes.RegistryAuthFromSpec, "": + ctnr = currentService.Spec.Task.GetContainer() + case apitypes.RegistryAuthFromPreviousSpec: + if currentService.PreviousSpec == nil { + return errors.New("service does not have a previous spec") + } + ctnr = currentService.PreviousSpec.Task.GetContainer() + default: + return errors.New("unsupported registryAuthFrom value") + } + if ctnr == nil { + return errors.New("service does not use container tasks") + } + newCtnr.PullOptions = ctnr.PullOptions + // update encodedAuth so it can be used to pin image by digest + if ctnr.PullOptions != nil { + encodedAuth = ctnr.PullOptions.RegistryAuth + } + } + + // retrieve auth config from encoded auth + authConfig := &apitypes.AuthConfig{} + if encodedAuth != "" { + if err := json.NewDecoder(base64.NewDecoder(base64.URLEncoding, strings.NewReader(encodedAuth))).Decode(authConfig); err != nil { + logrus.Warnf("invalid authconfig: %v", err) + } + } + + // pin image by digest for API versions < 1.30 + // TODO(nishanttotla): The check on "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE" + // should be removed in the future. Since integration tests only use the + // latest API version, so this is no longer required. + if os.Getenv("DOCKER_SERVICE_PREFER_OFFLINE_IMAGE") != "1" && queryRegistry { + digestImage, err := c.imageWithDigestString(ctx, newCtnr.Image, authConfig) + if err != nil { + logrus.Warnf("unable to pin image %s to digest: %s", newCtnr.Image, err.Error()) + // warning in the client response should be concise + resp.Warnings = append(resp.Warnings, digestWarning(newCtnr.Image)) + } else if newCtnr.Image != digestImage { + logrus.Debugf("pinning image %s by digest: %s", newCtnr.Image, digestImage) + newCtnr.Image = digestImage + } else { + logrus.Debugf("updating service using supplied digest reference %s", newCtnr.Image) + } + + // Replace the context with a fresh one. + // If we timed out while communicating with the + // registry, then "ctx" will already be expired, which + // would cause UpdateService below to fail. Reusing + // "ctx" could make it impossible to update a service + // if the registry is slow or unresponsive. + var cancel func() + ctx, cancel = c.getRequestContext() + defer cancel() + } } var rollback swarmapi.UpdateServiceRequest_Rollback diff --git a/components/engine/daemon/cluster/tasks.go b/components/engine/daemon/cluster/tasks.go index 47cd5563b9..f0d6621dc5 100644 --- a/components/engine/daemon/cluster/tasks.go +++ b/components/engine/daemon/cluster/tasks.go @@ -19,7 +19,7 @@ func (c *Cluster) GetTasks(options apitypes.TaskListOptions) ([]types.Task, erro return nil, c.errNoManager(state) } - byName := func(filter filters.Args) error { + filterTransform := func(filter filters.Args) error { if filter.Include("service") { serviceFilters := filter.Get("service") for _, serviceFilter := range serviceFilters { @@ -42,10 +42,15 @@ func (c *Cluster) GetTasks(options apitypes.TaskListOptions) ([]types.Task, erro filter.Add("node", node.ID) } } + if !filter.Include("runtime") { + // default to only showing container tasks + filter.Add("runtime", "container") + filter.Add("runtime", "") + } return nil } - filters, err := newListTasksFilters(options.Filters, byName) + filters, err := newListTasksFilters(options.Filters, filterTransform) if err != nil { return nil, err } @@ -61,11 +66,12 @@ func (c *Cluster) GetTasks(options apitypes.TaskListOptions) ([]types.Task, erro } tasks := make([]types.Task, 0, len(r.Tasks)) - for _, task := range r.Tasks { - if task.Spec.GetContainer() != nil { - tasks = append(tasks, convert.TaskFromGRPC(*task)) + t, err := convert.TaskFromGRPC(*task) + if err != nil { + return nil, err } + tasks = append(tasks, t) } return tasks, nil } @@ -83,5 +89,5 @@ func (c *Cluster) GetTask(input string) (types.Task, error) { }); err != nil { return types.Task{}, err } - return convert.TaskFromGRPC(*task), nil + return convert.TaskFromGRPC(*task) } diff --git a/components/engine/docs/api/version-history.md b/components/engine/docs/api/version-history.md index 26f7585860..4a9dddbfde 100644 --- a/components/engine/docs/api/version-history.md +++ b/components/engine/docs/api/version-history.md @@ -28,6 +28,7 @@ keywords: "API, Docker, rcli, REST, documentation" * `GET /images/(name)/get` now includes an `ImageMetadata` field which contains image metadata that is local to the engine and not part of the image config. * `POST /swarm/init` now accepts a `DataPathAddr` property to set the IP-address or network interface to use for data traffic * `POST /swarm/join` now accepts a `DataPathAddr` property to set the IP-address or network interface to use for data traffic +* `POST /services/create` now accepts a `PluginSpec` when `TaskTemplate.Runtime` is set to `plugin` ## v1.30 API changes diff --git a/components/engine/integration-cli/daemon/daemon_swarm.go b/components/engine/integration-cli/daemon/daemon_swarm.go index a7058c512d..ba414066cc 100644 --- a/components/engine/integration-cli/daemon/daemon_swarm.go +++ b/components/engine/integration-cli/daemon/daemon_swarm.go @@ -1,6 +1,7 @@ package daemon import ( + "context" "encoding/json" "fmt" "net/http" @@ -10,6 +11,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/client" "github.com/docker/docker/integration-cli/checker" "github.com/go-check/check" "github.com/pkg/errors" @@ -124,20 +126,29 @@ type ConfigConstructor func(*swarm.Config) // SpecConstructor defines a swarm spec constructor type SpecConstructor func(*swarm.Spec) -// CreateService creates a swarm service given the specified service constructor -func (d *Swarm) CreateService(c *check.C, f ...ServiceConstructor) string { +// CreateServiceWithOptions creates a swarm service given the specified service constructors +// and auth config +func (d *Swarm) CreateServiceWithOptions(c *check.C, opts types.ServiceCreateOptions, f ...ServiceConstructor) string { + cl, err := client.NewClient(d.Sock(), "", nil, nil) + c.Assert(err, checker.IsNil, check.Commentf("failed to create client")) + defer cl.Close() + var service swarm.Service for _, fn := range f { fn(&service) } - status, out, err := d.SockRequest("POST", "/services/create", service.Spec) - c.Assert(err, checker.IsNil, check.Commentf(string(out))) - c.Assert(status, checker.Equals, http.StatusCreated, check.Commentf("output: %q", string(out))) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() - var scr types.ServiceCreateResponse - c.Assert(json.Unmarshal(out, &scr), checker.IsNil) - return scr.ID + res, err := cl.ServiceCreate(ctx, service.Spec, opts) + c.Assert(err, checker.IsNil) + return res.ID +} + +// CreateService creates a swarm service given the specified service constructor +func (d *Swarm) CreateService(c *check.C, f ...ServiceConstructor) string { + return d.CreateServiceWithOptions(c, types.ServiceCreateOptions{}, f...) } // GetService returns the swarm service corresponding to the specified id @@ -200,6 +211,37 @@ func (d *Swarm) CheckServiceUpdateState(service string) func(*check.C) (interfac } } +// CheckPluginRunning returns the runtime state of the plugin +func (d *Swarm) CheckPluginRunning(plugin string) func(c *check.C) (interface{}, check.CommentInterface) { + return func(c *check.C) (interface{}, check.CommentInterface) { + status, out, err := d.SockRequest("GET", "/plugins/"+plugin+"/json", nil) + c.Assert(err, checker.IsNil, check.Commentf(string(out))) + if status != http.StatusOK { + return false, nil + } + + var p types.Plugin + c.Assert(json.Unmarshal(out, &p), checker.IsNil, check.Commentf(string(out))) + + return p.Enabled, check.Commentf("%+v", p) + } +} + +// CheckPluginImage returns the runtime state of the plugin +func (d *Swarm) CheckPluginImage(plugin string) func(c *check.C) (interface{}, check.CommentInterface) { + return func(c *check.C) (interface{}, check.CommentInterface) { + status, out, err := d.SockRequest("GET", "/plugins/"+plugin+"/json", nil) + c.Assert(err, checker.IsNil, check.Commentf(string(out))) + if status != http.StatusOK { + return false, nil + } + + var p types.Plugin + c.Assert(json.Unmarshal(out, &p), checker.IsNil, check.Commentf(string(out))) + return p.PluginReference, check.Commentf("%+v", p) + } +} + // CheckServiceTasks returns the number of tasks for the specified service func (d *Swarm) CheckServiceTasks(service string) func(*check.C) (interface{}, check.CommentInterface) { return func(c *check.C) (interface{}, check.CommentInterface) { @@ -247,7 +289,7 @@ func (d *Swarm) CheckRunningTaskImages(c *check.C) (interface{}, check.CommentIn result := make(map[string]int) for _, task := range tasks { - if task.Status.State == swarm.TaskStateRunning { + if task.Status.State == swarm.TaskStateRunning && task.Spec.ContainerSpec != nil { result[task.Spec.ContainerSpec.Image]++ } } diff --git a/components/engine/integration-cli/docker_api_swarm_service_test.go b/components/engine/integration-cli/docker_api_swarm_service_test.go index 5360949362..66e17c4def 100644 --- a/components/engine/integration-cli/docker_api_swarm_service_test.go +++ b/components/engine/integration-cli/docker_api_swarm_service_test.go @@ -4,15 +4,19 @@ package main import ( "fmt" + "path" "strconv" "strings" "syscall" "time" "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/api/types/swarm/runtime" "github.com/docker/docker/integration-cli/checker" "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/integration-cli/fixtures/plugin" "github.com/go-check/check" + "golang.org/x/net/context" ) func setPortConfig(portConfig []swarm.PortConfig) daemon.ServiceConstructor { @@ -596,3 +600,77 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) { } } } + +// Test plugins deployed via swarm services +func (s *DockerSwarmSuite) TestAPISwarmServicesPlugin(c *check.C) { + testRequires(c, DaemonIsLinux, IsAmd64) + reg := setupRegistry(c, false, "", "") + defer reg.Close() + + repo := path.Join(privateRegistryURL, "swarm", "test:v1") + repo2 := path.Join(privateRegistryURL, "swarm", "test:v2") + name := "test" + + err := plugin.CreateInRegistry(context.Background(), repo, nil) + c.Assert(err, checker.IsNil, check.Commentf("failed to create plugin")) + err = plugin.CreateInRegistry(context.Background(), repo2, nil) + c.Assert(err, checker.IsNil, check.Commentf("failed to create plugin")) + + d1 := s.AddDaemon(c, true, true) + d2 := s.AddDaemon(c, true, true) + d3 := s.AddDaemon(c, true, false) + + makePlugin := func(repo, name string, constraints []string) func(*swarm.Service) { + return func(s *swarm.Service) { + s.Spec.TaskTemplate.Runtime = "plugin" + s.Spec.TaskTemplate.PluginSpec = &runtime.PluginSpec{ + Name: name, + Remote: repo, + } + if constraints != nil { + s.Spec.TaskTemplate.Placement = &swarm.Placement{ + Constraints: constraints, + } + } + } + } + + id := d1.CreateService(c, makePlugin(repo, name, nil)) + waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.True) + waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.True) + waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.True) + + service := d1.GetService(c, id) + d1.UpdateService(c, service, makePlugin(repo2, name, nil)) + waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginImage(name), checker.Equals, repo2) + waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginImage(name), checker.Equals, repo2) + waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginImage(name), checker.Equals, repo2) + waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.True) + waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.True) + waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.True) + + d1.RemoveService(c, id) + waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.False) + waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.False) + waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.False) + + // constrain to managers only + id = d1.CreateService(c, makePlugin(repo, name, []string{"node.role==manager"})) + waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.True) + waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.True) + waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.False) // Not a manager, not running it + d1.RemoveService(c, id) + waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.False) + waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.False) + waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.False) + + // with no name + id = d1.CreateService(c, makePlugin(repo, "", nil)) + waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(repo), checker.True) + waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(repo), checker.True) + waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(repo), checker.True) + d1.RemoveService(c, id) + waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(repo), checker.False) + waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(repo), checker.False) + waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(repo), checker.False) +} diff --git a/components/engine/integration-cli/docker_api_swarm_test.go b/components/engine/integration-cli/docker_api_swarm_test.go index a146bcc519..9d24757b4f 100644 --- a/components/engine/integration-cli/docker_api_swarm_test.go +++ b/components/engine/integration-cli/docker_api_swarm_test.go @@ -560,7 +560,7 @@ func simpleTestService(s *swarm.Service) { s.Spec = swarm.ServiceSpec{ TaskTemplate: swarm.TaskSpec{ - ContainerSpec: swarm.ContainerSpec{ + ContainerSpec: &swarm.ContainerSpec{ Image: "busybox:latest", Command: []string{"/bin/top"}, }, @@ -583,7 +583,7 @@ func serviceForUpdate(s *swarm.Service) { s.Spec = swarm.ServiceSpec{ TaskTemplate: swarm.TaskSpec{ - ContainerSpec: swarm.ContainerSpec{ + ContainerSpec: &swarm.ContainerSpec{ Image: "busybox:latest", Command: []string{"/bin/top"}, }, @@ -641,6 +641,9 @@ func setRollbackOrder(order string) daemon.ServiceConstructor { func setImage(image string) daemon.ServiceConstructor { return func(s *swarm.Service) { + if s.Spec.TaskTemplate.ContainerSpec == nil { + s.Spec.TaskTemplate.ContainerSpec = &swarm.ContainerSpec{} + } s.Spec.TaskTemplate.ContainerSpec.Image = image } } @@ -921,6 +924,9 @@ func (s *DockerSwarmSuite) TestAPISwarmHealthcheckNone(c *check.C) { instances := 1 d.CreateService(c, simpleTestService, setInstances(instances), func(s *swarm.Service) { + if s.Spec.TaskTemplate.ContainerSpec == nil { + s.Spec.TaskTemplate.ContainerSpec = &swarm.ContainerSpec{} + } s.Spec.TaskTemplate.ContainerSpec.Healthcheck = &container.HealthConfig{} s.Spec.TaskTemplate.Networks = []swarm.NetworkAttachmentConfig{ {Target: "lb"}, diff --git a/components/engine/integration-cli/fixtures/plugin/basic/basic.go b/components/engine/integration-cli/fixtures/plugin/basic/basic.go new file mode 100644 index 0000000000..892272826f --- /dev/null +++ b/components/engine/integration-cli/fixtures/plugin/basic/basic.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + "net" + "net/http" + "os" + "path/filepath" +) + +func main() { + p, err := filepath.Abs(filepath.Join("run", "docker", "plugins")) + if err != nil { + panic(err) + } + if err := os.MkdirAll(p, 0755); err != nil { + panic(err) + } + l, err := net.Listen("unix", filepath.Join(p, "basic.sock")) + if err != nil { + panic(err) + } + + mux := http.NewServeMux() + server := http.Server{ + Addr: l.Addr().String(), + Handler: http.NewServeMux(), + } + mux.HandleFunc("/Plugin.Activate", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1.1+json") + fmt.Println(w, `{"Implements": ["dummy"]}`) + }) + server.Serve(l) +} diff --git a/components/engine/integration-cli/fixtures/plugin/plugin.go b/components/engine/integration-cli/fixtures/plugin/plugin.go new file mode 100644 index 0000000000..1be6169735 --- /dev/null +++ b/components/engine/integration-cli/fixtures/plugin/plugin.go @@ -0,0 +1,183 @@ +package plugin + +import ( + "encoding/json" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/libcontainerd" + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/plugin" + "github.com/docker/docker/registry" + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +// CreateOpt is is passed used to change the defualt plugin config before +// creating it +type CreateOpt func(*Config) + +// Config wraps types.PluginConfig to provide some extra state for options +// extra customizations on the plugin details, such as using a custom binary to +// create the plugin with. +type Config struct { + *types.PluginConfig + binPath string +} + +// WithBinary is a CreateOpt to set an custom binary to create the plugin with. +// This binary must be statically compiled. +func WithBinary(bin string) CreateOpt { + return func(cfg *Config) { + cfg.binPath = bin + } +} + +// CreateClient is the interface used for `BuildPlugin` to interact with the +// daemon. +type CreateClient interface { + PluginCreate(context.Context, io.Reader, types.PluginCreateOptions) error +} + +// Create creates a new plugin with the specified name +func Create(ctx context.Context, c CreateClient, name string, opts ...CreateOpt) error { + tmpDir, err := ioutil.TempDir("", "create-test-plugin") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + tar, err := makePluginBundle(tmpDir, opts...) + if err != nil { + return err + } + defer tar.Close() + + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + return c.PluginCreate(ctx, tar, types.PluginCreateOptions{RepoName: name}) +} + +// TODO(@cpuguy83): we really shouldn't have to do this... +// The manager panics on init when `Executor` is not set. +type dummyExecutor struct{} + +func (dummyExecutor) Client(libcontainerd.Backend) (libcontainerd.Client, error) { return nil, nil } +func (dummyExecutor) Cleanup() {} +func (dummyExecutor) UpdateOptions(...libcontainerd.RemoteOption) error { return nil } + +// CreateInRegistry makes a plugin (locally) and pushes it to a registry. +// This does not use a dockerd instance to create or push the plugin. +// If you just want to create a plugin in some daemon, use `Create`. +// +// This can be useful when testing plugins on swarm where you don't really want +// the plugin to exist on any of the daemons (immediately) and there needs to be +// some way to distribute the plugin. +func CreateInRegistry(ctx context.Context, repo string, auth *types.AuthConfig, opts ...CreateOpt) error { + tmpDir, err := ioutil.TempDir("", "create-test-plugin-local") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + inPath := filepath.Join(tmpDir, "plugin") + if err := os.MkdirAll(inPath, 0755); err != nil { + return errors.Wrap(err, "error creating plugin root") + } + + tar, err := makePluginBundle(inPath, opts...) + if err != nil { + return err + } + defer tar.Close() + + managerConfig := plugin.ManagerConfig{ + Store: plugin.NewStore(), + RegistryService: registry.NewService(registry.ServiceOptions{V2Only: true}), + Root: filepath.Join(tmpDir, "root"), + ExecRoot: "/run/docker", // manager init fails if not set + Executor: dummyExecutor{}, + LogPluginEvent: func(id, name, action string) {}, // panics when not set + } + manager, err := plugin.NewManager(managerConfig) + if err != nil { + return errors.Wrap(err, "error creating plugin manager") + } + + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := manager.CreateFromContext(ctx, tar, &types.PluginCreateOptions{RepoName: repo}); err != nil { + return err + } + + if auth == nil { + auth = &types.AuthConfig{} + } + err = manager.Push(ctx, repo, nil, auth, ioutil.Discard) + return errors.Wrap(err, "error pushing plugin") +} + +func makePluginBundle(inPath string, opts ...CreateOpt) (io.ReadCloser, error) { + p := &types.PluginConfig{ + Interface: types.PluginConfigInterface{ + Socket: "basic.sock", + Types: []types.PluginInterfaceType{{Capability: "docker.dummy/1.0"}}, + }, + Entrypoint: []string{"/basic"}, + } + cfg := &Config{ + PluginConfig: p, + } + for _, o := range opts { + o(cfg) + } + if cfg.binPath == "" { + binPath, err := ensureBasicPluginBin() + if err != nil { + return nil, err + } + cfg.binPath = binPath + } + + configJSON, err := json.Marshal(p) + if err != nil { + return nil, err + } + if err := ioutil.WriteFile(filepath.Join(inPath, "config.json"), configJSON, 0644); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Join(inPath, "rootfs", filepath.Dir(p.Entrypoint[0])), 0755); err != nil { + return nil, errors.Wrap(err, "error creating plugin rootfs dir") + } + if err := archive.NewDefaultArchiver().CopyFileWithTar(cfg.binPath, filepath.Join(inPath, "rootfs", p.Entrypoint[0])); err != nil { + return nil, errors.Wrap(err, "error copying plugin binary to rootfs path") + } + tar, err := archive.Tar(inPath, archive.Uncompressed) + return tar, errors.Wrap(err, "error making plugin archive") +} + +func ensureBasicPluginBin() (string, error) { + name := "docker-basic-plugin" + p, err := exec.LookPath(name) + if err == nil { + return p, nil + } + + goBin, err := exec.LookPath("go") + if err != nil { + return "", err + } + installPath := filepath.Join(os.Getenv("GOPATH"), "bin", name) + cmd := exec.Command(goBin, "build", "-o", installPath, "./"+filepath.Join("fixtures", "plugin", "basic")) + cmd.Env = append(cmd.Env, "CGO_ENABLED=0") + if out, err := cmd.CombinedOutput(); err != nil { + return "", errors.Wrapf(err, "error building basic plugin bin: %s", string(out)) + } + return installPath, nil +} diff --git a/components/engine/pkg/pubsub/publisher.go b/components/engine/pkg/pubsub/publisher.go index 09364617e4..8e30d16ae5 100644 --- a/components/engine/pkg/pubsub/publisher.go +++ b/components/engine/pkg/pubsub/publisher.go @@ -53,6 +53,16 @@ func (p *Publisher) SubscribeTopic(topic topicFunc) chan interface{} { return ch } +// SubscribeTopicWithBuffer adds a new subscriber that filters messages sent by a topic. +// The returned channel has a buffer of the specified size. +func (p *Publisher) SubscribeTopicWithBuffer(topic topicFunc, buffer int) chan interface{} { + ch := make(chan interface{}, buffer) + p.m.Lock() + p.subscribers[ch] = topic + p.m.Unlock() + return ch +} + // Evict removes the specified subscriber from receiving any more messages. func (p *Publisher) Evict(sub chan interface{}) { p.m.Lock() diff --git a/components/engine/plugin/backend_linux.go b/components/engine/plugin/backend_linux.go index d65b90a2b7..055b8e3107 100644 --- a/components/engine/plugin/backend_linux.go +++ b/components/engine/plugin/backend_linux.go @@ -67,6 +67,7 @@ func (pm *Manager) Disable(refOrID string, config *types.PluginDisableConfig) er if err := pm.disable(p, c); err != nil { return err } + pm.publisher.Publish(EventDisable{Plugin: p.PluginObj}) pm.config.LogPluginEvent(p.GetID(), refOrID, "disable") return nil } @@ -82,6 +83,7 @@ func (pm *Manager) Enable(refOrID string, config *types.PluginEnableConfig) erro if err := pm.enable(p, c, false); err != nil { return err } + pm.publisher.Publish(EventEnable{Plugin: p.PluginObj}) pm.config.LogPluginEvent(p.GetID(), refOrID, "enable") return nil } @@ -296,7 +298,7 @@ func (pm *Manager) Upgrade(ctx context.Context, ref reference.Named, name string } // Pull pulls a plugin, check if the correct privileges are provided and install the plugin. -func (pm *Manager) Pull(ctx context.Context, ref reference.Named, name string, metaHeader http.Header, authConfig *types.AuthConfig, privileges types.PluginPrivileges, outStream io.Writer) (err error) { +func (pm *Manager) Pull(ctx context.Context, ref reference.Named, name string, metaHeader http.Header, authConfig *types.AuthConfig, privileges types.PluginPrivileges, outStream io.Writer, opts ...CreateOpt) (err error) { pm.muGC.RLock() defer pm.muGC.RUnlock() @@ -340,12 +342,19 @@ func (pm *Manager) Pull(ctx context.Context, ref reference.Named, name string, m return err } - p, err := pm.createPlugin(name, dm.configDigest, dm.blobs, tmpRootFSDir, &privileges) + refOpt := func(p *v2.Plugin) { + p.PluginObj.PluginReference = ref.String() + } + optsList := make([]CreateOpt, 0, len(opts)+1) + optsList = append(optsList, opts...) + optsList = append(optsList, refOpt) + + p, err := pm.createPlugin(name, dm.configDigest, dm.blobs, tmpRootFSDir, &privileges, optsList...) if err != nil { return err } - p.PluginObj.PluginReference = ref.String() + pm.publisher.Publish(EventCreate{Plugin: p.PluginObj}) return nil } @@ -640,6 +649,7 @@ func (pm *Manager) Remove(name string, config *types.PluginRmConfig) error { } pm.config.Store.Remove(p) pm.config.LogPluginEvent(id, name, "remove") + pm.publisher.Publish(EventRemove{Plugin: p.PluginObj}) return nil } @@ -771,6 +781,7 @@ func (pm *Manager) CreateFromContext(ctx context.Context, tarCtx io.ReadCloser, } p.PluginObj.PluginReference = name + pm.publisher.Publish(EventCreate{Plugin: p.PluginObj}) pm.config.LogPluginEvent(p.PluginObj.ID, name, "create") return nil diff --git a/components/engine/plugin/backend_unsupported.go b/components/engine/plugin/backend_unsupported.go index 2d4850eeba..e69bb883d0 100644 --- a/components/engine/plugin/backend_unsupported.go +++ b/components/engine/plugin/backend_unsupported.go @@ -36,7 +36,7 @@ func (pm *Manager) Privileges(ctx context.Context, ref reference.Named, metaHead } // Pull pulls a plugin, check if the correct privileges are provided and install the plugin. -func (pm *Manager) Pull(ctx context.Context, ref reference.Named, name string, metaHeader http.Header, authConfig *types.AuthConfig, privileges types.PluginPrivileges, out io.Writer) error { +func (pm *Manager) Pull(ctx context.Context, ref reference.Named, name string, metaHeader http.Header, authConfig *types.AuthConfig, privileges types.PluginPrivileges, out io.Writer, opts ...CreateOpt) error { return errNotSupported } diff --git a/components/engine/plugin/defs.go b/components/engine/plugin/defs.go index cf44c97ec8..3e930de048 100644 --- a/components/engine/plugin/defs.go +++ b/components/engine/plugin/defs.go @@ -24,3 +24,14 @@ func NewStore() *Store { handlers: make(map[string][]func(string, *plugins.Client)), } } + +// CreateOpt is used to configure specific plugin details when created +type CreateOpt func(p *v2.Plugin) + +// WithSwarmService is a CreateOpt that flags the passed in a plugin as a plugin +// managed by swarm +func WithSwarmService(id string) CreateOpt { + return func(p *v2.Plugin) { + p.SwarmServiceID = id + } +} diff --git a/components/engine/plugin/events.go b/components/engine/plugin/events.go new file mode 100644 index 0000000000..92e603850d --- /dev/null +++ b/components/engine/plugin/events.go @@ -0,0 +1,111 @@ +package plugin + +import ( + "fmt" + "reflect" + + "github.com/docker/docker/api/types" +) + +// Event is emitted for actions performed on the plugin manager +type Event interface { + matches(Event) bool +} + +// EventCreate is an event which is emitted when a plugin is created +// This is either by pull or create from context. +// +// Use the `Interfaces` field to match only plugins that implement a specific +// interface. +// These are matched against using "or" logic. +// If no interfaces are listed, all are matched. +type EventCreate struct { + Interfaces map[string]bool + Plugin types.Plugin +} + +func (e EventCreate) matches(observed Event) bool { + oe, ok := observed.(EventCreate) + if !ok { + return false + } + if len(e.Interfaces) == 0 { + return true + } + + var ifaceMatch bool + for _, in := range oe.Plugin.Config.Interface.Types { + if e.Interfaces[in.Capability] { + ifaceMatch = true + break + } + } + return ifaceMatch +} + +// EventRemove is an event which is emitted when a plugin is removed +// It maches on the passed in plugin's ID only. +type EventRemove struct { + Plugin types.Plugin +} + +func (e EventRemove) matches(observed Event) bool { + oe, ok := observed.(EventRemove) + if !ok { + return false + } + return e.Plugin.ID == oe.Plugin.ID +} + +// EventDisable is an event that is emitted when a plugin is disabled +// It maches on the passed in plugin's ID only. +type EventDisable struct { + Plugin types.Plugin +} + +func (e EventDisable) matches(observed Event) bool { + oe, ok := observed.(EventDisable) + if !ok { + return false + } + return e.Plugin.ID == oe.Plugin.ID +} + +// EventEnable is an event that is emitted when a plugin is disabled +// It maches on the passed in plugin's ID only. +type EventEnable struct { + Plugin types.Plugin +} + +func (e EventEnable) matches(observed Event) bool { + oe, ok := observed.(EventEnable) + if !ok { + return false + } + return e.Plugin.ID == oe.Plugin.ID +} + +// SubscribeEvents provides an event channel to listen for structured events from +// the plugin manager actions, CRUD operations. +// The caller must call the returned `cancel()` function once done with the channel +// or this will leak resources. +func (pm *Manager) SubscribeEvents(buffer int, watchEvents ...Event) (eventCh <-chan interface{}, cancel func()) { + topic := func(i interface{}) bool { + observed, ok := i.(Event) + if !ok { + panic(fmt.Sprintf("unexpected type passed to event channel: %v", reflect.TypeOf(i))) + } + for _, e := range watchEvents { + if e.matches(observed) { + return true + } + } + // If no specific events are specified always assume a matched event + // If some events were specified and none matched above, then the event + // doesn't match + return watchEvents == nil + } + ch := pm.publisher.SubscribeTopicWithBuffer(topic, buffer) + cancelFunc := func() { pm.publisher.Evict(ch) } + return ch, cancelFunc +} diff --git a/components/engine/plugin/manager.go b/components/engine/plugin/manager.go index 43b1e676ff..fada0d667a 100644 --- a/components/engine/plugin/manager.go +++ b/components/engine/plugin/manager.go @@ -22,6 +22,7 @@ import ( "github.com/docker/docker/pkg/authorization" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/mount" + "github.com/docker/docker/pkg/pubsub" "github.com/docker/docker/pkg/system" "github.com/docker/docker/plugin/v2" "github.com/docker/docker/registry" @@ -63,6 +64,7 @@ type Manager struct { cMap map[*v2.Plugin]*controller containerdClient libcontainerd.Client blobStore *basicBlobStore + publisher *pubsub.Publisher } // controller represents the manager's control on a plugin. @@ -117,6 +119,8 @@ func NewManager(config ManagerConfig) (*Manager, error) { if err := manager.reload(); err != nil { return nil, errors.Wrap(err, "failed to restore plugins") } + + manager.publisher = pubsub.NewPublisher(0, 0) return manager, nil } @@ -268,6 +272,11 @@ func (pm *Manager) reload() error { // todo: restore return nil } +// Get looks up the requested plugin in the store. +func (pm *Manager) Get(idOrName string) (*v2.Plugin, error) { + return pm.config.Store.GetV2Plugin(idOrName) +} + func (pm *Manager) loadPlugin(id string) (*v2.Plugin, error) { p := filepath.Join(pm.config.Root, id, configFileName) dt, err := ioutil.ReadFile(p) diff --git a/components/engine/plugin/manager_linux.go b/components/engine/plugin/manager_linux.go index 678be84b3f..7f79e6900e 100644 --- a/components/engine/plugin/manager_linux.go +++ b/components/engine/plugin/manager_linux.go @@ -274,7 +274,7 @@ func (pm *Manager) setupNewPlugin(configDigest digest.Digest, blobsums []digest. } // createPlugin creates a new plugin. take lock before calling. -func (pm *Manager) createPlugin(name string, configDigest digest.Digest, blobsums []digest.Digest, rootFSDir string, privileges *types.PluginPrivileges) (p *v2.Plugin, err error) { +func (pm *Manager) createPlugin(name string, configDigest digest.Digest, blobsums []digest.Digest, rootFSDir string, privileges *types.PluginPrivileges, opts ...CreateOpt) (p *v2.Plugin, err error) { if err := pm.config.Store.validateName(name); err != nil { // todo: this check is wrong. remove store return nil, err } @@ -294,6 +294,9 @@ func (pm *Manager) createPlugin(name string, configDigest digest.Digest, blobsum Blobsums: blobsums, } p.InitEmptySettings() + for _, o := range opts { + o(p) + } pdir := filepath.Join(pm.config.Root, p.PluginObj.ID) if err := os.MkdirAll(pdir, 0700); err != nil { diff --git a/components/engine/plugin/v2/plugin.go b/components/engine/plugin/v2/plugin.go index 74ff64080a..b77536c986 100644 --- a/components/engine/plugin/v2/plugin.go +++ b/components/engine/plugin/v2/plugin.go @@ -22,6 +22,8 @@ type Plugin struct { Config digest.Digest Blobsums []digest.Digest + + SwarmServiceID string } const defaultPluginRuntimeDestination = "/run/docker/plugins" From 754d5e0397336aab06a681c00c6367b3f8092590 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 6 Jul 2017 11:06:29 -0700 Subject: [PATCH 29/56] Remove old "get.docker.com" install script This script is part of the Docker product, and no longer maintained in this repository. The script has been updated to install Docker CE packages from https://download.docker.com, and is now located in the https://github.com/docker/docker-install repository. Signed-off-by: Sebastiaan van Stijn Upstream-commit: e038f9f81f871237e9ec7a2188eb6b401bdd8809 Component: engine --- components/engine/hack/README.md | 8 - components/engine/hack/install.sh | 544 ------------------ components/engine/hack/make/install-script | 63 -- components/engine/hack/make/test-deb-install | 71 --- .../engine/hack/make/test-install-script | 31 - components/engine/hack/release.sh | 8 - 6 files changed, 725 deletions(-) delete mode 100644 components/engine/hack/install.sh delete mode 100644 components/engine/hack/make/install-script delete mode 100755 components/engine/hack/make/test-deb-install delete mode 100755 components/engine/hack/make/test-install-script diff --git a/components/engine/hack/README.md b/components/engine/hack/README.md index 326e35e169..802395d533 100644 --- a/components/engine/hack/README.md +++ b/components/engine/hack/README.md @@ -18,14 +18,6 @@ Generates AUTHORS; a file with all the names and corresponding emails of individual contributors. AUTHORS can be found in the home directory of this repository. -## Install (install.sh) - -Executable install script for installing Docker. If updates to this are -desired, please use hack/release.sh during a normal release. The following -one-liner may be used for script hotfixes: - -- `aws s3 cp --acl public-read hack/install.sh s3://get.docker.com/index` - ## Make There are two make files, each with different extensions. Neither are supposed diff --git a/components/engine/hack/install.sh b/components/engine/hack/install.sh deleted file mode 100644 index 12a9b3e933..0000000000 --- a/components/engine/hack/install.sh +++ /dev/null @@ -1,544 +0,0 @@ -#!/bin/sh -set -e -# -# This script is meant for quick & easy install via: -# 'curl -sSL https://get.docker.com/ | sh' -# or: -# 'wget -qO- https://get.docker.com/ | sh' -# -# For test builds (ie. release candidates): -# 'curl -fsSL https://test.docker.com/ | sh' -# or: -# 'wget -qO- https://test.docker.com/ | sh' -# -# For experimental builds: -# 'curl -fsSL https://experimental.docker.com/ | sh' -# or: -# 'wget -qO- https://experimental.docker.com/ | sh' -# -# Docker Maintainers: -# To update this script on https://get.docker.com, -# use hack/release.sh during a normal release, -# or the following one-liner for script hotfixes: -# aws s3 cp --acl public-read hack/install.sh s3://get.docker.com/index -# - -url="https://get.docker.com/" -apt_url="https://apt.dockerproject.org" -yum_url="https://yum.dockerproject.org" - -docker_key="-----BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1 - -mQINBFWln24BEADrBl5p99uKh8+rpvqJ48u4eTtjeXAWbslJotmC/CakbNSqOb9o -ddfzRvGVeJVERt/Q/mlvEqgnyTQy+e6oEYN2Y2kqXceUhXagThnqCoxcEJ3+KM4R -mYdoe/BJ/J/6rHOjq7Omk24z2qB3RU1uAv57iY5VGw5p45uZB4C4pNNsBJXoCvPn -TGAs/7IrekFZDDgVraPx/hdiwopQ8NltSfZCyu/jPpWFK28TR8yfVlzYFwibj5WK -dHM7ZTqlA1tHIG+agyPf3Rae0jPMsHR6q+arXVwMccyOi+ULU0z8mHUJ3iEMIrpT -X+80KaN/ZjibfsBOCjcfiJSB/acn4nxQQgNZigna32velafhQivsNREFeJpzENiG -HOoyC6qVeOgKrRiKxzymj0FIMLru/iFF5pSWcBQB7PYlt8J0G80lAcPr6VCiN+4c -NKv03SdvA69dCOj79PuO9IIvQsJXsSq96HB+TeEmmL+xSdpGtGdCJHHM1fDeCqkZ -hT+RtBGQL2SEdWjxbF43oQopocT8cHvyX6Zaltn0svoGs+wX3Z/H6/8P5anog43U -65c0A+64Jj00rNDr8j31izhtQMRo892kGeQAaaxg4Pz6HnS7hRC+cOMHUU4HA7iM -zHrouAdYeTZeZEQOA7SxtCME9ZnGwe2grxPXh/U/80WJGkzLFNcTKdv+rwARAQAB -tDdEb2NrZXIgUmVsZWFzZSBUb29sIChyZWxlYXNlZG9ja2VyKSA8ZG9ja2VyQGRv -Y2tlci5jb20+iQIcBBABCgAGBQJWw7vdAAoJEFyzYeVS+w0QHysP/i37m4SyoOCV -cnybl18vzwBEcp4VCRbXvHvOXty1gccVIV8/aJqNKgBV97lY3vrpOyiIeB8ETQeg -srxFE7t/Gz0rsLObqfLEHdmn5iBJRkhLfCpzjeOnyB3Z0IJB6UogO/msQVYe5CXJ -l6uwr0AmoiCBLrVlDAktxVh9RWch0l0KZRX2FpHu8h+uM0/zySqIidlYfLa3y5oH -scU+nGU1i6ImwDTD3ysZC5jp9aVfvUmcESyAb4vvdcAHR+bXhA/RW8QHeeMFliWw -7Z2jYHyuHmDnWG2yUrnCqAJTrWV+OfKRIzzJFBs4e88ru5h2ZIXdRepw/+COYj34 -LyzxR2cxr2u/xvxwXCkSMe7F4KZAphD+1ws61FhnUMi/PERMYfTFuvPrCkq4gyBj -t3fFpZ2NR/fKW87QOeVcn1ivXl9id3MMs9KXJsg7QasT7mCsee2VIFsxrkFQ2jNp -D+JAERRn9Fj4ArHL5TbwkkFbZZvSi6fr5h2GbCAXIGhIXKnjjorPY/YDX6X8AaHO -W1zblWy/CFr6VFl963jrjJgag0G6tNtBZLrclZgWhOQpeZZ5Lbvz2ZA5CqRrfAVc -wPNW1fObFIRtqV6vuVluFOPCMAAnOnqR02w9t17iVQjO3oVN0mbQi9vjuExXh1Yo -ScVetiO6LSmlQfVEVRTqHLMgXyR/EMo7iQIcBBABCgAGBQJXSWBlAAoJEFyzYeVS -+w0QeH0QAI6btAfYwYPuAjfRUy9qlnPhZ+xt1rnwsUzsbmo8K3XTNh+l/R08nu0d -sczw30Q1wju28fh1N8ay223+69f0+yICaXqR18AbGgFGKX7vo0gfEVaxdItUN3eH -NydGFzmeOKbAlrxIMECnSTG/TkFVYO9Ntlv9vSN2BupmTagTRErxLZKnVsWRzp+X -elwlgU5BCZ6U6Ze8+bIc6F1bZstf17X8i6XNV/rOCLx2yP0hn1osoljoLPpW8nzk -wvqYsYbCA28lMt1aqe0UWvRCqR0zxlKn17NZQqjbxcajEMCajoQ01MshmO5GWePV -iv2abCZ/iaC5zKqVT3deMJHLq7lum6qhA41E9gJH9QoqT+qgadheeFfoC1QP7cke -+tXmYg2R39p3l5Hmm+JQbP4f9V5mpWExvHGCSbcatr35tnakIJZugq2ogzsm1djC -Sz9222RXl9OoFqsm1bNzA78+/cOt5N2cyhU0bM2T/zgh42YbDD+JDU/HSmxUIpU+ -wrGvZGM2FU/up0DRxOC4U1fL6HHlj8liNJWfEg3vhougOh66gGF9ik5j4eIlNoz6 -lst+gmvlZQ9/9hRDeoG+AbhZeIlQ4CCw+Y1j/+fUxIzKHPVK+aFJd+oJVNvbojJW -/SgDdSMtFwqOvXyYcHl30Ws0gZUeDyAmNGZeJ3kFklnApDmeKK+OiQIiBBABCgAM -BQJXe5zTBYMHhh+AAAoJEDG4FaMBBnSp7YMQAJqrXoBonZAq07B6qUaT3aBCgnY4 -JshbXmFb/XrrS75f7YJDPx2fJJdqrbYDIHHgOjzxvp3ngPpOpJzI5sYmkaugeoCO -/KHu/+39XqgTB7fguzapRfbvuWp+qzPcHSdb9opnagfzKAze3DQnnLiwCPlsyvGp -zC4KzXgV2ze/4raaOye1kK7O0cHyapmn/q/TR3S8YapyXq5VpLThwJAw1SRDu0Yx -eXIAQiIfaSxT79EktoioW2CSV8/djt+gBjXnKYJJA8P1zzX7GNt/Rc2YG0Ot4v6t -BW16xqFTg+n5JzbeK5cZ1jbIXXfCcaZJyiM2MzYGhSJ9+EV7JYF05OAIWE4SGTRj -XMquQ2oMLSwMCPQHm+FCD9PXQ0tHYx6tKT34wksdmoWsdejl/n3NS+178mG1WI/l -N079h3im2gRwOykMou/QWs3vGw/xDoOYHPV2gJ7To9BLVnVK/hROgdFLZFeyRScN -zwKm57HmYMFA74tX601OiHhk1ymP2UUc25oDWpLXlfcRULJJlo/KfZZF3pmKwIq3 -CilGayFUi1NNwuavG76EcAVtVFUVFFIITwkhkuRbBHIytzEHYosFgD5/acK0Pauq -JnwrwKv0nWq3aK7nKiALAD+iZvPNjFZau3/APqLEmvmRnAElmugcHsWREFxMMjMM -VgYFiYKUAJO8u46eiQI4BBMBAgAiBQJVpZ9uAhsvBgsJCAcDAgYVCAIJCgsEFgID -AQIeAQIXgAAKCRD3YiFXLFJgnbRfEAC9Uai7Rv20QIDlDogRzd+Vebg4ahyoUdj0 -CH+nAk40RIoq6G26u1e+sdgjpCa8jF6vrx+smpgd1HeJdmpahUX0XN3X9f9qU9oj -9A4I1WDalRWJh+tP5WNv2ySy6AwcP9QnjuBMRTnTK27pk1sEMg9oJHK5p+ts8hlS -C4SluyMKH5NMVy9c+A9yqq9NF6M6d6/ehKfBFFLG9BX+XLBATvf1ZemGVHQusCQe -bTGv0C0V9yqtdPdRWVIEhHxyNHATaVYOafTj/EF0lDxLl6zDT6trRV5n9F1VCEh4 -Aal8L5MxVPcIZVO7NHT2EkQgn8CvWjV3oKl2GopZF8V4XdJRl90U/WDv/6cmfI08 -GkzDYBHhS8ULWRFwGKobsSTyIvnbk4NtKdnTGyTJCQ8+6i52s+C54PiNgfj2ieNn -6oOR7d+bNCcG1CdOYY+ZXVOcsjl73UYvtJrO0Rl/NpYERkZ5d/tzw4jZ6FCXgggA -/Zxcjk6Y1ZvIm8Mt8wLRFH9Nww+FVsCtaCXJLP8DlJLASMD9rl5QS9Ku3u7ZNrr5 -HWXPHXITX660jglyshch6CWeiUATqjIAzkEQom/kEnOrvJAtkypRJ59vYQOedZ1s -FVELMXg2UCkD/FwojfnVtjzYaTCeGwFQeqzHmM241iuOmBYPeyTY5veF49aBJA1g -EJOQTvBR8Q== -=Yhur ------END PGP PUBLIC KEY BLOCK----- -" - -mirror='' -while [ $# -gt 0 ]; do - case "$1" in - --mirror) - mirror="$2" - shift - ;; - *) - echo "Illegal option $1" - ;; - esac - shift $(( $# > 0 ? 1 : 0 )) -done - -case "$mirror" in - AzureChinaCloud) - apt_url="https://mirror.azure.cn/docker-engine/apt" - yum_url="https://mirror.azure.cn/docker-engine/yum" - ;; - Aliyun) - apt_url="https://mirrors.aliyun.com/docker-engine/apt" - yum_url="https://mirrors.aliyun.com/docker-engine/yum" - ;; -esac - -command_exists() { - command -v "$@" > /dev/null 2>&1 -} - -echo_docker_as_nonroot() { - if command_exists docker && [ -e /var/run/docker.sock ]; then - ( - set -x - $sh_c 'docker version' - ) || true - fi - your_user=your-user - [ "$user" != 'root' ] && your_user="$user" - # intentionally mixed spaces and tabs here -- tabs are stripped by "<<-EOF", spaces are kept in the output - cat <<-EOF - - If you would like to use Docker as a non-root user, you should now consider - adding your user to the "docker" group with something like: - - sudo usermod -aG docker $your_user - - Remember that you will have to log out and back in for this to take effect! - - WARNING: Adding a user to the "docker" group will grant the ability to run - containers which can be used to obtain root privileges on the - docker host. - Refer to https://docs.docker.com/engine/security/security/#docker-daemon-attack-surface - for more information. - - EOF -} - -# Check if this is a forked Linux distro -check_forked() { - - # Check for lsb_release command existence, it usually exists in forked distros - if command_exists lsb_release; then - # Check if the `-u` option is supported - set +e - lsb_release -a -u > /dev/null 2>&1 - lsb_release_exit_code=$? - set -e - - # Check if the command has exited successfully, it means we're in a forked distro - if [ "$lsb_release_exit_code" = "0" ]; then - # Print info about current distro - cat <<-EOF - You're using '$lsb_dist' version '$dist_version'. - EOF - - # Get the upstream release info - lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[[:space:]]') - dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[[:space:]]') - - # Print info about upstream distro - cat <<-EOF - Upstream release is '$lsb_dist' version '$dist_version'. - EOF - else - if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then - # We're Debian and don't even know it! - lsb_dist=debian - dist_version="$(cat /etc/debian_version | sed 's/\/.*//' | sed 's/\..*//')" - case "$dist_version" in - 9) - dist_version="stretch" - ;; - 8|'Kali Linux 2') - dist_version="jessie" - ;; - 7) - dist_version="wheezy" - ;; - esac - fi - fi - fi -} - -semverParse() { - major="${1%%.*}" - minor="${1#$major.}" - minor="${minor%%.*}" - patch="${1#$major.$minor.}" - patch="${patch%%[-.]*}" -} - -do_install() { - architecture=$(uname -m) - case $architecture in - # officially supported - amd64|x86_64) - ;; - # unofficially supported with available repositories - armv6l|armv7l) - ;; - # unofficially supported without available repositories - aarch64|arm64|ppc64le|s390x) - cat 1>&2 <<-EOF - Error: This install script does not support $architecture, because no - $architecture package exists in Docker's repositories. - - Other install options include checking your distribution's package repository - for a version of Docker, or building Docker from source. - EOF - exit 1 - ;; - # not supported - *) - cat >&2 <<-EOF - Error: $architecture is not a recognized platform. - EOF - exit 1 - ;; - esac - - if command_exists docker; then - version="$(docker -v | cut -d ' ' -f3 | cut -d ',' -f1)" - MAJOR_W=1 - MINOR_W=10 - - semverParse $version - - shouldWarn=0 - if [ $major -lt $MAJOR_W ]; then - shouldWarn=1 - fi - - if [ $major -le $MAJOR_W ] && [ $minor -lt $MINOR_W ]; then - shouldWarn=1 - fi - - cat >&2 <<-'EOF' - Warning: the "docker" command appears to already exist on this system. - - If you already have Docker installed, this script can cause trouble, which is - why we're displaying this warning and provide the opportunity to cancel the - installation. - - If you installed the current Docker package using this script and are using it - EOF - - if [ $shouldWarn -eq 1 ]; then - cat >&2 <<-'EOF' - again to update Docker, we urge you to migrate your image store before upgrading - to v1.10+. - - You can find instructions for this here: - https://github.com/docker/docker/wiki/Engine-v1.10.0-content-addressability-migration - EOF - else - cat >&2 <<-'EOF' - again to update Docker, you can safely ignore this message. - EOF - fi - - cat >&2 <<-'EOF' - - You may press Ctrl+C now to abort this script. - EOF - ( set -x; sleep 20 ) - fi - - user="$(id -un 2>/dev/null || true)" - - sh_c='sh -c' - if [ "$user" != 'root' ]; then - if command_exists sudo; then - sh_c='sudo -E sh -c' - elif command_exists su; then - sh_c='su -c' - else - cat >&2 <<-'EOF' - Error: this installer needs the ability to run commands as root. - We are unable to find either "sudo" or "su" available to make this happen. - EOF - exit 1 - fi - fi - - curl='' - if command_exists curl; then - curl='curl -sSL' - elif command_exists wget; then - curl='wget -qO-' - elif command_exists busybox && busybox --list-modules | grep -q wget; then - curl='busybox wget -qO-' - fi - - # check to see which repo they are trying to install from - if [ -z "$repo" ]; then - repo='main' - if [ "https://test.docker.com/" = "$url" ]; then - repo='testing' - elif [ "https://experimental.docker.com/" = "$url" ]; then - repo='experimental' - fi - fi - - # perform some very rudimentary platform detection - lsb_dist='' - dist_version='' - if command_exists lsb_release; then - lsb_dist="$(lsb_release -si)" - fi - if [ -z "$lsb_dist" ] && [ -r /etc/lsb-release ]; then - lsb_dist="$(. /etc/lsb-release && echo "$DISTRIB_ID")" - fi - if [ -z "$lsb_dist" ] && [ -r /etc/debian_version ]; then - lsb_dist='debian' - fi - if [ -z "$lsb_dist" ] && [ -r /etc/fedora-release ]; then - lsb_dist='fedora' - fi - if [ -z "$lsb_dist" ] && [ -r /etc/oracle-release ]; then - lsb_dist='oracleserver' - fi - if [ -z "$lsb_dist" ] && [ -r /etc/centos-release ]; then - lsb_dist='centos' - fi - if [ -z "$lsb_dist" ] && [ -r /etc/redhat-release ]; then - lsb_dist='redhat' - fi - if [ -z "$lsb_dist" ] && [ -r /etc/photon-release ]; then - lsb_dist='photon' - fi - if [ -z "$lsb_dist" ] && [ -r /etc/os-release ]; then - lsb_dist="$(. /etc/os-release && echo "$ID")" - fi - - lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')" - - # Special case redhatenterpriseserver - if [ "${lsb_dist}" = "redhatenterpriseserver" ]; then - # Set it to redhat, it will be changed to centos below anyways - lsb_dist='redhat' - fi - - case "$lsb_dist" in - - ubuntu) - if command_exists lsb_release; then - dist_version="$(lsb_release --codename | cut -f2)" - fi - if [ -z "$dist_version" ] && [ -r /etc/lsb-release ]; then - dist_version="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")" - fi - ;; - - debian|raspbian) - dist_version="$(cat /etc/debian_version | sed 's/\/.*//' | sed 's/\..*//')" - case "$dist_version" in - 9) - dist_version="stretch" - ;; - 8) - dist_version="jessie" - ;; - 7) - dist_version="wheezy" - ;; - esac - ;; - - oracleserver) - # need to switch lsb_dist to match yum repo URL - lsb_dist="oraclelinux" - dist_version="$(rpm -q --whatprovides redhat-release --queryformat "%{VERSION}\n" | sed 's/\/.*//' | sed 's/\..*//' | sed 's/Server*//')" - ;; - - fedora|centos|redhat) - dist_version="$(rpm -q --whatprovides ${lsb_dist}-release --queryformat "%{VERSION}\n" | sed 's/\/.*//' | sed 's/\..*//' | sed 's/Server*//' | sort | tail -1)" - ;; - - "vmware photon") - lsb_dist="photon" - dist_version="$(. /etc/os-release && echo "$VERSION_ID")" - ;; - - *) - if command_exists lsb_release; then - dist_version="$(lsb_release --codename | cut -f2)" - fi - if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then - dist_version="$(. /etc/os-release && echo "$VERSION_ID")" - fi - ;; - - - esac - - # Check if this is a forked Linux distro - check_forked - - # Run setup for each distro accordingly - case "$lsb_dist" in - ubuntu|debian|raspbian) - export DEBIAN_FRONTEND=noninteractive - - did_apt_get_update= - apt_get_update() { - if [ -z "$did_apt_get_update" ]; then - ( set -x; $sh_c 'sleep 3; apt-get update' ) - did_apt_get_update=1 - fi - } - - if [ "$lsb_dist" != "raspbian" ]; then - # aufs is preferred over devicemapper; try to ensure the driver is available. - if ! grep -q aufs /proc/filesystems && ! $sh_c 'modprobe aufs'; then - if uname -r | grep -q -- '-generic' && dpkg -l 'linux-image-*-generic' | grep -qE '^ii|^hi' 2>/dev/null; then - kern_extras="linux-image-extra-$(uname -r) linux-image-extra-virtual" - - apt_get_update - ( set -x; $sh_c 'sleep 3; apt-get install -y -q '"$kern_extras" ) || true - - if ! grep -q aufs /proc/filesystems && ! $sh_c 'modprobe aufs'; then - echo >&2 'Warning: tried to install '"$kern_extras"' (for AUFS)' - echo >&2 ' but we still have no AUFS. Docker may not work. Proceeding anyways!' - ( set -x; sleep 10 ) - fi - else - echo >&2 'Warning: current kernel is not supported by the linux-image-extra-virtual' - echo >&2 ' package. We have no AUFS support. Consider installing the packages' - echo >&2 ' "linux-image-virtual" and "linux-image-extra-virtual" for AUFS support.' - ( set -x; sleep 10 ) - fi - fi - fi - - # install apparmor utils if they're missing and apparmor is enabled in the kernel - # otherwise Docker will fail to start - if [ "$(cat /sys/module/apparmor/parameters/enabled 2>/dev/null)" = 'Y' ]; then - if command -v apparmor_parser >/dev/null 2>&1; then - echo 'apparmor is enabled in the kernel and apparmor utils were already installed' - else - echo 'apparmor is enabled in the kernel, but apparmor_parser is missing. Trying to install it..' - apt_get_update - ( set -x; $sh_c 'sleep 3; apt-get install -y -q apparmor' ) - fi - fi - - if [ ! -e /usr/lib/apt/methods/https ]; then - apt_get_update - ( set -x; $sh_c 'sleep 3; apt-get install -y -q apt-transport-https ca-certificates' ) - fi - if [ -z "$curl" ]; then - apt_get_update - ( set -x; $sh_c 'sleep 3; apt-get install -y -q curl ca-certificates' ) - curl='curl -sSL' - fi - if ! command -v gpg > /dev/null; then - apt_get_update - ( set -x; $sh_c 'sleep 3; apt-get install -y -q gnupg2 || apt-get install -y -q gnupg' ) - fi - - # dirmngr is a separate package in ubuntu yakkety; see https://bugs.launchpad.net/ubuntu/+source/apt/+bug/1634464 - if ! command -v dirmngr > /dev/null; then - apt_get_update - ( set -x; $sh_c 'sleep 3; apt-get install -y -q dirmngr' ) - fi - - ( - set -x - echo "$docker_key" | $sh_c 'apt-key add -' - $sh_c "mkdir -p /etc/apt/sources.list.d" - $sh_c "echo deb \[arch=$(dpkg --print-architecture)\] ${apt_url}/repo ${lsb_dist}-${dist_version} ${repo} > /etc/apt/sources.list.d/docker.list" - $sh_c 'sleep 3; apt-get update; apt-get install -y -q docker-engine' - ) - echo_docker_as_nonroot - exit 0 - ;; - - fedora|centos|redhat|oraclelinux|photon) - if [ "${lsb_dist}" = "redhat" ]; then - # we use the centos repository for both redhat and centos releases - lsb_dist='centos' - fi - $sh_c "cat >/etc/yum.repos.d/docker-${repo}.repo" <<-EOF - [docker-${repo}-repo] - name=Docker ${repo} Repository - baseurl=${yum_url}/repo/${repo}/${lsb_dist}/${dist_version} - enabled=1 - gpgcheck=1 - gpgkey=${yum_url}/gpg - EOF - if [ "$lsb_dist" = "fedora" ] && [ "$dist_version" -ge "22" ]; then - ( - set -x - $sh_c 'sleep 3; dnf -y -q install docker-engine' - ) - elif [ "$lsb_dist" = "photon" ]; then - ( - set -x - $sh_c 'sleep 3; tdnf -y install docker-engine' - ) - else - ( - set -x - $sh_c 'sleep 3; yum -y -q install docker-engine' - ) - fi - echo_docker_as_nonroot - exit 0 - ;; - esac - - # intentionally mixed spaces and tabs here -- tabs are stripped by "<<-'EOF'", spaces are kept in the output - cat >&2 <<-'EOF' - - Either your platform is not easily detectable, is not supported by this - installer script (yet - PRs welcome! [hack/install.sh]), or does not yet have - a package for Docker. Please visit the following URL for more detailed - installation instructions: - - https://docs.docker.com/engine/installation/ - - EOF - exit 1 -} - -# wrapped up in a function so that we have some protection against only getting -# half the file during "curl | sh" -do_install diff --git a/components/engine/hack/make/install-script b/components/engine/hack/make/install-script deleted file mode 100644 index 558ae52e63..0000000000 --- a/components/engine/hack/make/install-script +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -set -e - -# This script modifies the install.sh script for domains and keys other than -# those used by the primary opensource releases. -# -# You can provide `url`, `yum_url`, `apt_url` and optionally `gpg_fingerprint` -# or `GPG_KEYID` as environment variables, or the defaults for open source are used. -# -# The lower-case variables are substituted into install.sh. -# -# gpg_fingerprint and GPG_KEYID are optional, defaulting to the opensource release -# key ("releasedocker"). Other GPG_KEYIDs will require you to mount a volume with -# the correct contents to /root/.gnupg. -# -# It outputs the modified `install.sh` file to $DOCKER_RELEASE_DIR (default: $DEST) -# -# Example usage: -# -# docker run \ -# --rm \ -# --privileged \ -# -e "GPG_KEYID=deadbeef" \ -# -e "GNUPGHOME=/root/.gnupg" \ -# -v $HOME/.gnupg:/root/.gnupg \ -# -v $(pwd):/go/src/github.com/docker/docker/bundles \ -# "$IMAGE_DOCKER" \ -# hack/make.sh install-script - -: ${DOCKER_RELEASE_DIR:=$DEST} -: ${GPG_KEYID:=releasedocker} - -DEFAULT_URL="https://get.docker.com/" -DEFAULT_APT_URL="https://apt.dockerproject.org" -DEFAULT_YUM_URL="https://yum.dockerproject.org" -DEFAULT_GPG_FINGERPRINT="58118E89F3A912897C070ADBF76221572C52609D" - -: ${url:=$DEFAULT_URL} -: ${apt_url:=$DEFAULT_APT_URL} -: ${yum_url:=$DEFAULT_YUM_URL} -if [[ "$GPG_KEYID" == "releasedocker" ]] ; then - : ${gpg_fingerprint:=$DEFAULT_GPG_FINGERPRINT} -fi - -DEST_FILE="$DOCKER_RELEASE_DIR/install.sh" - -bundle_install_script() { - mkdir -p "$DOCKER_RELEASE_DIR" - - if [[ -z "$gpg_fingerprint" ]] ; then - # NOTE: if no key matching key is in /root/.gnupg, this will fail - gpg_fingerprint=$(gpg --with-fingerprint -k "$GPG_KEYID" | grep "Key fingerprint" | awk -F "=" '{print $2};' | tr -d ' ') - fi - - cp hack/install.sh "$DEST_FILE" - sed -i.bak 's#^url=".*"$#url="'"$url"'"#' "$DEST_FILE" - sed -i.bak 's#^apt_url=".*"$#apt_url="'"$apt_url"'"#' "$DEST_FILE" - sed -i.bak 's#^yum_url=".*"$#yum_url="'"$yum_url"'"#' "$DEST_FILE" - sed -i.bak 's#^gpg_fingerprint=".*"$#gpg_fingerprint="'"$gpg_fingerprint"'"#' "$DEST_FILE" - rm "${DEST_FILE}.bak" -} - -bundle_install_script diff --git a/components/engine/hack/make/test-deb-install b/components/engine/hack/make/test-deb-install deleted file mode 100755 index bc8e40818d..0000000000 --- a/components/engine/hack/make/test-deb-install +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# This script is used for testing install.sh and that it works for -# each of component of our apt and yum repos -set -e - -: ${DEB_DIR:="$(pwd)/bundles/$(cat VERSION)/build-deb"} - -if [[ ! -d "${DEB_DIR}" ]]; then - echo "you must first run `make deb` or hack/make/build-deb" - exit 1 -fi - -test_deb_install(){ - # test for each Dockerfile in contrib/builder - - builderDir="contrib/builder/deb/${PACKAGE_ARCH}" - pkgs=( $(find "${builderDir}/"*/ -type d) ) - if [ ! -z "$DOCKER_BUILD_PKGS" ]; then - pkgs=() - for p in $DOCKER_BUILD_PKGS; do - pkgs+=( "$builderDir/$p" ) - done - fi - for dir in "${pkgs[@]}"; do - [ -d "$dir" ] || { echo >&2 "skipping nonexistent $dir"; continue; } - local from="$(awk 'toupper($1) == "FROM" { print $2; exit }' "$dir/Dockerfile")" - local dir=$(basename "$dir") - - if [[ ! -d "${DEB_DIR}/${dir}" ]]; then - echo "No deb found for ${dir}" - exit 1 - fi - - local script=$(mktemp /tmp/install-XXXXXXXXXX.sh) - cat <<-EOF > "${script}" - #!/bin/bash - set -e - set -x - - apt-get update && apt-get install -y apparmor - - dpkg -i /root/debs/*.deb || true - - apt-get install -yf - - /etc/init.d/apparmor start - - # this will do everything _except_ load the profile into the kernel - ( - cd /etc/apparmor.d - /sbin/apparmor_parser --skip-kernel-load docker-engine - ) - EOF - - chmod +x "${script}" - - echo "testing deb install for ${from}" - docker run --rm -i --privileged \ - -v ${DEB_DIR}/${dir}:/root/debs \ - -v ${script}:/install.sh \ - ${from} /install.sh - - rm -f ${script} - done -} - -( - bundle .integration-daemon-start - test_deb_install - bundle .integration-daemon-stop -) 2>&1 | tee -a "$DEST/test.log" diff --git a/components/engine/hack/make/test-install-script b/components/engine/hack/make/test-install-script deleted file mode 100755 index aec1287fd3..0000000000 --- a/components/engine/hack/make/test-install-script +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# This script is used for testing install.sh and that it works for -# each of component of our apt and yum repos -set -e - -test_install_script(){ - # these are equivalent to main, testing, experimental components - # in the repos, but its the url that will do the conversion - components=( experimental test get ) - - for component in "${components[@]}"; do - # change url to specific component for testing - local test_url=https://${component}.docker.com - local script=$(mktemp /tmp/install-XXXXXXXXXX.sh) - sed "s,url='https://get.docker.com/',url='${test_url}/'," hack/install.sh > "${script}" - - chmod +x "${script}" - - # test for each Dockerfile in contrib/builder - for dir in contrib/builder/*/*/; do - local from="$(awk 'toupper($1) == "FROM" { print $2; exit }' "$dir/Dockerfile")" - - echo "running install.sh for ${component} with ${from}" - docker run --rm -i -v ${script}:/install.sh ${from} /install.sh - done - - rm -f ${script} - done -} - -test_install_script diff --git a/components/engine/hack/release.sh b/components/engine/hack/release.sh index fbbcde52bc..5d7363044b 100755 --- a/components/engine/hack/release.sh +++ b/components/engine/hack/release.sh @@ -295,18 +295,10 @@ EOF fi } -# Upload the index script -release_index() { - echo "Releasing index" - url="$(s3_url)/" hack/make.sh install-script - write_to_s3 "s3://$BUCKET_PATH/index" < "bundles/$VERSION/install-script/install.sh" -} - main() { [ "$SKIP_RELEASE_BUILD" = '1' ] || build_all setup_s3 release_binaries - release_index } main From 455cc50b83b543c9b1d82c8044c4d0f57b6037b5 Mon Sep 17 00:00:00 2001 From: Madhan Raj Mookkandy Date: Mon, 12 Jun 2017 15:20:23 -0700 Subject: [PATCH 30/56] Include Endpoint List for Shared Endpoints Do not allow sharing of container network with hyperv containers Signed-off-by: Madhan Raj Mookkandy Upstream-commit: 349913ce9fde34d8acd08fad5ce866401f4d135e Component: engine --- components/engine/container/container.go | 3 +- .../engine/daemon/container_operations.go | 7 +++- .../daemon/container_operations_solaris.go | 3 +- .../daemon/container_operations_unix.go | 3 +- .../daemon/container_operations_windows.go | 39 ++++++++++++++++++- components/engine/daemon/start_windows.go | 4 ++ 6 files changed, 54 insertions(+), 5 deletions(-) diff --git a/components/engine/container/container.go b/components/engine/container/container.go index 854850f4fb..7d339c688f 100644 --- a/components/engine/container/container.go +++ b/components/engine/container/container.go @@ -107,7 +107,8 @@ type Container struct { NoNewPrivileges bool // Fields here are specific to Windows - NetworkSharedContainerID string + NetworkSharedContainerID string `json:"-"` + SharedEndpointList []string `json:"-"` } // NewBaseContainer creates a new container with its diff --git a/components/engine/daemon/container_operations.go b/components/engine/daemon/container_operations.go index 00ed2f4787..7c7dcc7ced 100644 --- a/components/engine/daemon/container_operations.go +++ b/components/engine/daemon/container_operations.go @@ -886,7 +886,12 @@ func (daemon *Daemon) initializeNetworking(container *container.Container) error if err != nil { return err } - initializeNetworkingPaths(container, nc) + + err = daemon.initializeNetworkingPaths(container, nc) + if err != nil { + return err + } + container.Config.Hostname = nc.Config.Hostname container.Config.Domainname = nc.Config.Domainname return nil diff --git a/components/engine/daemon/container_operations_solaris.go b/components/engine/daemon/container_operations_solaris.go index 1653948de1..c5728d0ee7 100644 --- a/components/engine/daemon/container_operations_solaris.go +++ b/components/engine/daemon/container_operations_solaris.go @@ -42,5 +42,6 @@ func setupPathsAndSandboxOptions(container *container.Container, sboxOptions *[] return nil } -func initializeNetworkingPaths(container *container.Container, nc *container.Container) { +func (daemon *Daemon) initializeNetworkingPaths(container *container.Container, nc *container.Container) error { + return nil } diff --git a/components/engine/daemon/container_operations_unix.go b/components/engine/daemon/container_operations_unix.go index 277c83e083..09c2b7df18 100644 --- a/components/engine/daemon/container_operations_unix.go +++ b/components/engine/daemon/container_operations_unix.go @@ -349,8 +349,9 @@ func setupPathsAndSandboxOptions(container *container.Container, sboxOptions *[] return nil } -func initializeNetworkingPaths(container *container.Container, nc *container.Container) { +func (daemon *Daemon) initializeNetworkingPaths(container *container.Container, nc *container.Container) error { container.HostnamePath = nc.HostnamePath container.HostsPath = nc.HostsPath container.ResolvConfPath = nc.ResolvConfPath + return nil } diff --git a/components/engine/daemon/container_operations_windows.go b/components/engine/daemon/container_operations_windows.go index 1c20ebd672..2788f1a7cd 100644 --- a/components/engine/daemon/container_operations_windows.go +++ b/components/engine/daemon/container_operations_windows.go @@ -160,6 +160,43 @@ func setupPathsAndSandboxOptions(container *container.Container, sboxOptions *[] return nil } -func initializeNetworkingPaths(container *container.Container, nc *container.Container) { +func (daemon *Daemon) initializeNetworkingPaths(container *container.Container, nc *container.Container) error { + + if nc.HostConfig.Isolation.IsHyperV() { + return fmt.Errorf("sharing of hyperv containers network is not supported") + } + container.NetworkSharedContainerID = nc.ID + + if nc.NetworkSettings != nil { + for n := range nc.NetworkSettings.Networks { + sn, err := daemon.FindNetwork(n) + if err != nil { + continue + } + + ep, err := nc.GetEndpointInNetwork(sn) + if err != nil { + continue + } + + data, err := ep.DriverInfo() + if err != nil { + continue + } + + if data["GW_INFO"] != nil { + gwInfo := data["GW_INFO"].(map[string]interface{}) + if gwInfo["hnsid"] != nil { + container.SharedEndpointList = append(container.SharedEndpointList, gwInfo["hnsid"].(string)) + } + } + + if data["hnsid"] != nil { + container.SharedEndpointList = append(container.SharedEndpointList, data["hnsid"].(string)) + } + } + } + + return nil } diff --git a/components/engine/daemon/start_windows.go b/components/engine/daemon/start_windows.go index da70a16112..74129bd612 100644 --- a/components/engine/daemon/start_windows.go +++ b/components/engine/daemon/start_windows.go @@ -150,7 +150,11 @@ func (daemon *Daemon) getLibcontainerdCreateOptions(container *container.Contain var networkSharedContainerID string if container.HostConfig.NetworkMode.IsContainer() { networkSharedContainerID = container.NetworkSharedContainerID + for _, ep := range container.SharedEndpointList { + epList = append(epList, ep) + } } + createOptions = append(createOptions, &libcontainerd.NetworkEndpointsOption{ Endpoints: epList, AllowUnqualifiedDNSQuery: AllowUnqualifiedDNSQuery, From f6e262f40264622b226c6444e05d13ff3c4fb202 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 6 Jul 2017 13:58:13 -0700 Subject: [PATCH 31/56] Un-fork coreos/etcd - bump to v3.2.1 Commit 077f08bf54dc89d382cbcbd797b2bd7c4867151d temporarily switch etcd to a fork, pending a pull-request to be merged, and a new release that contains the change. The pull request was merged, and included in etcd v3.2.0 This patch bumps etcd to v3.2.1, which contains some bug-fixes on top of v3.2.0 Signed-off-by: Sebastiaan van Stijn Upstream-commit: cbc480a40cb22653ac647a8d24957a0c33e3bffe Component: engine --- components/engine/vendor.conf | 2 +- .../vendor/github.com/coreos/etcd/README.md | 24 ++- .../github.com/coreos/etcd/client/client.go | 33 +++++ .../github.com/coreos/etcd/client/discover.go | 19 +++ .../github.com/coreos/etcd/client/srv.go | 65 -------- .../coreos/etcd/pkg/fileutil/fileutil.go | 11 +- .../coreos/etcd/pkg/fileutil/lock_linux.go | 3 +- .../coreos/etcd/pkg/fileutil/preallocate.go | 15 +- .../coreos/etcd/pkg/fileutil/purge.go | 4 +- .../github.com/coreos/etcd/pkg/idutil/id.go | 4 +- .../github.com/coreos/etcd/pkg/srv/srv.go | 140 ++++++++++++++++++ .../github.com/coreos/etcd/raft/README.md | 91 +++--------- .../coreos/etcd/raft/log_unstable.go | 20 +++ .../github.com/coreos/etcd/raft/node.go | 16 ++ .../github.com/coreos/etcd/raft/raft.go | 5 + .../coreos/etcd/raft/raftpb/raft.pb.go | 2 +- .../github.com/coreos/etcd/raft/read_only.go | 2 +- .../vendor/github.com/coreos/etcd/snap/db.go | 23 +-- .../coreos/etcd/snap/snappb/snap.pb.go | 2 +- .../coreos/etcd/snap/snapshotter.go | 10 +- .../github.com/coreos/etcd/version/version.go | 56 +++++++ .../github.com/coreos/etcd/wal/encoder.go | 2 +- .../coreos/etcd/wal/file_pipeline.go | 4 +- .../github.com/coreos/etcd/wal/repair.go | 6 +- .../vendor/github.com/coreos/etcd/wal/wal.go | 41 ++--- .../coreos/etcd/wal/walpb/record.pb.go | 2 +- 26 files changed, 387 insertions(+), 215 deletions(-) delete mode 100644 components/engine/vendor/github.com/coreos/etcd/client/srv.go create mode 100644 components/engine/vendor/github.com/coreos/etcd/pkg/srv/srv.go create mode 100644 components/engine/vendor/github.com/coreos/etcd/version/version.go diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 20460d5cc9..4310990131 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -42,7 +42,7 @@ github.com/vishvananda/netlink bd6d5de5ccef2d66b0a26177928d0d8895d7f969 github.com/BurntSushi/toml f706d00e3de6abe700c994cdd545a1a4915af060 github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374 github.com/deckarep/golang-set ef32fa3046d9f249d399f98ebaf9be944430fd1d -github.com/coreos/etcd ea5389a79f40206170582c1ea076191b8622cb8e https://github.com/aaronlehmann/etcd # for https://github.com/coreos/etcd/pull/7830 +github.com/coreos/etcd v3.2.1 github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065 github.com/hashicorp/consul v0.5.2 github.com/boltdb/bolt fff57c100f4dea1905678da7e90d92429dff2904 diff --git a/components/engine/vendor/github.com/coreos/etcd/README.md b/components/engine/vendor/github.com/coreos/etcd/README.md index e7d4e23321..8ab28492c7 100644 --- a/components/engine/vendor/github.com/coreos/etcd/README.md +++ b/components/engine/vendor/github.com/coreos/etcd/README.md @@ -11,7 +11,7 @@ ![etcd Logo](logos/etcd-horizontal-color.png) -etcd is a distributed, consistent key-value store for shared configuration and service discovery, with a focus on being: +etcd is a distributed reliable key-value store for the most critical data of a distributed system, with a focus on being: * *Simple*: well-defined, user-facing API (gRPC) * *Secure*: automatic TLS with optional client cert authentication @@ -37,13 +37,11 @@ See [etcdctl][etcdctl] for a simple command line client. ### Getting etcd -The easiest way to get etcd is to use one of the pre-built release binaries which are available for OSX, Linux, Windows, AppC (ACI), and Docker. Instructions for using these binaries are on the [GitHub releases page][github-release]. +The easiest way to get etcd is to use one of the pre-built release binaries which are available for OSX, Linux, Windows, [rkt][rkt], and Docker. Instructions for using these binaries are on the [GitHub releases page][github-release]. -For those wanting to try the very latest version, you can [build the latest version of etcd][dl-build] from the `master` branch. -You will first need [*Go*](https://golang.org/) installed on your machine (version 1.6+ is required). -All development occurs on `master`, including new features and bug fixes. -Bug fixes are first targeted at `master` and subsequently ported to release branches, as described in the [branch management][branch-management] guide. +For those wanting to try the very latest version, [build the latest version of etcd][dl-build] from the `master` branch. This first needs [*Go*](https://golang.org/) installed (version 1.8+ is required). All development occurs on `master`, including new features and bug fixes. Bug fixes are first targeted at `master` and subsequently ported to release branches, as described in the [branch management][branch-management] guide. +[rkt]: https://github.com/rkt/rkt/releases/ [github-release]: https://github.com/coreos/etcd/releases/ [branch-management]: ./Documentation/branch_management.md [dl-build]: ./Documentation/dl_build.md#build-the-latest-version @@ -75,9 +73,9 @@ That's it! etcd is now running and serving client requests. For more ### etcd TCP ports -The [official etcd ports][iana-ports] are 2379 for client requests, and 2380 for peer communication. +The [official etcd ports][iana-ports] are 2379 for client requests, and 2380 for peer communication. -[iana-ports]: https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=etcd +[iana-ports]: http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.txt ### Running a local etcd cluster @@ -95,7 +93,7 @@ Every cluster member and proxy accepts key value reads and key value writes. ### Running etcd on Kubernetes -If you want to run etcd cluster on Kubernetes, try [etcd operator](https://github.com/coreos/etcd-operator). +To run an etcd cluster on Kubernetes, try [etcd operator](https://github.com/coreos/etcd-operator). ### Next steps @@ -105,7 +103,7 @@ Now it's time to dig into the full etcd API and other guides. - Explore the full gRPC [API][api]. - Set up a [multi-machine cluster][clustering]. - Learn the [config format, env variables and flags][configuration]. -- Find [language bindings and tools][libraries-and-tools]. +- Find [language bindings and tools][integrations]. - Use TLS to [secure an etcd cluster][security]. - [Tune etcd][tuning]. @@ -113,7 +111,7 @@ Now it's time to dig into the full etcd API and other guides. [api]: ./Documentation/dev-guide/api_reference_v3.md [clustering]: ./Documentation/op-guide/clustering.md [configuration]: ./Documentation/op-guide/configuration.md -[libraries-and-tools]: ./Documentation/libraries-and-tools.md +[integrations]: ./Documentation/integrations.md [security]: ./Documentation/op-guide/security.md [tuning]: ./Documentation/tuning.md @@ -130,10 +128,8 @@ See [CONTRIBUTING](CONTRIBUTING.md) for details on submitting patches and the co ## Reporting bugs -See [reporting bugs](Documentation/reporting_bugs.md) for details about reporting any issue you may encounter. +See [reporting bugs](Documentation/reporting_bugs.md) for details about reporting any issues. ### License etcd is under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details. - - diff --git a/components/engine/vendor/github.com/coreos/etcd/client/client.go b/components/engine/vendor/github.com/coreos/etcd/client/client.go index f9131b4725..498dfbcc8f 100644 --- a/components/engine/vendor/github.com/coreos/etcd/client/client.go +++ b/components/engine/vendor/github.com/coreos/etcd/client/client.go @@ -15,6 +15,7 @@ package client import ( + "encoding/json" "errors" "fmt" "io/ioutil" @@ -27,6 +28,8 @@ import ( "sync" "time" + "github.com/coreos/etcd/version" + "golang.org/x/net/context" ) @@ -201,6 +204,9 @@ type Client interface { // returned SetEndpoints(eps []string) error + // GetVersion retrieves the current etcd server and cluster version + GetVersion(ctx context.Context) (*version.Versions, error) + httpClient } @@ -477,6 +483,33 @@ func (c *httpClusterClient) AutoSync(ctx context.Context, interval time.Duration } } +func (c *httpClusterClient) GetVersion(ctx context.Context) (*version.Versions, error) { + act := &getAction{Prefix: "/version"} + + resp, body, err := c.Do(ctx, act) + if err != nil { + return nil, err + } + + switch resp.StatusCode { + case http.StatusOK: + if len(body) == 0 { + return nil, ErrEmptyBody + } + var vresp version.Versions + if err := json.Unmarshal(body, &vresp); err != nil { + return nil, ErrInvalidJSON + } + return &vresp, nil + default: + var etcdErr Error + if err := json.Unmarshal(body, &etcdErr); err != nil { + return nil, ErrInvalidJSON + } + return nil, etcdErr + } +} + type roundTripResponse struct { resp *http.Response err error diff --git a/components/engine/vendor/github.com/coreos/etcd/client/discover.go b/components/engine/vendor/github.com/coreos/etcd/client/discover.go index bfd7aec93f..442e35fe54 100644 --- a/components/engine/vendor/github.com/coreos/etcd/client/discover.go +++ b/components/engine/vendor/github.com/coreos/etcd/client/discover.go @@ -14,8 +14,27 @@ package client +import ( + "github.com/coreos/etcd/pkg/srv" +) + // Discoverer is an interface that wraps the Discover method. type Discoverer interface { // Discover looks up the etcd servers for the domain. Discover(domain string) ([]string, error) } + +type srvDiscover struct{} + +// NewSRVDiscover constructs a new Discoverer that uses the stdlib to lookup SRV records. +func NewSRVDiscover() Discoverer { + return &srvDiscover{} +} + +func (d *srvDiscover) Discover(domain string) ([]string, error) { + srvs, err := srv.GetClient("etcd-client", domain) + if err != nil { + return nil, err + } + return srvs.Endpoints, nil +} diff --git a/components/engine/vendor/github.com/coreos/etcd/client/srv.go b/components/engine/vendor/github.com/coreos/etcd/client/srv.go deleted file mode 100644 index fdfa343592..0000000000 --- a/components/engine/vendor/github.com/coreos/etcd/client/srv.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2015 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package client - -import ( - "fmt" - "net" - "net/url" -) - -var ( - // indirection for testing - lookupSRV = net.LookupSRV -) - -type srvDiscover struct{} - -// NewSRVDiscover constructs a new Discoverer that uses the stdlib to lookup SRV records. -func NewSRVDiscover() Discoverer { - return &srvDiscover{} -} - -// Discover looks up the etcd servers for the domain. -func (d *srvDiscover) Discover(domain string) ([]string, error) { - var urls []*url.URL - - updateURLs := func(service, scheme string) error { - _, addrs, err := lookupSRV(service, "tcp", domain) - if err != nil { - return err - } - for _, srv := range addrs { - urls = append(urls, &url.URL{ - Scheme: scheme, - Host: net.JoinHostPort(srv.Target, fmt.Sprintf("%d", srv.Port)), - }) - } - return nil - } - - errHTTPS := updateURLs("etcd-client-ssl", "https") - errHTTP := updateURLs("etcd-client", "http") - - if errHTTPS != nil && errHTTP != nil { - return nil, fmt.Errorf("dns lookup errors: %s and %s", errHTTPS, errHTTP) - } - - endpoints := make([]string, len(urls)) - for i := range urls { - endpoints[i] = urls[i].String() - } - return endpoints, nil -} diff --git a/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/fileutil.go b/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/fileutil.go index 9585ed5e0e..fce5126c69 100644 --- a/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/fileutil.go +++ b/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/fileutil.go @@ -17,9 +17,10 @@ package fileutil import ( "fmt" + "io" "io/ioutil" "os" - "path" + "path/filepath" "sort" "github.com/coreos/pkg/capnslog" @@ -39,7 +40,7 @@ var ( // IsDirWriteable checks if dir is writable by writing and removing a file // to dir. It returns nil if dir is writable. func IsDirWriteable(dir string) error { - f := path.Join(dir, ".touch") + f := filepath.Join(dir, ".touch") if err := ioutil.WriteFile(f, []byte(""), PrivateFileMode); err != nil { return err } @@ -101,11 +102,11 @@ func Exist(name string) bool { // shorten the length of the file. func ZeroToEnd(f *os.File) error { // TODO: support FALLOC_FL_ZERO_RANGE - off, err := f.Seek(0, os.SEEK_CUR) + off, err := f.Seek(0, io.SeekCurrent) if err != nil { return err } - lenf, lerr := f.Seek(0, os.SEEK_END) + lenf, lerr := f.Seek(0, io.SeekEnd) if lerr != nil { return lerr } @@ -116,6 +117,6 @@ func ZeroToEnd(f *os.File) error { if err = Preallocate(f, lenf, true); err != nil { return err } - _, err = f.Seek(off, os.SEEK_SET) + _, err = f.Seek(off, io.SeekStart) return err } diff --git a/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/lock_linux.go b/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/lock_linux.go index dec25a1af4..939fea6238 100644 --- a/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/lock_linux.go +++ b/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/lock_linux.go @@ -17,6 +17,7 @@ package fileutil import ( + "io" "os" "syscall" ) @@ -36,7 +37,7 @@ const ( var ( wrlck = syscall.Flock_t{ Type: syscall.F_WRLCK, - Whence: int16(os.SEEK_SET), + Whence: int16(io.SeekStart), Start: 0, Len: 0, } diff --git a/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/preallocate.go b/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/preallocate.go index bb7f028123..c747b7cf81 100644 --- a/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/preallocate.go +++ b/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/preallocate.go @@ -14,7 +14,10 @@ package fileutil -import "os" +import ( + "io" + "os" +) // Preallocate tries to allocate the space for given // file. This operation is only supported on linux by a @@ -22,6 +25,10 @@ import "os" // If the operation is unsupported, no error will be returned. // Otherwise, the error encountered will be returned. func Preallocate(f *os.File, sizeInBytes int64, extendFile bool) error { + if sizeInBytes == 0 { + // fallocate will return EINVAL if length is 0; skip + return nil + } if extendFile { return preallocExtend(f, sizeInBytes) } @@ -29,15 +36,15 @@ func Preallocate(f *os.File, sizeInBytes int64, extendFile bool) error { } func preallocExtendTrunc(f *os.File, sizeInBytes int64) error { - curOff, err := f.Seek(0, os.SEEK_CUR) + curOff, err := f.Seek(0, io.SeekCurrent) if err != nil { return err } - size, err := f.Seek(sizeInBytes, os.SEEK_END) + size, err := f.Seek(sizeInBytes, io.SeekEnd) if err != nil { return err } - if _, err = f.Seek(curOff, os.SEEK_SET); err != nil { + if _, err = f.Seek(curOff, io.SeekStart); err != nil { return err } if sizeInBytes > size { diff --git a/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/purge.go b/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/purge.go index 53bda0c012..92fceab017 100644 --- a/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/purge.go +++ b/components/engine/vendor/github.com/coreos/etcd/pkg/fileutil/purge.go @@ -16,7 +16,7 @@ package fileutil import ( "os" - "path" + "path/filepath" "sort" "strings" "time" @@ -45,7 +45,7 @@ func purgeFile(dirname string, suffix string, max uint, interval time.Duration, sort.Strings(newfnames) fnames = newfnames for len(newfnames) > int(max) { - f := path.Join(dirname, newfnames[0]) + f := filepath.Join(dirname, newfnames[0]) l, err := TryLockFile(f, os.O_WRONLY, PrivateFileMode) if err != nil { break diff --git a/components/engine/vendor/github.com/coreos/etcd/pkg/idutil/id.go b/components/engine/vendor/github.com/coreos/etcd/pkg/idutil/id.go index 931beb2d05..2da2106265 100644 --- a/components/engine/vendor/github.com/coreos/etcd/pkg/idutil/id.go +++ b/components/engine/vendor/github.com/coreos/etcd/pkg/idutil/id.go @@ -32,8 +32,8 @@ const ( // a node member ID. // // The initial id is in this format: -// High order byte is memberID, next 5 bytes are from timestamp, -// and low order 2 bytes are 0s. +// High order 2 bytes are from memberID, next 5 bytes are from timestamp, +// and low order one byte is a counter. // | prefix | suffix | // | 2 bytes | 5 bytes | 1 byte | // | memberID | timestamp | cnt | diff --git a/components/engine/vendor/github.com/coreos/etcd/pkg/srv/srv.go b/components/engine/vendor/github.com/coreos/etcd/pkg/srv/srv.go new file mode 100644 index 0000000000..fefcbcb4b8 --- /dev/null +++ b/components/engine/vendor/github.com/coreos/etcd/pkg/srv/srv.go @@ -0,0 +1,140 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package srv looks up DNS SRV records. +package srv + +import ( + "fmt" + "net" + "net/url" + "strings" + + "github.com/coreos/etcd/pkg/types" +) + +var ( + // indirection for testing + lookupSRV = net.LookupSRV // net.DefaultResolver.LookupSRV when ctxs don't conflict + resolveTCPAddr = net.ResolveTCPAddr +) + +// GetCluster gets the cluster information via DNS discovery. +// Also sees each entry as a separate instance. +func GetCluster(service, name, dns string, apurls types.URLs) ([]string, error) { + tempName := int(0) + tcp2ap := make(map[string]url.URL) + + // First, resolve the apurls + for _, url := range apurls { + tcpAddr, err := resolveTCPAddr("tcp", url.Host) + if err != nil { + return nil, err + } + tcp2ap[tcpAddr.String()] = url + } + + stringParts := []string{} + updateNodeMap := func(service, scheme string) error { + _, addrs, err := lookupSRV(service, "tcp", dns) + if err != nil { + return err + } + for _, srv := range addrs { + port := fmt.Sprintf("%d", srv.Port) + host := net.JoinHostPort(srv.Target, port) + tcpAddr, terr := resolveTCPAddr("tcp", host) + if terr != nil { + err = terr + continue + } + n := "" + url, ok := tcp2ap[tcpAddr.String()] + if ok { + n = name + } + if n == "" { + n = fmt.Sprintf("%d", tempName) + tempName++ + } + // SRV records have a trailing dot but URL shouldn't. + shortHost := strings.TrimSuffix(srv.Target, ".") + urlHost := net.JoinHostPort(shortHost, port) + stringParts = append(stringParts, fmt.Sprintf("%s=%s://%s", n, scheme, urlHost)) + if ok && url.Scheme != scheme { + err = fmt.Errorf("bootstrap at %s from DNS for %s has scheme mismatch with expected peer %s", scheme+"://"+urlHost, service, url.String()) + } + } + if len(stringParts) == 0 { + return err + } + return nil + } + + failCount := 0 + err := updateNodeMap(service+"-ssl", "https") + srvErr := make([]string, 2) + if err != nil { + srvErr[0] = fmt.Sprintf("error querying DNS SRV records for _%s-ssl %s", service, err) + failCount++ + } + err = updateNodeMap(service, "http") + if err != nil { + srvErr[1] = fmt.Sprintf("error querying DNS SRV records for _%s %s", service, err) + failCount++ + } + if failCount == 2 { + return nil, fmt.Errorf("srv: too many errors querying DNS SRV records (%q, %q)", srvErr[0], srvErr[1]) + } + return stringParts, nil +} + +type SRVClients struct { + Endpoints []string + SRVs []*net.SRV +} + +// GetClient looks up the client endpoints for a service and domain. +func GetClient(service, domain string) (*SRVClients, error) { + var urls []*url.URL + var srvs []*net.SRV + + updateURLs := func(service, scheme string) error { + _, addrs, err := lookupSRV(service, "tcp", domain) + if err != nil { + return err + } + for _, srv := range addrs { + urls = append(urls, &url.URL{ + Scheme: scheme, + Host: net.JoinHostPort(srv.Target, fmt.Sprintf("%d", srv.Port)), + }) + } + srvs = append(srvs, addrs...) + return nil + } + + errHTTPS := updateURLs(service+"-ssl", "https") + errHTTP := updateURLs(service, "http") + + if errHTTPS != nil && errHTTP != nil { + return nil, fmt.Errorf("dns lookup errors: %s and %s", errHTTPS, errHTTP) + } + + endpoints := make([]string, len(urls)) + for i := range urls { + endpoints[i] = urls[i].String() + } + return &SRVClients{Endpoints: endpoints, SRVs: srvs}, nil +} diff --git a/components/engine/vendor/github.com/coreos/etcd/raft/README.md b/components/engine/vendor/github.com/coreos/etcd/raft/README.md index a724b95857..f485b83977 100644 --- a/components/engine/vendor/github.com/coreos/etcd/raft/README.md +++ b/components/engine/vendor/github.com/coreos/etcd/raft/README.md @@ -13,9 +13,7 @@ To keep the codebase small as well as provide flexibility, the library only impl In order to easily test the Raft library, its behavior should be deterministic. To achieve this determinism, the library models Raft as a state machine. The state machine takes a `Message` as input. A message can either be a local timer update or a network message sent from a remote peer. The state machine's output is a 3-tuple `{[]Messages, []LogEntries, NextState}` consisting of an array of `Messages`, `log entries`, and `Raft state changes`. For state machines with the same state, the same state machine input should always generate the same state machine output. -A simple example application, _raftexample_, is also available to help illustrate -how to use this package in practice: -https://github.com/coreos/etcd/tree/master/contrib/raftexample +A simple example application, _raftexample_, is also available to help illustrate how to use this package in practice: https://github.com/coreos/etcd/tree/master/contrib/raftexample # Features @@ -51,11 +49,11 @@ This raft implementation also includes a few optional enhancements: - [etcd](https://github.com/coreos/etcd) A distributed reliable key-value store - [tikv](https://github.com/pingcap/tikv) A Distributed transactional key value database powered by Rust and Raft - [swarmkit](https://github.com/docker/swarmkit) A toolkit for orchestrating distributed systems at any scale. +- [chain core](https://github.com/chain/chain) Software for operating permissioned, multi-asset blockchain networks ## Usage -The primary object in raft is a Node. You either start a Node from scratch -using raft.StartNode or start a Node from some initial state using raft.RestartNode. +The primary object in raft is a Node. Either start a Node from scratch using raft.StartNode or start a Node from some initial state using raft.RestartNode. To start a three-node cluster ```go @@ -73,7 +71,7 @@ To start a three-node cluster n := raft.StartNode(c, []raft.Peer{{ID: 0x02}, {ID: 0x03}}) ``` -You can start a single node cluster, like so: +Start a single node cluster, like so: ```go // Create storage and config as shown above. // Set peer list to itself, so this node can become the leader of this single-node cluster. @@ -81,7 +79,7 @@ You can start a single node cluster, like so: n := raft.StartNode(c, peers) ``` -To allow a new node to join this cluster, do not pass in any peers. First, you need add the node to the existing cluster by calling `ProposeConfChange` on any existing node inside the cluster. Then, you can start the node with empty peer list, like so: +To allow a new node to join this cluster, do not pass in any peers. First, add the node to the existing cluster by calling `ProposeConfChange` on any existing node inside the cluster. Then, start the node with an empty peer list, like so: ```go // Create storage and config as shown above. n := raft.StartNode(c, nil) @@ -110,46 +108,21 @@ To restart a node from previous state: n := raft.RestartNode(c) ``` -Now that you are holding onto a Node you have a few responsibilities: +After creating a Node, the user has a few responsibilities: -First, you must read from the Node.Ready() channel and process the updates -it contains. These steps may be performed in parallel, except as noted in step -2. +First, read from the Node.Ready() channel and process the updates it contains. These steps may be performed in parallel, except as noted in step 2. -1. Write HardState, Entries, and Snapshot to persistent storage if they are -not empty. Note that when writing an Entry with Index i, any -previously-persisted entries with Index >= i must be discarded. +1. Write HardState, Entries, and Snapshot to persistent storage if they are not empty. Note that when writing an Entry with Index i, any previously-persisted entries with Index >= i must be discarded. -2. Send all Messages to the nodes named in the To field. It is important that -no messages be sent until the latest HardState has been persisted to disk, -and all Entries written by any previous Ready batch (Messages may be sent while -entries from the same batch are being persisted). To reduce the I/O latency, an -optimization can be applied to make leader write to disk in parallel with its -followers (as explained at section 10.2.1 in Raft thesis). If any Message has type -MsgSnap, call Node.ReportSnapshot() after it has been sent (these messages may be -large). Note: Marshalling messages is not thread-safe; it is important that you -make sure that no new entries are persisted while marshalling. -The easiest way to achieve this is to serialise the messages directly inside -your main raft loop. +2. Send all Messages to the nodes named in the To field. It is important that no messages be sent until the latest HardState has been persisted to disk, and all Entries written by any previous Ready batch (Messages may be sent while entries from the same batch are being persisted). To reduce the I/O latency, an optimization can be applied to make leader write to disk in parallel with its followers (as explained at section 10.2.1 in Raft thesis). If any Message has type MsgSnap, call Node.ReportSnapshot() after it has been sent (these messages may be large). Note: Marshalling messages is not thread-safe; it is important to make sure that no new entries are persisted while marshalling. The easiest way to achieve this is to serialise the messages directly inside the main raft loop. -3. Apply Snapshot (if any) and CommittedEntries to the state machine. -If any committed Entry has Type EntryConfChange, call Node.ApplyConfChange() -to apply it to the node. The configuration change may be cancelled at this point -by setting the NodeID field to zero before calling ApplyConfChange -(but ApplyConfChange must be called one way or the other, and the decision to cancel -must be based solely on the state machine and not external information such as -the observed health of the node). +3. Apply Snapshot (if any) and CommittedEntries to the state machine. If any committed Entry has Type EntryConfChange, call Node.ApplyConfChange() to apply it to the node. The configuration change may be cancelled at this point by setting the NodeID field to zero before calling ApplyConfChange (but ApplyConfChange must be called one way or the other, and the decision to cancel must be based solely on the state machine and not external information such as the observed health of the node). -4. Call Node.Advance() to signal readiness for the next batch of updates. -This may be done at any time after step 1, although all updates must be processed -in the order they were returned by Ready. +4. Call Node.Advance() to signal readiness for the next batch of updates. This may be done at any time after step 1, although all updates must be processed in the order they were returned by Ready. -Second, all persisted log entries must be made available via an -implementation of the Storage interface. The provided MemoryStorage -type can be used for this (if you repopulate its state upon a -restart), or you can supply your own disk-backed implementation. +Second, all persisted log entries must be made available via an implementation of the Storage interface. The provided MemoryStorage type can be used for this (if repopulating its state upon a restart), or a custom disk-backed implementation can be supplied. -Third, when you receive a message from another node, pass it to Node.Step: +Third, after receiving a message from another node, pass it to Node.Step: ```go func recvRaftRPC(ctx context.Context, m raftpb.Message) { @@ -157,10 +130,7 @@ Third, when you receive a message from another node, pass it to Node.Step: } ``` -Finally, you need to call `Node.Tick()` at regular intervals (probably -via a `time.Ticker`). Raft has two important timeouts: heartbeat and the -election timeout. However, internally to the raft package time is -represented by an abstract "tick". +Finally, call `Node.Tick()` at regular intervals (probably via a `time.Ticker`). Raft has two important timeouts: heartbeat and the election timeout. However, internally to the raft package time is represented by an abstract "tick". The total state machine handling loop will look something like this: @@ -190,16 +160,13 @@ The total state machine handling loop will look something like this: } ``` -To propose changes to the state machine from your node take your application -data, serialize it into a byte slice and call: +To propose changes to the state machine from the node to take application data, serialize it into a byte slice and call: ```go n.Propose(ctx, data) ``` -If the proposal is committed, data will appear in committed entries with type -raftpb.EntryNormal. There is no guarantee that a proposed command will be -committed; you may have to re-propose after a timeout. +If the proposal is committed, data will appear in committed entries with type raftpb.EntryNormal. There is no guarantee that a proposed command will be committed; the command may have to be reproposed after a timeout. To add or remove node in a cluster, build ConfChange struct 'cc' and call: @@ -207,8 +174,7 @@ To add or remove node in a cluster, build ConfChange struct 'cc' and call: n.ProposeConfChange(ctx, cc) ``` -After config change is committed, some committed entry with type -raftpb.EntryConfChange will be returned. You must apply it to node through: +After config change is committed, some committed entry with type raftpb.EntryConfChange will be returned. This must be applied to node through: ```go var cc raftpb.ConfChange @@ -223,25 +189,8 @@ may be reused. Node IDs must be non-zero. ## Implementation notes -This implementation is up to date with the final Raft thesis -(https://ramcloud.stanford.edu/~ongaro/thesis.pdf), although our -implementation of the membership change protocol differs somewhat from -that described in chapter 4. The key invariant that membership changes -happen one node at a time is preserved, but in our implementation the -membership change takes effect when its entry is applied, not when it -is added to the log (so the entry is committed under the old -membership instead of the new). This is equivalent in terms of safety, -since the old and new configurations are guaranteed to overlap. +This implementation is up to date with the final Raft thesis (https://ramcloud.stanford.edu/~ongaro/thesis.pdf), although this implementation of the membership change protocol differs somewhat from that described in chapter 4. The key invariant that membership changes happen one node at a time is preserved, but in our implementation the membership change takes effect when its entry is applied, not when it is added to the log (so the entry is committed under the old membership instead of the new). This is equivalent in terms of safety, since the old and new configurations are guaranteed to overlap. -To ensure that we do not attempt to commit two membership changes at -once by matching log positions (which would be unsafe since they -should have different quorum requirements), we simply disallow any -proposed membership change while any uncommitted change appears in -the leader's log. +To ensure there is no attempt to commit two membership changes at once by matching log positions (which would be unsafe since they should have different quorum requirements), any proposed membership change is simply disallowed while any uncommitted change appears in the leader's log. -This approach introduces a problem when you try to remove a member -from a two-member cluster: If one of the members dies before the -other one receives the commit of the confchange entry, then the member -cannot be removed any more since the cluster cannot make progress. -For this reason it is highly recommended to use three or more nodes in -every cluster. +This approach introduces a problem when removing a member from a two-member cluster: If one of the members dies before the other one receives the commit of the confchange entry, then the member cannot be removed any more since the cluster cannot make progress. For this reason it is highly recommended to use three or more nodes in every cluster. diff --git a/components/engine/vendor/github.com/coreos/etcd/raft/log_unstable.go b/components/engine/vendor/github.com/coreos/etcd/raft/log_unstable.go index 8ae301c3d8..263af9ce40 100644 --- a/components/engine/vendor/github.com/coreos/etcd/raft/log_unstable.go +++ b/components/engine/vendor/github.com/coreos/etcd/raft/log_unstable.go @@ -85,6 +85,26 @@ func (u *unstable) stableTo(i, t uint64) { if gt == t && i >= u.offset { u.entries = u.entries[i+1-u.offset:] u.offset = i + 1 + u.shrinkEntriesArray() + } +} + +// shrinkEntriesArray discards the underlying array used by the entries slice +// if most of it isn't being used. This avoids holding references to a bunch of +// potentially large entries that aren't needed anymore. Simply clearing the +// entries wouldn't be safe because clients might still be using them. +func (u *unstable) shrinkEntriesArray() { + // We replace the array if we're using less than half of the space in + // it. This number is fairly arbitrary, chosen as an attempt to balance + // memory usage vs number of allocations. It could probably be improved + // with some focused tuning. + const lenMultiple = 2 + if len(u.entries) == 0 { + u.entries = nil + } else if len(u.entries)*lenMultiple < cap(u.entries) { + newEntries := make([]pb.Entry, len(u.entries)) + copy(newEntries, u.entries) + u.entries = newEntries } } diff --git a/components/engine/vendor/github.com/coreos/etcd/raft/node.go b/components/engine/vendor/github.com/coreos/etcd/raft/node.go index c8410fdc77..5da1c1193b 100644 --- a/components/engine/vendor/github.com/coreos/etcd/raft/node.go +++ b/components/engine/vendor/github.com/coreos/etcd/raft/node.go @@ -83,6 +83,10 @@ type Ready struct { // If it contains a MsgSnap message, the application MUST report back to raft // when the snapshot has been received or has failed by calling ReportSnapshot. Messages []pb.Message + + // MustSync indicates whether the HardState and Entries must be synchronously + // written to disk or if an asynchronous write is permissible. + MustSync bool } func isHardStateEqual(a, b pb.HardState) bool { @@ -517,5 +521,17 @@ func newReady(r *raft, prevSoftSt *SoftState, prevHardSt pb.HardState) Ready { if len(r.readStates) != 0 { rd.ReadStates = r.readStates } + rd.MustSync = MustSync(rd.HardState, prevHardSt, len(rd.Entries)) return rd } + +// MustSync returns true if the hard state and count of Raft entries indicate +// that a synchronous write to persistent storage is required. +func MustSync(st, prevst pb.HardState, entsnum int) bool { + // Persistent state on all servers: + // (Updated on stable storage before responding to RPCs) + // currentTerm + // votedFor + // log entries[] + return entsnum != 0 || st.Vote != prevst.Vote || st.Term != prevst.Term +} diff --git a/components/engine/vendor/github.com/coreos/etcd/raft/raft.go b/components/engine/vendor/github.com/coreos/etcd/raft/raft.go index 633cc147f5..29f2039820 100644 --- a/components/engine/vendor/github.com/coreos/etcd/raft/raft.go +++ b/components/engine/vendor/github.com/coreos/etcd/raft/raft.go @@ -823,6 +823,11 @@ func stepLeader(r *raft, m pb.Message) { return case pb.MsgReadIndex: if r.quorum() > 1 { + if r.raftLog.zeroTermOnErrCompacted(r.raftLog.term(r.raftLog.committed)) != r.Term { + // Reject read only request when this leader has not committed any log entry at its term. + return + } + // thinking: use an interally defined context instead of the user given context. // We can express this in terms of the term and index instead of a user-supplied value. // This would allow multiple reads to piggyback on the same message. diff --git a/components/engine/vendor/github.com/coreos/etcd/raft/raftpb/raft.pb.go b/components/engine/vendor/github.com/coreos/etcd/raft/raftpb/raft.pb.go index 86ad312070..3c45eef003 100644 --- a/components/engine/vendor/github.com/coreos/etcd/raft/raftpb/raft.pb.go +++ b/components/engine/vendor/github.com/coreos/etcd/raft/raftpb/raft.pb.go @@ -1847,7 +1847,7 @@ func init() { proto.RegisterFile("raft.proto", fileDescriptorRaft) } var fileDescriptorRaft = []byte{ // 790 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x64, 0x54, 0xcd, 0x6e, 0xdb, 0x46, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x54, 0xcd, 0x6e, 0xdb, 0x46, 0x10, 0x16, 0x29, 0xea, 0x6f, 0x28, 0xcb, 0xab, 0xb5, 0x5a, 0x2c, 0x0c, 0x43, 0x55, 0x85, 0x1e, 0x04, 0x17, 0x76, 0x5b, 0x1d, 0x7a, 0xe8, 0xcd, 0x96, 0x0a, 0x58, 0x40, 0x65, 0xb8, 0xb2, 0xdc, 0x43, 0x83, 0x20, 0x58, 0x8b, 0x2b, 0x4a, 0x89, 0xc9, 0x25, 0x96, 0x2b, 0xc7, 0xbe, 0x04, 0x79, diff --git a/components/engine/vendor/github.com/coreos/etcd/raft/read_only.go b/components/engine/vendor/github.com/coreos/etcd/raft/read_only.go index 05a21dabd1..d0085237e3 100644 --- a/components/engine/vendor/github.com/coreos/etcd/raft/read_only.go +++ b/components/engine/vendor/github.com/coreos/etcd/raft/read_only.go @@ -100,7 +100,7 @@ func (ro *readOnly) advance(m pb.Message) []*readIndexStatus { if found { ro.readIndexQueue = ro.readIndexQueue[i:] for _, rs := range rss { - delete(ro.pendingReadIndex, string(rs.req.Context)) + delete(ro.pendingReadIndex, string(rs.req.Entries[0].Data)) } return rss } diff --git a/components/engine/vendor/github.com/coreos/etcd/snap/db.go b/components/engine/vendor/github.com/coreos/etcd/snap/db.go index 743deac1e2..01d897ae86 100644 --- a/components/engine/vendor/github.com/coreos/etcd/snap/db.go +++ b/components/engine/vendor/github.com/coreos/etcd/snap/db.go @@ -15,15 +15,18 @@ package snap import ( + "errors" "fmt" "io" "io/ioutil" "os" - "path" + "path/filepath" "github.com/coreos/etcd/pkg/fileutil" ) +var ErrNoDBSnapshot = errors.New("snap: snapshot file doesn't exist") + // SaveDBFrom saves snapshot of the database from the given reader. It // guarantees the save operation is atomic. func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { @@ -41,7 +44,7 @@ func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { os.Remove(f.Name()) return n, err } - fn := path.Join(s.dir, fmt.Sprintf("%016x.snap.db", id)) + fn := s.dbFilePath(id) if fileutil.Exist(fn) { os.Remove(f.Name()) return n, nil @@ -60,15 +63,15 @@ func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { // DBFilePath returns the file path for the snapshot of the database with // given id. If the snapshot does not exist, it returns error. func (s *Snapshotter) DBFilePath(id uint64) (string, error) { - fns, err := fileutil.ReadDir(s.dir) - if err != nil { + if _, err := fileutil.ReadDir(s.dir); err != nil { return "", err } - wfn := fmt.Sprintf("%016x.snap.db", id) - for _, fn := range fns { - if fn == wfn { - return path.Join(s.dir, fn), nil - } + if fn := s.dbFilePath(id); fileutil.Exist(fn) { + return fn, nil } - return "", fmt.Errorf("snap: snapshot file doesn't exist") + return "", ErrNoDBSnapshot +} + +func (s *Snapshotter) dbFilePath(id uint64) string { + return filepath.Join(s.dir, fmt.Sprintf("%016x.snap.db", id)) } diff --git a/components/engine/vendor/github.com/coreos/etcd/snap/snappb/snap.pb.go b/components/engine/vendor/github.com/coreos/etcd/snap/snappb/snap.pb.go index 130e2277c8..05a77ff9d0 100644 --- a/components/engine/vendor/github.com/coreos/etcd/snap/snappb/snap.pb.go +++ b/components/engine/vendor/github.com/coreos/etcd/snap/snappb/snap.pb.go @@ -342,7 +342,7 @@ func init() { proto.RegisterFile("snap.proto", fileDescriptorSnap) } var fileDescriptorSnap = []byte{ // 126 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x2a, 0xce, 0x4b, 0x2c, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2a, 0xce, 0x4b, 0x2c, 0xd0, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x03, 0xb1, 0x0b, 0x92, 0xa4, 0x44, 0xd2, 0xf3, 0xd3, 0xf3, 0xc1, 0x42, 0xfa, 0x20, 0x16, 0x44, 0x56, 0xc9, 0x8c, 0x8b, 0x03, 0x24, 0x5f, 0x9c, 0x91, 0x5f, 0x22, 0x24, 0xc6, 0xc5, 0x9c, 0x5c, 0x94, 0x2c, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0xeb, diff --git a/components/engine/vendor/github.com/coreos/etcd/snap/snapshotter.go b/components/engine/vendor/github.com/coreos/etcd/snap/snapshotter.go index 50d09dda14..0075559212 100644 --- a/components/engine/vendor/github.com/coreos/etcd/snap/snapshotter.go +++ b/components/engine/vendor/github.com/coreos/etcd/snap/snapshotter.go @@ -21,7 +21,7 @@ import ( "hash/crc32" "io/ioutil" "os" - "path" + "path/filepath" "sort" "strings" "time" @@ -84,13 +84,13 @@ func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error { marshallingDurations.Observe(float64(time.Since(start)) / float64(time.Second)) } - err = pioutil.WriteAndSyncFile(path.Join(s.dir, fname), d, 0666) + err = pioutil.WriteAndSyncFile(filepath.Join(s.dir, fname), d, 0666) if err == nil { saveDurations.Observe(float64(time.Since(start)) / float64(time.Second)) } else { - err1 := os.Remove(path.Join(s.dir, fname)) + err1 := os.Remove(filepath.Join(s.dir, fname)) if err1 != nil { - plog.Errorf("failed to remove broken snapshot file %s", path.Join(s.dir, fname)) + plog.Errorf("failed to remove broken snapshot file %s", filepath.Join(s.dir, fname)) } } return err @@ -114,7 +114,7 @@ func (s *Snapshotter) Load() (*raftpb.Snapshot, error) { } func loadSnap(dir, name string) (*raftpb.Snapshot, error) { - fpath := path.Join(dir, name) + fpath := filepath.Join(dir, name) snap, err := Read(fpath) if err != nil { renameBroken(fpath) diff --git a/components/engine/vendor/github.com/coreos/etcd/version/version.go b/components/engine/vendor/github.com/coreos/etcd/version/version.go new file mode 100644 index 0000000000..84bdd09f1d --- /dev/null +++ b/components/engine/vendor/github.com/coreos/etcd/version/version.go @@ -0,0 +1,56 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package version implements etcd version parsing and contains latest version +// information. +package version + +import ( + "fmt" + "strings" + + "github.com/coreos/go-semver/semver" +) + +var ( + // MinClusterVersion is the min cluster version this etcd binary is compatible with. + MinClusterVersion = "3.0.0" + Version = "3.2.1" + APIVersion = "unknown" + + // Git SHA Value will be set during build + GitSHA = "Not provided (use ./build instead of go build)" +) + +func init() { + ver, err := semver.NewVersion(Version) + if err == nil { + APIVersion = fmt.Sprintf("%d.%d", ver.Major, ver.Minor) + } +} + +type Versions struct { + Server string `json:"etcdserver"` + Cluster string `json:"etcdcluster"` + // TODO: raft state machine version +} + +// Cluster only keeps the major.minor. +func Cluster(v string) string { + vs := strings.Split(v, ".") + if len(vs) <= 2 { + return v + } + return fmt.Sprintf("%s.%s", vs[0], vs[1]) +} diff --git a/components/engine/vendor/github.com/coreos/etcd/wal/encoder.go b/components/engine/vendor/github.com/coreos/etcd/wal/encoder.go index efe58928cc..aac1e197e5 100644 --- a/components/engine/vendor/github.com/coreos/etcd/wal/encoder.go +++ b/components/engine/vendor/github.com/coreos/etcd/wal/encoder.go @@ -52,7 +52,7 @@ func newEncoder(w io.Writer, prevCrc uint32, pageOffset int) *encoder { // newFileEncoder creates a new encoder with current file offset for the page writer. func newFileEncoder(f *os.File, prevCrc uint32) (*encoder, error) { - offset, err := f.Seek(0, os.SEEK_CUR) + offset, err := f.Seek(0, io.SeekCurrent) if err != nil { return nil, err } diff --git a/components/engine/vendor/github.com/coreos/etcd/wal/file_pipeline.go b/components/engine/vendor/github.com/coreos/etcd/wal/file_pipeline.go index 3412210a35..5e32a0693c 100644 --- a/components/engine/vendor/github.com/coreos/etcd/wal/file_pipeline.go +++ b/components/engine/vendor/github.com/coreos/etcd/wal/file_pipeline.go @@ -17,7 +17,7 @@ package wal import ( "fmt" "os" - "path" + "path/filepath" "github.com/coreos/etcd/pkg/fileutil" ) @@ -65,7 +65,7 @@ func (fp *filePipeline) Close() error { func (fp *filePipeline) alloc() (f *fileutil.LockedFile, err error) { // count % 2 so this file isn't the same as the one last published - fpath := path.Join(fp.dir, fmt.Sprintf("%d.tmp", fp.count%2)) + fpath := filepath.Join(fp.dir, fmt.Sprintf("%d.tmp", fp.count%2)) if f, err = fileutil.LockFile(fpath, os.O_CREATE|os.O_WRONLY, fileutil.PrivateFileMode); err != nil { return nil, err } diff --git a/components/engine/vendor/github.com/coreos/etcd/wal/repair.go b/components/engine/vendor/github.com/coreos/etcd/wal/repair.go index 0a920e2d8b..091036b57b 100644 --- a/components/engine/vendor/github.com/coreos/etcd/wal/repair.go +++ b/components/engine/vendor/github.com/coreos/etcd/wal/repair.go @@ -17,7 +17,7 @@ package wal import ( "io" "os" - "path" + "path/filepath" "github.com/coreos/etcd/pkg/fileutil" "github.com/coreos/etcd/wal/walpb" @@ -62,7 +62,7 @@ func Repair(dirpath string) bool { } defer bf.Close() - if _, err = f.Seek(0, os.SEEK_SET); err != nil { + if _, err = f.Seek(0, io.SeekStart); err != nil { plog.Errorf("could not repair %v, failed to read file", f.Name()) return false } @@ -94,6 +94,6 @@ func openLast(dirpath string) (*fileutil.LockedFile, error) { if err != nil { return nil, err } - last := path.Join(dirpath, names[len(names)-1]) + last := filepath.Join(dirpath, names[len(names)-1]) return fileutil.LockFile(last, os.O_RDWR, fileutil.PrivateFileMode) } diff --git a/components/engine/vendor/github.com/coreos/etcd/wal/wal.go b/components/engine/vendor/github.com/coreos/etcd/wal/wal.go index 69ed6b2390..2cac25c1c9 100644 --- a/components/engine/vendor/github.com/coreos/etcd/wal/wal.go +++ b/components/engine/vendor/github.com/coreos/etcd/wal/wal.go @@ -21,7 +21,7 @@ import ( "hash/crc32" "io" "os" - "path" + "path/filepath" "sync" "time" @@ -97,7 +97,7 @@ func Create(dirpath string, metadata []byte) (*WAL, error) { } // keep temporary wal directory so WAL initialization appears atomic - tmpdirpath := path.Clean(dirpath) + ".tmp" + tmpdirpath := filepath.Clean(dirpath) + ".tmp" if fileutil.Exist(tmpdirpath) { if err := os.RemoveAll(tmpdirpath); err != nil { return nil, err @@ -107,12 +107,12 @@ func Create(dirpath string, metadata []byte) (*WAL, error) { return nil, err } - p := path.Join(tmpdirpath, walName(0, 0)) + p := filepath.Join(tmpdirpath, walName(0, 0)) f, err := fileutil.LockFile(p, os.O_WRONLY|os.O_CREATE, fileutil.PrivateFileMode) if err != nil { return nil, err } - if _, err = f.Seek(0, os.SEEK_END); err != nil { + if _, err = f.Seek(0, io.SeekEnd); err != nil { return nil, err } if err = fileutil.Preallocate(f.File, SegmentSizeBytes, true); err != nil { @@ -143,7 +143,7 @@ func Create(dirpath string, metadata []byte) (*WAL, error) { } // directory was renamed; sync parent dir to persist rename - pdir, perr := fileutil.OpenDir(path.Dir(w.dir)) + pdir, perr := fileutil.OpenDir(filepath.Dir(w.dir)) if perr != nil { return nil, perr } @@ -196,7 +196,7 @@ func openAtIndex(dirpath string, snap walpb.Snapshot, write bool) (*WAL, error) rs := make([]io.Reader, 0) ls := make([]*fileutil.LockedFile, 0) for _, name := range names[nameIndex:] { - p := path.Join(dirpath, name) + p := filepath.Join(dirpath, name) if write { l, err := fileutil.TryLockFile(p, os.O_RDWR, fileutil.PrivateFileMode) if err != nil { @@ -232,7 +232,7 @@ func openAtIndex(dirpath string, snap walpb.Snapshot, write bool) (*WAL, error) // write reuses the file descriptors from read; don't close so // WAL can append without dropping the file lock w.readClose = nil - if _, _, err := parseWalName(path.Base(w.tail().Name())); err != nil { + if _, _, err := parseWalName(filepath.Base(w.tail().Name())); err != nil { closer() return nil, err } @@ -322,7 +322,7 @@ func (w *WAL) ReadAll() (metadata []byte, state raftpb.HardState, ents []raftpb. // not all, will cause CRC errors on WAL open. Since the records // were never fully synced to disk in the first place, it's safe // to zero them out to avoid any CRC errors from new writes. - if _, err = w.tail().Seek(w.decoder.lastOffset(), os.SEEK_SET); err != nil { + if _, err = w.tail().Seek(w.decoder.lastOffset(), io.SeekStart); err != nil { return nil, state, nil, err } if err = fileutil.ZeroToEnd(w.tail().File); err != nil { @@ -361,7 +361,7 @@ func (w *WAL) ReadAll() (metadata []byte, state raftpb.HardState, ents []raftpb. // Then cut atomically rename temp wal file to a wal file. func (w *WAL) cut() error { // close old wal file; truncate to avoid wasting space if an early cut - off, serr := w.tail().Seek(0, os.SEEK_CUR) + off, serr := w.tail().Seek(0, io.SeekCurrent) if serr != nil { return serr } @@ -372,7 +372,7 @@ func (w *WAL) cut() error { return err } - fpath := path.Join(w.dir, walName(w.seq()+1, w.enti+1)) + fpath := filepath.Join(w.dir, walName(w.seq()+1, w.enti+1)) // create a temp wal file with name sequence + 1, or truncate the existing one newTail, err := w.fp.Open() @@ -401,7 +401,7 @@ func (w *WAL) cut() error { return err } - off, err = w.tail().Seek(0, os.SEEK_CUR) + off, err = w.tail().Seek(0, io.SeekCurrent) if err != nil { return err } @@ -418,7 +418,7 @@ func (w *WAL) cut() error { if newTail, err = fileutil.LockFile(fpath, os.O_WRONLY, fileutil.PrivateFileMode); err != nil { return err } - if _, err = newTail.Seek(off, os.SEEK_SET); err != nil { + if _, err = newTail.Seek(off, io.SeekStart); err != nil { return err } @@ -464,7 +464,7 @@ func (w *WAL) ReleaseLockTo(index uint64) error { found := false for i, l := range w.locks { - _, lockIndex, err := parseWalName(path.Base(l.Name())) + _, lockIndex, err := parseWalName(filepath.Base(l.Name())) if err != nil { return err } @@ -552,7 +552,7 @@ func (w *WAL) Save(st raftpb.HardState, ents []raftpb.Entry) error { return nil } - mustSync := mustSync(st, w.state, len(ents)) + mustSync := raft.MustSync(st, w.state, len(ents)) // TODO(xiangli): no more reference operator for i := range ents { @@ -564,7 +564,7 @@ func (w *WAL) Save(st raftpb.HardState, ents []raftpb.Entry) error { return err } - curOff, err := w.tail().Seek(0, os.SEEK_CUR) + curOff, err := w.tail().Seek(0, io.SeekCurrent) if err != nil { return err } @@ -611,22 +611,13 @@ func (w *WAL) seq() uint64 { if t == nil { return 0 } - seq, _, err := parseWalName(path.Base(t.Name())) + seq, _, err := parseWalName(filepath.Base(t.Name())) if err != nil { plog.Fatalf("bad wal name %s (%v)", t.Name(), err) } return seq } -func mustSync(st, prevst raftpb.HardState, entsnum int) bool { - // Persistent state on all servers: - // (Updated on stable storage before responding to RPCs) - // currentTerm - // votedFor - // log entries[] - return entsnum != 0 || st.Vote != prevst.Vote || st.Term != prevst.Term -} - func closeAll(rcs ...io.ReadCloser) error { for _, f := range rcs { if err := f.Close(); err != nil { diff --git a/components/engine/vendor/github.com/coreos/etcd/wal/walpb/record.pb.go b/components/engine/vendor/github.com/coreos/etcd/wal/walpb/record.pb.go index e1a77d5e51..664fae1305 100644 --- a/components/engine/vendor/github.com/coreos/etcd/wal/walpb/record.pb.go +++ b/components/engine/vendor/github.com/coreos/etcd/wal/walpb/record.pb.go @@ -506,7 +506,7 @@ func init() { proto.RegisterFile("record.proto", fileDescriptorRecord) } var fileDescriptorRecord = []byte{ // 186 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x4a, 0x4d, 0xce, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x4a, 0x4d, 0xce, 0x2f, 0x4a, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x2d, 0x4f, 0xcc, 0x29, 0x48, 0x92, 0x12, 0x49, 0xcf, 0x4f, 0xcf, 0x07, 0x8b, 0xe8, 0x83, 0x58, 0x10, 0x49, 0x25, 0x3f, 0x2e, 0xb6, 0x20, 0xb0, 0x62, 0x21, 0x09, 0x2e, 0x96, 0x92, 0xca, 0x82, 0x54, 0x09, 0x46, 0x05, 0x46, 0x0d, From eb145cf9d737d9d6ef9c744a1e61c483a25ece8d Mon Sep 17 00:00:00 2001 From: John Howard Date: Thu, 6 Jul 2017 14:29:43 -0700 Subject: [PATCH 32/56] Revendor jhowardmsft/opengcs v0.0.9 Signed-off-by: John Howard Upstream-commit: 7643e8909675fb7b1d52a8c7566fb674db340a39 Component: engine --- components/engine/vendor.conf | 2 +- .../opengcs/gogcs/client/config.go | 2 - .../opengcs/gogcs/client/createsandbox.go | 19 ++------ .../opengcs/gogcs/client/hotaddvhd.go | 2 - .../opengcs/gogcs/client/hotremovevhd.go | 2 - .../opengcs/gogcs/client/layervhddetails.go | 2 - .../opengcs/gogcs/client/process.go | 43 ++++++++++++++++++- .../opengcs/gogcs/client/tartovhd.go | 6 +-- .../opengcs/gogcs/client/unsupported.go | 2 - .../opengcs/gogcs/client/utilities.go | 12 +++--- .../opengcs/gogcs/client/vhdtotar.go | 2 - 11 files changed, 54 insertions(+), 40 deletions(-) diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 20460d5cc9..ece20e6462 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -8,7 +8,7 @@ github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git github.com/gorilla/context v1.1 github.com/gorilla/mux v1.1 -github.com/jhowardmsft/opengcs v0.0.7 +github.com/jhowardmsft/opengcs v0.0.9 github.com/kr/pty 5cf931ef8f github.com/mattn/go-shellwords v1.0.3 github.com/tchap/go-patricia v2.2.6 diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/config.go b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/config.go index 49f3e0b1c5..fedb563abe 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/config.go +++ b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/config.go @@ -2,8 +2,6 @@ package client -// TODO @jhowardmsft - This will move to Microsoft/opengcs soon - import ( "encoding/json" "fmt" diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/createsandbox.go b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/createsandbox.go index 4fd9cc907a..65971601a1 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/createsandbox.go +++ b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/createsandbox.go @@ -2,19 +2,15 @@ package client -// TODO @jhowardmsft - This will move to Microsoft/opengcs soon - import ( "fmt" "os" - "sync" "github.com/Sirupsen/logrus" ) -var sandboxCacheLock sync.Mutex - -// CreateSandbox does what it says on the tin. This is done by copying a prebuilt-sandbox from the ServiceVM +// CreateSandbox does what it says on the tin. This is done by copying a prebuilt-sandbox from the ServiceVM. +// It is the responsibility of the caller to synchronise simultaneous attempts to create the cache file. // TODO: @jhowardmsft maxSizeInMB isn't hooked up in GCS. Needs a platform change which is in flight. func (config *Config) CreateSandbox(destFile string, maxSizeInMB uint32, cacheFile string) error { // Smallest we can accept is the default sandbox size as we can't size down, only expand. @@ -26,17 +22,13 @@ func (config *Config) CreateSandbox(destFile string, maxSizeInMB uint32, cacheFi // Retrieve from cache if the default size and already on disk if cacheFile != "" && maxSizeInMB == DefaultSandboxSizeMB { - sandboxCacheLock.Lock() if _, err := os.Stat(cacheFile); err == nil { - if err := copyFile(cacheFile, destFile); err != nil { - sandboxCacheLock.Unlock() + if err := CopyFile(cacheFile, destFile, false); err != nil { return fmt.Errorf("opengcs: CreateSandbox: Failed to copy cached sandbox '%s' to '%s': %s", cacheFile, destFile, err) } - sandboxCacheLock.Unlock() logrus.Debugf("opengcs: CreateSandbox: %s fulfilled from cache", destFile) return nil } - sandboxCacheLock.Unlock() } if config.Uvm == nil { @@ -62,15 +54,12 @@ func (config *Config) CreateSandbox(destFile string, maxSizeInMB uint32, cacheFi // Populate the cache if cacheFile != "" && maxSizeInMB == DefaultSandboxSizeMB { - sandboxCacheLock.Lock() // It may already exist due to being created on another thread, in which case no copy back needed. if _, err := os.Stat(cacheFile); os.IsNotExist(err) { - if err := copyFile(destFile, cacheFile); err != nil { - sandboxCacheLock.Unlock() + if err := CopyFile(destFile, cacheFile, false); err != nil { return fmt.Errorf("opengcs: CreateSandbox: Failed to seed sandbox cache '%s' from '%s': %s", destFile, cacheFile, err) } } - sandboxCacheLock.Unlock() } logrus.Debugf("opengcs: CreateSandbox: %s created (non-cache)", destFile) diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotaddvhd.go b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotaddvhd.go index 62de79016a..c0545925f7 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotaddvhd.go +++ b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotaddvhd.go @@ -2,8 +2,6 @@ package client -// TODO @jhowardmsft - This will move to Microsoft/opengcs soon - import ( "fmt" diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotremovevhd.go b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotremovevhd.go index 71167d00ee..1f3d21aa81 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotremovevhd.go +++ b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/hotremovevhd.go @@ -2,8 +2,6 @@ package client -// TODO @jhowardmsft - This will move to Microsoft/opengcs soon - import ( "fmt" diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/layervhddetails.go b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/layervhddetails.go index 6ce29cdc56..010780069a 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/layervhddetails.go +++ b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/layervhddetails.go @@ -2,8 +2,6 @@ package client -// TODO @jhowardmsft - This will move to Microsoft/opengcs soon - import ( "fmt" "os" diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/process.go b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/process.go index 6592ab6a03..02655227a9 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/process.go +++ b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/process.go @@ -2,8 +2,6 @@ package client -// TODO @jhowardmsft - This will move to Microsoft/opengcs soon - import ( "fmt" "io" @@ -59,3 +57,44 @@ func (config *Config) createUtilsProcess(commandLine string) (process, error) { logrus.Debugf("opengcs: createUtilsProcess success: pid %d", proc.Process.Pid()) return proc, nil } + +// RunProcess runs the given command line program in the utilityVM. It takes in +// an input to the reader to feed into stdin and returns stdout to output. +func (config *Config) RunProcess(commandLine string, input io.Reader, output io.Writer) error { + logrus.Debugf("opengcs: RunProcess: %s", commandLine) + process, err := config.createUtilsProcess(commandLine) + if err != nil { + return err + } + defer process.Process.Close() + + // Send the data into the process's stdin + if input != nil { + if _, err = copyWithTimeout(process.Stdin, + input, + 0, + config.UvmTimeoutSeconds, + fmt.Sprintf("send to stdin of %s", commandLine)); err != nil { + return err + } + + // Don't need stdin now we've sent everything. This signals GCS that we are finished sending data. + if err := process.Process.CloseStdin(); err != nil { + return err + } + } + + if output != nil { + // Copy the data over to the writer. + if _, err := copyWithTimeout(output, + process.Stdout, + 0, + config.UvmTimeoutSeconds, + fmt.Sprintf("RunProcess: copy back from %s", commandLine)); err != nil { + return err + } + } + + logrus.Debugf("opengcs: runProcess success: %s", commandLine) + return nil +} diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/tartovhd.go b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/tartovhd.go index f6ce79ecb6..3560245b49 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/tartovhd.go +++ b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/tartovhd.go @@ -2,8 +2,6 @@ package client -// TODO @jhowardmsft - This will move to Microsoft/opengcs soon - import ( "fmt" "io" @@ -26,7 +24,7 @@ func (config *Config) TarToVhd(targetVHDFile string, reader io.Reader) (int64, e defer process.Process.Close() // Send the tarstream into the `tar2vhd`s stdin - if _, err = copyWithTimeout(process.Stdin, reader, 0, config.UvmTimeoutSeconds, fmt.Sprintf("send %s, to stdin of tar2vhd", targetVHDFile)); err != nil { + if _, err = copyWithTimeout(process.Stdin, reader, 0, config.UvmTimeoutSeconds, fmt.Sprintf("stdin of tar2vhd for generating %s", targetVHDFile)); err != nil { return 0, fmt.Errorf("opengcs: TarToVhd: %s: failed to send to tar2vhd in uvm: %s", targetVHDFile, err) } @@ -36,7 +34,7 @@ func (config *Config) TarToVhd(targetVHDFile string, reader io.Reader) (int64, e } // Write stdout contents of `tar2vhd` to the VHD file - payloadSize, err := writeFileFromReader(targetVHDFile, process.Stdout, config.UvmTimeoutSeconds, fmt.Sprintf("output of tar2vhd to %s", targetVHDFile)) + payloadSize, err := writeFileFromReader(targetVHDFile, process.Stdout, config.UvmTimeoutSeconds, fmt.Sprintf("stdout of tar2vhd to %s", targetVHDFile)) if err != nil { return 0, fmt.Errorf("opengcs: TarToVhd: %s: failed writing VHD file: %s", targetVHDFile, err) } diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/unsupported.go b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/unsupported.go index 4a7abab3b0..63b7067dce 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/unsupported.go +++ b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/unsupported.go @@ -1,5 +1,3 @@ // +build !windows package client - -// TODO @jhowardmsft - This will move to Microsoft/opengcs soon diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/utilities.go b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/utilities.go index 4ebd9bb156..8976dc0af9 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/utilities.go +++ b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/utilities.go @@ -2,8 +2,6 @@ package client -// TODO @jhowardmsft - This will move to Microsoft/opengcs soon - import ( "fmt" "io" @@ -72,10 +70,13 @@ func copyWithTimeout(dst io.Writer, src io.Reader, size int64, timeoutSeconds in return result.bytes, nil } -// copyFile is a utility for copying a file - used for the sandbox cache. +// CopyFile is a utility for copying a file - used for the sandbox cache. // Uses CopyFileW win32 API for performance -func copyFile(srcFile, destFile string) error { +func CopyFile(srcFile, destFile string, overwrite bool) error { var bFailIfExists uint32 = 1 + if overwrite { + bFailIfExists = 0 + } lpExistingFileName, err := syscall.UTF16PtrFromString(srcFile) if err != nil { @@ -92,8 +93,7 @@ func copyFile(srcFile, destFile string) error { uintptr(unsafe.Pointer(lpNewFileName)), uintptr(bFailIfExists)) if r1 == 0 { - return fmt.Errorf("failed CopyFileW Win32 call from '%s' to %s: %s", srcFile, destFile, err) + return fmt.Errorf("failed CopyFileW Win32 call from '%s' to '%s': %s", srcFile, destFile, err) } return nil - } diff --git a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/vhdtotar.go b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/vhdtotar.go index 9504fb7c84..bdbd381929 100644 --- a/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/vhdtotar.go +++ b/components/engine/vendor/github.com/jhowardmsft/opengcs/gogcs/client/vhdtotar.go @@ -2,8 +2,6 @@ package client -// TODO @jhowardmsft - This will move to Microsoft/opengcs soon - import ( "fmt" "io" From 23b4fdf3bc23eb2bfffc63bec16ebfb390f43455 Mon Sep 17 00:00:00 2001 From: Yassine TIJANI Date: Wed, 24 May 2017 17:22:09 +0200 Subject: [PATCH 33/56] removing TODO, and explain why we can't filter by id on the matchevent method. Signed-off-by: Yassine TIJANI Upstream-commit: be9aa09246534a1fdf7c08cf287dbe1e6de375c9 Component: engine --- .../cluster/executor/container/controller.go | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/components/engine/daemon/cluster/executor/container/controller.go b/components/engine/daemon/cluster/executor/container/controller.go index 8e95816138..29d55faf65 100644 --- a/components/engine/daemon/cluster/executor/container/controller.go +++ b/components/engine/daemon/cluster/executor/container/controller.go @@ -28,11 +28,10 @@ const defaultGossipConvergeDelay = 2 * time.Second // Most operations against docker's API are done through the container name, // which is unique to the task. type controller struct { - task *api.Task - adapter *containerAdapter - closed chan struct{} - err error - + task *api.Task + adapter *containerAdapter + closed chan struct{} + err error pulled chan struct{} // closed after pull cancelPull func() // cancels pull context if not nil pullErr error // pull error, only read after pulled closed @@ -146,7 +145,6 @@ func (r *controller) Prepare(ctx context.Context) error { } } } - if err := r.adapter.create(ctx); err != nil { if isContainerCreateNameConflict(err) { if _, err := r.adapter.inspect(ctx); err != nil { @@ -564,15 +562,8 @@ func (r *controller) matchevent(event events.Message) bool { if event.Type != events.ContainerEventType { return false } - - // TODO(stevvooe): Filter based on ID matching, in addition to name. - - // Make sure the events are for this container. - if event.Actor.Attributes["name"] != r.adapter.container.name() { - return false - } - - return true + // we can't filter using id since it will have huge chances to introduce a deadlock. see #33377. + return event.Actor.Attributes["name"] == r.adapter.container.name() } func (r *controller) checkClosed() error { From dd3dcdbd0db7fdcccb51a997db9cb4b2da43602e Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Fri, 7 Jul 2017 16:42:14 -0700 Subject: [PATCH 34/56] vendor: add go-semver Signed-off-by: Aaron Lehmann Upstream-commit: 733f672100a436b272ea871c4ba427f07d7bfa8a Component: engine --- components/engine/vendor.conf | 1 + .../github.com/coreos/go-semver/LICENSE | 202 +++++++++++++ .../github.com/coreos/go-semver/README.md | 28 ++ .../coreos/go-semver/semver/semver.go | 268 ++++++++++++++++++ .../coreos/go-semver/semver/sort.go | 38 +++ 5 files changed, 537 insertions(+) create mode 100644 components/engine/vendor/github.com/coreos/go-semver/LICENSE create mode 100644 components/engine/vendor/github.com/coreos/go-semver/README.md create mode 100644 components/engine/vendor/github.com/coreos/go-semver/semver/semver.go create mode 100644 components/engine/vendor/github.com/coreos/go-semver/semver/sort.go diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 4310990131..9f12d43417 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -43,6 +43,7 @@ github.com/BurntSushi/toml f706d00e3de6abe700c994cdd545a1a4915af060 github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374 github.com/deckarep/golang-set ef32fa3046d9f249d399f98ebaf9be944430fd1d github.com/coreos/etcd v3.2.1 +github.com/coreos/go-semver v0.2.0 github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065 github.com/hashicorp/consul v0.5.2 github.com/boltdb/bolt fff57c100f4dea1905678da7e90d92429dff2904 diff --git a/components/engine/vendor/github.com/coreos/go-semver/LICENSE b/components/engine/vendor/github.com/coreos/go-semver/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/components/engine/vendor/github.com/coreos/go-semver/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/components/engine/vendor/github.com/coreos/go-semver/README.md b/components/engine/vendor/github.com/coreos/go-semver/README.md new file mode 100644 index 0000000000..5bc9263cfb --- /dev/null +++ b/components/engine/vendor/github.com/coreos/go-semver/README.md @@ -0,0 +1,28 @@ +# go-semver - Semantic Versioning Library + +[![Build Status](https://travis-ci.org/coreos/go-semver.svg?branch=master)](https://travis-ci.org/coreos/go-semver) +[![GoDoc](https://godoc.org/github.com/coreos/go-semver/semver?status.svg)](https://godoc.org/github.com/coreos/go-semver/semver) + +go-semver is a [semantic versioning][semver] library for Go. It lets you parse +and compare two semantic version strings. + +[semver]: http://semver.org/ + +## Usage + +```go +vA := semver.New("1.2.3") +vB := semver.New("3.2.1") + +fmt.Printf("%s < %s == %t\n", vA, vB, vA.LessThan(*vB)) +``` + +## Example Application + +``` +$ go run example.go 1.2.3 3.2.1 +1.2.3 < 3.2.1 == true + +$ go run example.go 5.2.3 3.2.1 +5.2.3 < 3.2.1 == false +``` diff --git a/components/engine/vendor/github.com/coreos/go-semver/semver/semver.go b/components/engine/vendor/github.com/coreos/go-semver/semver/semver.go new file mode 100644 index 0000000000..110fc23e15 --- /dev/null +++ b/components/engine/vendor/github.com/coreos/go-semver/semver/semver.go @@ -0,0 +1,268 @@ +// Copyright 2013-2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Semantic Versions http://semver.org +package semver + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "strings" +) + +type Version struct { + Major int64 + Minor int64 + Patch int64 + PreRelease PreRelease + Metadata string +} + +type PreRelease string + +func splitOff(input *string, delim string) (val string) { + parts := strings.SplitN(*input, delim, 2) + + if len(parts) == 2 { + *input = parts[0] + val = parts[1] + } + + return val +} + +func New(version string) *Version { + return Must(NewVersion(version)) +} + +func NewVersion(version string) (*Version, error) { + v := Version{} + + if err := v.Set(version); err != nil { + return nil, err + } + + return &v, nil +} + +// Must is a helper for wrapping NewVersion and will panic if err is not nil. +func Must(v *Version, err error) *Version { + if err != nil { + panic(err) + } + return v +} + +// Set parses and updates v from the given version string. Implements flag.Value +func (v *Version) Set(version string) error { + metadata := splitOff(&version, "+") + preRelease := PreRelease(splitOff(&version, "-")) + dotParts := strings.SplitN(version, ".", 3) + + if len(dotParts) != 3 { + return fmt.Errorf("%s is not in dotted-tri format", version) + } + + parsed := make([]int64, 3, 3) + + for i, v := range dotParts[:3] { + val, err := strconv.ParseInt(v, 10, 64) + parsed[i] = val + if err != nil { + return err + } + } + + v.Metadata = metadata + v.PreRelease = preRelease + v.Major = parsed[0] + v.Minor = parsed[1] + v.Patch = parsed[2] + return nil +} + +func (v Version) String() string { + var buffer bytes.Buffer + + fmt.Fprintf(&buffer, "%d.%d.%d", v.Major, v.Minor, v.Patch) + + if v.PreRelease != "" { + fmt.Fprintf(&buffer, "-%s", v.PreRelease) + } + + if v.Metadata != "" { + fmt.Fprintf(&buffer, "+%s", v.Metadata) + } + + return buffer.String() +} + +func (v *Version) UnmarshalYAML(unmarshal func(interface{}) error) error { + var data string + if err := unmarshal(&data); err != nil { + return err + } + return v.Set(data) +} + +func (v Version) MarshalJSON() ([]byte, error) { + return []byte(`"` + v.String() + `"`), nil +} + +func (v *Version) UnmarshalJSON(data []byte) error { + l := len(data) + if l == 0 || string(data) == `""` { + return nil + } + if l < 2 || data[0] != '"' || data[l-1] != '"' { + return errors.New("invalid semver string") + } + return v.Set(string(data[1 : l-1])) +} + +// Compare tests if v is less than, equal to, or greater than versionB, +// returning -1, 0, or +1 respectively. +func (v Version) Compare(versionB Version) int { + if cmp := recursiveCompare(v.Slice(), versionB.Slice()); cmp != 0 { + return cmp + } + return preReleaseCompare(v, versionB) +} + +// Equal tests if v is equal to versionB. +func (v Version) Equal(versionB Version) bool { + return v.Compare(versionB) == 0 +} + +// LessThan tests if v is less than versionB. +func (v Version) LessThan(versionB Version) bool { + return v.Compare(versionB) < 0 +} + +// Slice converts the comparable parts of the semver into a slice of integers. +func (v Version) Slice() []int64 { + return []int64{v.Major, v.Minor, v.Patch} +} + +func (p PreRelease) Slice() []string { + preRelease := string(p) + return strings.Split(preRelease, ".") +} + +func preReleaseCompare(versionA Version, versionB Version) int { + a := versionA.PreRelease + b := versionB.PreRelease + + /* Handle the case where if two versions are otherwise equal it is the + * one without a PreRelease that is greater */ + if len(a) == 0 && (len(b) > 0) { + return 1 + } else if len(b) == 0 && (len(a) > 0) { + return -1 + } + + // If there is a prerelease, check and compare each part. + return recursivePreReleaseCompare(a.Slice(), b.Slice()) +} + +func recursiveCompare(versionA []int64, versionB []int64) int { + if len(versionA) == 0 { + return 0 + } + + a := versionA[0] + b := versionB[0] + + if a > b { + return 1 + } else if a < b { + return -1 + } + + return recursiveCompare(versionA[1:], versionB[1:]) +} + +func recursivePreReleaseCompare(versionA []string, versionB []string) int { + // A larger set of pre-release fields has a higher precedence than a smaller set, + // if all of the preceding identifiers are equal. + if len(versionA) == 0 { + if len(versionB) > 0 { + return -1 + } + return 0 + } else if len(versionB) == 0 { + // We're longer than versionB so return 1. + return 1 + } + + a := versionA[0] + b := versionB[0] + + aInt := false + bInt := false + + aI, err := strconv.Atoi(versionA[0]) + if err == nil { + aInt = true + } + + bI, err := strconv.Atoi(versionB[0]) + if err == nil { + bInt = true + } + + // Handle Integer Comparison + if aInt && bInt { + if aI > bI { + return 1 + } else if aI < bI { + return -1 + } + } + + // Handle String Comparison + if a > b { + return 1 + } else if a < b { + return -1 + } + + return recursivePreReleaseCompare(versionA[1:], versionB[1:]) +} + +// BumpMajor increments the Major field by 1 and resets all other fields to their default values +func (v *Version) BumpMajor() { + v.Major += 1 + v.Minor = 0 + v.Patch = 0 + v.PreRelease = PreRelease("") + v.Metadata = "" +} + +// BumpMinor increments the Minor field by 1 and resets all other fields to their default values +func (v *Version) BumpMinor() { + v.Minor += 1 + v.Patch = 0 + v.PreRelease = PreRelease("") + v.Metadata = "" +} + +// BumpPatch increments the Patch field by 1 and resets all other fields to their default values +func (v *Version) BumpPatch() { + v.Patch += 1 + v.PreRelease = PreRelease("") + v.Metadata = "" +} diff --git a/components/engine/vendor/github.com/coreos/go-semver/semver/sort.go b/components/engine/vendor/github.com/coreos/go-semver/semver/sort.go new file mode 100644 index 0000000000..e256b41a5d --- /dev/null +++ b/components/engine/vendor/github.com/coreos/go-semver/semver/sort.go @@ -0,0 +1,38 @@ +// Copyright 2013-2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package semver + +import ( + "sort" +) + +type Versions []*Version + +func (s Versions) Len() int { + return len(s) +} + +func (s Versions) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s Versions) Less(i, j int) bool { + return s[i].LessThan(*s[j]) +} + +// Sort sorts the given slice of Version +func Sort(versions []*Version) { + sort.Sort(Versions(versions)) +} From f4101eb2fb71e60aae17760b48ad90b4d7138346 Mon Sep 17 00:00:00 2001 From: allencloud Date: Thu, 6 Jul 2017 16:47:21 +0800 Subject: [PATCH 35/56] return prune data when context canceled Signed-off-by: allencloud Upstream-commit: 87b4dc2002d50e245f432bbd97e9dee2db25cf96 Component: engine --- .../api/server/backend/build/backend.go | 2 +- components/engine/builder/fscache/fscache.go | 13 ++++++++++--- .../engine/builder/fscache/fscache_test.go | 2 +- components/engine/daemon/prune.go | 19 +++++++++++-------- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/components/engine/api/server/backend/build/backend.go b/components/engine/api/server/backend/build/backend.go index b5a758c387..f93fba93b7 100644 --- a/components/engine/api/server/backend/build/backend.go +++ b/components/engine/api/server/backend/build/backend.go @@ -70,7 +70,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string // PruneCache removes all cached build sources func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) { - size, err := b.fsCache.Prune() + size, err := b.fsCache.Prune(ctx) if err != nil { return nil, errors.Wrap(err, "failed to prune build cache") } diff --git a/components/engine/builder/fscache/fscache.go b/components/engine/builder/fscache/fscache.go index 802db96de0..63331091a6 100644 --- a/components/engine/builder/fscache/fscache.go +++ b/components/engine/builder/fscache/fscache.go @@ -154,8 +154,8 @@ func (fsc *FSCache) DiskUsage() (int64, error) { } // Prune allows manually cleaning up the cache -func (fsc *FSCache) Prune() (uint64, error) { - return fsc.store.Prune() +func (fsc *FSCache) Prune(ctx context.Context) (uint64, error) { + return fsc.store.Prune(ctx) } // Close stops the gc and closes the persistent db @@ -396,12 +396,19 @@ func (s *fsCacheStore) DiskUsage() (int64, error) { } // Prune allows manually cleaning up the cache -func (s *fsCacheStore) Prune() (uint64, error) { +func (s *fsCacheStore) Prune(ctx context.Context) (uint64, error) { s.mu.Lock() defer s.mu.Unlock() var size uint64 for id, snap := range s.sources { + select { + case <-ctx.Done(): + logrus.Debugf("Cache prune operation cancelled, pruned size: %d", size) + // when the context is cancelled, only return current size and nil + return size, nil + default: + } if len(snap.refs) == 0 { ss, err := snap.getSize() if err != nil { diff --git a/components/engine/builder/fscache/fscache_test.go b/components/engine/builder/fscache/fscache_test.go index c7c0531f27..2532a218c8 100644 --- a/components/engine/builder/fscache/fscache_test.go +++ b/components/engine/builder/fscache/fscache_test.go @@ -97,7 +97,7 @@ func TestFSCache(t *testing.T) { assert.Equal(t, s, int64(8)) // prune deletes everything - released, err := fscache.Prune() + released, err := fscache.Prune(context.TODO()) assert.Nil(t, err) assert.Equal(t, released, uint64(8)) diff --git a/components/engine/daemon/prune.go b/components/engine/daemon/prune.go index 1d8686b36f..9f8a545e7c 100644 --- a/components/engine/daemon/prune.go +++ b/components/engine/daemon/prune.go @@ -74,8 +74,8 @@ func (daemon *Daemon) ContainersPrune(ctx context.Context, pruneFilters filters. for _, c := range allContainers { select { case <-ctx.Done(): - logrus.Warnf("ContainersPrune operation cancelled: %#v", *rep) - return rep, ctx.Err() + logrus.Debugf("ContainersPrune operation cancelled: %#v", *rep) + return rep, nil default: } @@ -121,7 +121,7 @@ func (daemon *Daemon) VolumesPrune(ctx context.Context, pruneFilters filters.Arg pruneVols := func(v volume.Volume) error { select { case <-ctx.Done(): - logrus.Warnf("VolumesPrune operation cancelled: %#v", *rep) + logrus.Debugf("VolumesPrune operation cancelled: %#v", *rep) return ctx.Err() default: } @@ -153,6 +153,9 @@ func (daemon *Daemon) VolumesPrune(ctx context.Context, pruneFilters filters.Arg } err = daemon.traverseLocalVolumes(pruneVols) + if err == context.Canceled { + return rep, nil + } return rep, err } @@ -303,8 +306,7 @@ deleteImagesLoop: } if canceled { - logrus.Warnf("ImagesPrune operation cancelled: %#v", *rep) - return nil, ctx.Err() + logrus.Debugf("ImagesPrune operation cancelled: %#v", *rep) } return rep, nil @@ -320,6 +322,7 @@ func (daemon *Daemon) localNetworksPrune(ctx context.Context, pruneFilters filte l := func(nw libnetwork.Network) bool { select { case <-ctx.Done(): + // context cancelled return true default: } @@ -370,7 +373,7 @@ func (daemon *Daemon) clusterNetworksPrune(ctx context.Context, pruneFilters fil for _, nw := range networks { select { case <-ctx.Done(): - return rep, ctx.Err() + return rep, nil default: if nw.Ingress { // Routing-mesh network removal has to be explicitly invoked by user @@ -427,8 +430,8 @@ func (daemon *Daemon) NetworksPrune(ctx context.Context, pruneFilters filters.Ar select { case <-ctx.Done(): - logrus.Warnf("NetworksPrune operation cancelled: %#v", *rep) - return nil, ctx.Err() + logrus.Debugf("NetworksPrune operation cancelled: %#v", *rep) + return rep, nil default: } From 9e646d838643b47cec02f7785804af56b20c79e8 Mon Sep 17 00:00:00 2001 From: Yuanhong Peng Date: Fri, 7 Jul 2017 15:33:45 +0800 Subject: [PATCH 36/56] Return an empty stats if "container not found" If we get "container not found" error from containerd, it's possibly because that this container has already been stopped. It will be ok to ignore this error and just return an empty stats. Signed-off-by: Yuanhong Peng Upstream-commit: 4a6cbf9bcb78d38c48ef963f585f0fadf733e101 Component: engine --- components/engine/daemon/daemon_unix.go | 3 +++ components/engine/daemon/daemon_windows.go | 3 +++ components/engine/daemon/errors.go | 12 +++++++++ components/engine/daemon/stats/collector.go | 30 ++++++++++++--------- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/components/engine/daemon/daemon_unix.go b/components/engine/daemon/daemon_unix.go index a7ba9c37b2..ff9eadf75b 100644 --- a/components/engine/daemon/daemon_unix.go +++ b/components/engine/daemon/daemon_unix.go @@ -1165,6 +1165,9 @@ func (daemon *Daemon) stats(c *container.Container) (*types.StatsJSON, error) { } stats, err := daemon.containerd.Stats(c.ID) if err != nil { + if strings.Contains(err.Error(), "container not found") { + return nil, errNotFound{c.ID} + } return nil, err } s := &types.StatsJSON{} diff --git a/components/engine/daemon/daemon_windows.go b/components/engine/daemon/daemon_windows.go index bb9c85b80e..798ba39971 100644 --- a/components/engine/daemon/daemon_windows.go +++ b/components/engine/daemon/daemon_windows.go @@ -525,6 +525,9 @@ func (daemon *Daemon) stats(c *container.Container) (*types.StatsJSON, error) { // Obtain the stats from HCS via libcontainerd stats, err := daemon.containerd.Stats(c.ID) if err != nil { + if strings.Contains(err.Error(), "container not found") { + return nil, errNotFound{c.ID} + } return nil, err } diff --git a/components/engine/daemon/errors.go b/components/engine/daemon/errors.go index 96c30df2ea..5050f87e4c 100644 --- a/components/engine/daemon/errors.go +++ b/components/engine/daemon/errors.go @@ -39,3 +39,15 @@ func errExecPaused(id string) error { err := fmt.Errorf("Container %s is paused, unpause the container before exec", id) return errors.NewRequestConflictError(err) } + +type errNotFound struct { + containerID string +} + +func (e errNotFound) Error() string { + return fmt.Sprintf("Container %s is not found", e.containerID) +} + +func (e errNotFound) ContainerNotFound() bool { + return true +} diff --git a/components/engine/daemon/stats/collector.go b/components/engine/daemon/stats/collector.go index 71bcff446e..0520efa238 100644 --- a/components/engine/daemon/stats/collector.go +++ b/components/engine/daemon/stats/collector.go @@ -88,24 +88,25 @@ func (s *Collector) Run() { for _, pair := range pairs { stats, err := s.supervisor.GetContainerStats(pair.container) - if err != nil { - if _, ok := err.(notRunningErr); !ok { - logrus.Errorf("collecting stats for %s: %v", pair.container.ID, err) - continue - } - // publish empty stats containing only name and ID if not running + switch err.(type) { + case nil: + // FIXME: move to containerd on Linux (not Windows) + stats.CPUStats.SystemUsage = systemUsage + stats.CPUStats.OnlineCPUs = onlineCPUs + + pair.publisher.Publish(*stats) + + case notRunningErr, notFoundErr: + // publish empty stats containing only name and ID if not running or not found pair.publisher.Publish(types.StatsJSON{ Name: pair.container.Name, ID: pair.container.ID, }) - continue - } - // FIXME: move to containerd on Linux (not Windows) - stats.CPUStats.SystemUsage = systemUsage - stats.CPUStats.OnlineCPUs = onlineCPUs - pair.publisher.Publish(*stats) + default: + logrus.Errorf("collecting stats for %s: %v", pair.container.ID, err) + } } } } @@ -114,3 +115,8 @@ type notRunningErr interface { error ContainerIsRunning() bool } + +type notFoundErr interface { + error + ContainerNotFound() bool +} From b8ebf7be046401911c9138ed35c8b8209c91225b Mon Sep 17 00:00:00 2001 From: Justin Cormack Date: Mon, 10 Jul 2017 14:10:43 +0100 Subject: [PATCH 37/56] Split homedir files by operating system libcontainer/user does not build at all on Windows any more, and this was breaking the client on Windows with upstream `runc`. As these functions are not used anyway, just split out and stop checking `runtime`. Signed-off-by: Justin Cormack Upstream-commit: b7bd959294b5dc5b6e18b3fed5dde18f4f7f1d20 Component: engine --- .../homedir/{homedir.go => homedir_unix.go} | 11 +++------ .../engine/pkg/homedir/homedir_windows.go | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 8 deletions(-) rename components/engine/pkg/homedir/{homedir.go => homedir_unix.go} (77%) create mode 100644 components/engine/pkg/homedir/homedir_windows.go diff --git a/components/engine/pkg/homedir/homedir.go b/components/engine/pkg/homedir/homedir_unix.go similarity index 77% rename from components/engine/pkg/homedir/homedir.go rename to components/engine/pkg/homedir/homedir_unix.go index 8154e83f0c..f2a20ea8f8 100644 --- a/components/engine/pkg/homedir/homedir.go +++ b/components/engine/pkg/homedir/homedir_unix.go @@ -1,8 +1,9 @@ +// +build !windows + package homedir import ( "os" - "runtime" "github.com/opencontainers/runc/libcontainer/user" ) @@ -10,9 +11,6 @@ import ( // Key returns the env var name for the user's home dir based on // the platform being run on func Key() string { - if runtime.GOOS == "windows" { - return "USERPROFILE" - } return "HOME" } @@ -21,7 +19,7 @@ func Key() string { // Returned path should be used with "path/filepath" to form new paths. func Get() string { home := os.Getenv(Key()) - if home == "" && runtime.GOOS != "windows" { + if home == "" { if u, err := user.CurrentUser(); err == nil { return u.Home } @@ -32,8 +30,5 @@ func Get() string { // GetShortcutString returns the string that is shortcut to user's home directory // in the native shell of the platform running on. func GetShortcutString() string { - if runtime.GOOS == "windows" { - return "%USERPROFILE%" // be careful while using in format functions - } return "~" } diff --git a/components/engine/pkg/homedir/homedir_windows.go b/components/engine/pkg/homedir/homedir_windows.go new file mode 100644 index 0000000000..fafdb2bbf9 --- /dev/null +++ b/components/engine/pkg/homedir/homedir_windows.go @@ -0,0 +1,24 @@ +package homedir + +import ( + "os" +) + +// Key returns the env var name for the user's home dir based on +// the platform being run on +func Key() string { + return "USERPROFILE" +} + +// Get returns the home directory of the current user with the help of +// environment variables depending on the target operating system. +// Returned path should be used with "path/filepath" to form new paths. +func Get() string { + return os.Getenv(Key()) +} + +// GetShortcutString returns the string that is shortcut to user's home directory +// in the native shell of the platform running on. +func GetShortcutString() string { + return "%USERPROFILE%" // be careful while using in format functions +} From 30a181615aaae1293e82c99a42049976338d5d7c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 10 Jul 2017 14:05:13 -0700 Subject: [PATCH 38/56] Bump vndr to 9909bb2b8a0b7ea464527b376dc50389c90df587 This bumps vndr to 9909bb2b8a0b7ea464527b376dc50389c90df587 and revendors dependencies. Includes a change that prunes go files with `+build ignore` Signed-off-by: Sebastiaan van Stijn Upstream-commit: 63f4bc5237dc928125ac62aa5468f6d8b28272c7 Component: engine --- .../engine/hack/dockerfile/binaries-commits | 2 +- .../Microsoft/hcsshim/mksyscall_windows.go | 934 --- .../engine/vendor/github.com/kr/pty/types.go | 10 - .../vendor/github.com/kr/pty/types_freebsd.go | 15 - .../github.com/miekg/dns/types_generate.go | 266 - .../vendor/golang.org/x/sys/unix/mkpost.go | 62 - .../golang.org/x/sys/unix/types_darwin.go | 250 - .../golang.org/x/sys/unix/types_dragonfly.go | 242 - .../golang.org/x/sys/unix/types_freebsd.go | 353 - .../golang.org/x/sys/unix/types_linux.go | 450 -- .../golang.org/x/sys/unix/types_netbsd.go | 232 - .../golang.org/x/sys/unix/types_openbsd.go | 244 - .../golang.org/x/sys/unix/types_solaris.go | 262 - .../golang.org/x/text/internal/gen/code.go | 351 - .../golang.org/x/text/internal/gen/gen.go | 281 - .../x/text/internal/triegen/compact.go | 58 - .../x/text/internal/triegen/print.go | 251 - .../x/text/internal/triegen/triegen.go | 494 -- .../golang.org/x/text/internal/ucd/ucd.go | 376 -- .../golang.org/x/text/unicode/bidi/gen.go | 133 - .../x/text/unicode/bidi/gen_ranges.go | 57 - .../x/text/unicode/bidi/gen_trieval.go | 64 - .../golang.org/x/text/unicode/cldr/base.go | 100 - .../golang.org/x/text/unicode/cldr/cldr.go | 130 - .../golang.org/x/text/unicode/cldr/collate.go | 359 -- .../golang.org/x/text/unicode/cldr/decode.go | 171 - .../golang.org/x/text/unicode/cldr/makexml.go | 400 -- .../golang.org/x/text/unicode/cldr/resolve.go | 602 -- .../golang.org/x/text/unicode/cldr/slice.go | 144 - .../golang.org/x/text/unicode/cldr/xml.go | 1456 ----- .../x/text/unicode/norm/maketables.go | 978 --- .../golang.org/x/text/unicode/norm/triegen.go | 117 - .../x/text/unicode/rangetable/gen.go | 113 - .../x/text/unicode/rangetable/merge.go | 260 - .../x/text/unicode/rangetable/rangetable.go | 70 - .../x/text/unicode/rangetable/tables.go | 5735 ----------------- 36 files changed, 1 insertion(+), 16021 deletions(-) delete mode 100644 components/engine/vendor/github.com/Microsoft/hcsshim/mksyscall_windows.go delete mode 100644 components/engine/vendor/github.com/kr/pty/types.go delete mode 100644 components/engine/vendor/github.com/kr/pty/types_freebsd.go delete mode 100644 components/engine/vendor/github.com/miekg/dns/types_generate.go delete mode 100644 components/engine/vendor/golang.org/x/sys/unix/mkpost.go delete mode 100644 components/engine/vendor/golang.org/x/sys/unix/types_darwin.go delete mode 100644 components/engine/vendor/golang.org/x/sys/unix/types_dragonfly.go delete mode 100644 components/engine/vendor/golang.org/x/sys/unix/types_freebsd.go delete mode 100644 components/engine/vendor/golang.org/x/sys/unix/types_linux.go delete mode 100644 components/engine/vendor/golang.org/x/sys/unix/types_netbsd.go delete mode 100644 components/engine/vendor/golang.org/x/sys/unix/types_openbsd.go delete mode 100644 components/engine/vendor/golang.org/x/sys/unix/types_solaris.go delete mode 100644 components/engine/vendor/golang.org/x/text/internal/gen/code.go delete mode 100644 components/engine/vendor/golang.org/x/text/internal/gen/gen.go delete mode 100644 components/engine/vendor/golang.org/x/text/internal/triegen/compact.go delete mode 100644 components/engine/vendor/golang.org/x/text/internal/triegen/print.go delete mode 100644 components/engine/vendor/golang.org/x/text/internal/triegen/triegen.go delete mode 100644 components/engine/vendor/golang.org/x/text/internal/ucd/ucd.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/bidi/gen.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/cldr/base.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/cldr/cldr.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/cldr/collate.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/cldr/decode.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/cldr/makexml.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/cldr/resolve.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/cldr/slice.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/cldr/xml.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/norm/maketables.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/norm/triegen.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/rangetable/gen.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/rangetable/merge.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/rangetable/rangetable.go delete mode 100644 components/engine/vendor/golang.org/x/text/unicode/rangetable/tables.go diff --git a/components/engine/hack/dockerfile/binaries-commits b/components/engine/hack/dockerfile/binaries-commits index 98344bc2fa..84c5c0faae 100644 --- a/components/engine/hack/dockerfile/binaries-commits +++ b/components/engine/hack/dockerfile/binaries-commits @@ -7,7 +7,7 @@ RUNC_COMMIT=2d41c047c83e09a6d61d464906feb2a2f3c52aa4 CONTAINERD_COMMIT=3addd840653146c90a254301d6c3a663c7fd6429 TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574 LIBNETWORK_COMMIT=7b2b1feb1de4817d522cc372af149ff48d25028e -VNDR_COMMIT=c56e082291115e369f77601f9c071dd0b87c7120 +VNDR_COMMIT=9909bb2b8a0b7ea464527b376dc50389c90df587 # CLI DOCKERCLI_REPO=https://github.com/docker/cli diff --git a/components/engine/vendor/github.com/Microsoft/hcsshim/mksyscall_windows.go b/components/engine/vendor/github.com/Microsoft/hcsshim/mksyscall_windows.go deleted file mode 100644 index 82393ca367..0000000000 --- a/components/engine/vendor/github.com/Microsoft/hcsshim/mksyscall_windows.go +++ /dev/null @@ -1,934 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -/* -mksyscall_windows generates windows system call bodies - -It parses all files specified on command line containing function -prototypes (like syscall_windows.go) and prints system call bodies -to standard output. - -The prototypes are marked by lines beginning with "//sys" and read -like func declarations if //sys is replaced by func, but: - -* The parameter lists must give a name for each argument. This - includes return parameters. - -* The parameter lists must give a type for each argument: - the (x, y, z int) shorthand is not allowed. - -* If the return parameter is an error number, it must be named err. - -* If go func name needs to be different from it's winapi dll name, - the winapi name could be specified at the end, after "=" sign, like - //sys LoadLibrary(libname string) (handle uint32, err error) = LoadLibraryA - -* Each function that returns err needs to supply a condition, that - return value of winapi will be tested against to detect failure. - This would set err to windows "last-error", otherwise it will be nil. - The value can be provided at end of //sys declaration, like - //sys LoadLibrary(libname string) (handle uint32, err error) [failretval==-1] = LoadLibraryA - and is [failretval==0] by default. - -Usage: - mksyscall_windows [flags] [path ...] - -The flags are: - -output - Specify output file name (outputs to console if blank). - -trace - Generate print statement after every syscall. -*/ -package main - -import ( - "bufio" - "bytes" - "errors" - "flag" - "fmt" - "go/format" - "go/parser" - "go/token" - "io" - "io/ioutil" - "log" - "os" - "path/filepath" - "runtime" - "sort" - "strconv" - "strings" - "text/template" -) - -var ( - filename = flag.String("output", "", "output file name (standard output if omitted)") - printTraceFlag = flag.Bool("trace", false, "generate print statement after every syscall") - systemDLL = flag.Bool("systemdll", true, "whether all DLLs should be loaded from the Windows system directory") -) - -func trim(s string) string { - return strings.Trim(s, " \t") -} - -var packageName string - -func packagename() string { - return packageName -} - -func syscalldot() string { - if packageName == "syscall" { - return "" - } - return "syscall." -} - -// Param is function parameter -type Param struct { - Name string - Type string - fn *Fn - tmpVarIdx int -} - -// tmpVar returns temp variable name that will be used to represent p during syscall. -func (p *Param) tmpVar() string { - if p.tmpVarIdx < 0 { - p.tmpVarIdx = p.fn.curTmpVarIdx - p.fn.curTmpVarIdx++ - } - return fmt.Sprintf("_p%d", p.tmpVarIdx) -} - -// BoolTmpVarCode returns source code for bool temp variable. -func (p *Param) BoolTmpVarCode() string { - const code = `var %s uint32 - if %s { - %s = 1 - } else { - %s = 0 - }` - tmp := p.tmpVar() - return fmt.Sprintf(code, tmp, p.Name, tmp, tmp) -} - -// SliceTmpVarCode returns source code for slice temp variable. -func (p *Param) SliceTmpVarCode() string { - const code = `var %s *%s - if len(%s) > 0 { - %s = &%s[0] - }` - tmp := p.tmpVar() - return fmt.Sprintf(code, tmp, p.Type[2:], p.Name, tmp, p.Name) -} - -// StringTmpVarCode returns source code for string temp variable. -func (p *Param) StringTmpVarCode() string { - errvar := p.fn.Rets.ErrorVarName() - if errvar == "" { - errvar = "_" - } - tmp := p.tmpVar() - const code = `var %s %s - %s, %s = %s(%s)` - s := fmt.Sprintf(code, tmp, p.fn.StrconvType(), tmp, errvar, p.fn.StrconvFunc(), p.Name) - if errvar == "-" { - return s - } - const morecode = ` - if %s != nil { - return - }` - return s + fmt.Sprintf(morecode, errvar) -} - -// TmpVarCode returns source code for temp variable. -func (p *Param) TmpVarCode() string { - switch { - case p.Type == "bool": - return p.BoolTmpVarCode() - case strings.HasPrefix(p.Type, "[]"): - return p.SliceTmpVarCode() - default: - return "" - } -} - -// TmpVarHelperCode returns source code for helper's temp variable. -func (p *Param) TmpVarHelperCode() string { - if p.Type != "string" { - return "" - } - return p.StringTmpVarCode() -} - -// SyscallArgList returns source code fragments representing p parameter -// in syscall. Slices are translated into 2 syscall parameters: pointer to -// the first element and length. -func (p *Param) SyscallArgList() []string { - t := p.HelperType() - var s string - switch { - case t[0] == '*': - s = fmt.Sprintf("unsafe.Pointer(%s)", p.Name) - case t == "bool": - s = p.tmpVar() - case strings.HasPrefix(t, "[]"): - return []string{ - fmt.Sprintf("uintptr(unsafe.Pointer(%s))", p.tmpVar()), - fmt.Sprintf("uintptr(len(%s))", p.Name), - } - default: - s = p.Name - } - return []string{fmt.Sprintf("uintptr(%s)", s)} -} - -// IsError determines if p parameter is used to return error. -func (p *Param) IsError() bool { - return p.Name == "err" && p.Type == "error" -} - -// HelperType returns type of parameter p used in helper function. -func (p *Param) HelperType() string { - if p.Type == "string" { - return p.fn.StrconvType() - } - return p.Type -} - -// join concatenates parameters ps into a string with sep separator. -// Each parameter is converted into string by applying fn to it -// before conversion. -func join(ps []*Param, fn func(*Param) string, sep string) string { - if len(ps) == 0 { - return "" - } - a := make([]string, 0) - for _, p := range ps { - a = append(a, fn(p)) - } - return strings.Join(a, sep) -} - -// Rets describes function return parameters. -type Rets struct { - Name string - Type string - ReturnsError bool - FailCond string -} - -// ErrorVarName returns error variable name for r. -func (r *Rets) ErrorVarName() string { - if r.ReturnsError { - return "err" - } - if r.Type == "error" { - return r.Name - } - return "" -} - -// ToParams converts r into slice of *Param. -func (r *Rets) ToParams() []*Param { - ps := make([]*Param, 0) - if len(r.Name) > 0 { - ps = append(ps, &Param{Name: r.Name, Type: r.Type}) - } - if r.ReturnsError { - ps = append(ps, &Param{Name: "err", Type: "error"}) - } - return ps -} - -// List returns source code of syscall return parameters. -func (r *Rets) List() string { - s := join(r.ToParams(), func(p *Param) string { return p.Name + " " + p.Type }, ", ") - if len(s) > 0 { - s = "(" + s + ")" - } - return s -} - -// PrintList returns source code of trace printing part correspondent -// to syscall return values. -func (r *Rets) PrintList() string { - return join(r.ToParams(), func(p *Param) string { return fmt.Sprintf(`"%s=", %s, `, p.Name, p.Name) }, `", ", `) -} - -// SetReturnValuesCode returns source code that accepts syscall return values. -func (r *Rets) SetReturnValuesCode() string { - if r.Name == "" && !r.ReturnsError { - return "" - } - retvar := "r0" - if r.Name == "" { - retvar = "r1" - } - errvar := "_" - if r.ReturnsError { - errvar = "e1" - } - return fmt.Sprintf("%s, _, %s := ", retvar, errvar) -} - -func (r *Rets) useLongHandleErrorCode(retvar string) string { - const code = `if %s { - if e1 != 0 { - err = errnoErr(e1) - } else { - err = %sEINVAL - } - }` - cond := retvar + " == 0" - if r.FailCond != "" { - cond = strings.Replace(r.FailCond, "failretval", retvar, 1) - } - return fmt.Sprintf(code, cond, syscalldot()) -} - -// SetErrorCode returns source code that sets return parameters. -func (r *Rets) SetErrorCode() string { - const code = `if r0 != 0 { - %s = %sErrno(r0) - }` - const hrCode = `if int32(r0) < 0 { - %s = %sErrno(win32FromHresult(r0)) - }` - if r.Name == "" && !r.ReturnsError { - return "" - } - if r.Name == "" { - return r.useLongHandleErrorCode("r1") - } - if r.Type == "error" { - if r.Name == "hr" { - return fmt.Sprintf(hrCode, r.Name, syscalldot()) - } else { - return fmt.Sprintf(code, r.Name, syscalldot()) - } - } - s := "" - switch { - case r.Type[0] == '*': - s = fmt.Sprintf("%s = (%s)(unsafe.Pointer(r0))", r.Name, r.Type) - case r.Type == "bool": - s = fmt.Sprintf("%s = r0 != 0", r.Name) - default: - s = fmt.Sprintf("%s = %s(r0)", r.Name, r.Type) - } - if !r.ReturnsError { - return s - } - return s + "\n\t" + r.useLongHandleErrorCode(r.Name) -} - -// Fn describes syscall function. -type Fn struct { - Name string - Params []*Param - Rets *Rets - PrintTrace bool - confirmproc bool - dllname string - dllfuncname string - src string - // TODO: get rid of this field and just use parameter index instead - curTmpVarIdx int // insure tmp variables have uniq names -} - -// extractParams parses s to extract function parameters. -func extractParams(s string, f *Fn) ([]*Param, error) { - s = trim(s) - if s == "" { - return nil, nil - } - a := strings.Split(s, ",") - ps := make([]*Param, len(a)) - for i := range ps { - s2 := trim(a[i]) - b := strings.Split(s2, " ") - if len(b) != 2 { - b = strings.Split(s2, "\t") - if len(b) != 2 { - return nil, errors.New("Could not extract function parameter from \"" + s2 + "\"") - } - } - ps[i] = &Param{ - Name: trim(b[0]), - Type: trim(b[1]), - fn: f, - tmpVarIdx: -1, - } - } - return ps, nil -} - -// extractSection extracts text out of string s starting after start -// and ending just before end. found return value will indicate success, -// and prefix, body and suffix will contain correspondent parts of string s. -func extractSection(s string, start, end rune) (prefix, body, suffix string, found bool) { - s = trim(s) - if strings.HasPrefix(s, string(start)) { - // no prefix - body = s[1:] - } else { - a := strings.SplitN(s, string(start), 2) - if len(a) != 2 { - return "", "", s, false - } - prefix = a[0] - body = a[1] - } - a := strings.SplitN(body, string(end), 2) - if len(a) != 2 { - return "", "", "", false - } - return prefix, a[0], a[1], true -} - -// newFn parses string s and return created function Fn. -func newFn(s string) (*Fn, error) { - s = trim(s) - f := &Fn{ - Rets: &Rets{}, - src: s, - PrintTrace: *printTraceFlag, - } - // function name and args - prefix, body, s, found := extractSection(s, '(', ')') - if !found || prefix == "" { - return nil, errors.New("Could not extract function name and parameters from \"" + f.src + "\"") - } - f.Name = prefix - var err error - f.Params, err = extractParams(body, f) - if err != nil { - return nil, err - } - // return values - _, body, s, found = extractSection(s, '(', ')') - if found { - r, err := extractParams(body, f) - if err != nil { - return nil, err - } - switch len(r) { - case 0: - case 1: - if r[0].IsError() { - f.Rets.ReturnsError = true - } else { - f.Rets.Name = r[0].Name - f.Rets.Type = r[0].Type - } - case 2: - if !r[1].IsError() { - return nil, errors.New("Only last windows error is allowed as second return value in \"" + f.src + "\"") - } - f.Rets.ReturnsError = true - f.Rets.Name = r[0].Name - f.Rets.Type = r[0].Type - default: - return nil, errors.New("Too many return values in \"" + f.src + "\"") - } - } - // fail condition - _, body, s, found = extractSection(s, '[', ']') - if found { - f.Rets.FailCond = body - } - // dll and dll function names - s = trim(s) - if s == "" { - return f, nil - } - if !strings.HasPrefix(s, "=") { - return nil, errors.New("Could not extract dll name from \"" + f.src + "\"") - } - s = trim(s[1:]) - a := strings.Split(s, ".") - switch len(a) { - case 1: - f.dllfuncname = a[0] - case 2: - f.dllname = a[0] - f.dllfuncname = a[1] - default: - return nil, errors.New("Could not extract dll name from \"" + f.src + "\"") - } - if f.dllfuncname[len(f.dllfuncname)-1] == '?' { - f.confirmproc = true - f.dllfuncname = f.dllfuncname[0 : len(f.dllfuncname)-1] - } - return f, nil -} - -// DLLName returns DLL name for function f. -func (f *Fn) DLLName() string { - if f.dllname == "" { - return "kernel32" - } - return f.dllname -} - -// DLLName returns DLL function name for function f. -func (f *Fn) DLLFuncName() string { - if f.dllfuncname == "" { - return f.Name - } - return f.dllfuncname -} - -func (f *Fn) ConfirmProc() bool { - return f.confirmproc -} - -// ParamList returns source code for function f parameters. -func (f *Fn) ParamList() string { - return join(f.Params, func(p *Param) string { return p.Name + " " + p.Type }, ", ") -} - -// HelperParamList returns source code for helper function f parameters. -func (f *Fn) HelperParamList() string { - return join(f.Params, func(p *Param) string { return p.Name + " " + p.HelperType() }, ", ") -} - -// ParamPrintList returns source code of trace printing part correspondent -// to syscall input parameters. -func (f *Fn) ParamPrintList() string { - return join(f.Params, func(p *Param) string { return fmt.Sprintf(`"%s=", %s, `, p.Name, p.Name) }, `", ", `) -} - -// ParamCount return number of syscall parameters for function f. -func (f *Fn) ParamCount() int { - n := 0 - for _, p := range f.Params { - n += len(p.SyscallArgList()) - } - return n -} - -// SyscallParamCount determines which version of Syscall/Syscall6/Syscall9/... -// to use. It returns parameter count for correspondent SyscallX function. -func (f *Fn) SyscallParamCount() int { - n := f.ParamCount() - switch { - case n <= 3: - return 3 - case n <= 6: - return 6 - case n <= 9: - return 9 - case n <= 12: - return 12 - case n <= 15: - return 15 - default: - panic("too many arguments to system call") - } -} - -// Syscall determines which SyscallX function to use for function f. -func (f *Fn) Syscall() string { - c := f.SyscallParamCount() - if c == 3 { - return syscalldot() + "Syscall" - } - return syscalldot() + "Syscall" + strconv.Itoa(c) -} - -// SyscallParamList returns source code for SyscallX parameters for function f. -func (f *Fn) SyscallParamList() string { - a := make([]string, 0) - for _, p := range f.Params { - a = append(a, p.SyscallArgList()...) - } - for len(a) < f.SyscallParamCount() { - a = append(a, "0") - } - return strings.Join(a, ", ") -} - -// HelperCallParamList returns source code of call into function f helper. -func (f *Fn) HelperCallParamList() string { - a := make([]string, 0, len(f.Params)) - for _, p := range f.Params { - s := p.Name - if p.Type == "string" { - s = p.tmpVar() - } - a = append(a, s) - } - return strings.Join(a, ", ") -} - -// IsUTF16 is true, if f is W (utf16) function. It is false -// for all A (ascii) functions. -func (_ *Fn) IsUTF16() bool { - return true -} - -// StrconvFunc returns name of Go string to OS string function for f. -func (f *Fn) StrconvFunc() string { - if f.IsUTF16() { - return syscalldot() + "UTF16PtrFromString" - } - return syscalldot() + "BytePtrFromString" -} - -// StrconvType returns Go type name used for OS string for f. -func (f *Fn) StrconvType() string { - if f.IsUTF16() { - return "*uint16" - } - return "*byte" -} - -// HasStringParam is true, if f has at least one string parameter. -// Otherwise it is false. -func (f *Fn) HasStringParam() bool { - for _, p := range f.Params { - if p.Type == "string" { - return true - } - } - return false -} - -var uniqDllFuncName = make(map[string]bool) - -// IsNotDuplicate is true if f is not a duplicated function -func (f *Fn) IsNotDuplicate() bool { - funcName := f.DLLFuncName() - if uniqDllFuncName[funcName] == false { - uniqDllFuncName[funcName] = true - return true - } - return false -} - -// HelperName returns name of function f helper. -func (f *Fn) HelperName() string { - if !f.HasStringParam() { - return f.Name - } - return "_" + f.Name -} - -// Source files and functions. -type Source struct { - Funcs []*Fn - Files []string - StdLibImports []string - ExternalImports []string -} - -func (src *Source) Import(pkg string) { - src.StdLibImports = append(src.StdLibImports, pkg) - sort.Strings(src.StdLibImports) -} - -func (src *Source) ExternalImport(pkg string) { - src.ExternalImports = append(src.ExternalImports, pkg) - sort.Strings(src.ExternalImports) -} - -// ParseFiles parses files listed in fs and extracts all syscall -// functions listed in sys comments. It returns source files -// and functions collection *Source if successful. -func ParseFiles(fs []string) (*Source, error) { - src := &Source{ - Funcs: make([]*Fn, 0), - Files: make([]string, 0), - StdLibImports: []string{ - "unsafe", - }, - ExternalImports: make([]string, 0), - } - for _, file := range fs { - if err := src.ParseFile(file); err != nil { - return nil, err - } - } - return src, nil -} - -// DLLs return dll names for a source set src. -func (src *Source) DLLs() []string { - uniq := make(map[string]bool) - r := make([]string, 0) - for _, f := range src.Funcs { - name := f.DLLName() - if _, found := uniq[name]; !found { - uniq[name] = true - r = append(r, name) - } - } - return r -} - -// ParseFile adds additional file path to a source set src. -func (src *Source) ParseFile(path string) error { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - s := bufio.NewScanner(file) - for s.Scan() { - t := trim(s.Text()) - if len(t) < 7 { - continue - } - if !strings.HasPrefix(t, "//sys") { - continue - } - t = t[5:] - if !(t[0] == ' ' || t[0] == '\t') { - continue - } - f, err := newFn(t[1:]) - if err != nil { - return err - } - src.Funcs = append(src.Funcs, f) - } - if err := s.Err(); err != nil { - return err - } - src.Files = append(src.Files, path) - - // get package name - fset := token.NewFileSet() - _, err = file.Seek(0, 0) - if err != nil { - return err - } - pkg, err := parser.ParseFile(fset, "", file, parser.PackageClauseOnly) - if err != nil { - return err - } - packageName = pkg.Name.Name - - return nil -} - -// IsStdRepo returns true if src is part of standard library. -func (src *Source) IsStdRepo() (bool, error) { - if len(src.Files) == 0 { - return false, errors.New("no input files provided") - } - abspath, err := filepath.Abs(src.Files[0]) - if err != nil { - return false, err - } - goroot := runtime.GOROOT() - if runtime.GOOS == "windows" { - abspath = strings.ToLower(abspath) - goroot = strings.ToLower(goroot) - } - sep := string(os.PathSeparator) - if !strings.HasSuffix(goroot, sep) { - goroot += sep - } - return strings.HasPrefix(abspath, goroot), nil -} - -// Generate output source file from a source set src. -func (src *Source) Generate(w io.Writer) error { - const ( - pkgStd = iota // any package in std library - pkgXSysWindows // x/sys/windows package - pkgOther - ) - isStdRepo, err := src.IsStdRepo() - if err != nil { - return err - } - var pkgtype int - switch { - case isStdRepo: - pkgtype = pkgStd - case packageName == "windows": - // TODO: this needs better logic than just using package name - pkgtype = pkgXSysWindows - default: - pkgtype = pkgOther - } - if *systemDLL { - switch pkgtype { - case pkgStd: - src.Import("internal/syscall/windows/sysdll") - case pkgXSysWindows: - default: - src.ExternalImport("golang.org/x/sys/windows") - } - } - src.ExternalImport("github.com/Microsoft/go-winio") - if packageName != "syscall" { - src.Import("syscall") - } - funcMap := template.FuncMap{ - "packagename": packagename, - "syscalldot": syscalldot, - "newlazydll": func(dll string) string { - arg := "\"" + dll + ".dll\"" - if !*systemDLL { - return syscalldot() + "NewLazyDLL(" + arg + ")" - } - switch pkgtype { - case pkgStd: - return syscalldot() + "NewLazyDLL(sysdll.Add(" + arg + "))" - case pkgXSysWindows: - return "NewLazySystemDLL(" + arg + ")" - default: - return "windows.NewLazySystemDLL(" + arg + ")" - } - }, - } - t := template.Must(template.New("main").Funcs(funcMap).Parse(srcTemplate)) - err = t.Execute(w, src) - if err != nil { - return errors.New("Failed to execute template: " + err.Error()) - } - return nil -} - -func usage() { - fmt.Fprintf(os.Stderr, "usage: mksyscall_windows [flags] [path ...]\n") - flag.PrintDefaults() - os.Exit(1) -} - -func main() { - flag.Usage = usage - flag.Parse() - if len(flag.Args()) <= 0 { - fmt.Fprintf(os.Stderr, "no files to parse provided\n") - usage() - } - - src, err := ParseFiles(flag.Args()) - if err != nil { - log.Fatal(err) - } - - var buf bytes.Buffer - if err := src.Generate(&buf); err != nil { - log.Fatal(err) - } - - data, err := format.Source(buf.Bytes()) - if err != nil { - log.Fatal(err) - } - if *filename == "" { - _, err = os.Stdout.Write(data) - } else { - err = ioutil.WriteFile(*filename, data, 0644) - } - if err != nil { - log.Fatal(err) - } -} - -// TODO: use println instead to print in the following template -const srcTemplate = ` - -{{define "main"}}// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT - -package {{packagename}} - -import ( -{{range .StdLibImports}}"{{.}}" -{{end}} - -{{range .ExternalImports}}"{{.}}" -{{end}} -) - -var _ unsafe.Pointer - -// Do the interface allocations only once for common -// Errno values. -const ( - errnoERROR_IO_PENDING = 997 -) - -var ( - errERROR_IO_PENDING error = {{syscalldot}}Errno(errnoERROR_IO_PENDING) -) - -// errnoErr returns common boxed Errno values, to prevent -// allocations at runtime. -func errnoErr(e {{syscalldot}}Errno) error { - switch e { - case 0: - return nil - case errnoERROR_IO_PENDING: - return errERROR_IO_PENDING - } - // TODO: add more here, after collecting data on the common - // error values see on Windows. (perhaps when running - // all.bat?) - return e -} - -var ( -{{template "dlls" .}} -{{template "funcnames" .}}) -{{range .Funcs}}{{if .HasStringParam}}{{template "helperbody" .}}{{end}}{{template "funcbody" .}}{{end}} -{{end}} - -{{/* help functions */}} - -{{define "dlls"}}{{range .DLLs}} mod{{.}} = {{newlazydll .}} -{{end}}{{end}} - -{{define "funcnames"}}{{range .Funcs}}{{if .IsNotDuplicate}} proc{{.DLLFuncName}} = mod{{.DLLName}}.NewProc("{{.DLLFuncName}}"){{end}} -{{end}}{{end}} - -{{define "helperbody"}} -func {{.Name}}({{.ParamList}}) {{template "results" .}}{ -{{template "helpertmpvars" .}} return {{.HelperName}}({{.HelperCallParamList}}) -} -{{end}} - -{{define "funcbody"}} -func {{.HelperName}}({{.HelperParamList}}) {{template "results" .}}{ -{{template "tmpvars" .}} {{template "syscallcheck" .}}{{template "syscall" .}} -{{template "seterror" .}}{{template "printtrace" .}} return -} -{{end}} - -{{define "helpertmpvars"}}{{range .Params}}{{if .TmpVarHelperCode}} {{.TmpVarHelperCode}} -{{end}}{{end}}{{end}} - -{{define "tmpvars"}}{{range .Params}}{{if .TmpVarCode}} {{.TmpVarCode}} -{{end}}{{end}}{{end}} - -{{define "results"}}{{if .Rets.List}}{{.Rets.List}} {{end}}{{end}} - -{{define "syscall"}}{{.Rets.SetReturnValuesCode}}{{.Syscall}}(proc{{.DLLFuncName}}.Addr(), {{.ParamCount}}, {{.SyscallParamList}}){{end}} - -{{define "syscallcheck"}}{{if .ConfirmProc}}if {{.Rets.ErrorVarName}} = proc{{.DLLFuncName}}.Find(); {{.Rets.ErrorVarName}} != nil { - return -} -{{end}}{{end}} - - -{{define "seterror"}}{{if .Rets.SetErrorCode}} {{.Rets.SetErrorCode}} -{{end}}{{end}} - -{{define "printtrace"}}{{if .PrintTrace}} print("SYSCALL: {{.Name}}(", {{.ParamPrintList}}") (", {{.Rets.PrintList}}")\n") -{{end}}{{end}} - -` diff --git a/components/engine/vendor/github.com/kr/pty/types.go b/components/engine/vendor/github.com/kr/pty/types.go deleted file mode 100644 index 5aecb6bcdc..0000000000 --- a/components/engine/vendor/github.com/kr/pty/types.go +++ /dev/null @@ -1,10 +0,0 @@ -// +build ignore - -package pty - -import "C" - -type ( - _C_int C.int - _C_uint C.uint -) diff --git a/components/engine/vendor/github.com/kr/pty/types_freebsd.go b/components/engine/vendor/github.com/kr/pty/types_freebsd.go deleted file mode 100644 index ce3eb95181..0000000000 --- a/components/engine/vendor/github.com/kr/pty/types_freebsd.go +++ /dev/null @@ -1,15 +0,0 @@ -// +build ignore - -package pty - -/* -#include -#include -*/ -import "C" - -const ( - _C_SPECNAMELEN = C.SPECNAMELEN /* max length of devicename */ -) - -type fiodgnameArg C.struct_fiodgname_arg diff --git a/components/engine/vendor/github.com/miekg/dns/types_generate.go b/components/engine/vendor/github.com/miekg/dns/types_generate.go deleted file mode 100644 index 53690141a1..0000000000 --- a/components/engine/vendor/github.com/miekg/dns/types_generate.go +++ /dev/null @@ -1,266 +0,0 @@ -//+build ignore - -// types_generate.go is meant to run with go generate. It will use -// go/{importer,types} to track down all the RR struct types. Then for each type -// it will generate conversion tables (TypeToRR and TypeToString) and banal -// methods (len, Header, copy) based on the struct tags. The generated source is -// written to ztypes.go, and is meant to be checked into git. -package main - -import ( - "bytes" - "fmt" - "go/format" - "go/importer" - "go/types" - "log" - "os" - "strings" - "text/template" -) - -var skipLen = map[string]struct{}{ - "NSEC": struct{}{}, - "NSEC3": struct{}{}, - "OPT": struct{}{}, - "WKS": struct{}{}, - "IPSECKEY": struct{}{}, -} - -var packageHdr = ` -// *** DO NOT MODIFY *** -// AUTOGENERATED BY go generate - -package dns - -import ( - "encoding/base64" - "net" -) - -` - -var TypeToRR = template.Must(template.New("TypeToRR").Parse(` -// TypeToRR is a map of constructors for each RR type. -var TypeToRR = map[uint16]func() RR{ -{{range .}}{{if ne . "RFC3597"}} Type{{.}}: func() RR { return new({{.}}) }, -{{end}}{{end}} } - -`)) - -var typeToString = template.Must(template.New("typeToString").Parse(` -// TypeToString is a map of strings for each RR type. -var TypeToString = map[uint16]string{ -{{range .}}{{if ne . "NSAPPTR"}} Type{{.}}: "{{.}}", -{{end}}{{end}} TypeNSAPPTR: "NSAP-PTR", -} - -`)) - -var headerFunc = template.Must(template.New("headerFunc").Parse(` -// Header() functions -{{range .}} func (rr *{{.}}) Header() *RR_Header { return &rr.Hdr } -{{end}} - -`)) - -// getTypeStruct will take a type and the package scope, and return the -// (innermost) struct if the type is considered a RR type (currently defined as -// those structs beginning with a RR_Header, could be redefined as implementing -// the RR interface). The bool return value indicates if embedded structs were -// resolved. -func getTypeStruct(t types.Type, scope *types.Scope) (*types.Struct, bool) { - st, ok := t.Underlying().(*types.Struct) - if !ok { - return nil, false - } - if st.Field(0).Type() == scope.Lookup("RR_Header").Type() { - return st, false - } - if st.Field(0).Anonymous() { - st, _ := getTypeStruct(st.Field(0).Type(), scope) - return st, true - } - return nil, false -} - -func main() { - // Import and type-check the package - pkg, err := importer.Default().Import("github.com/miekg/dns") - fatalIfErr(err) - scope := pkg.Scope() - - // Collect constants like TypeX - var numberedTypes []string - for _, name := range scope.Names() { - o := scope.Lookup(name) - if o == nil || !o.Exported() { - continue - } - b, ok := o.Type().(*types.Basic) - if !ok || b.Kind() != types.Uint16 { - continue - } - if !strings.HasPrefix(o.Name(), "Type") { - continue - } - name := strings.TrimPrefix(o.Name(), "Type") - if name == "PrivateRR" { - continue - } - numberedTypes = append(numberedTypes, name) - } - - // Collect actual types (*X) - var namedTypes []string - for _, name := range scope.Names() { - o := scope.Lookup(name) - if o == nil || !o.Exported() { - continue - } - if st, _ := getTypeStruct(o.Type(), scope); st == nil { - continue - } - if name == "PrivateRR" { - continue - } - - // Check if corresponding TypeX exists - if scope.Lookup("Type"+o.Name()) == nil && o.Name() != "RFC3597" { - log.Fatalf("Constant Type%s does not exist.", o.Name()) - } - - namedTypes = append(namedTypes, o.Name()) - } - - b := &bytes.Buffer{} - b.WriteString(packageHdr) - - // Generate TypeToRR - fatalIfErr(TypeToRR.Execute(b, namedTypes)) - - // Generate typeToString - fatalIfErr(typeToString.Execute(b, numberedTypes)) - - // Generate headerFunc - fatalIfErr(headerFunc.Execute(b, namedTypes)) - - // Generate len() - fmt.Fprint(b, "// len() functions\n") - for _, name := range namedTypes { - if _, ok := skipLen[name]; ok { - continue - } - o := scope.Lookup(name) - st, isEmbedded := getTypeStruct(o.Type(), scope) - if isEmbedded { - continue - } - fmt.Fprintf(b, "func (rr *%s) len() int {\n", name) - fmt.Fprintf(b, "l := rr.Hdr.len()\n") - for i := 1; i < st.NumFields(); i++ { - o := func(s string) { fmt.Fprintf(b, s, st.Field(i).Name()) } - - if _, ok := st.Field(i).Type().(*types.Slice); ok { - switch st.Tag(i) { - case `dns:"-"`: - // ignored - case `dns:"cdomain-name"`, `dns:"domain-name"`, `dns:"txt"`: - o("for _, x := range rr.%s { l += len(x) + 1 }\n") - default: - log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) - } - continue - } - - switch st.Tag(i) { - case `dns:"-"`: - // ignored - case `dns:"cdomain-name"`, `dns:"domain-name"`: - o("l += len(rr.%s) + 1\n") - case `dns:"octet"`: - o("l += len(rr.%s)\n") - case `dns:"base64"`: - o("l += base64.StdEncoding.DecodedLen(len(rr.%s))\n") - case `dns:"size-hex"`, `dns:"hex"`: - o("l += len(rr.%s)/2 + 1\n") - case `dns:"a"`: - o("l += net.IPv4len // %s\n") - case `dns:"aaaa"`: - o("l += net.IPv6len // %s\n") - case `dns:"txt"`: - o("for _, t := range rr.%s { l += len(t) + 1 }\n") - case `dns:"uint48"`: - o("l += 6 // %s\n") - case "": - switch st.Field(i).Type().(*types.Basic).Kind() { - case types.Uint8: - o("l += 1 // %s\n") - case types.Uint16: - o("l += 2 // %s\n") - case types.Uint32: - o("l += 4 // %s\n") - case types.Uint64: - o("l += 8 // %s\n") - case types.String: - o("l += len(rr.%s) + 1\n") - default: - log.Fatalln(name, st.Field(i).Name()) - } - default: - log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) - } - } - fmt.Fprintf(b, "return l }\n") - } - - // Generate copy() - fmt.Fprint(b, "// copy() functions\n") - for _, name := range namedTypes { - o := scope.Lookup(name) - st, isEmbedded := getTypeStruct(o.Type(), scope) - if isEmbedded { - continue - } - fmt.Fprintf(b, "func (rr *%s) copy() RR {\n", name) - fields := []string{"*rr.Hdr.copyHeader()"} - for i := 1; i < st.NumFields(); i++ { - f := st.Field(i).Name() - if sl, ok := st.Field(i).Type().(*types.Slice); ok { - t := sl.Underlying().String() - t = strings.TrimPrefix(t, "[]") - t = strings.TrimPrefix(t, "github.com/miekg/dns.") - fmt.Fprintf(b, "%s := make([]%s, len(rr.%s)); copy(%s, rr.%s)\n", - f, t, f, f, f) - fields = append(fields, f) - continue - } - if st.Field(i).Type().String() == "net.IP" { - fields = append(fields, "copyIP(rr."+f+")") - continue - } - fields = append(fields, "rr."+f) - } - fmt.Fprintf(b, "return &%s{%s}\n", name, strings.Join(fields, ",")) - fmt.Fprintf(b, "}\n") - } - - // gofmt - res, err := format.Source(b.Bytes()) - if err != nil { - b.WriteTo(os.Stderr) - log.Fatal(err) - } - - // write result - f, err := os.Create("ztypes.go") - fatalIfErr(err) - defer f.Close() - f.Write(res) -} - -func fatalIfErr(err error) { - if err != nil { - log.Fatal(err) - } -} diff --git a/components/engine/vendor/golang.org/x/sys/unix/mkpost.go b/components/engine/vendor/golang.org/x/sys/unix/mkpost.go deleted file mode 100644 index ed50d902af..0000000000 --- a/components/engine/vendor/golang.org/x/sys/unix/mkpost.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -// mkpost processes the output of cgo -godefs to -// modify the generated types. It is used to clean up -// the sys API in an architecture specific manner. -// -// mkpost is run after cgo -godefs by mkall.sh. -package main - -import ( - "fmt" - "go/format" - "io/ioutil" - "log" - "os" - "regexp" -) - -func main() { - b, err := ioutil.ReadAll(os.Stdin) - if err != nil { - log.Fatal(err) - } - s := string(b) - - goarch := os.Getenv("GOARCH") - goos := os.Getenv("GOOS") - if goarch == "s390x" && goos == "linux" { - // Export the types of PtraceRegs fields. - re := regexp.MustCompile("ptrace(Psw|Fpregs|Per)") - s = re.ReplaceAllString(s, "Ptrace$1") - - // Replace padding fields inserted by cgo with blank identifiers. - re = regexp.MustCompile("Pad_cgo[A-Za-z0-9_]*") - s = re.ReplaceAllString(s, "_") - - // Replace other unwanted fields with blank identifiers. - re = regexp.MustCompile("X_[A-Za-z0-9_]*") - s = re.ReplaceAllString(s, "_") - - // Replace the control_regs union with a blank identifier for now. - re = regexp.MustCompile("(Control_regs)\\s+\\[0\\]uint64") - s = re.ReplaceAllString(s, "_ [0]uint64") - } - - // gofmt - b, err = format.Source([]byte(s)) - if err != nil { - log.Fatal(err) - } - - // Append this command to the header to show where the new file - // came from. - re := regexp.MustCompile("(cgo -godefs [a-zA-Z0-9_]+\\.go.*)") - b = re.ReplaceAll(b, []byte("$1 | go run mkpost.go")) - - fmt.Printf("%s", b) -} diff --git a/components/engine/vendor/golang.org/x/sys/unix/types_darwin.go b/components/engine/vendor/golang.org/x/sys/unix/types_darwin.go deleted file mode 100644 index 1153261822..0000000000 --- a/components/engine/vendor/golang.org/x/sys/unix/types_darwin.go +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -/* -Input to cgo -godefs. See also mkerrors.sh and mkall.sh -*/ - -// +godefs map struct_in_addr [4]byte /* in_addr */ -// +godefs map struct_in6_addr [16]byte /* in6_addr */ - -package unix - -/* -#define __DARWIN_UNIX03 0 -#define KERNEL -#define _DARWIN_USE_64_BIT_INODE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum { - sizeofPtr = sizeof(void*), -}; - -union sockaddr_all { - struct sockaddr s1; // this one gets used for fields - struct sockaddr_in s2; // these pad it out - struct sockaddr_in6 s3; - struct sockaddr_un s4; - struct sockaddr_dl s5; -}; - -struct sockaddr_any { - struct sockaddr addr; - char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; -}; - -*/ -import "C" - -// Machine characteristics; for internal use. - -const ( - sizeofPtr = C.sizeofPtr - sizeofShort = C.sizeof_short - sizeofInt = C.sizeof_int - sizeofLong = C.sizeof_long - sizeofLongLong = C.sizeof_longlong -) - -// Basic types - -type ( - _C_short C.short - _C_int C.int - _C_long C.long - _C_long_long C.longlong -) - -// Time - -type Timespec C.struct_timespec - -type Timeval C.struct_timeval - -type Timeval32 C.struct_timeval32 - -// Processes - -type Rusage C.struct_rusage - -type Rlimit C.struct_rlimit - -type _Gid_t C.gid_t - -// Files - -type Stat_t C.struct_stat64 - -type Statfs_t C.struct_statfs64 - -type Flock_t C.struct_flock - -type Fstore_t C.struct_fstore - -type Radvisory_t C.struct_radvisory - -type Fbootstraptransfer_t C.struct_fbootstraptransfer - -type Log2phys_t C.struct_log2phys - -type Fsid C.struct_fsid - -type Dirent C.struct_dirent - -// Sockets - -type RawSockaddrInet4 C.struct_sockaddr_in - -type RawSockaddrInet6 C.struct_sockaddr_in6 - -type RawSockaddrUnix C.struct_sockaddr_un - -type RawSockaddrDatalink C.struct_sockaddr_dl - -type RawSockaddr C.struct_sockaddr - -type RawSockaddrAny C.struct_sockaddr_any - -type _Socklen C.socklen_t - -type Linger C.struct_linger - -type Iovec C.struct_iovec - -type IPMreq C.struct_ip_mreq - -type IPv6Mreq C.struct_ipv6_mreq - -type Msghdr C.struct_msghdr - -type Cmsghdr C.struct_cmsghdr - -type Inet4Pktinfo C.struct_in_pktinfo - -type Inet6Pktinfo C.struct_in6_pktinfo - -type IPv6MTUInfo C.struct_ip6_mtuinfo - -type ICMPv6Filter C.struct_icmp6_filter - -const ( - SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in - SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 - SizeofSockaddrAny = C.sizeof_struct_sockaddr_any - SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un - SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl - SizeofLinger = C.sizeof_struct_linger - SizeofIPMreq = C.sizeof_struct_ip_mreq - SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq - SizeofMsghdr = C.sizeof_struct_msghdr - SizeofCmsghdr = C.sizeof_struct_cmsghdr - SizeofInet4Pktinfo = C.sizeof_struct_in_pktinfo - SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo - SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo - SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter -) - -// Ptrace requests - -const ( - PTRACE_TRACEME = C.PT_TRACE_ME - PTRACE_CONT = C.PT_CONTINUE - PTRACE_KILL = C.PT_KILL -) - -// Events (kqueue, kevent) - -type Kevent_t C.struct_kevent - -// Select - -type FdSet C.fd_set - -// Routing and interface messages - -const ( - SizeofIfMsghdr = C.sizeof_struct_if_msghdr - SizeofIfData = C.sizeof_struct_if_data - SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr - SizeofIfmaMsghdr = C.sizeof_struct_ifma_msghdr - SizeofIfmaMsghdr2 = C.sizeof_struct_ifma_msghdr2 - SizeofRtMsghdr = C.sizeof_struct_rt_msghdr - SizeofRtMetrics = C.sizeof_struct_rt_metrics -) - -type IfMsghdr C.struct_if_msghdr - -type IfData C.struct_if_data - -type IfaMsghdr C.struct_ifa_msghdr - -type IfmaMsghdr C.struct_ifma_msghdr - -type IfmaMsghdr2 C.struct_ifma_msghdr2 - -type RtMsghdr C.struct_rt_msghdr - -type RtMetrics C.struct_rt_metrics - -// Berkeley packet filter - -const ( - SizeofBpfVersion = C.sizeof_struct_bpf_version - SizeofBpfStat = C.sizeof_struct_bpf_stat - SizeofBpfProgram = C.sizeof_struct_bpf_program - SizeofBpfInsn = C.sizeof_struct_bpf_insn - SizeofBpfHdr = C.sizeof_struct_bpf_hdr -) - -type BpfVersion C.struct_bpf_version - -type BpfStat C.struct_bpf_stat - -type BpfProgram C.struct_bpf_program - -type BpfInsn C.struct_bpf_insn - -type BpfHdr C.struct_bpf_hdr - -// Terminal handling - -type Termios C.struct_termios - -// fchmodat-like syscalls. - -const ( - AT_FDCWD = C.AT_FDCWD - AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW -) diff --git a/components/engine/vendor/golang.org/x/sys/unix/types_dragonfly.go b/components/engine/vendor/golang.org/x/sys/unix/types_dragonfly.go deleted file mode 100644 index f3c971dffd..0000000000 --- a/components/engine/vendor/golang.org/x/sys/unix/types_dragonfly.go +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -/* -Input to cgo -godefs. See also mkerrors.sh and mkall.sh -*/ - -// +godefs map struct_in_addr [4]byte /* in_addr */ -// +godefs map struct_in6_addr [16]byte /* in6_addr */ - -package unix - -/* -#define KERNEL -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum { - sizeofPtr = sizeof(void*), -}; - -union sockaddr_all { - struct sockaddr s1; // this one gets used for fields - struct sockaddr_in s2; // these pad it out - struct sockaddr_in6 s3; - struct sockaddr_un s4; - struct sockaddr_dl s5; -}; - -struct sockaddr_any { - struct sockaddr addr; - char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; -}; - -*/ -import "C" - -// Machine characteristics; for internal use. - -const ( - sizeofPtr = C.sizeofPtr - sizeofShort = C.sizeof_short - sizeofInt = C.sizeof_int - sizeofLong = C.sizeof_long - sizeofLongLong = C.sizeof_longlong -) - -// Basic types - -type ( - _C_short C.short - _C_int C.int - _C_long C.long - _C_long_long C.longlong -) - -// Time - -type Timespec C.struct_timespec - -type Timeval C.struct_timeval - -// Processes - -type Rusage C.struct_rusage - -type Rlimit C.struct_rlimit - -type _Gid_t C.gid_t - -// Files - -const ( // Directory mode bits - S_IFMT = C.S_IFMT - S_IFIFO = C.S_IFIFO - S_IFCHR = C.S_IFCHR - S_IFDIR = C.S_IFDIR - S_IFBLK = C.S_IFBLK - S_IFREG = C.S_IFREG - S_IFLNK = C.S_IFLNK - S_IFSOCK = C.S_IFSOCK - S_ISUID = C.S_ISUID - S_ISGID = C.S_ISGID - S_ISVTX = C.S_ISVTX - S_IRUSR = C.S_IRUSR - S_IWUSR = C.S_IWUSR - S_IXUSR = C.S_IXUSR -) - -type Stat_t C.struct_stat - -type Statfs_t C.struct_statfs - -type Flock_t C.struct_flock - -type Dirent C.struct_dirent - -type Fsid C.struct_fsid - -// Sockets - -type RawSockaddrInet4 C.struct_sockaddr_in - -type RawSockaddrInet6 C.struct_sockaddr_in6 - -type RawSockaddrUnix C.struct_sockaddr_un - -type RawSockaddrDatalink C.struct_sockaddr_dl - -type RawSockaddr C.struct_sockaddr - -type RawSockaddrAny C.struct_sockaddr_any - -type _Socklen C.socklen_t - -type Linger C.struct_linger - -type Iovec C.struct_iovec - -type IPMreq C.struct_ip_mreq - -type IPv6Mreq C.struct_ipv6_mreq - -type Msghdr C.struct_msghdr - -type Cmsghdr C.struct_cmsghdr - -type Inet6Pktinfo C.struct_in6_pktinfo - -type IPv6MTUInfo C.struct_ip6_mtuinfo - -type ICMPv6Filter C.struct_icmp6_filter - -const ( - SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in - SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 - SizeofSockaddrAny = C.sizeof_struct_sockaddr_any - SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un - SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl - SizeofLinger = C.sizeof_struct_linger - SizeofIPMreq = C.sizeof_struct_ip_mreq - SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq - SizeofMsghdr = C.sizeof_struct_msghdr - SizeofCmsghdr = C.sizeof_struct_cmsghdr - SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo - SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo - SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter -) - -// Ptrace requests - -const ( - PTRACE_TRACEME = C.PT_TRACE_ME - PTRACE_CONT = C.PT_CONTINUE - PTRACE_KILL = C.PT_KILL -) - -// Events (kqueue, kevent) - -type Kevent_t C.struct_kevent - -// Select - -type FdSet C.fd_set - -// Routing and interface messages - -const ( - SizeofIfMsghdr = C.sizeof_struct_if_msghdr - SizeofIfData = C.sizeof_struct_if_data - SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr - SizeofIfmaMsghdr = C.sizeof_struct_ifma_msghdr - SizeofIfAnnounceMsghdr = C.sizeof_struct_if_announcemsghdr - SizeofRtMsghdr = C.sizeof_struct_rt_msghdr - SizeofRtMetrics = C.sizeof_struct_rt_metrics -) - -type IfMsghdr C.struct_if_msghdr - -type IfData C.struct_if_data - -type IfaMsghdr C.struct_ifa_msghdr - -type IfmaMsghdr C.struct_ifma_msghdr - -type IfAnnounceMsghdr C.struct_if_announcemsghdr - -type RtMsghdr C.struct_rt_msghdr - -type RtMetrics C.struct_rt_metrics - -// Berkeley packet filter - -const ( - SizeofBpfVersion = C.sizeof_struct_bpf_version - SizeofBpfStat = C.sizeof_struct_bpf_stat - SizeofBpfProgram = C.sizeof_struct_bpf_program - SizeofBpfInsn = C.sizeof_struct_bpf_insn - SizeofBpfHdr = C.sizeof_struct_bpf_hdr -) - -type BpfVersion C.struct_bpf_version - -type BpfStat C.struct_bpf_stat - -type BpfProgram C.struct_bpf_program - -type BpfInsn C.struct_bpf_insn - -type BpfHdr C.struct_bpf_hdr - -// Terminal handling - -type Termios C.struct_termios diff --git a/components/engine/vendor/golang.org/x/sys/unix/types_freebsd.go b/components/engine/vendor/golang.org/x/sys/unix/types_freebsd.go deleted file mode 100644 index ae24557ad1..0000000000 --- a/components/engine/vendor/golang.org/x/sys/unix/types_freebsd.go +++ /dev/null @@ -1,353 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -/* -Input to cgo -godefs. See also mkerrors.sh and mkall.sh -*/ - -// +godefs map struct_in_addr [4]byte /* in_addr */ -// +godefs map struct_in6_addr [16]byte /* in6_addr */ - -package unix - -/* -#define KERNEL -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum { - sizeofPtr = sizeof(void*), -}; - -union sockaddr_all { - struct sockaddr s1; // this one gets used for fields - struct sockaddr_in s2; // these pad it out - struct sockaddr_in6 s3; - struct sockaddr_un s4; - struct sockaddr_dl s5; -}; - -struct sockaddr_any { - struct sockaddr addr; - char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; -}; - -// This structure is a duplicate of stat on FreeBSD 8-STABLE. -// See /usr/include/sys/stat.h. -struct stat8 { -#undef st_atimespec st_atim -#undef st_mtimespec st_mtim -#undef st_ctimespec st_ctim -#undef st_birthtimespec st_birthtim - __dev_t st_dev; - ino_t st_ino; - mode_t st_mode; - nlink_t st_nlink; - uid_t st_uid; - gid_t st_gid; - __dev_t st_rdev; -#if __BSD_VISIBLE - struct timespec st_atimespec; - struct timespec st_mtimespec; - struct timespec st_ctimespec; -#else - time_t st_atime; - long __st_atimensec; - time_t st_mtime; - long __st_mtimensec; - time_t st_ctime; - long __st_ctimensec; -#endif - off_t st_size; - blkcnt_t st_blocks; - blksize_t st_blksize; - fflags_t st_flags; - __uint32_t st_gen; - __int32_t st_lspare; -#if __BSD_VISIBLE - struct timespec st_birthtimespec; - unsigned int :(8 / 2) * (16 - (int)sizeof(struct timespec)); - unsigned int :(8 / 2) * (16 - (int)sizeof(struct timespec)); -#else - time_t st_birthtime; - long st_birthtimensec; - unsigned int :(8 / 2) * (16 - (int)sizeof(struct __timespec)); - unsigned int :(8 / 2) * (16 - (int)sizeof(struct __timespec)); -#endif -}; - -// This structure is a duplicate of if_data on FreeBSD 8-STABLE. -// See /usr/include/net/if.h. -struct if_data8 { - u_char ifi_type; - u_char ifi_physical; - u_char ifi_addrlen; - u_char ifi_hdrlen; - u_char ifi_link_state; - u_char ifi_spare_char1; - u_char ifi_spare_char2; - u_char ifi_datalen; - u_long ifi_mtu; - u_long ifi_metric; - u_long ifi_baudrate; - u_long ifi_ipackets; - u_long ifi_ierrors; - u_long ifi_opackets; - u_long ifi_oerrors; - u_long ifi_collisions; - u_long ifi_ibytes; - u_long ifi_obytes; - u_long ifi_imcasts; - u_long ifi_omcasts; - u_long ifi_iqdrops; - u_long ifi_noproto; - u_long ifi_hwassist; - time_t ifi_epoch; - struct timeval ifi_lastchange; -}; - -// This structure is a duplicate of if_msghdr on FreeBSD 8-STABLE. -// See /usr/include/net/if.h. -struct if_msghdr8 { - u_short ifm_msglen; - u_char ifm_version; - u_char ifm_type; - int ifm_addrs; - int ifm_flags; - u_short ifm_index; - struct if_data8 ifm_data; -}; -*/ -import "C" - -// Machine characteristics; for internal use. - -const ( - sizeofPtr = C.sizeofPtr - sizeofShort = C.sizeof_short - sizeofInt = C.sizeof_int - sizeofLong = C.sizeof_long - sizeofLongLong = C.sizeof_longlong -) - -// Basic types - -type ( - _C_short C.short - _C_int C.int - _C_long C.long - _C_long_long C.longlong -) - -// Time - -type Timespec C.struct_timespec - -type Timeval C.struct_timeval - -// Processes - -type Rusage C.struct_rusage - -type Rlimit C.struct_rlimit - -type _Gid_t C.gid_t - -// Files - -const ( // Directory mode bits - S_IFMT = C.S_IFMT - S_IFIFO = C.S_IFIFO - S_IFCHR = C.S_IFCHR - S_IFDIR = C.S_IFDIR - S_IFBLK = C.S_IFBLK - S_IFREG = C.S_IFREG - S_IFLNK = C.S_IFLNK - S_IFSOCK = C.S_IFSOCK - S_ISUID = C.S_ISUID - S_ISGID = C.S_ISGID - S_ISVTX = C.S_ISVTX - S_IRUSR = C.S_IRUSR - S_IWUSR = C.S_IWUSR - S_IXUSR = C.S_IXUSR -) - -type Stat_t C.struct_stat8 - -type Statfs_t C.struct_statfs - -type Flock_t C.struct_flock - -type Dirent C.struct_dirent - -type Fsid C.struct_fsid - -// Advice to Fadvise - -const ( - FADV_NORMAL = C.POSIX_FADV_NORMAL - FADV_RANDOM = C.POSIX_FADV_RANDOM - FADV_SEQUENTIAL = C.POSIX_FADV_SEQUENTIAL - FADV_WILLNEED = C.POSIX_FADV_WILLNEED - FADV_DONTNEED = C.POSIX_FADV_DONTNEED - FADV_NOREUSE = C.POSIX_FADV_NOREUSE -) - -// Sockets - -type RawSockaddrInet4 C.struct_sockaddr_in - -type RawSockaddrInet6 C.struct_sockaddr_in6 - -type RawSockaddrUnix C.struct_sockaddr_un - -type RawSockaddrDatalink C.struct_sockaddr_dl - -type RawSockaddr C.struct_sockaddr - -type RawSockaddrAny C.struct_sockaddr_any - -type _Socklen C.socklen_t - -type Linger C.struct_linger - -type Iovec C.struct_iovec - -type IPMreq C.struct_ip_mreq - -type IPMreqn C.struct_ip_mreqn - -type IPv6Mreq C.struct_ipv6_mreq - -type Msghdr C.struct_msghdr - -type Cmsghdr C.struct_cmsghdr - -type Inet6Pktinfo C.struct_in6_pktinfo - -type IPv6MTUInfo C.struct_ip6_mtuinfo - -type ICMPv6Filter C.struct_icmp6_filter - -const ( - SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in - SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 - SizeofSockaddrAny = C.sizeof_struct_sockaddr_any - SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un - SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl - SizeofLinger = C.sizeof_struct_linger - SizeofIPMreq = C.sizeof_struct_ip_mreq - SizeofIPMreqn = C.sizeof_struct_ip_mreqn - SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq - SizeofMsghdr = C.sizeof_struct_msghdr - SizeofCmsghdr = C.sizeof_struct_cmsghdr - SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo - SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo - SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter -) - -// Ptrace requests - -const ( - PTRACE_TRACEME = C.PT_TRACE_ME - PTRACE_CONT = C.PT_CONTINUE - PTRACE_KILL = C.PT_KILL -) - -// Events (kqueue, kevent) - -type Kevent_t C.struct_kevent - -// Select - -type FdSet C.fd_set - -// Routing and interface messages - -const ( - sizeofIfMsghdr = C.sizeof_struct_if_msghdr - SizeofIfMsghdr = C.sizeof_struct_if_msghdr8 - sizeofIfData = C.sizeof_struct_if_data - SizeofIfData = C.sizeof_struct_if_data8 - SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr - SizeofIfmaMsghdr = C.sizeof_struct_ifma_msghdr - SizeofIfAnnounceMsghdr = C.sizeof_struct_if_announcemsghdr - SizeofRtMsghdr = C.sizeof_struct_rt_msghdr - SizeofRtMetrics = C.sizeof_struct_rt_metrics -) - -type ifMsghdr C.struct_if_msghdr - -type IfMsghdr C.struct_if_msghdr8 - -type ifData C.struct_if_data - -type IfData C.struct_if_data8 - -type IfaMsghdr C.struct_ifa_msghdr - -type IfmaMsghdr C.struct_ifma_msghdr - -type IfAnnounceMsghdr C.struct_if_announcemsghdr - -type RtMsghdr C.struct_rt_msghdr - -type RtMetrics C.struct_rt_metrics - -// Berkeley packet filter - -const ( - SizeofBpfVersion = C.sizeof_struct_bpf_version - SizeofBpfStat = C.sizeof_struct_bpf_stat - SizeofBpfZbuf = C.sizeof_struct_bpf_zbuf - SizeofBpfProgram = C.sizeof_struct_bpf_program - SizeofBpfInsn = C.sizeof_struct_bpf_insn - SizeofBpfHdr = C.sizeof_struct_bpf_hdr - SizeofBpfZbufHeader = C.sizeof_struct_bpf_zbuf_header -) - -type BpfVersion C.struct_bpf_version - -type BpfStat C.struct_bpf_stat - -type BpfZbuf C.struct_bpf_zbuf - -type BpfProgram C.struct_bpf_program - -type BpfInsn C.struct_bpf_insn - -type BpfHdr C.struct_bpf_hdr - -type BpfZbufHeader C.struct_bpf_zbuf_header - -// Terminal handling - -type Termios C.struct_termios diff --git a/components/engine/vendor/golang.org/x/sys/unix/types_linux.go b/components/engine/vendor/golang.org/x/sys/unix/types_linux.go deleted file mode 100644 index 7dea79a8ef..0000000000 --- a/components/engine/vendor/golang.org/x/sys/unix/types_linux.go +++ /dev/null @@ -1,450 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -/* -Input to cgo -godefs. See also mkerrors.sh and mkall.sh -*/ - -// +godefs map struct_in_addr [4]byte /* in_addr */ -// +godefs map struct_in6_addr [16]byte /* in6_addr */ - -package unix - -/* -#define _LARGEFILE_SOURCE -#define _LARGEFILE64_SOURCE -#define _FILE_OFFSET_BITS 64 -#define _GNU_SOURCE - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef TCSETS2 -// On systems that have "struct termios2" use this as type Termios. -typedef struct termios2 termios_t; -#else -typedef struct termios termios_t; -#endif - -enum { - sizeofPtr = sizeof(void*), -}; - -union sockaddr_all { - struct sockaddr s1; // this one gets used for fields - struct sockaddr_in s2; // these pad it out - struct sockaddr_in6 s3; - struct sockaddr_un s4; - struct sockaddr_ll s5; - struct sockaddr_nl s6; -}; - -struct sockaddr_any { - struct sockaddr addr; - char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; -}; - -// copied from /usr/include/linux/un.h -struct my_sockaddr_un { - sa_family_t sun_family; -#if defined(__ARM_EABI__) || defined(__powerpc64__) - // on ARM char is by default unsigned - signed char sun_path[108]; -#else - char sun_path[108]; -#endif -}; - -#ifdef __ARM_EABI__ -typedef struct user_regs PtraceRegs; -#elif defined(__aarch64__) -typedef struct user_pt_regs PtraceRegs; -#elif defined(__powerpc64__) -typedef struct pt_regs PtraceRegs; -#elif defined(__mips__) -typedef struct user PtraceRegs; -#elif defined(__s390x__) -typedef struct _user_regs_struct PtraceRegs; -#else -typedef struct user_regs_struct PtraceRegs; -#endif - -#if defined(__s390x__) -typedef struct _user_psw_struct ptracePsw; -typedef struct _user_fpregs_struct ptraceFpregs; -typedef struct _user_per_struct ptracePer; -#else -typedef struct {} ptracePsw; -typedef struct {} ptraceFpregs; -typedef struct {} ptracePer; -#endif - -// The real epoll_event is a union, and godefs doesn't handle it well. -struct my_epoll_event { - uint32_t events; -#if defined(__ARM_EABI__) || defined(__aarch64__) - // padding is not specified in linux/eventpoll.h but added to conform to the - // alignment requirements of EABI - int32_t padFd; -#elif defined(__powerpc64__) || defined(__s390x__) - int32_t _padFd; -#endif - int32_t fd; - int32_t pad; -}; - -*/ -import "C" - -// Machine characteristics; for internal use. - -const ( - sizeofPtr = C.sizeofPtr - sizeofShort = C.sizeof_short - sizeofInt = C.sizeof_int - sizeofLong = C.sizeof_long - sizeofLongLong = C.sizeof_longlong - PathMax = C.PATH_MAX -) - -// Basic types - -type ( - _C_short C.short - _C_int C.int - _C_long C.long - _C_long_long C.longlong -) - -// Time - -type Timespec C.struct_timespec - -type Timeval C.struct_timeval - -type Timex C.struct_timex - -type Time_t C.time_t - -type Tms C.struct_tms - -type Utimbuf C.struct_utimbuf - -// Processes - -type Rusage C.struct_rusage - -type Rlimit C.struct_rlimit - -type _Gid_t C.gid_t - -// Files - -type Stat_t C.struct_stat - -type Statfs_t C.struct_statfs - -type Dirent C.struct_dirent - -type Fsid C.fsid_t - -type Flock_t C.struct_flock - -// Advice to Fadvise - -const ( - FADV_NORMAL = C.POSIX_FADV_NORMAL - FADV_RANDOM = C.POSIX_FADV_RANDOM - FADV_SEQUENTIAL = C.POSIX_FADV_SEQUENTIAL - FADV_WILLNEED = C.POSIX_FADV_WILLNEED - FADV_DONTNEED = C.POSIX_FADV_DONTNEED - FADV_NOREUSE = C.POSIX_FADV_NOREUSE -) - -// Sockets - -type RawSockaddrInet4 C.struct_sockaddr_in - -type RawSockaddrInet6 C.struct_sockaddr_in6 - -type RawSockaddrUnix C.struct_my_sockaddr_un - -type RawSockaddrLinklayer C.struct_sockaddr_ll - -type RawSockaddrNetlink C.struct_sockaddr_nl - -type RawSockaddrHCI C.struct_sockaddr_hci - -type RawSockaddr C.struct_sockaddr - -type RawSockaddrAny C.struct_sockaddr_any - -type _Socklen C.socklen_t - -type Linger C.struct_linger - -type Iovec C.struct_iovec - -type IPMreq C.struct_ip_mreq - -type IPMreqn C.struct_ip_mreqn - -type IPv6Mreq C.struct_ipv6_mreq - -type Msghdr C.struct_msghdr - -type Cmsghdr C.struct_cmsghdr - -type Inet4Pktinfo C.struct_in_pktinfo - -type Inet6Pktinfo C.struct_in6_pktinfo - -type IPv6MTUInfo C.struct_ip6_mtuinfo - -type ICMPv6Filter C.struct_icmp6_filter - -type Ucred C.struct_ucred - -type TCPInfo C.struct_tcp_info - -const ( - SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in - SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 - SizeofSockaddrAny = C.sizeof_struct_sockaddr_any - SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un - SizeofSockaddrLinklayer = C.sizeof_struct_sockaddr_ll - SizeofSockaddrNetlink = C.sizeof_struct_sockaddr_nl - SizeofSockaddrHCI = C.sizeof_struct_sockaddr_hci - SizeofLinger = C.sizeof_struct_linger - SizeofIPMreq = C.sizeof_struct_ip_mreq - SizeofIPMreqn = C.sizeof_struct_ip_mreqn - SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq - SizeofMsghdr = C.sizeof_struct_msghdr - SizeofCmsghdr = C.sizeof_struct_cmsghdr - SizeofInet4Pktinfo = C.sizeof_struct_in_pktinfo - SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo - SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo - SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter - SizeofUcred = C.sizeof_struct_ucred - SizeofTCPInfo = C.sizeof_struct_tcp_info -) - -// Netlink routing and interface messages - -const ( - IFA_UNSPEC = C.IFA_UNSPEC - IFA_ADDRESS = C.IFA_ADDRESS - IFA_LOCAL = C.IFA_LOCAL - IFA_LABEL = C.IFA_LABEL - IFA_BROADCAST = C.IFA_BROADCAST - IFA_ANYCAST = C.IFA_ANYCAST - IFA_CACHEINFO = C.IFA_CACHEINFO - IFA_MULTICAST = C.IFA_MULTICAST - IFLA_UNSPEC = C.IFLA_UNSPEC - IFLA_ADDRESS = C.IFLA_ADDRESS - IFLA_BROADCAST = C.IFLA_BROADCAST - IFLA_IFNAME = C.IFLA_IFNAME - IFLA_MTU = C.IFLA_MTU - IFLA_LINK = C.IFLA_LINK - IFLA_QDISC = C.IFLA_QDISC - IFLA_STATS = C.IFLA_STATS - IFLA_COST = C.IFLA_COST - IFLA_PRIORITY = C.IFLA_PRIORITY - IFLA_MASTER = C.IFLA_MASTER - IFLA_WIRELESS = C.IFLA_WIRELESS - IFLA_PROTINFO = C.IFLA_PROTINFO - IFLA_TXQLEN = C.IFLA_TXQLEN - IFLA_MAP = C.IFLA_MAP - IFLA_WEIGHT = C.IFLA_WEIGHT - IFLA_OPERSTATE = C.IFLA_OPERSTATE - IFLA_LINKMODE = C.IFLA_LINKMODE - IFLA_LINKINFO = C.IFLA_LINKINFO - IFLA_NET_NS_PID = C.IFLA_NET_NS_PID - IFLA_IFALIAS = C.IFLA_IFALIAS - IFLA_MAX = C.IFLA_MAX - RT_SCOPE_UNIVERSE = C.RT_SCOPE_UNIVERSE - RT_SCOPE_SITE = C.RT_SCOPE_SITE - RT_SCOPE_LINK = C.RT_SCOPE_LINK - RT_SCOPE_HOST = C.RT_SCOPE_HOST - RT_SCOPE_NOWHERE = C.RT_SCOPE_NOWHERE - RT_TABLE_UNSPEC = C.RT_TABLE_UNSPEC - RT_TABLE_COMPAT = C.RT_TABLE_COMPAT - RT_TABLE_DEFAULT = C.RT_TABLE_DEFAULT - RT_TABLE_MAIN = C.RT_TABLE_MAIN - RT_TABLE_LOCAL = C.RT_TABLE_LOCAL - RT_TABLE_MAX = C.RT_TABLE_MAX - RTA_UNSPEC = C.RTA_UNSPEC - RTA_DST = C.RTA_DST - RTA_SRC = C.RTA_SRC - RTA_IIF = C.RTA_IIF - RTA_OIF = C.RTA_OIF - RTA_GATEWAY = C.RTA_GATEWAY - RTA_PRIORITY = C.RTA_PRIORITY - RTA_PREFSRC = C.RTA_PREFSRC - RTA_METRICS = C.RTA_METRICS - RTA_MULTIPATH = C.RTA_MULTIPATH - RTA_FLOW = C.RTA_FLOW - RTA_CACHEINFO = C.RTA_CACHEINFO - RTA_TABLE = C.RTA_TABLE - RTN_UNSPEC = C.RTN_UNSPEC - RTN_UNICAST = C.RTN_UNICAST - RTN_LOCAL = C.RTN_LOCAL - RTN_BROADCAST = C.RTN_BROADCAST - RTN_ANYCAST = C.RTN_ANYCAST - RTN_MULTICAST = C.RTN_MULTICAST - RTN_BLACKHOLE = C.RTN_BLACKHOLE - RTN_UNREACHABLE = C.RTN_UNREACHABLE - RTN_PROHIBIT = C.RTN_PROHIBIT - RTN_THROW = C.RTN_THROW - RTN_NAT = C.RTN_NAT - RTN_XRESOLVE = C.RTN_XRESOLVE - RTNLGRP_NONE = C.RTNLGRP_NONE - RTNLGRP_LINK = C.RTNLGRP_LINK - RTNLGRP_NOTIFY = C.RTNLGRP_NOTIFY - RTNLGRP_NEIGH = C.RTNLGRP_NEIGH - RTNLGRP_TC = C.RTNLGRP_TC - RTNLGRP_IPV4_IFADDR = C.RTNLGRP_IPV4_IFADDR - RTNLGRP_IPV4_MROUTE = C.RTNLGRP_IPV4_MROUTE - RTNLGRP_IPV4_ROUTE = C.RTNLGRP_IPV4_ROUTE - RTNLGRP_IPV4_RULE = C.RTNLGRP_IPV4_RULE - RTNLGRP_IPV6_IFADDR = C.RTNLGRP_IPV6_IFADDR - RTNLGRP_IPV6_MROUTE = C.RTNLGRP_IPV6_MROUTE - RTNLGRP_IPV6_ROUTE = C.RTNLGRP_IPV6_ROUTE - RTNLGRP_IPV6_IFINFO = C.RTNLGRP_IPV6_IFINFO - RTNLGRP_IPV6_PREFIX = C.RTNLGRP_IPV6_PREFIX - RTNLGRP_IPV6_RULE = C.RTNLGRP_IPV6_RULE - RTNLGRP_ND_USEROPT = C.RTNLGRP_ND_USEROPT - SizeofNlMsghdr = C.sizeof_struct_nlmsghdr - SizeofNlMsgerr = C.sizeof_struct_nlmsgerr - SizeofRtGenmsg = C.sizeof_struct_rtgenmsg - SizeofNlAttr = C.sizeof_struct_nlattr - SizeofRtAttr = C.sizeof_struct_rtattr - SizeofIfInfomsg = C.sizeof_struct_ifinfomsg - SizeofIfAddrmsg = C.sizeof_struct_ifaddrmsg - SizeofRtMsg = C.sizeof_struct_rtmsg - SizeofRtNexthop = C.sizeof_struct_rtnexthop -) - -type NlMsghdr C.struct_nlmsghdr - -type NlMsgerr C.struct_nlmsgerr - -type RtGenmsg C.struct_rtgenmsg - -type NlAttr C.struct_nlattr - -type RtAttr C.struct_rtattr - -type IfInfomsg C.struct_ifinfomsg - -type IfAddrmsg C.struct_ifaddrmsg - -type RtMsg C.struct_rtmsg - -type RtNexthop C.struct_rtnexthop - -// Linux socket filter - -const ( - SizeofSockFilter = C.sizeof_struct_sock_filter - SizeofSockFprog = C.sizeof_struct_sock_fprog -) - -type SockFilter C.struct_sock_filter - -type SockFprog C.struct_sock_fprog - -// Inotify - -type InotifyEvent C.struct_inotify_event - -const SizeofInotifyEvent = C.sizeof_struct_inotify_event - -// Ptrace - -// Register structures -type PtraceRegs C.PtraceRegs - -// Structures contained in PtraceRegs on s390x (exported by mkpost.go) -type ptracePsw C.ptracePsw - -type ptraceFpregs C.ptraceFpregs - -type ptracePer C.ptracePer - -// Misc - -type FdSet C.fd_set - -type Sysinfo_t C.struct_sysinfo - -type Utsname C.struct_utsname - -type Ustat_t C.struct_ustat - -type EpollEvent C.struct_my_epoll_event - -const ( - AT_FDCWD = C.AT_FDCWD - AT_REMOVEDIR = C.AT_REMOVEDIR - AT_SYMLINK_FOLLOW = C.AT_SYMLINK_FOLLOW - AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW -) - -type PollFd C.struct_pollfd - -const ( - POLLIN = C.POLLIN - POLLPRI = C.POLLPRI - POLLOUT = C.POLLOUT - POLLRDHUP = C.POLLRDHUP - POLLERR = C.POLLERR - POLLHUP = C.POLLHUP - POLLNVAL = C.POLLNVAL -) - -type Sigset_t C.sigset_t - -// Terminal handling - -type Termios C.termios_t diff --git a/components/engine/vendor/golang.org/x/sys/unix/types_netbsd.go b/components/engine/vendor/golang.org/x/sys/unix/types_netbsd.go deleted file mode 100644 index d15f93d192..0000000000 --- a/components/engine/vendor/golang.org/x/sys/unix/types_netbsd.go +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -/* -Input to cgo -godefs. See also mkerrors.sh and mkall.sh -*/ - -// +godefs map struct_in_addr [4]byte /* in_addr */ -// +godefs map struct_in6_addr [16]byte /* in6_addr */ - -package unix - -/* -#define KERNEL -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum { - sizeofPtr = sizeof(void*), -}; - -union sockaddr_all { - struct sockaddr s1; // this one gets used for fields - struct sockaddr_in s2; // these pad it out - struct sockaddr_in6 s3; - struct sockaddr_un s4; - struct sockaddr_dl s5; -}; - -struct sockaddr_any { - struct sockaddr addr; - char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; -}; - -*/ -import "C" - -// Machine characteristics; for internal use. - -const ( - sizeofPtr = C.sizeofPtr - sizeofShort = C.sizeof_short - sizeofInt = C.sizeof_int - sizeofLong = C.sizeof_long - sizeofLongLong = C.sizeof_longlong -) - -// Basic types - -type ( - _C_short C.short - _C_int C.int - _C_long C.long - _C_long_long C.longlong -) - -// Time - -type Timespec C.struct_timespec - -type Timeval C.struct_timeval - -// Processes - -type Rusage C.struct_rusage - -type Rlimit C.struct_rlimit - -type _Gid_t C.gid_t - -// Files - -type Stat_t C.struct_stat - -type Statfs_t C.struct_statfs - -type Flock_t C.struct_flock - -type Dirent C.struct_dirent - -type Fsid C.fsid_t - -// Sockets - -type RawSockaddrInet4 C.struct_sockaddr_in - -type RawSockaddrInet6 C.struct_sockaddr_in6 - -type RawSockaddrUnix C.struct_sockaddr_un - -type RawSockaddrDatalink C.struct_sockaddr_dl - -type RawSockaddr C.struct_sockaddr - -type RawSockaddrAny C.struct_sockaddr_any - -type _Socklen C.socklen_t - -type Linger C.struct_linger - -type Iovec C.struct_iovec - -type IPMreq C.struct_ip_mreq - -type IPv6Mreq C.struct_ipv6_mreq - -type Msghdr C.struct_msghdr - -type Cmsghdr C.struct_cmsghdr - -type Inet6Pktinfo C.struct_in6_pktinfo - -type IPv6MTUInfo C.struct_ip6_mtuinfo - -type ICMPv6Filter C.struct_icmp6_filter - -const ( - SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in - SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 - SizeofSockaddrAny = C.sizeof_struct_sockaddr_any - SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un - SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl - SizeofLinger = C.sizeof_struct_linger - SizeofIPMreq = C.sizeof_struct_ip_mreq - SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq - SizeofMsghdr = C.sizeof_struct_msghdr - SizeofCmsghdr = C.sizeof_struct_cmsghdr - SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo - SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo - SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter -) - -// Ptrace requests - -const ( - PTRACE_TRACEME = C.PT_TRACE_ME - PTRACE_CONT = C.PT_CONTINUE - PTRACE_KILL = C.PT_KILL -) - -// Events (kqueue, kevent) - -type Kevent_t C.struct_kevent - -// Select - -type FdSet C.fd_set - -// Routing and interface messages - -const ( - SizeofIfMsghdr = C.sizeof_struct_if_msghdr - SizeofIfData = C.sizeof_struct_if_data - SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr - SizeofIfAnnounceMsghdr = C.sizeof_struct_if_announcemsghdr - SizeofRtMsghdr = C.sizeof_struct_rt_msghdr - SizeofRtMetrics = C.sizeof_struct_rt_metrics -) - -type IfMsghdr C.struct_if_msghdr - -type IfData C.struct_if_data - -type IfaMsghdr C.struct_ifa_msghdr - -type IfAnnounceMsghdr C.struct_if_announcemsghdr - -type RtMsghdr C.struct_rt_msghdr - -type RtMetrics C.struct_rt_metrics - -type Mclpool C.struct_mclpool - -// Berkeley packet filter - -const ( - SizeofBpfVersion = C.sizeof_struct_bpf_version - SizeofBpfStat = C.sizeof_struct_bpf_stat - SizeofBpfProgram = C.sizeof_struct_bpf_program - SizeofBpfInsn = C.sizeof_struct_bpf_insn - SizeofBpfHdr = C.sizeof_struct_bpf_hdr -) - -type BpfVersion C.struct_bpf_version - -type BpfStat C.struct_bpf_stat - -type BpfProgram C.struct_bpf_program - -type BpfInsn C.struct_bpf_insn - -type BpfHdr C.struct_bpf_hdr - -type BpfTimeval C.struct_bpf_timeval - -// Terminal handling - -type Termios C.struct_termios - -// Sysctl - -type Sysctlnode C.struct_sysctlnode diff --git a/components/engine/vendor/golang.org/x/sys/unix/types_openbsd.go b/components/engine/vendor/golang.org/x/sys/unix/types_openbsd.go deleted file mode 100644 index b66fe25f73..0000000000 --- a/components/engine/vendor/golang.org/x/sys/unix/types_openbsd.go +++ /dev/null @@ -1,244 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -/* -Input to cgo -godefs. See also mkerrors.sh and mkall.sh -*/ - -// +godefs map struct_in_addr [4]byte /* in_addr */ -// +godefs map struct_in6_addr [16]byte /* in6_addr */ - -package unix - -/* -#define KERNEL -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum { - sizeofPtr = sizeof(void*), -}; - -union sockaddr_all { - struct sockaddr s1; // this one gets used for fields - struct sockaddr_in s2; // these pad it out - struct sockaddr_in6 s3; - struct sockaddr_un s4; - struct sockaddr_dl s5; -}; - -struct sockaddr_any { - struct sockaddr addr; - char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; -}; - -*/ -import "C" - -// Machine characteristics; for internal use. - -const ( - sizeofPtr = C.sizeofPtr - sizeofShort = C.sizeof_short - sizeofInt = C.sizeof_int - sizeofLong = C.sizeof_long - sizeofLongLong = C.sizeof_longlong -) - -// Basic types - -type ( - _C_short C.short - _C_int C.int - _C_long C.long - _C_long_long C.longlong -) - -// Time - -type Timespec C.struct_timespec - -type Timeval C.struct_timeval - -// Processes - -type Rusage C.struct_rusage - -type Rlimit C.struct_rlimit - -type _Gid_t C.gid_t - -// Files - -const ( // Directory mode bits - S_IFMT = C.S_IFMT - S_IFIFO = C.S_IFIFO - S_IFCHR = C.S_IFCHR - S_IFDIR = C.S_IFDIR - S_IFBLK = C.S_IFBLK - S_IFREG = C.S_IFREG - S_IFLNK = C.S_IFLNK - S_IFSOCK = C.S_IFSOCK - S_ISUID = C.S_ISUID - S_ISGID = C.S_ISGID - S_ISVTX = C.S_ISVTX - S_IRUSR = C.S_IRUSR - S_IWUSR = C.S_IWUSR - S_IXUSR = C.S_IXUSR -) - -type Stat_t C.struct_stat - -type Statfs_t C.struct_statfs - -type Flock_t C.struct_flock - -type Dirent C.struct_dirent - -type Fsid C.fsid_t - -// Sockets - -type RawSockaddrInet4 C.struct_sockaddr_in - -type RawSockaddrInet6 C.struct_sockaddr_in6 - -type RawSockaddrUnix C.struct_sockaddr_un - -type RawSockaddrDatalink C.struct_sockaddr_dl - -type RawSockaddr C.struct_sockaddr - -type RawSockaddrAny C.struct_sockaddr_any - -type _Socklen C.socklen_t - -type Linger C.struct_linger - -type Iovec C.struct_iovec - -type IPMreq C.struct_ip_mreq - -type IPv6Mreq C.struct_ipv6_mreq - -type Msghdr C.struct_msghdr - -type Cmsghdr C.struct_cmsghdr - -type Inet6Pktinfo C.struct_in6_pktinfo - -type IPv6MTUInfo C.struct_ip6_mtuinfo - -type ICMPv6Filter C.struct_icmp6_filter - -const ( - SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in - SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 - SizeofSockaddrAny = C.sizeof_struct_sockaddr_any - SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un - SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl - SizeofLinger = C.sizeof_struct_linger - SizeofIPMreq = C.sizeof_struct_ip_mreq - SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq - SizeofMsghdr = C.sizeof_struct_msghdr - SizeofCmsghdr = C.sizeof_struct_cmsghdr - SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo - SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo - SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter -) - -// Ptrace requests - -const ( - PTRACE_TRACEME = C.PT_TRACE_ME - PTRACE_CONT = C.PT_CONTINUE - PTRACE_KILL = C.PT_KILL -) - -// Events (kqueue, kevent) - -type Kevent_t C.struct_kevent - -// Select - -type FdSet C.fd_set - -// Routing and interface messages - -const ( - SizeofIfMsghdr = C.sizeof_struct_if_msghdr - SizeofIfData = C.sizeof_struct_if_data - SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr - SizeofIfAnnounceMsghdr = C.sizeof_struct_if_announcemsghdr - SizeofRtMsghdr = C.sizeof_struct_rt_msghdr - SizeofRtMetrics = C.sizeof_struct_rt_metrics -) - -type IfMsghdr C.struct_if_msghdr - -type IfData C.struct_if_data - -type IfaMsghdr C.struct_ifa_msghdr - -type IfAnnounceMsghdr C.struct_if_announcemsghdr - -type RtMsghdr C.struct_rt_msghdr - -type RtMetrics C.struct_rt_metrics - -type Mclpool C.struct_mclpool - -// Berkeley packet filter - -const ( - SizeofBpfVersion = C.sizeof_struct_bpf_version - SizeofBpfStat = C.sizeof_struct_bpf_stat - SizeofBpfProgram = C.sizeof_struct_bpf_program - SizeofBpfInsn = C.sizeof_struct_bpf_insn - SizeofBpfHdr = C.sizeof_struct_bpf_hdr -) - -type BpfVersion C.struct_bpf_version - -type BpfStat C.struct_bpf_stat - -type BpfProgram C.struct_bpf_program - -type BpfInsn C.struct_bpf_insn - -type BpfHdr C.struct_bpf_hdr - -type BpfTimeval C.struct_bpf_timeval - -// Terminal handling - -type Termios C.struct_termios diff --git a/components/engine/vendor/golang.org/x/sys/unix/types_solaris.go b/components/engine/vendor/golang.org/x/sys/unix/types_solaris.go deleted file mode 100644 index c5d5c8f16a..0000000000 --- a/components/engine/vendor/golang.org/x/sys/unix/types_solaris.go +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -/* -Input to cgo -godefs. See also mkerrors.sh and mkall.sh -*/ - -// +godefs map struct_in_addr [4]byte /* in_addr */ -// +godefs map struct_in6_addr [16]byte /* in6_addr */ - -package unix - -/* -#define KERNEL -// These defines ensure that builds done on newer versions of Solaris are -// backwards-compatible with older versions of Solaris and -// OpenSolaris-based derivatives. -#define __USE_SUNOS_SOCKETS__ // msghdr -#define __USE_LEGACY_PROTOTYPES__ // iovec -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum { - sizeofPtr = sizeof(void*), -}; - -union sockaddr_all { - struct sockaddr s1; // this one gets used for fields - struct sockaddr_in s2; // these pad it out - struct sockaddr_in6 s3; - struct sockaddr_un s4; - struct sockaddr_dl s5; -}; - -struct sockaddr_any { - struct sockaddr addr; - char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; -}; - -*/ -import "C" - -// Machine characteristics; for internal use. - -const ( - sizeofPtr = C.sizeofPtr - sizeofShort = C.sizeof_short - sizeofInt = C.sizeof_int - sizeofLong = C.sizeof_long - sizeofLongLong = C.sizeof_longlong - PathMax = C.PATH_MAX - MaxHostNameLen = C.MAXHOSTNAMELEN -) - -// Basic types - -type ( - _C_short C.short - _C_int C.int - _C_long C.long - _C_long_long C.longlong -) - -// Time - -type Timespec C.struct_timespec - -type Timeval C.struct_timeval - -type Timeval32 C.struct_timeval32 - -type Tms C.struct_tms - -type Utimbuf C.struct_utimbuf - -// Processes - -type Rusage C.struct_rusage - -type Rlimit C.struct_rlimit - -type _Gid_t C.gid_t - -// Files - -const ( // Directory mode bits - S_IFMT = C.S_IFMT - S_IFIFO = C.S_IFIFO - S_IFCHR = C.S_IFCHR - S_IFDIR = C.S_IFDIR - S_IFBLK = C.S_IFBLK - S_IFREG = C.S_IFREG - S_IFLNK = C.S_IFLNK - S_IFSOCK = C.S_IFSOCK - S_ISUID = C.S_ISUID - S_ISGID = C.S_ISGID - S_ISVTX = C.S_ISVTX - S_IRUSR = C.S_IRUSR - S_IWUSR = C.S_IWUSR - S_IXUSR = C.S_IXUSR -) - -type Stat_t C.struct_stat - -type Flock_t C.struct_flock - -type Dirent C.struct_dirent - -// Sockets - -type RawSockaddrInet4 C.struct_sockaddr_in - -type RawSockaddrInet6 C.struct_sockaddr_in6 - -type RawSockaddrUnix C.struct_sockaddr_un - -type RawSockaddrDatalink C.struct_sockaddr_dl - -type RawSockaddr C.struct_sockaddr - -type RawSockaddrAny C.struct_sockaddr_any - -type _Socklen C.socklen_t - -type Linger C.struct_linger - -type Iovec C.struct_iovec - -type IPMreq C.struct_ip_mreq - -type IPv6Mreq C.struct_ipv6_mreq - -type Msghdr C.struct_msghdr - -type Cmsghdr C.struct_cmsghdr - -type Inet6Pktinfo C.struct_in6_pktinfo - -type IPv6MTUInfo C.struct_ip6_mtuinfo - -type ICMPv6Filter C.struct_icmp6_filter - -const ( - SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in - SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 - SizeofSockaddrAny = C.sizeof_struct_sockaddr_any - SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un - SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl - SizeofLinger = C.sizeof_struct_linger - SizeofIPMreq = C.sizeof_struct_ip_mreq - SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq - SizeofMsghdr = C.sizeof_struct_msghdr - SizeofCmsghdr = C.sizeof_struct_cmsghdr - SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo - SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo - SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter -) - -// Select - -type FdSet C.fd_set - -// Misc - -type Utsname C.struct_utsname - -type Ustat_t C.struct_ustat - -const ( - AT_FDCWD = C.AT_FDCWD - AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW - AT_SYMLINK_FOLLOW = C.AT_SYMLINK_FOLLOW - AT_REMOVEDIR = C.AT_REMOVEDIR - AT_EACCESS = C.AT_EACCESS -) - -// Routing and interface messages - -const ( - SizeofIfMsghdr = C.sizeof_struct_if_msghdr - SizeofIfData = C.sizeof_struct_if_data - SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr - SizeofRtMsghdr = C.sizeof_struct_rt_msghdr - SizeofRtMetrics = C.sizeof_struct_rt_metrics -) - -type IfMsghdr C.struct_if_msghdr - -type IfData C.struct_if_data - -type IfaMsghdr C.struct_ifa_msghdr - -type RtMsghdr C.struct_rt_msghdr - -type RtMetrics C.struct_rt_metrics - -// Berkeley packet filter - -const ( - SizeofBpfVersion = C.sizeof_struct_bpf_version - SizeofBpfStat = C.sizeof_struct_bpf_stat - SizeofBpfProgram = C.sizeof_struct_bpf_program - SizeofBpfInsn = C.sizeof_struct_bpf_insn - SizeofBpfHdr = C.sizeof_struct_bpf_hdr -) - -type BpfVersion C.struct_bpf_version - -type BpfStat C.struct_bpf_stat - -type BpfProgram C.struct_bpf_program - -type BpfInsn C.struct_bpf_insn - -type BpfTimeval C.struct_bpf_timeval - -type BpfHdr C.struct_bpf_hdr - -// sysconf information - -const _SC_PAGESIZE = C._SC_PAGESIZE - -// Terminal handling - -type Termios C.struct_termios - -type Termio C.struct_termio - -type Winsize C.struct_winsize diff --git a/components/engine/vendor/golang.org/x/text/internal/gen/code.go b/components/engine/vendor/golang.org/x/text/internal/gen/code.go deleted file mode 100644 index d7031b6945..0000000000 --- a/components/engine/vendor/golang.org/x/text/internal/gen/code.go +++ /dev/null @@ -1,351 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package gen - -import ( - "bytes" - "encoding/gob" - "fmt" - "hash" - "hash/fnv" - "io" - "log" - "os" - "reflect" - "strings" - "unicode" - "unicode/utf8" -) - -// This file contains utilities for generating code. - -// TODO: other write methods like: -// - slices, maps, types, etc. - -// CodeWriter is a utility for writing structured code. It computes the content -// hash and size of written content. It ensures there are newlines between -// written code blocks. -type CodeWriter struct { - buf bytes.Buffer - Size int - Hash hash.Hash32 // content hash - gob *gob.Encoder - // For comments we skip the usual one-line separator if they are followed by - // a code block. - skipSep bool -} - -func (w *CodeWriter) Write(p []byte) (n int, err error) { - return w.buf.Write(p) -} - -// NewCodeWriter returns a new CodeWriter. -func NewCodeWriter() *CodeWriter { - h := fnv.New32() - return &CodeWriter{Hash: h, gob: gob.NewEncoder(h)} -} - -// WriteGoFile appends the buffer with the total size of all created structures -// and writes it as a Go file to the the given file with the given package name. -func (w *CodeWriter) WriteGoFile(filename, pkg string) { - f, err := os.Create(filename) - if err != nil { - log.Fatalf("Could not create file %s: %v", filename, err) - } - defer f.Close() - if _, err = w.WriteGo(f, pkg); err != nil { - log.Fatalf("Error writing file %s: %v", filename, err) - } -} - -// WriteGo appends the buffer with the total size of all created structures and -// writes it as a Go file to the the given writer with the given package name. -func (w *CodeWriter) WriteGo(out io.Writer, pkg string) (n int, err error) { - sz := w.Size - w.WriteComment("Total table size %d bytes (%dKiB); checksum: %X\n", sz, sz/1024, w.Hash.Sum32()) - defer w.buf.Reset() - return WriteGo(out, pkg, w.buf.Bytes()) -} - -func (w *CodeWriter) printf(f string, x ...interface{}) { - fmt.Fprintf(w, f, x...) -} - -func (w *CodeWriter) insertSep() { - if w.skipSep { - w.skipSep = false - return - } - // Use at least two newlines to ensure a blank space between the previous - // block. WriteGoFile will remove extraneous newlines. - w.printf("\n\n") -} - -// WriteComment writes a comment block. All line starts are prefixed with "//". -// Initial empty lines are gobbled. The indentation for the first line is -// stripped from consecutive lines. -func (w *CodeWriter) WriteComment(comment string, args ...interface{}) { - s := fmt.Sprintf(comment, args...) - s = strings.Trim(s, "\n") - - // Use at least two newlines to ensure a blank space between the previous - // block. WriteGoFile will remove extraneous newlines. - w.printf("\n\n// ") - w.skipSep = true - - // strip first indent level. - sep := "\n" - for ; len(s) > 0 && (s[0] == '\t' || s[0] == ' '); s = s[1:] { - sep += s[:1] - } - - strings.NewReplacer(sep, "\n// ", "\n", "\n// ").WriteString(w, s) - - w.printf("\n") -} - -func (w *CodeWriter) writeSizeInfo(size int) { - w.printf("// Size: %d bytes\n", size) -} - -// WriteConst writes a constant of the given name and value. -func (w *CodeWriter) WriteConst(name string, x interface{}) { - w.insertSep() - v := reflect.ValueOf(x) - - switch v.Type().Kind() { - case reflect.String: - w.printf("const %s %s = ", name, typeName(x)) - w.WriteString(v.String()) - w.printf("\n") - default: - w.printf("const %s = %#v\n", name, x) - } -} - -// WriteVar writes a variable of the given name and value. -func (w *CodeWriter) WriteVar(name string, x interface{}) { - w.insertSep() - v := reflect.ValueOf(x) - oldSize := w.Size - sz := int(v.Type().Size()) - w.Size += sz - - switch v.Type().Kind() { - case reflect.String: - w.printf("var %s %s = ", name, typeName(x)) - w.WriteString(v.String()) - case reflect.Struct: - w.gob.Encode(x) - fallthrough - case reflect.Slice, reflect.Array: - w.printf("var %s = ", name) - w.writeValue(v) - w.writeSizeInfo(w.Size - oldSize) - default: - w.printf("var %s %s = ", name, typeName(x)) - w.gob.Encode(x) - w.writeValue(v) - w.writeSizeInfo(w.Size - oldSize) - } - w.printf("\n") -} - -func (w *CodeWriter) writeValue(v reflect.Value) { - x := v.Interface() - switch v.Kind() { - case reflect.String: - w.WriteString(v.String()) - case reflect.Array: - // Don't double count: callers of WriteArray count on the size being - // added, so we need to discount it here. - w.Size -= int(v.Type().Size()) - w.writeSlice(x, true) - case reflect.Slice: - w.writeSlice(x, false) - case reflect.Struct: - w.printf("%s{\n", typeName(v.Interface())) - t := v.Type() - for i := 0; i < v.NumField(); i++ { - w.printf("%s: ", t.Field(i).Name) - w.writeValue(v.Field(i)) - w.printf(",\n") - } - w.printf("}") - default: - w.printf("%#v", x) - } -} - -// WriteString writes a string literal. -func (w *CodeWriter) WriteString(s string) { - s = strings.Replace(s, `\`, `\\`, -1) - io.WriteString(w.Hash, s) // content hash - w.Size += len(s) - - const maxInline = 40 - if len(s) <= maxInline { - w.printf("%q", s) - return - } - - // We will render the string as a multi-line string. - const maxWidth = 80 - 4 - len(`"`) - len(`" +`) - - // When starting on its own line, go fmt indents line 2+ an extra level. - n, max := maxWidth, maxWidth-4 - - // As per https://golang.org/issue/18078, the compiler has trouble - // compiling the concatenation of many strings, s0 + s1 + s2 + ... + sN, - // for large N. We insert redundant, explicit parentheses to work around - // that, lowering the N at any given step: (s0 + s1 + ... + s63) + (s64 + - // ... + s127) + etc + (etc + ... + sN). - explicitParens, extraComment := len(s) > 128*1024, "" - if explicitParens { - w.printf(`(`) - extraComment = "; the redundant, explicit parens are for https://golang.org/issue/18078" - } - - // Print "" +\n, if a string does not start on its own line. - b := w.buf.Bytes() - if p := len(bytes.TrimRight(b, " \t")); p > 0 && b[p-1] != '\n' { - w.printf("\"\" + // Size: %d bytes%s\n", len(s), extraComment) - n, max = maxWidth, maxWidth - } - - w.printf(`"`) - - for sz, p, nLines := 0, 0, 0; p < len(s); { - var r rune - r, sz = utf8.DecodeRuneInString(s[p:]) - out := s[p : p+sz] - chars := 1 - if !unicode.IsPrint(r) || r == utf8.RuneError || r == '"' { - switch sz { - case 1: - out = fmt.Sprintf("\\x%02x", s[p]) - case 2, 3: - out = fmt.Sprintf("\\u%04x", r) - case 4: - out = fmt.Sprintf("\\U%08x", r) - } - chars = len(out) - } - if n -= chars; n < 0 { - nLines++ - if explicitParens && nLines&63 == 63 { - w.printf("\") + (\"") - } - w.printf("\" +\n\"") - n = max - len(out) - } - w.printf("%s", out) - p += sz - } - w.printf(`"`) - if explicitParens { - w.printf(`)`) - } -} - -// WriteSlice writes a slice value. -func (w *CodeWriter) WriteSlice(x interface{}) { - w.writeSlice(x, false) -} - -// WriteArray writes an array value. -func (w *CodeWriter) WriteArray(x interface{}) { - w.writeSlice(x, true) -} - -func (w *CodeWriter) writeSlice(x interface{}, isArray bool) { - v := reflect.ValueOf(x) - w.gob.Encode(v.Len()) - w.Size += v.Len() * int(v.Type().Elem().Size()) - name := typeName(x) - if isArray { - name = fmt.Sprintf("[%d]%s", v.Len(), name[strings.Index(name, "]")+1:]) - } - if isArray { - w.printf("%s{\n", name) - } else { - w.printf("%s{ // %d elements\n", name, v.Len()) - } - - switch kind := v.Type().Elem().Kind(); kind { - case reflect.String: - for _, s := range x.([]string) { - w.WriteString(s) - w.printf(",\n") - } - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - // nLine and nBlock are the number of elements per line and block. - nLine, nBlock, format := 8, 64, "%d," - switch kind { - case reflect.Uint8: - format = "%#02x," - case reflect.Uint16: - format = "%#04x," - case reflect.Uint32: - nLine, nBlock, format = 4, 32, "%#08x," - case reflect.Uint, reflect.Uint64: - nLine, nBlock, format = 4, 32, "%#016x," - case reflect.Int8: - nLine = 16 - } - n := nLine - for i := 0; i < v.Len(); i++ { - if i%nBlock == 0 && v.Len() > nBlock { - w.printf("// Entry %X - %X\n", i, i+nBlock-1) - } - x := v.Index(i).Interface() - w.gob.Encode(x) - w.printf(format, x) - if n--; n == 0 { - n = nLine - w.printf("\n") - } - } - w.printf("\n") - case reflect.Struct: - zero := reflect.Zero(v.Type().Elem()).Interface() - for i := 0; i < v.Len(); i++ { - x := v.Index(i).Interface() - w.gob.EncodeValue(v) - if !reflect.DeepEqual(zero, x) { - line := fmt.Sprintf("%#v,\n", x) - line = line[strings.IndexByte(line, '{'):] - w.printf("%d: ", i) - w.printf(line) - } - } - case reflect.Array: - for i := 0; i < v.Len(); i++ { - w.printf("%d: %#v,\n", i, v.Index(i).Interface()) - } - default: - panic("gen: slice elem type not supported") - } - w.printf("}") -} - -// WriteType writes a definition of the type of the given value and returns the -// type name. -func (w *CodeWriter) WriteType(x interface{}) string { - t := reflect.TypeOf(x) - w.printf("type %s struct {\n", t.Name()) - for i := 0; i < t.NumField(); i++ { - w.printf("\t%s %s\n", t.Field(i).Name, t.Field(i).Type) - } - w.printf("}\n") - return t.Name() -} - -// typeName returns the name of the go type of x. -func typeName(x interface{}) string { - t := reflect.ValueOf(x).Type() - return strings.Replace(fmt.Sprint(t), "main.", "", 1) -} diff --git a/components/engine/vendor/golang.org/x/text/internal/gen/gen.go b/components/engine/vendor/golang.org/x/text/internal/gen/gen.go deleted file mode 100644 index 84c699faa9..0000000000 --- a/components/engine/vendor/golang.org/x/text/internal/gen/gen.go +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package gen contains common code for the various code generation tools in the -// text repository. Its usage ensures consistency between tools. -// -// This package defines command line flags that are common to most generation -// tools. The flags allow for specifying specific Unicode and CLDR versions -// in the public Unicode data repository (http://www.unicode.org/Public). -// -// A local Unicode data mirror can be set through the flag -local or the -// environment variable UNICODE_DIR. The former takes precedence. The local -// directory should follow the same structure as the public repository. -// -// IANA data can also optionally be mirrored by putting it in the iana directory -// rooted at the top of the local mirror. Beware, though, that IANA data is not -// versioned. So it is up to the developer to use the right version. -package gen // import "golang.org/x/text/internal/gen" - -import ( - "bytes" - "flag" - "fmt" - "go/build" - "go/format" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "path" - "path/filepath" - "sync" - "unicode" - - "golang.org/x/text/unicode/cldr" -) - -var ( - url = flag.String("url", - "http://www.unicode.org/Public", - "URL of Unicode database directory") - iana = flag.String("iana", - "http://www.iana.org", - "URL of the IANA repository") - unicodeVersion = flag.String("unicode", - getEnv("UNICODE_VERSION", unicode.Version), - "unicode version to use") - cldrVersion = flag.String("cldr", - getEnv("CLDR_VERSION", cldr.Version), - "cldr version to use") -) - -func getEnv(name, def string) string { - if v := os.Getenv(name); v != "" { - return v - } - return def -} - -// Init performs common initialization for a gen command. It parses the flags -// and sets up the standard logging parameters. -func Init() { - log.SetPrefix("") - log.SetFlags(log.Lshortfile) - flag.Parse() -} - -const header = `// This file was generated by go generate; DO NOT EDIT - -package %s - -` - -// UnicodeVersion reports the requested Unicode version. -func UnicodeVersion() string { - return *unicodeVersion -} - -// UnicodeVersion reports the requested CLDR version. -func CLDRVersion() string { - return *cldrVersion -} - -// IsLocal reports whether data files are available locally. -func IsLocal() bool { - dir, err := localReadmeFile() - if err != nil { - return false - } - if _, err = os.Stat(dir); err != nil { - return false - } - return true -} - -// OpenUCDFile opens the requested UCD file. The file is specified relative to -// the public Unicode root directory. It will call log.Fatal if there are any -// errors. -func OpenUCDFile(file string) io.ReadCloser { - return openUnicode(path.Join(*unicodeVersion, "ucd", file)) -} - -// OpenCLDRCoreZip opens the CLDR core zip file. It will call log.Fatal if there -// are any errors. -func OpenCLDRCoreZip() io.ReadCloser { - return OpenUnicodeFile("cldr", *cldrVersion, "core.zip") -} - -// OpenUnicodeFile opens the requested file of the requested category from the -// root of the Unicode data archive. The file is specified relative to the -// public Unicode root directory. If version is "", it will use the default -// Unicode version. It will call log.Fatal if there are any errors. -func OpenUnicodeFile(category, version, file string) io.ReadCloser { - if version == "" { - version = UnicodeVersion() - } - return openUnicode(path.Join(category, version, file)) -} - -// OpenIANAFile opens the requested IANA file. The file is specified relative -// to the IANA root, which is typically either http://www.iana.org or the -// iana directory in the local mirror. It will call log.Fatal if there are any -// errors. -func OpenIANAFile(path string) io.ReadCloser { - return Open(*iana, "iana", path) -} - -var ( - dirMutex sync.Mutex - localDir string -) - -const permissions = 0755 - -func localReadmeFile() (string, error) { - p, err := build.Import("golang.org/x/text", "", build.FindOnly) - if err != nil { - return "", fmt.Errorf("Could not locate package: %v", err) - } - return filepath.Join(p.Dir, "DATA", "README"), nil -} - -func getLocalDir() string { - dirMutex.Lock() - defer dirMutex.Unlock() - - readme, err := localReadmeFile() - if err != nil { - log.Fatal(err) - } - dir := filepath.Dir(readme) - if _, err := os.Stat(readme); err != nil { - if err := os.MkdirAll(dir, permissions); err != nil { - log.Fatalf("Could not create directory: %v", err) - } - ioutil.WriteFile(readme, []byte(readmeTxt), permissions) - } - return dir -} - -const readmeTxt = `Generated by golang.org/x/text/internal/gen. DO NOT EDIT. - -This directory contains downloaded files used to generate the various tables -in the golang.org/x/text subrepo. - -Note that the language subtag repo (iana/assignments/language-subtag-registry) -and all other times in the iana subdirectory are not versioned and will need -to be periodically manually updated. The easiest way to do this is to remove -the entire iana directory. This is mostly of concern when updating the language -package. -` - -// Open opens subdir/path if a local directory is specified and the file exists, -// where subdir is a directory relative to the local root, or fetches it from -// urlRoot/path otherwise. It will call log.Fatal if there are any errors. -func Open(urlRoot, subdir, path string) io.ReadCloser { - file := filepath.Join(getLocalDir(), subdir, filepath.FromSlash(path)) - return open(file, urlRoot, path) -} - -func openUnicode(path string) io.ReadCloser { - file := filepath.Join(getLocalDir(), filepath.FromSlash(path)) - return open(file, *url, path) -} - -// TODO: automatically periodically update non-versioned files. - -func open(file, urlRoot, path string) io.ReadCloser { - if f, err := os.Open(file); err == nil { - return f - } - r := get(urlRoot, path) - defer r.Close() - b, err := ioutil.ReadAll(r) - if err != nil { - log.Fatalf("Could not download file: %v", err) - } - os.MkdirAll(filepath.Dir(file), permissions) - if err := ioutil.WriteFile(file, b, permissions); err != nil { - log.Fatalf("Could not create file: %v", err) - } - return ioutil.NopCloser(bytes.NewReader(b)) -} - -func get(root, path string) io.ReadCloser { - url := root + "/" + path - fmt.Printf("Fetching %s...", url) - defer fmt.Println(" done.") - resp, err := http.Get(url) - if err != nil { - log.Fatalf("HTTP GET: %v", err) - } - if resp.StatusCode != 200 { - log.Fatalf("Bad GET status for %q: %q", url, resp.Status) - } - return resp.Body -} - -// TODO: use Write*Version in all applicable packages. - -// WriteUnicodeVersion writes a constant for the Unicode version from which the -// tables are generated. -func WriteUnicodeVersion(w io.Writer) { - fmt.Fprintf(w, "// UnicodeVersion is the Unicode version from which the tables in this package are derived.\n") - fmt.Fprintf(w, "const UnicodeVersion = %q\n\n", UnicodeVersion()) -} - -// WriteCLDRVersion writes a constant for the CLDR version from which the -// tables are generated. -func WriteCLDRVersion(w io.Writer) { - fmt.Fprintf(w, "// CLDRVersion is the CLDR version from which the tables in this package are derived.\n") - fmt.Fprintf(w, "const CLDRVersion = %q\n\n", CLDRVersion()) -} - -// WriteGoFile prepends a standard file comment and package statement to the -// given bytes, applies gofmt, and writes them to a file with the given name. -// It will call log.Fatal if there are any errors. -func WriteGoFile(filename, pkg string, b []byte) { - w, err := os.Create(filename) - if err != nil { - log.Fatalf("Could not create file %s: %v", filename, err) - } - defer w.Close() - if _, err = WriteGo(w, pkg, b); err != nil { - log.Fatalf("Error writing file %s: %v", filename, err) - } -} - -// WriteGo prepends a standard file comment and package statement to the given -// bytes, applies gofmt, and writes them to w. -func WriteGo(w io.Writer, pkg string, b []byte) (n int, err error) { - src := []byte(fmt.Sprintf(header, pkg)) - src = append(src, b...) - formatted, err := format.Source(src) - if err != nil { - // Print the generated code even in case of an error so that the - // returned error can be meaningfully interpreted. - n, _ = w.Write(src) - return n, err - } - return w.Write(formatted) -} - -// Repackage rewrites a Go file from belonging to package main to belonging to -// the given package. -func Repackage(inFile, outFile, pkg string) { - src, err := ioutil.ReadFile(inFile) - if err != nil { - log.Fatalf("reading %s: %v", inFile, err) - } - const toDelete = "package main\n\n" - i := bytes.Index(src, []byte(toDelete)) - if i < 0 { - log.Fatalf("Could not find %q in %s.", toDelete, inFile) - } - w := &bytes.Buffer{} - w.Write(src[i+len(toDelete):]) - WriteGoFile(outFile, pkg, w.Bytes()) -} diff --git a/components/engine/vendor/golang.org/x/text/internal/triegen/compact.go b/components/engine/vendor/golang.org/x/text/internal/triegen/compact.go deleted file mode 100644 index 397b975c1b..0000000000 --- a/components/engine/vendor/golang.org/x/text/internal/triegen/compact.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package triegen - -// This file defines Compacter and its implementations. - -import "io" - -// A Compacter generates an alternative, more space-efficient way to store a -// trie value block. A trie value block holds all possible values for the last -// byte of a UTF-8 encoded rune. Excluding ASCII characters, a trie value block -// always has 64 values, as a UTF-8 encoding ends with a byte in [0x80, 0xC0). -type Compacter interface { - // Size returns whether the Compacter could encode the given block as well - // as its size in case it can. len(v) is always 64. - Size(v []uint64) (sz int, ok bool) - - // Store stores the block using the Compacter's compression method. - // It returns a handle with which the block can be retrieved. - // len(v) is always 64. - Store(v []uint64) uint32 - - // Print writes the data structures associated to the given store to w. - Print(w io.Writer) error - - // Handler returns the name of a function that gets called during trie - // lookup for blocks generated by the Compacter. The function should be of - // the form func (n uint32, b byte) uint64, where n is the index returned by - // the Compacter's Store method and b is the last byte of the UTF-8 - // encoding, where 0x80 <= b < 0xC0, for which to do the lookup in the - // block. - Handler() string -} - -// simpleCompacter is the default Compacter used by builder. It implements a -// normal trie block. -type simpleCompacter builder - -func (b *simpleCompacter) Size([]uint64) (sz int, ok bool) { - return blockSize * b.ValueSize, true -} - -func (b *simpleCompacter) Store(v []uint64) uint32 { - h := uint32(len(b.ValueBlocks) - blockOffset) - b.ValueBlocks = append(b.ValueBlocks, v) - return h -} - -func (b *simpleCompacter) Print(io.Writer) error { - // Structures are printed in print.go. - return nil -} - -func (b *simpleCompacter) Handler() string { - panic("Handler should be special-cased for this Compacter") -} diff --git a/components/engine/vendor/golang.org/x/text/internal/triegen/print.go b/components/engine/vendor/golang.org/x/text/internal/triegen/print.go deleted file mode 100644 index 8d9f120bcd..0000000000 --- a/components/engine/vendor/golang.org/x/text/internal/triegen/print.go +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package triegen - -import ( - "bytes" - "fmt" - "io" - "strings" - "text/template" -) - -// print writes all the data structures as well as the code necessary to use the -// trie to w. -func (b *builder) print(w io.Writer) error { - b.Stats.NValueEntries = len(b.ValueBlocks) * blockSize - b.Stats.NValueBytes = len(b.ValueBlocks) * blockSize * b.ValueSize - b.Stats.NIndexEntries = len(b.IndexBlocks) * blockSize - b.Stats.NIndexBytes = len(b.IndexBlocks) * blockSize * b.IndexSize - b.Stats.NHandleBytes = len(b.Trie) * 2 * b.IndexSize - - // If we only have one root trie, all starter blocks are at position 0 and - // we can access the arrays directly. - if len(b.Trie) == 1 { - // At this point we cannot refer to the generated tables directly. - b.ASCIIBlock = b.Name + "Values" - b.StarterBlock = b.Name + "Index" - } else { - // Otherwise we need to have explicit starter indexes in the trie - // structure. - b.ASCIIBlock = "t.ascii" - b.StarterBlock = "t.utf8Start" - } - - b.SourceType = "[]byte" - if err := lookupGen.Execute(w, b); err != nil { - return err - } - - b.SourceType = "string" - if err := lookupGen.Execute(w, b); err != nil { - return err - } - - if err := trieGen.Execute(w, b); err != nil { - return err - } - - for _, c := range b.Compactions { - if err := c.c.Print(w); err != nil { - return err - } - } - - return nil -} - -func printValues(n int, values []uint64) string { - w := &bytes.Buffer{} - boff := n * blockSize - fmt.Fprintf(w, "\t// Block %#x, offset %#x", n, boff) - var newline bool - for i, v := range values { - if i%6 == 0 { - newline = true - } - if v != 0 { - if newline { - fmt.Fprintf(w, "\n") - newline = false - } - fmt.Fprintf(w, "\t%#02x:%#04x, ", boff+i, v) - } - } - return w.String() -} - -func printIndex(b *builder, nr int, n *node) string { - w := &bytes.Buffer{} - boff := nr * blockSize - fmt.Fprintf(w, "\t// Block %#x, offset %#x", nr, boff) - var newline bool - for i, c := range n.children { - if i%8 == 0 { - newline = true - } - if c != nil { - v := b.Compactions[c.index.compaction].Offset + uint32(c.index.index) - if v != 0 { - if newline { - fmt.Fprintf(w, "\n") - newline = false - } - fmt.Fprintf(w, "\t%#02x:%#02x, ", boff+i, v) - } - } - } - return w.String() -} - -var ( - trieGen = template.Must(template.New("trie").Funcs(template.FuncMap{ - "printValues": printValues, - "printIndex": printIndex, - "title": strings.Title, - "dec": func(x int) int { return x - 1 }, - "psize": func(n int) string { - return fmt.Sprintf("%d bytes (%.2f KiB)", n, float64(n)/1024) - }, - }).Parse(trieTemplate)) - lookupGen = template.Must(template.New("lookup").Parse(lookupTemplate)) -) - -// TODO: consider the return type of lookup. It could be uint64, even if the -// internal value type is smaller. We will have to verify this with the -// performance of unicode/norm, which is very sensitive to such changes. -const trieTemplate = `{{$b := .}}{{$multi := gt (len .Trie) 1}} -// {{.Name}}Trie. Total size: {{psize .Size}}. Checksum: {{printf "%08x" .Checksum}}. -type {{.Name}}Trie struct { {{if $multi}} - ascii []{{.ValueType}} // index for ASCII bytes - utf8Start []{{.IndexType}} // index for UTF-8 bytes >= 0xC0 -{{end}}} - -func new{{title .Name}}Trie(i int) *{{.Name}}Trie { {{if $multi}} - h := {{.Name}}TrieHandles[i] - return &{{.Name}}Trie{ {{.Name}}Values[uint32(h.ascii)<<6:], {{.Name}}Index[uint32(h.multi)<<6:] } -} - -type {{.Name}}TrieHandle struct { - ascii, multi {{.IndexType}} -} - -// {{.Name}}TrieHandles: {{len .Trie}} handles, {{.Stats.NHandleBytes}} bytes -var {{.Name}}TrieHandles = [{{len .Trie}}]{{.Name}}TrieHandle{ -{{range .Trie}} { {{.ASCIIIndex}}, {{.StarterIndex}} }, // {{printf "%08x" .Checksum}}: {{.Name}} -{{end}}}{{else}} - return &{{.Name}}Trie{} -} -{{end}} -// lookupValue determines the type of block n and looks up the value for b. -func (t *{{.Name}}Trie) lookupValue(n uint32, b byte) {{.ValueType}}{{$last := dec (len .Compactions)}} { - switch { {{range $i, $c := .Compactions}} - {{if eq $i $last}}default{{else}}case n < {{$c.Cutoff}}{{end}}:{{if ne $i 0}} - n -= {{$c.Offset}}{{end}} - return {{print $b.ValueType}}({{$c.Handler}}){{end}} - } -} - -// {{.Name}}Values: {{len .ValueBlocks}} blocks, {{.Stats.NValueEntries}} entries, {{.Stats.NValueBytes}} bytes -// The third block is the zero block. -var {{.Name}}Values = [{{.Stats.NValueEntries}}]{{.ValueType}} { -{{range $i, $v := .ValueBlocks}}{{printValues $i $v}} -{{end}}} - -// {{.Name}}Index: {{len .IndexBlocks}} blocks, {{.Stats.NIndexEntries}} entries, {{.Stats.NIndexBytes}} bytes -// Block 0 is the zero block. -var {{.Name}}Index = [{{.Stats.NIndexEntries}}]{{.IndexType}} { -{{range $i, $v := .IndexBlocks}}{{printIndex $b $i $v}} -{{end}}} -` - -// TODO: consider allowing zero-length strings after evaluating performance with -// unicode/norm. -const lookupTemplate = ` -// lookup{{if eq .SourceType "string"}}String{{end}} returns the trie value for the first UTF-8 encoding in s and -// the width in bytes of this encoding. The size will be 0 if s does not -// hold enough bytes to complete the encoding. len(s) must be greater than 0. -func (t *{{.Name}}Trie) lookup{{if eq .SourceType "string"}}String{{end}}(s {{.SourceType}}) (v {{.ValueType}}, sz int) { - c0 := s[0] - switch { - case c0 < 0x80: // is ASCII - return {{.ASCIIBlock}}[c0], 1 - case c0 < 0xC2: - return 0, 1 // Illegal UTF-8: not a starter, not ASCII. - case c0 < 0xE0: // 2-byte UTF-8 - if len(s) < 2 { - return 0, 0 - } - i := {{.StarterBlock}}[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c1), 2 - case c0 < 0xF0: // 3-byte UTF-8 - if len(s) < 3 { - return 0, 0 - } - i := {{.StarterBlock}}[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = {{.Name}}Index[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c2), 3 - case c0 < 0xF8: // 4-byte UTF-8 - if len(s) < 4 { - return 0, 0 - } - i := {{.StarterBlock}}[c0] - c1 := s[1] - if c1 < 0x80 || 0xC0 <= c1 { - return 0, 1 // Illegal UTF-8: not a continuation byte. - } - o := uint32(i)<<6 + uint32(c1) - i = {{.Name}}Index[o] - c2 := s[2] - if c2 < 0x80 || 0xC0 <= c2 { - return 0, 2 // Illegal UTF-8: not a continuation byte. - } - o = uint32(i)<<6 + uint32(c2) - i = {{.Name}}Index[o] - c3 := s[3] - if c3 < 0x80 || 0xC0 <= c3 { - return 0, 3 // Illegal UTF-8: not a continuation byte. - } - return t.lookupValue(uint32(i), c3), 4 - } - // Illegal rune - return 0, 1 -} - -// lookup{{if eq .SourceType "string"}}String{{end}}Unsafe returns the trie value for the first UTF-8 encoding in s. -// s must start with a full and valid UTF-8 encoded rune. -func (t *{{.Name}}Trie) lookup{{if eq .SourceType "string"}}String{{end}}Unsafe(s {{.SourceType}}) {{.ValueType}} { - c0 := s[0] - if c0 < 0x80 { // is ASCII - return {{.ASCIIBlock}}[c0] - } - i := {{.StarterBlock}}[c0] - if c0 < 0xE0 { // 2-byte UTF-8 - return t.lookupValue(uint32(i), s[1]) - } - i = {{.Name}}Index[uint32(i)<<6+uint32(s[1])] - if c0 < 0xF0 { // 3-byte UTF-8 - return t.lookupValue(uint32(i), s[2]) - } - i = {{.Name}}Index[uint32(i)<<6+uint32(s[2])] - if c0 < 0xF8 { // 4-byte UTF-8 - return t.lookupValue(uint32(i), s[3]) - } - return 0 -} -` diff --git a/components/engine/vendor/golang.org/x/text/internal/triegen/triegen.go b/components/engine/vendor/golang.org/x/text/internal/triegen/triegen.go deleted file mode 100644 index adb0108124..0000000000 --- a/components/engine/vendor/golang.org/x/text/internal/triegen/triegen.go +++ /dev/null @@ -1,494 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package triegen implements a code generator for a trie for associating -// unsigned integer values with UTF-8 encoded runes. -// -// Many of the go.text packages use tries for storing per-rune information. A -// trie is especially useful if many of the runes have the same value. If this -// is the case, many blocks can be expected to be shared allowing for -// information on many runes to be stored in little space. -// -// As most of the lookups are done directly on []byte slices, the tries use the -// UTF-8 bytes directly for the lookup. This saves a conversion from UTF-8 to -// runes and contributes a little bit to better performance. It also naturally -// provides a fast path for ASCII. -// -// Space is also an issue. There are many code points defined in Unicode and as -// a result tables can get quite large. So every byte counts. The triegen -// package automatically chooses the smallest integer values to represent the -// tables. Compacters allow further compression of the trie by allowing for -// alternative representations of individual trie blocks. -// -// triegen allows generating multiple tries as a single structure. This is -// useful when, for example, one wants to generate tries for several languages -// that have a lot of values in common. Some existing libraries for -// internationalization store all per-language data as a dynamically loadable -// chunk. The go.text packages are designed with the assumption that the user -// typically wants to compile in support for all supported languages, in line -// with the approach common to Go to create a single standalone binary. The -// multi-root trie approach can give significant storage savings in this -// scenario. -// -// triegen generates both tables and code. The code is optimized to use the -// automatically chosen data types. The following code is generated for a Trie -// or multiple Tries named "foo": -// - type fooTrie -// The trie type. -// -// - func newFooTrie(x int) *fooTrie -// Trie constructor, where x is the index of the trie passed to Gen. -// -// - func (t *fooTrie) lookup(s []byte) (v uintX, sz int) -// The lookup method, where uintX is automatically chosen. -// -// - func lookupString, lookupUnsafe and lookupStringUnsafe -// Variants of the above. -// -// - var fooValues and fooIndex and any tables generated by Compacters. -// The core trie data. -// -// - var fooTrieHandles -// Indexes of starter blocks in case of multiple trie roots. -// -// It is recommended that users test the generated trie by checking the returned -// value for every rune. Such exhaustive tests are possible as the the number of -// runes in Unicode is limited. -package triegen // import "golang.org/x/text/internal/triegen" - -// TODO: Arguably, the internally optimized data types would not have to be -// exposed in the generated API. We could also investigate not generating the -// code, but using it through a package. We would have to investigate the impact -// on performance of making such change, though. For packages like unicode/norm, -// small changes like this could tank performance. - -import ( - "encoding/binary" - "fmt" - "hash/crc64" - "io" - "log" - "unicode/utf8" -) - -// builder builds a set of tries for associating values with runes. The set of -// tries can share common index and value blocks. -type builder struct { - Name string - - // ValueType is the type of the trie values looked up. - ValueType string - - // ValueSize is the byte size of the ValueType. - ValueSize int - - // IndexType is the type of trie index values used for all UTF-8 bytes of - // a rune except the last one. - IndexType string - - // IndexSize is the byte size of the IndexType. - IndexSize int - - // SourceType is used when generating the lookup functions. If the user - // requests StringSupport, all lookup functions will be generated for - // string input as well. - SourceType string - - Trie []*Trie - - IndexBlocks []*node - ValueBlocks [][]uint64 - Compactions []compaction - Checksum uint64 - - ASCIIBlock string - StarterBlock string - - indexBlockIdx map[uint64]int - valueBlockIdx map[uint64]nodeIndex - asciiBlockIdx map[uint64]int - - // Stats are used to fill out the template. - Stats struct { - NValueEntries int - NValueBytes int - NIndexEntries int - NIndexBytes int - NHandleBytes int - } - - err error -} - -// A nodeIndex encodes the index of a node, which is defined by the compaction -// which stores it and an index within the compaction. For internal nodes, the -// compaction is always 0. -type nodeIndex struct { - compaction int - index int -} - -// compaction keeps track of stats used for the compaction. -type compaction struct { - c Compacter - blocks []*node - maxHandle uint32 - totalSize int - - // Used by template-based generator and thus exported. - Cutoff uint32 - Offset uint32 - Handler string -} - -func (b *builder) setError(err error) { - if b.err == nil { - b.err = err - } -} - -// An Option can be passed to Gen. -type Option func(b *builder) error - -// Compact configures the trie generator to use the given Compacter. -func Compact(c Compacter) Option { - return func(b *builder) error { - b.Compactions = append(b.Compactions, compaction{ - c: c, - Handler: c.Handler() + "(n, b)"}) - return nil - } -} - -// Gen writes Go code for a shared trie lookup structure to w for the given -// Tries. The generated trie type will be called nameTrie. newNameTrie(x) will -// return the *nameTrie for tries[x]. A value can be looked up by using one of -// the various lookup methods defined on nameTrie. It returns the table size of -// the generated trie. -func Gen(w io.Writer, name string, tries []*Trie, opts ...Option) (sz int, err error) { - // The index contains two dummy blocks, followed by the zero block. The zero - // block is at offset 0x80, so that the offset for the zero block for - // continuation bytes is 0. - b := &builder{ - Name: name, - Trie: tries, - IndexBlocks: []*node{{}, {}, {}}, - Compactions: []compaction{{ - Handler: name + "Values[n<<6+uint32(b)]", - }}, - // The 0 key in indexBlockIdx and valueBlockIdx is the hash of the zero - // block. - indexBlockIdx: map[uint64]int{0: 0}, - valueBlockIdx: map[uint64]nodeIndex{0: {}}, - asciiBlockIdx: map[uint64]int{}, - } - b.Compactions[0].c = (*simpleCompacter)(b) - - for _, f := range opts { - if err := f(b); err != nil { - return 0, err - } - } - b.build() - if b.err != nil { - return 0, b.err - } - if err = b.print(w); err != nil { - return 0, err - } - return b.Size(), nil -} - -// A Trie represents a single root node of a trie. A builder may build several -// overlapping tries at once. -type Trie struct { - root *node - - hiddenTrie -} - -// hiddenTrie contains values we want to be visible to the template generator, -// but hidden from the API documentation. -type hiddenTrie struct { - Name string - Checksum uint64 - ASCIIIndex int - StarterIndex int -} - -// NewTrie returns a new trie root. -func NewTrie(name string) *Trie { - return &Trie{ - &node{ - children: make([]*node, blockSize), - values: make([]uint64, utf8.RuneSelf), - }, - hiddenTrie{Name: name}, - } -} - -// Gen is a convenience wrapper around the Gen func passing t as the only trie -// and uses the name passed to NewTrie. It returns the size of the generated -// tables. -func (t *Trie) Gen(w io.Writer, opts ...Option) (sz int, err error) { - return Gen(w, t.Name, []*Trie{t}, opts...) -} - -// node is a node of the intermediate trie structure. -type node struct { - // children holds this node's children. It is always of length 64. - // A child node may be nil. - children []*node - - // values contains the values of this node. If it is non-nil, this node is - // either a root or leaf node: - // For root nodes, len(values) == 128 and it maps the bytes in [0x00, 0x7F]. - // For leaf nodes, len(values) == 64 and it maps the bytes in [0x80, 0xBF]. - values []uint64 - - index nodeIndex -} - -// Insert associates value with the given rune. Insert will panic if a non-zero -// value is passed for an invalid rune. -func (t *Trie) Insert(r rune, value uint64) { - if value == 0 { - return - } - s := string(r) - if []rune(s)[0] != r && value != 0 { - // Note: The UCD tables will always assign what amounts to a zero value - // to a surrogate. Allowing a zero value for an illegal rune allows - // users to iterate over [0..MaxRune] without having to explicitly - // exclude surrogates, which would be tedious. - panic(fmt.Sprintf("triegen: non-zero value for invalid rune %U", r)) - } - if len(s) == 1 { - // It is a root node value (ASCII). - t.root.values[s[0]] = value - return - } - - n := t.root - for ; len(s) > 1; s = s[1:] { - if n.children == nil { - n.children = make([]*node, blockSize) - } - p := s[0] % blockSize - c := n.children[p] - if c == nil { - c = &node{} - n.children[p] = c - } - if len(s) > 2 && c.values != nil { - log.Fatalf("triegen: insert(%U): found internal node with values", r) - } - n = c - } - if n.values == nil { - n.values = make([]uint64, blockSize) - } - if n.children != nil { - log.Fatalf("triegen: insert(%U): found leaf node that also has child nodes", r) - } - n.values[s[0]-0x80] = value -} - -// Size returns the number of bytes the generated trie will take to store. It -// needs to be exported as it is used in the templates. -func (b *builder) Size() int { - // Index blocks. - sz := len(b.IndexBlocks) * blockSize * b.IndexSize - - // Skip the first compaction, which represents the normal value blocks, as - // its totalSize does not account for the ASCII blocks, which are managed - // separately. - sz += len(b.ValueBlocks) * blockSize * b.ValueSize - for _, c := range b.Compactions[1:] { - sz += c.totalSize - } - - // TODO: this computation does not account for the fixed overhead of a using - // a compaction, either code or data. As for data, though, the typical - // overhead of data is in the order of bytes (2 bytes for cases). Further, - // the savings of using a compaction should anyway be substantial for it to - // be worth it. - - // For multi-root tries, we also need to account for the handles. - if len(b.Trie) > 1 { - sz += 2 * b.IndexSize * len(b.Trie) - } - return sz -} - -func (b *builder) build() { - // Compute the sizes of the values. - var vmax uint64 - for _, t := range b.Trie { - vmax = maxValue(t.root, vmax) - } - b.ValueType, b.ValueSize = getIntType(vmax) - - // Compute all block allocations. - // TODO: first compute the ASCII blocks for all tries and then the other - // nodes. ASCII blocks are more restricted in placement, as they require two - // blocks to be placed consecutively. Processing them first may improve - // sharing (at least one zero block can be expected to be saved.) - for _, t := range b.Trie { - b.Checksum += b.buildTrie(t) - } - - // Compute the offsets for all the Compacters. - offset := uint32(0) - for i := range b.Compactions { - c := &b.Compactions[i] - c.Offset = offset - offset += c.maxHandle + 1 - c.Cutoff = offset - } - - // Compute the sizes of indexes. - // TODO: different byte positions could have different sizes. So far we have - // not found a case where this is beneficial. - imax := uint64(b.Compactions[len(b.Compactions)-1].Cutoff) - for _, ib := range b.IndexBlocks { - if x := uint64(ib.index.index); x > imax { - imax = x - } - } - b.IndexType, b.IndexSize = getIntType(imax) -} - -func maxValue(n *node, max uint64) uint64 { - if n == nil { - return max - } - for _, c := range n.children { - max = maxValue(c, max) - } - for _, v := range n.values { - if max < v { - max = v - } - } - return max -} - -func getIntType(v uint64) (string, int) { - switch { - case v < 1<<8: - return "uint8", 1 - case v < 1<<16: - return "uint16", 2 - case v < 1<<32: - return "uint32", 4 - } - return "uint64", 8 -} - -const ( - blockSize = 64 - - // Subtract two blocks to offset 0x80, the first continuation byte. - blockOffset = 2 - - // Subtract three blocks to offset 0xC0, the first non-ASCII starter. - rootBlockOffset = 3 -) - -var crcTable = crc64.MakeTable(crc64.ISO) - -func (b *builder) buildTrie(t *Trie) uint64 { - n := t.root - - // Get the ASCII offset. For the first trie, the ASCII block will be at - // position 0. - hasher := crc64.New(crcTable) - binary.Write(hasher, binary.BigEndian, n.values) - hash := hasher.Sum64() - - v, ok := b.asciiBlockIdx[hash] - if !ok { - v = len(b.ValueBlocks) - b.asciiBlockIdx[hash] = v - - b.ValueBlocks = append(b.ValueBlocks, n.values[:blockSize], n.values[blockSize:]) - if v == 0 { - // Add the zero block at position 2 so that it will be assigned a - // zero reference in the lookup blocks. - // TODO: always do this? This would allow us to remove a check from - // the trie lookup, but at the expense of extra space. Analyze - // performance for unicode/norm. - b.ValueBlocks = append(b.ValueBlocks, make([]uint64, blockSize)) - } - } - t.ASCIIIndex = v - - // Compute remaining offsets. - t.Checksum = b.computeOffsets(n, true) - // We already subtracted the normal blockOffset from the index. Subtract the - // difference for starter bytes. - t.StarterIndex = n.index.index - (rootBlockOffset - blockOffset) - return t.Checksum -} - -func (b *builder) computeOffsets(n *node, root bool) uint64 { - // For the first trie, the root lookup block will be at position 3, which is - // the offset for UTF-8 non-ASCII starter bytes. - first := len(b.IndexBlocks) == rootBlockOffset - if first { - b.IndexBlocks = append(b.IndexBlocks, n) - } - - // We special-case the cases where all values recursively are 0. This allows - // for the use of a zero block to which all such values can be directed. - hash := uint64(0) - if n.children != nil || n.values != nil { - hasher := crc64.New(crcTable) - for _, c := range n.children { - var v uint64 - if c != nil { - v = b.computeOffsets(c, false) - } - binary.Write(hasher, binary.BigEndian, v) - } - binary.Write(hasher, binary.BigEndian, n.values) - hash = hasher.Sum64() - } - - if first { - b.indexBlockIdx[hash] = rootBlockOffset - blockOffset - } - - // Compacters don't apply to internal nodes. - if n.children != nil { - v, ok := b.indexBlockIdx[hash] - if !ok { - v = len(b.IndexBlocks) - blockOffset - b.IndexBlocks = append(b.IndexBlocks, n) - b.indexBlockIdx[hash] = v - } - n.index = nodeIndex{0, v} - } else { - h, ok := b.valueBlockIdx[hash] - if !ok { - bestI, bestSize := 0, blockSize*b.ValueSize - for i, c := range b.Compactions[1:] { - if sz, ok := c.c.Size(n.values); ok && bestSize > sz { - bestI, bestSize = i+1, sz - } - } - c := &b.Compactions[bestI] - c.totalSize += bestSize - v := c.c.Store(n.values) - if c.maxHandle < v { - c.maxHandle = v - } - h = nodeIndex{bestI, int(v)} - b.valueBlockIdx[hash] = h - } - n.index = h - } - return hash -} diff --git a/components/engine/vendor/golang.org/x/text/internal/ucd/ucd.go b/components/engine/vendor/golang.org/x/text/internal/ucd/ucd.go deleted file mode 100644 index 309e8d8b16..0000000000 --- a/components/engine/vendor/golang.org/x/text/internal/ucd/ucd.go +++ /dev/null @@ -1,376 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package ucd provides a parser for Unicode Character Database files, the -// format of which is defined in http://www.unicode.org/reports/tr44/. See -// http://www.unicode.org/Public/UCD/latest/ucd/ for example files. -// -// It currently does not support substitutions of missing fields. -package ucd // import "golang.org/x/text/internal/ucd" - -import ( - "bufio" - "bytes" - "errors" - "io" - "log" - "regexp" - "strconv" - "strings" -) - -// UnicodeData.txt fields. -const ( - CodePoint = iota - Name - GeneralCategory - CanonicalCombiningClass - BidiClass - DecompMapping - DecimalValue - DigitValue - NumericValue - BidiMirrored - Unicode1Name - ISOComment - SimpleUppercaseMapping - SimpleLowercaseMapping - SimpleTitlecaseMapping -) - -// Parse calls f for each entry in the given reader of a UCD file. It will close -// the reader upon return. It will call log.Fatal if any error occurred. -// -// This implements the most common usage pattern of using Parser. -func Parse(r io.ReadCloser, f func(p *Parser)) { - defer r.Close() - - p := New(r) - for p.Next() { - f(p) - } - if err := p.Err(); err != nil { - r.Close() // os.Exit will cause defers not to be called. - log.Fatal(err) - } -} - -// An Option is used to configure a Parser. -type Option func(p *Parser) - -func keepRanges(p *Parser) { - p.keepRanges = true -} - -var ( - // KeepRanges prevents the expansion of ranges. The raw ranges can be - // obtained by calling Range(0) on the parser. - KeepRanges Option = keepRanges -) - -// The Part option register a handler for lines starting with a '@'. The text -// after a '@' is available as the first field. Comments are handled as usual. -func Part(f func(p *Parser)) Option { - return func(p *Parser) { - p.partHandler = f - } -} - -// The CommentHandler option passes comments that are on a line by itself to -// a given handler. -func CommentHandler(f func(s string)) Option { - return func(p *Parser) { - p.commentHandler = f - } -} - -// A Parser parses Unicode Character Database (UCD) files. -type Parser struct { - scanner *bufio.Scanner - - keepRanges bool // Don't expand rune ranges in field 0. - - err error - comment []byte - field [][]byte - // parsedRange is needed in case Range(0) is called more than once for one - // field. In some cases this requires scanning ahead. - parsedRange bool - rangeStart, rangeEnd rune - - partHandler func(p *Parser) - commentHandler func(s string) -} - -func (p *Parser) setError(err error) { - if p.err == nil { - p.err = err - } -} - -func (p *Parser) getField(i int) []byte { - if i >= len(p.field) { - return nil - } - return p.field[i] -} - -// Err returns a non-nil error if any error occurred during parsing. -func (p *Parser) Err() error { - return p.err -} - -// New returns a Parser for the given Reader. -func New(r io.Reader, o ...Option) *Parser { - p := &Parser{ - scanner: bufio.NewScanner(r), - } - for _, f := range o { - f(p) - } - return p -} - -// Next parses the next line in the file. It returns true if a line was parsed -// and false if it reached the end of the file. -func (p *Parser) Next() bool { - if !p.keepRanges && p.rangeStart < p.rangeEnd { - p.rangeStart++ - return true - } - p.comment = nil - p.field = p.field[:0] - p.parsedRange = false - - for p.scanner.Scan() { - b := p.scanner.Bytes() - if len(b) == 0 { - continue - } - if b[0] == '#' { - if p.commentHandler != nil { - p.commentHandler(strings.TrimSpace(string(b[1:]))) - } - continue - } - - // Parse line - if i := bytes.IndexByte(b, '#'); i != -1 { - p.comment = bytes.TrimSpace(b[i+1:]) - b = b[:i] - } - if b[0] == '@' { - if p.partHandler != nil { - p.field = append(p.field, bytes.TrimSpace(b[1:])) - p.partHandler(p) - p.field = p.field[:0] - } - p.comment = nil - continue - } - for { - i := bytes.IndexByte(b, ';') - if i == -1 { - p.field = append(p.field, bytes.TrimSpace(b)) - break - } - p.field = append(p.field, bytes.TrimSpace(b[:i])) - b = b[i+1:] - } - if !p.keepRanges { - p.rangeStart, p.rangeEnd = p.getRange(0) - } - return true - } - p.setError(p.scanner.Err()) - return false -} - -func parseRune(b []byte) (rune, error) { - if len(b) > 2 && b[0] == 'U' && b[1] == '+' { - b = b[2:] - } - x, err := strconv.ParseUint(string(b), 16, 32) - return rune(x), err -} - -func (p *Parser) parseRune(b []byte) rune { - x, err := parseRune(b) - p.setError(err) - return x -} - -// Rune parses and returns field i as a rune. -func (p *Parser) Rune(i int) rune { - if i > 0 || p.keepRanges { - return p.parseRune(p.getField(i)) - } - return p.rangeStart -} - -// Runes interprets and returns field i as a sequence of runes. -func (p *Parser) Runes(i int) (runes []rune) { - add := func(b []byte) { - if b = bytes.TrimSpace(b); len(b) > 0 { - runes = append(runes, p.parseRune(b)) - } - } - for b := p.getField(i); ; { - i := bytes.IndexByte(b, ' ') - if i == -1 { - add(b) - break - } - add(b[:i]) - b = b[i+1:] - } - return -} - -var ( - errIncorrectLegacyRange = errors.New("ucd: unmatched <* First>") - - // reRange matches one line of a legacy rune range. - reRange = regexp.MustCompile("^([0-9A-F]*);<([^,]*), ([^>]*)>(.*)$") -) - -// Range parses and returns field i as a rune range. A range is inclusive at -// both ends. If the field only has one rune, first and last will be identical. -// It supports the legacy format for ranges used in UnicodeData.txt. -func (p *Parser) Range(i int) (first, last rune) { - if !p.keepRanges { - return p.rangeStart, p.rangeStart - } - return p.getRange(i) -} - -func (p *Parser) getRange(i int) (first, last rune) { - b := p.getField(i) - if k := bytes.Index(b, []byte("..")); k != -1 { - return p.parseRune(b[:k]), p.parseRune(b[k+2:]) - } - // The first field may not be a rune, in which case we may ignore any error - // and set the range as 0..0. - x, err := parseRune(b) - if err != nil { - // Disable range parsing henceforth. This ensures that an error will be - // returned if the user subsequently will try to parse this field as - // a Rune. - p.keepRanges = true - } - // Special case for UnicodeData that was retained for backwards compatibility. - if i == 0 && len(p.field) > 1 && bytes.HasSuffix(p.field[1], []byte("First>")) { - if p.parsedRange { - return p.rangeStart, p.rangeEnd - } - mf := reRange.FindStringSubmatch(p.scanner.Text()) - if mf == nil || !p.scanner.Scan() { - p.setError(errIncorrectLegacyRange) - return x, x - } - // Using Bytes would be more efficient here, but Text is a lot easier - // and this is not a frequent case. - ml := reRange.FindStringSubmatch(p.scanner.Text()) - if ml == nil || mf[2] != ml[2] || ml[3] != "Last" || mf[4] != ml[4] { - p.setError(errIncorrectLegacyRange) - return x, x - } - p.rangeStart, p.rangeEnd = x, p.parseRune(p.scanner.Bytes()[:len(ml[1])]) - p.parsedRange = true - return p.rangeStart, p.rangeEnd - } - return x, x -} - -// bools recognizes all valid UCD boolean values. -var bools = map[string]bool{ - "": false, - "N": false, - "No": false, - "F": false, - "False": false, - "Y": true, - "Yes": true, - "T": true, - "True": true, -} - -// Bool parses and returns field i as a boolean value. -func (p *Parser) Bool(i int) bool { - b := p.getField(i) - for s, v := range bools { - if bstrEq(b, s) { - return v - } - } - p.setError(strconv.ErrSyntax) - return false -} - -// Int parses and returns field i as an integer value. -func (p *Parser) Int(i int) int { - x, err := strconv.ParseInt(string(p.getField(i)), 10, 64) - p.setError(err) - return int(x) -} - -// Uint parses and returns field i as an unsigned integer value. -func (p *Parser) Uint(i int) uint { - x, err := strconv.ParseUint(string(p.getField(i)), 10, 64) - p.setError(err) - return uint(x) -} - -// Float parses and returns field i as a decimal value. -func (p *Parser) Float(i int) float64 { - x, err := strconv.ParseFloat(string(p.getField(i)), 64) - p.setError(err) - return x -} - -// String parses and returns field i as a string value. -func (p *Parser) String(i int) string { - return string(p.getField(i)) -} - -// Strings parses and returns field i as a space-separated list of strings. -func (p *Parser) Strings(i int) []string { - ss := strings.Split(string(p.getField(i)), " ") - for i, s := range ss { - ss[i] = strings.TrimSpace(s) - } - return ss -} - -// Comment returns the comments for the current line. -func (p *Parser) Comment() string { - return string(p.comment) -} - -var errUndefinedEnum = errors.New("ucd: undefined enum value") - -// Enum interprets and returns field i as a value that must be one of the values -// in enum. -func (p *Parser) Enum(i int, enum ...string) string { - b := p.getField(i) - for _, s := range enum { - if bstrEq(b, s) { - return s - } - } - p.setError(errUndefinedEnum) - return "" -} - -func bstrEq(b []byte, s string) bool { - if len(b) != len(s) { - return false - } - for i, c := range b { - if c != s[i] { - return false - } - } - return true -} diff --git a/components/engine/vendor/golang.org/x/text/unicode/bidi/gen.go b/components/engine/vendor/golang.org/x/text/unicode/bidi/gen.go deleted file mode 100644 index 040f3013d5..0000000000 --- a/components/engine/vendor/golang.org/x/text/unicode/bidi/gen.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -import ( - "flag" - "log" - - "golang.org/x/text/internal/gen" - "golang.org/x/text/internal/triegen" - "golang.org/x/text/internal/ucd" -) - -var outputFile = flag.String("out", "tables.go", "output file") - -func main() { - gen.Init() - gen.Repackage("gen_trieval.go", "trieval.go", "bidi") - gen.Repackage("gen_ranges.go", "ranges_test.go", "bidi") - - genTables() -} - -// bidiClass names and codes taken from class "bc" in -// http://www.unicode.org/Public/8.0.0/ucd/PropertyValueAliases.txt -var bidiClass = map[string]Class{ - "AL": AL, // ArabicLetter - "AN": AN, // ArabicNumber - "B": B, // ParagraphSeparator - "BN": BN, // BoundaryNeutral - "CS": CS, // CommonSeparator - "EN": EN, // EuropeanNumber - "ES": ES, // EuropeanSeparator - "ET": ET, // EuropeanTerminator - "L": L, // LeftToRight - "NSM": NSM, // NonspacingMark - "ON": ON, // OtherNeutral - "R": R, // RightToLeft - "S": S, // SegmentSeparator - "WS": WS, // WhiteSpace - - "FSI": Control, - "PDF": Control, - "PDI": Control, - "LRE": Control, - "LRI": Control, - "LRO": Control, - "RLE": Control, - "RLI": Control, - "RLO": Control, -} - -func genTables() { - if numClass > 0x0F { - log.Fatalf("Too many Class constants (%#x > 0x0F).", numClass) - } - w := gen.NewCodeWriter() - defer w.WriteGoFile(*outputFile, "bidi") - - gen.WriteUnicodeVersion(w) - - t := triegen.NewTrie("bidi") - - // Build data about bracket mapping. These bits need to be or-ed with - // any other bits. - orMask := map[rune]uint64{} - - xorMap := map[rune]int{} - xorMasks := []rune{0} // First value is no-op. - - ucd.Parse(gen.OpenUCDFile("BidiBrackets.txt"), func(p *ucd.Parser) { - r1 := p.Rune(0) - r2 := p.Rune(1) - xor := r1 ^ r2 - if _, ok := xorMap[xor]; !ok { - xorMap[xor] = len(xorMasks) - xorMasks = append(xorMasks, xor) - } - entry := uint64(xorMap[xor]) << xorMaskShift - switch p.String(2) { - case "o": - entry |= openMask - case "c", "n": - default: - log.Fatalf("Unknown bracket class %q.", p.String(2)) - } - orMask[r1] = entry - }) - - w.WriteComment(` - xorMasks contains masks to be xor-ed with brackets to get the reverse - version.`) - w.WriteVar("xorMasks", xorMasks) - - done := map[rune]bool{} - - insert := func(r rune, c Class) { - if !done[r] { - t.Insert(r, orMask[r]|uint64(c)) - done[r] = true - } - } - - // Insert the derived BiDi properties. - ucd.Parse(gen.OpenUCDFile("extracted/DerivedBidiClass.txt"), func(p *ucd.Parser) { - r := p.Rune(0) - class, ok := bidiClass[p.String(1)] - if !ok { - log.Fatalf("%U: Unknown BiDi class %q", r, p.String(1)) - } - insert(r, class) - }) - visitDefaults(insert) - - // TODO: use sparse blocks. This would reduce table size considerably - // from the looks of it. - - sz, err := t.Gen(w) - if err != nil { - log.Fatal(err) - } - w.Size += sz -} - -// dummy values to make methods in gen_common compile. The real versions -// will be generated by this file to tables.go. -var ( - xorMasks []rune -) diff --git a/components/engine/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go b/components/engine/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go deleted file mode 100644 index 51bd68fa7f..0000000000 --- a/components/engine/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -import ( - "unicode" - - "golang.org/x/text/internal/gen" - "golang.org/x/text/internal/ucd" - "golang.org/x/text/unicode/rangetable" -) - -// These tables are hand-extracted from: -// http://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedBidiClass.txt -func visitDefaults(fn func(r rune, c Class)) { - // first write default values for ranges listed above. - visitRunes(fn, AL, []rune{ - 0x0600, 0x07BF, // Arabic - 0x08A0, 0x08FF, // Arabic Extended-A - 0xFB50, 0xFDCF, // Arabic Presentation Forms - 0xFDF0, 0xFDFF, - 0xFE70, 0xFEFF, - 0x0001EE00, 0x0001EEFF, // Arabic Mathematical Alpha Symbols - }) - visitRunes(fn, R, []rune{ - 0x0590, 0x05FF, // Hebrew - 0x07C0, 0x089F, // Nko et al. - 0xFB1D, 0xFB4F, - 0x00010800, 0x00010FFF, // Cypriot Syllabary et. al. - 0x0001E800, 0x0001EDFF, - 0x0001EF00, 0x0001EFFF, - }) - visitRunes(fn, ET, []rune{ // European Terminator - 0x20A0, 0x20Cf, // Currency symbols - }) - rangetable.Visit(unicode.Noncharacter_Code_Point, func(r rune) { - fn(r, BN) // Boundary Neutral - }) - ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) { - if p.String(1) == "Default_Ignorable_Code_Point" { - fn(p.Rune(0), BN) // Boundary Neutral - } - }) -} - -func visitRunes(fn func(r rune, c Class), c Class, runes []rune) { - for i := 0; i < len(runes); i += 2 { - lo, hi := runes[i], runes[i+1] - for j := lo; j <= hi; j++ { - fn(j, c) - } - } -} diff --git a/components/engine/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go b/components/engine/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go deleted file mode 100644 index 9cb9942894..0000000000 --- a/components/engine/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -// Class is the Unicode BiDi class. Each rune has a single class. -type Class uint - -const ( - L Class = iota // LeftToRight - R // RightToLeft - EN // EuropeanNumber - ES // EuropeanSeparator - ET // EuropeanTerminator - AN // ArabicNumber - CS // CommonSeparator - B // ParagraphSeparator - S // SegmentSeparator - WS // WhiteSpace - ON // OtherNeutral - BN // BoundaryNeutral - NSM // NonspacingMark - AL // ArabicLetter - Control // Control LRO - PDI - - numClass - - LRO // LeftToRightOverride - RLO // RightToLeftOverride - LRE // LeftToRightEmbedding - RLE // RightToLeftEmbedding - PDF // PopDirectionalFormat - LRI // LeftToRightIsolate - RLI // RightToLeftIsolate - FSI // FirstStrongIsolate - PDI // PopDirectionalIsolate - - unknownClass = ^Class(0) -) - -var controlToClass = map[rune]Class{ - 0x202D: LRO, // LeftToRightOverride, - 0x202E: RLO, // RightToLeftOverride, - 0x202A: LRE, // LeftToRightEmbedding, - 0x202B: RLE, // RightToLeftEmbedding, - 0x202C: PDF, // PopDirectionalFormat, - 0x2066: LRI, // LeftToRightIsolate, - 0x2067: RLI, // RightToLeftIsolate, - 0x2068: FSI, // FirstStrongIsolate, - 0x2069: PDI, // PopDirectionalIsolate, -} - -// A trie entry has the following bits: -// 7..5 XOR mask for brackets -// 4 1: Bracket open, 0: Bracket close -// 3..0 Class type - -const ( - openMask = 0x10 - xorMaskShift = 5 -) diff --git a/components/engine/vendor/golang.org/x/text/unicode/cldr/base.go b/components/engine/vendor/golang.org/x/text/unicode/cldr/base.go deleted file mode 100644 index 2382f4d6da..0000000000 --- a/components/engine/vendor/golang.org/x/text/unicode/cldr/base.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package cldr - -import ( - "encoding/xml" - "regexp" - "strconv" -) - -// Elem is implemented by every XML element. -type Elem interface { - setEnclosing(Elem) - setName(string) - enclosing() Elem - - GetCommon() *Common -} - -type hidden struct { - CharData string `xml:",chardata"` - Alias *struct { - Common - Source string `xml:"source,attr"` - Path string `xml:"path,attr"` - } `xml:"alias"` - Def *struct { - Common - Choice string `xml:"choice,attr,omitempty"` - Type string `xml:"type,attr,omitempty"` - } `xml:"default"` -} - -// Common holds several of the most common attributes and sub elements -// of an XML element. -type Common struct { - XMLName xml.Name - name string - enclElem Elem - Type string `xml:"type,attr,omitempty"` - Reference string `xml:"reference,attr,omitempty"` - Alt string `xml:"alt,attr,omitempty"` - ValidSubLocales string `xml:"validSubLocales,attr,omitempty"` - Draft string `xml:"draft,attr,omitempty"` - hidden -} - -// Default returns the default type to select from the enclosed list -// or "" if no default value is specified. -func (e *Common) Default() string { - if e.Def == nil { - return "" - } - if e.Def.Choice != "" { - return e.Def.Choice - } else if e.Def.Type != "" { - // Type is still used by the default element in collation. - return e.Def.Type - } - return "" -} - -// GetCommon returns e. It is provided such that Common implements Elem. -func (e *Common) GetCommon() *Common { - return e -} - -// Data returns the character data accumulated for this element. -func (e *Common) Data() string { - e.CharData = charRe.ReplaceAllStringFunc(e.CharData, replaceUnicode) - return e.CharData -} - -func (e *Common) setName(s string) { - e.name = s -} - -func (e *Common) enclosing() Elem { - return e.enclElem -} - -func (e *Common) setEnclosing(en Elem) { - e.enclElem = en -} - -// Escape characters that can be escaped without further escaping the string. -var charRe = regexp.MustCompile(`&#x[0-9a-fA-F]*;|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|\\x[0-9a-fA-F]{2}|\\[0-7]{3}|\\[abtnvfr]`) - -// replaceUnicode converts hexadecimal Unicode codepoint notations to a one-rune string. -// It assumes the input string is correctly formatted. -func replaceUnicode(s string) string { - if s[1] == '#' { - r, _ := strconv.ParseInt(s[3:len(s)-1], 16, 32) - return string(r) - } - r, _, _, _ := strconv.UnquoteChar(s, 0) - return string(r) -} diff --git a/components/engine/vendor/golang.org/x/text/unicode/cldr/cldr.go b/components/engine/vendor/golang.org/x/text/unicode/cldr/cldr.go deleted file mode 100644 index 2197f8ac26..0000000000 --- a/components/engine/vendor/golang.org/x/text/unicode/cldr/cldr.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:generate go run makexml.go -output xml.go - -// Package cldr provides a parser for LDML and related XML formats. -// This package is intended to be used by the table generation tools -// for the various internationalization-related packages. -// As the XML types are generated from the CLDR DTD, and as the CLDR standard -// is periodically amended, this package may change considerably over time. -// This mostly means that data may appear and disappear between versions. -// That is, old code should keep compiling for newer versions, but data -// may have moved or changed. -// CLDR version 22 is the first version supported by this package. -// Older versions may not work. -package cldr // import "golang.org/x/text/unicode/cldr" - -import ( - "fmt" - "sort" -) - -// CLDR provides access to parsed data of the Unicode Common Locale Data Repository. -type CLDR struct { - parent map[string][]string - locale map[string]*LDML - resolved map[string]*LDML - bcp47 *LDMLBCP47 - supp *SupplementalData -} - -func makeCLDR() *CLDR { - return &CLDR{ - parent: make(map[string][]string), - locale: make(map[string]*LDML), - resolved: make(map[string]*LDML), - bcp47: &LDMLBCP47{}, - supp: &SupplementalData{}, - } -} - -// BCP47 returns the parsed BCP47 LDML data. If no such data was parsed, nil is returned. -func (cldr *CLDR) BCP47() *LDMLBCP47 { - return nil -} - -// Draft indicates the draft level of an element. -type Draft int - -const ( - Approved Draft = iota - Contributed - Provisional - Unconfirmed -) - -var drafts = []string{"unconfirmed", "provisional", "contributed", "approved", ""} - -// ParseDraft returns the Draft value corresponding to the given string. The -// empty string corresponds to Approved. -func ParseDraft(level string) (Draft, error) { - if level == "" { - return Approved, nil - } - for i, s := range drafts { - if level == s { - return Unconfirmed - Draft(i), nil - } - } - return Approved, fmt.Errorf("cldr: unknown draft level %q", level) -} - -func (d Draft) String() string { - return drafts[len(drafts)-1-int(d)] -} - -// SetDraftLevel sets which draft levels to include in the evaluated LDML. -// Any draft element for which the draft level is higher than lev will be excluded. -// If multiple draft levels are available for a single element, the one with the -// lowest draft level will be selected, unless preferDraft is true, in which case -// the highest draft will be chosen. -// It is assumed that the underlying LDML is canonicalized. -func (cldr *CLDR) SetDraftLevel(lev Draft, preferDraft bool) { - // TODO: implement - cldr.resolved = make(map[string]*LDML) -} - -// RawLDML returns the LDML XML for id in unresolved form. -// id must be one of the strings returned by Locales. -func (cldr *CLDR) RawLDML(loc string) *LDML { - return cldr.locale[loc] -} - -// LDML returns the fully resolved LDML XML for loc, which must be one of -// the strings returned by Locales. -func (cldr *CLDR) LDML(loc string) (*LDML, error) { - return cldr.resolve(loc) -} - -// Supplemental returns the parsed supplemental data. If no such data was parsed, -// nil is returned. -func (cldr *CLDR) Supplemental() *SupplementalData { - return cldr.supp -} - -// Locales returns the locales for which there exist files. -// Valid sublocales for which there is no file are not included. -// The root locale is always sorted first. -func (cldr *CLDR) Locales() []string { - loc := []string{"root"} - hasRoot := false - for l, _ := range cldr.locale { - if l == "root" { - hasRoot = true - continue - } - loc = append(loc, l) - } - sort.Strings(loc[1:]) - if !hasRoot { - return loc[1:] - } - return loc -} - -// Get fills in the fields of x based on the XPath path. -func Get(e Elem, path string) (res Elem, err error) { - return walkXPath(e, path) -} diff --git a/components/engine/vendor/golang.org/x/text/unicode/cldr/collate.go b/components/engine/vendor/golang.org/x/text/unicode/cldr/collate.go deleted file mode 100644 index 80ee28d795..0000000000 --- a/components/engine/vendor/golang.org/x/text/unicode/cldr/collate.go +++ /dev/null @@ -1,359 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package cldr - -import ( - "bufio" - "encoding/xml" - "errors" - "fmt" - "strconv" - "strings" - "unicode" - "unicode/utf8" -) - -// RuleProcessor can be passed to Collator's Process method, which -// parses the rules and calls the respective method for each rule found. -type RuleProcessor interface { - Reset(anchor string, before int) error - Insert(level int, str, context, extend string) error - Index(id string) -} - -const ( - // cldrIndex is a Unicode-reserved sentinel value used to mark the start - // of a grouping within an index. - // We ignore any rule that starts with this rune. - // See http://unicode.org/reports/tr35/#Collation_Elements for details. - cldrIndex = "\uFDD0" - - // specialAnchor is the format in which to represent logical reset positions, - // such as "first tertiary ignorable". - specialAnchor = "<%s/>" -) - -// Process parses the rules for the tailorings of this collation -// and calls the respective methods of p for each rule found. -func (c Collation) Process(p RuleProcessor) (err error) { - if len(c.Cr) > 0 { - if len(c.Cr) > 1 { - return fmt.Errorf("multiple cr elements, want 0 or 1") - } - return processRules(p, c.Cr[0].Data()) - } - if c.Rules.Any != nil { - return c.processXML(p) - } - return errors.New("no tailoring data") -} - -// processRules parses rules in the Collation Rule Syntax defined in -// http://www.unicode.org/reports/tr35/tr35-collation.html#Collation_Tailorings. -func processRules(p RuleProcessor, s string) (err error) { - chk := func(s string, e error) string { - if err == nil { - err = e - } - return s - } - i := 0 // Save the line number for use after the loop. - scanner := bufio.NewScanner(strings.NewReader(s)) - for ; scanner.Scan() && err == nil; i++ { - for s := skipSpace(scanner.Text()); s != "" && s[0] != '#'; s = skipSpace(s) { - level := 5 - var ch byte - switch ch, s = s[0], s[1:]; ch { - case '&': // followed by or '[' ']' - if s = skipSpace(s); consume(&s, '[') { - s = chk(parseSpecialAnchor(p, s)) - } else { - s = chk(parseAnchor(p, 0, s)) - } - case '<': // sort relation '<'{1,4}, optionally followed by '*'. - for level = 1; consume(&s, '<'); level++ { - } - if level > 4 { - err = fmt.Errorf("level %d > 4", level) - } - fallthrough - case '=': // identity relation, optionally followed by *. - if consume(&s, '*') { - s = chk(parseSequence(p, level, s)) - } else { - s = chk(parseOrder(p, level, s)) - } - default: - chk("", fmt.Errorf("illegal operator %q", ch)) - break - } - } - } - if chk("", scanner.Err()); err != nil { - return fmt.Errorf("%d: %v", i, err) - } - return nil -} - -// parseSpecialAnchor parses the anchor syntax which is either of the form -// ['before' ] -// or -// [