diff --git a/components/engine/Makefile b/components/engine/Makefile index 32988150c7..6f5145a026 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -53,7 +53,8 @@ DOCKER_ENVS := \ -e http_proxy \ -e https_proxy \ -e no_proxy \ - -e VERSION + -e VERSION \ + -e PLATFORM # note: we _cannot_ add "-e DOCKER_BUILDTAGS" here because even if it's unset in the shell, that would shadow the "ENV DOCKER_BUILDTAGS" set in our Dockerfile, which is very important for our official builds # to allow `make BIND_DIR=. shell` or `make BIND_DIR= test` diff --git a/components/engine/api/server/router/system/system_routes.go b/components/engine/api/server/router/system/system_routes.go index 8f6aecd77a..535956d65f 100644 --- a/components/engine/api/server/router/system/system_routes.go +++ b/components/engine/api/server/router/system/system_routes.go @@ -6,7 +6,6 @@ import ( "net/http" "time" - "github.com/docker/docker/api" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/events" @@ -65,7 +64,6 @@ func (s *systemRouter) getInfo(ctx context.Context, w http.ResponseWriter, r *ht func (s *systemRouter) getVersion(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { info := s.backend.SystemVersion() - info.APIVersion = api.DefaultVersion return httputils.WriteJSON(w, http.StatusOK, info) } diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index b7ed3b8731..b1533f6b71 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -254,6 +254,7 @@ definitions: properties: Propagation: description: "A propagation mode with the value `[r]private`, `[r]shared`, or `[r]slave`." + type: "string" enum: - "private" - "rprivate" @@ -823,9 +824,7 @@ definitions: type: "string" Cmd: description: "Command to run specified as a string or an array of strings." - type: - - "array" - - "string" + type: "array" items: type: "string" Healthcheck: @@ -853,9 +852,7 @@ definitions: The entry point for the container as a string or an array of strings. If the array consists of exactly one empty string (`[""]`) then the entry point is reset to system default (i.e., the entry point used by docker when there is no `ENTRYPOINT` instruction in the `Dockerfile`). - type: - - "array" - - "string" + type: "array" items: type: "string" NetworkDisabled: @@ -6764,6 +6761,28 @@ paths: schema: type: "object" properties: + Platform: + type: "object" + required: [Name] + properties: + Name: + type: "string" + Components: + type: "array" + items: + type: "object" + x-go-name: ComponentVersion + required: [Name, Version] + properties: + Name: + type: "string" + Version: + type: "string" + x-nullable: false + Details: + type: "object" + x-nullable: true + Version: type: "string" ApiVersion: diff --git a/components/engine/api/types/types.go b/components/engine/api/types/types.go index f7ac772971..7814e6b934 100644 --- a/components/engine/api/types/types.go +++ b/components/engine/api/types/types.go @@ -107,9 +107,21 @@ type Ping struct { Experimental bool } +// ComponentVersion describes the version information for a specific component. +type ComponentVersion struct { + Name string + Version string + Details map[string]string `json:",omitempty"` +} + // Version contains response of Engine API: // GET "/version" type Version struct { + Platform struct{ Name string } `json:",omitempty"` + Components []ComponentVersion `json:",omitempty"` + + // The following fields are deprecated, they relate to the Engine component and are kept for backwards compatibility + Version string APIVersion string `json:"ApiVersion"` MinAPIVersion string `json:"MinAPIVersion,omitempty"` diff --git a/components/engine/cmd/dockerd/config.go b/components/engine/cmd/dockerd/config.go index b9d586a4ba..c4ae197335 100644 --- a/components/engine/cmd/dockerd/config.go +++ b/components/engine/cmd/dockerd/config.go @@ -59,6 +59,8 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) { flags.IntVar(&maxConcurrentDownloads, "max-concurrent-downloads", config.DefaultMaxConcurrentDownloads, "Set the max concurrent downloads for each pull") flags.IntVar(&maxConcurrentUploads, "max-concurrent-uploads", config.DefaultMaxConcurrentUploads, "Set the max concurrent uploads for each push") flags.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", defaultShutdownTimeout, "Set the default shutdown timeout") + flags.IntVar(&conf.NetworkDiagnosticPort, "network-diagnostic-port", 0, "TCP port number of the network diagnostic server") + flags.MarkHidden("network-diagnostic-port") flags.StringVar(&conf.SwarmDefaultAdvertiseAddr, "swarm-default-advertise-addr", "", "Set default address or interface for swarm advertised address") flags.BoolVar(&conf.Experimental, "experimental", false, "Enable experimental features") diff --git a/components/engine/daemon/cluster/swarm.go b/components/engine/daemon/cluster/swarm.go index 61223691bd..ec527ce59f 100644 --- a/components/engine/daemon/cluster/swarm.go +++ b/components/engine/daemon/cluster/swarm.go @@ -228,6 +228,13 @@ func (c *Cluster) Update(version uint64, spec types.Spec, flags types.UpdateFlag return err } + // Validate spec name. + if spec.Annotations.Name == "" { + spec.Annotations.Name = "default" + } else if spec.Annotations.Name != "default" { + return validationError{errors.New(`swarm spec must be named "default"`)} + } + // In update, client should provide the complete spec of the swarm, including // Name and Labels. If a field is specified with 0 or nil, then the default value // will be used to swarmkit. diff --git a/components/engine/daemon/config/config.go b/components/engine/daemon/config/config.go index 1e22a6fea7..199fae6cf1 100644 --- a/components/engine/daemon/config/config.go +++ b/components/engine/daemon/config/config.go @@ -85,26 +85,27 @@ type CommonTLSOptions struct { // It includes json tags to deserialize configuration from a file // using the same names that the flags in the command line use. type CommonConfig struct { - AuthzMiddleware *authorization.Middleware `json:"-"` - AuthorizationPlugins []string `json:"authorization-plugins,omitempty"` // AuthorizationPlugins holds list of authorization plugins - AutoRestart bool `json:"-"` - Context map[string][]string `json:"-"` - DisableBridge bool `json:"-"` - DNS []string `json:"dns,omitempty"` - DNSOptions []string `json:"dns-opts,omitempty"` - DNSSearch []string `json:"dns-search,omitempty"` - ExecOptions []string `json:"exec-opts,omitempty"` - GraphDriver string `json:"storage-driver,omitempty"` - GraphOptions []string `json:"storage-opts,omitempty"` - Labels []string `json:"labels,omitempty"` - Mtu int `json:"mtu,omitempty"` - Pidfile string `json:"pidfile,omitempty"` - RawLogs bool `json:"raw-logs,omitempty"` - RootDeprecated string `json:"graph,omitempty"` - Root string `json:"data-root,omitempty"` - ExecRoot string `json:"exec-root,omitempty"` - SocketGroup string `json:"group,omitempty"` - CorsHeaders string `json:"api-cors-header,omitempty"` + AuthzMiddleware *authorization.Middleware `json:"-"` + AuthorizationPlugins []string `json:"authorization-plugins,omitempty"` // AuthorizationPlugins holds list of authorization plugins + AutoRestart bool `json:"-"` + Context map[string][]string `json:"-"` + DisableBridge bool `json:"-"` + DNS []string `json:"dns,omitempty"` + DNSOptions []string `json:"dns-opts,omitempty"` + DNSSearch []string `json:"dns-search,omitempty"` + ExecOptions []string `json:"exec-opts,omitempty"` + GraphDriver string `json:"storage-driver,omitempty"` + GraphOptions []string `json:"storage-opts,omitempty"` + Labels []string `json:"labels,omitempty"` + Mtu int `json:"mtu,omitempty"` + NetworkDiagnosticPort int `json:"network-diagnostic-port,omitempty"` + Pidfile string `json:"pidfile,omitempty"` + RawLogs bool `json:"raw-logs,omitempty"` + RootDeprecated string `json:"graph,omitempty"` + Root string `json:"data-root,omitempty"` + ExecRoot string `json:"exec-root,omitempty"` + SocketGroup string `json:"group,omitempty"` + CorsHeaders string `json:"api-cors-header,omitempty"` // TrustKeyPath is used to generate the daemon ID and for signing schema 1 manifests // when pushing to a registry which does not support schema 2. This field is marked as diff --git a/components/engine/daemon/graphdriver/aufs/aufs.go b/components/engine/daemon/graphdriver/aufs/aufs.go index 11e763d025..248b8bf88d 100644 --- a/components/engine/daemon/graphdriver/aufs/aufs.go +++ b/components/engine/daemon/graphdriver/aufs/aufs.go @@ -89,7 +89,16 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, graphdriver.ErrNotSupported } - fsMagic, err := graphdriver.GetFSMagic(root) + // Perform feature detection on /var/lib/docker/aufs if it's an existing directory. + // This covers situations where /var/lib/docker/aufs is a mount, and on a different + // filesystem than /var/lib/docker. + // If the path does not exist, fall back to using /var/lib/docker for feature detection. + testdir := root + if _, err := os.Stat(testdir); os.IsNotExist(err) { + testdir = filepath.Dir(testdir) + } + + fsMagic, err := graphdriver.GetFSMagic(testdir) if err != nil { return nil, err } diff --git a/components/engine/daemon/graphdriver/btrfs/btrfs.go b/components/engine/daemon/graphdriver/btrfs/btrfs.go index 0dabf711dd..57313c94c6 100644 --- a/components/engine/daemon/graphdriver/btrfs/btrfs.go +++ b/components/engine/daemon/graphdriver/btrfs/btrfs.go @@ -51,7 +51,16 @@ type btrfsOptions struct { // An error is returned if BTRFS is not supported. func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { - fsMagic, err := graphdriver.GetFSMagic(home) + // Perform feature detection on /var/lib/docker/btrfs if it's an existing directory. + // This covers situations where /var/lib/docker/btrfs is a mount, and on a different + // filesystem than /var/lib/docker. + // If the path does not exist, fall back to using /var/lib/docker for feature detection. + testdir := home + if _, err := os.Stat(testdir); os.IsNotExist(err) { + testdir = filepath.Dir(testdir) + } + + fsMagic, err := graphdriver.GetFSMagic(testdir) if err != nil { return nil, err } diff --git a/components/engine/daemon/graphdriver/driver.go b/components/engine/daemon/graphdriver/driver.go index d08c6dc5b7..ceb49b9818 100644 --- a/components/engine/daemon/graphdriver/driver.go +++ b/components/engine/daemon/graphdriver/driver.go @@ -1,7 +1,6 @@ package graphdriver import ( - "errors" "fmt" "io" "os" @@ -28,13 +27,6 @@ const ( var ( // All registered drivers drivers map[string]InitFunc - - // ErrNotSupported returned when driver is not supported. - ErrNotSupported = errors.New("driver not supported") - // ErrPrerequisites returned when driver does not meet prerequisites. - ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)") - // ErrIncompatibleFS returned when file system is not supported. - ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver") ) //CreateOpts contains optional arguments for Create() and CreateReadWrite() @@ -248,7 +240,7 @@ func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, err for _, name := range list { driver, err := getBuiltinDriver(name, config.Root, config.DriverOptions, config.UIDMaps, config.GIDMaps) if err != nil { - if isDriverNotSupported(err) { + if IsDriverNotSupported(err) { continue } return nil, err @@ -260,7 +252,7 @@ func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, err for name, initFunc := range drivers { driver, err := initFunc(filepath.Join(config.Root, name), config.DriverOptions, config.UIDMaps, config.GIDMaps) if err != nil { - if isDriverNotSupported(err) { + if IsDriverNotSupported(err) { continue } return nil, err @@ -270,12 +262,6 @@ func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, err return nil, fmt.Errorf("No supported storage backend found") } -// isDriverNotSupported returns true if the error initializing -// the graph driver is a non-supported error. -func isDriverNotSupported(err error) bool { - return err == ErrNotSupported || err == ErrPrerequisites || err == ErrIncompatibleFS -} - // scanPriorDrivers returns an un-ordered scan of directories of prior storage drivers func scanPriorDrivers(root string) map[string]bool { driversMap := make(map[string]bool) @@ -291,6 +277,18 @@ func scanPriorDrivers(root string) map[string]bool { return driversMap } +// IsInitialized checks if the driver's home-directory exists and is non-empty. +func IsInitialized(driverHome string) bool { + _, err := os.Stat(driverHome) + if os.IsNotExist(err) { + return false + } + if err != nil { + logrus.Warnf("graphdriver.IsInitialized: stat failed: %v", err) + } + return !isEmptyDir(driverHome) +} + // isEmptyDir checks if a directory is empty. It is used to check if prior // storage-driver directories exist. If an error occurs, it also assumes the // directory is not empty (which preserves the behavior _before_ this check diff --git a/components/engine/daemon/graphdriver/driver_linux.go b/components/engine/daemon/graphdriver/driver_linux.go index aa3cfc9f79..f59862db85 100644 --- a/components/engine/daemon/graphdriver/driver_linux.go +++ b/components/engine/daemon/graphdriver/driver_linux.go @@ -3,8 +3,6 @@ package graphdriver import ( - "path/filepath" - "github.com/docker/docker/pkg/mount" "golang.org/x/sys/unix" ) @@ -82,7 +80,7 @@ var ( // GetFSMagic returns the filesystem id given the path. func GetFSMagic(rootpath string) (FsMagic, error) { var buf unix.Statfs_t - if err := unix.Statfs(filepath.Dir(rootpath), &buf); err != nil { + if err := unix.Statfs(rootpath, &buf); err != nil { return 0, err } return FsMagic(buf.Type), nil diff --git a/components/engine/daemon/graphdriver/errors.go b/components/engine/daemon/graphdriver/errors.go new file mode 100644 index 0000000000..dd52ee4ada --- /dev/null +++ b/components/engine/daemon/graphdriver/errors.go @@ -0,0 +1,36 @@ +package graphdriver + +const ( + // ErrNotSupported returned when driver is not supported. + ErrNotSupported NotSupportedError = "driver not supported" + // ErrPrerequisites returned when driver does not meet prerequisites. + ErrPrerequisites NotSupportedError = "prerequisites for driver not satisfied (wrong filesystem?)" + // ErrIncompatibleFS returned when file system is not supported. + ErrIncompatibleFS NotSupportedError = "backing file system is unsupported for this graph driver" +) + +// ErrUnSupported signals that the graph-driver is not supported on the current configuration +type ErrUnSupported interface { + NotSupported() +} + +// NotSupportedError signals that the graph-driver is not supported on the current configuration +type NotSupportedError string + +func (e NotSupportedError) Error() string { + return string(e) +} + +// NotSupported signals that a graph-driver is not supported. +func (e NotSupportedError) NotSupported() {} + +// IsDriverNotSupported returns true if the error initializing +// the graph driver is a non-supported error. +func IsDriverNotSupported(err error) bool { + switch err.(type) { + case ErrUnSupported: + return true + default: + return false + } +} diff --git a/components/engine/daemon/graphdriver/graphtest/graphtest_unix.go b/components/engine/daemon/graphdriver/graphtest/graphtest_unix.go index c25d4826fc..da9443e098 100644 --- a/components/engine/daemon/graphdriver/graphtest/graphtest_unix.go +++ b/components/engine/daemon/graphdriver/graphtest/graphtest_unix.go @@ -42,7 +42,7 @@ func newDriver(t testing.TB, name string, options []string) *Driver { d, err := graphdriver.GetDriver(name, nil, graphdriver.Options{DriverOptions: options, Root: root}) if err != nil { t.Logf("graphdriver: %v\n", err) - if err == graphdriver.ErrNotSupported || err == graphdriver.ErrPrerequisites || err == graphdriver.ErrIncompatibleFS { + if graphdriver.IsDriverNotSupported(err) { t.Skipf("Driver %s not supported", name) } t.Fatal(err) diff --git a/components/engine/daemon/graphdriver/overlay/overlay.go b/components/engine/daemon/graphdriver/overlay/overlay.go index 647a86d8ae..bcb21d87e4 100644 --- a/components/engine/daemon/graphdriver/overlay/overlay.go +++ b/components/engine/daemon/graphdriver/overlay/overlay.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "strconv" "github.com/docker/docker/daemon/graphdriver" @@ -119,7 +120,16 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, graphdriver.ErrNotSupported } - fsMagic, err := graphdriver.GetFSMagic(home) + // Perform feature detection on /var/lib/docker/overlay if it's an existing directory. + // This covers situations where /var/lib/docker/overlay is a mount, and on a different + // filesystem than /var/lib/docker. + // If the path does not exist, fall back to using /var/lib/docker for feature detection. + testdir := home + if _, err := os.Stat(testdir); os.IsNotExist(err) { + testdir = filepath.Dir(testdir) + } + + fsMagic, err := graphdriver.GetFSMagic(testdir) if err != nil { return nil, err } @@ -128,11 +138,23 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap } switch fsMagic { - case graphdriver.FsMagicAufs, graphdriver.FsMagicBtrfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicZfs, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs: + case graphdriver.FsMagicAufs, graphdriver.FsMagicBtrfs, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs, graphdriver.FsMagicOverlay, graphdriver.FsMagicZfs: logrus.Errorf("'overlay' is not supported over %s", backingFs) return nil, graphdriver.ErrIncompatibleFS } + supportsDType, err := fsutils.SupportsDType(testdir) + if err != nil { + return nil, err + } + if !supportsDType { + if !graphdriver.IsInitialized(home) { + return nil, overlayutils.ErrDTypeNotSupported("overlay", backingFs) + } + // allow running without d_type only for existing setups (#27443) + logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs)) + } + rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) if err != nil { return nil, err @@ -146,15 +168,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - supportsDType, err := fsutils.SupportsDType(home) - if err != nil { - return nil, err - } - if !supportsDType { - // not a fatal error until v17.12 (#27443) - logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs)) - } - d := &Driver{ home: home, uidMaps: uidMaps, diff --git a/components/engine/daemon/graphdriver/overlay2/overlay.go b/components/engine/daemon/graphdriver/overlay2/overlay.go index e660d8058a..f1731ea935 100644 --- a/components/engine/daemon/graphdriver/overlay2/overlay.go +++ b/components/engine/daemon/graphdriver/overlay2/overlay.go @@ -136,7 +136,16 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - fsMagic, err := graphdriver.GetFSMagic(home) + // Perform feature detection on /var/lib/docker/overlay2 if it's an existing directory. + // This covers situations where /var/lib/docker/overlay2 is a mount, and on a different + // filesystem than /var/lib/docker. + // If the path does not exist, fall back to using /var/lib/docker for feature detection. + testdir := home + if _, err := os.Stat(testdir); os.IsNotExist(err) { + testdir = filepath.Dir(testdir) + } + + fsMagic, err := graphdriver.GetFSMagic(testdir) if err != nil { return nil, err } @@ -144,9 +153,8 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap backingFs = fsName } - // check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs switch fsMagic { - case graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs: + case graphdriver.FsMagicAufs, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs, graphdriver.FsMagicOverlay, graphdriver.FsMagicZfs: logrus.Errorf("'overlay2' is not supported over %s", backingFs) return nil, graphdriver.ErrIncompatibleFS case graphdriver.FsMagicBtrfs: @@ -165,12 +173,23 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap if opts.overrideKernelCheck { logrus.Warn("Using pre-4.0.0 kernel for overlay2, mount failures may require kernel update") } else { - if err := supportsMultipleLowerDir(filepath.Dir(home)); err != nil { + if err := supportsMultipleLowerDir(testdir); err != nil { logrus.Debugf("Multiple lower dirs not supported: %v", err) return nil, graphdriver.ErrNotSupported } } } + supportsDType, err := fsutils.SupportsDType(testdir) + if err != nil { + return nil, err + } + if !supportsDType { + if !graphdriver.IsInitialized(home) { + return nil, overlayutils.ErrDTypeNotSupported("overlay2", backingFs) + } + // allow running without d_type only for existing setups (#27443) + logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay2", backingFs)) + } rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) if err != nil { @@ -185,15 +204,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - supportsDType, err := fsutils.SupportsDType(home) - if err != nil { - return nil, err - } - if !supportsDType { - // not a fatal error until v17.12 (#27443) - logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay2", backingFs)) - } - d := &Driver{ home: home, uidMaps: uidMaps, diff --git a/components/engine/daemon/graphdriver/overlayutils/overlayutils.go b/components/engine/daemon/graphdriver/overlayutils/overlayutils.go index 7491c3457c..9f71c60d51 100644 --- a/components/engine/daemon/graphdriver/overlayutils/overlayutils.go +++ b/components/engine/daemon/graphdriver/overlayutils/overlayutils.go @@ -3,8 +3,9 @@ package overlayutils import ( - "errors" "fmt" + + "github.com/docker/docker/daemon/graphdriver" ) // ErrDTypeNotSupported denotes that the backing filesystem doesn't support d_type. @@ -13,6 +14,7 @@ func ErrDTypeNotSupported(driver, backingFs string) error { if backingFs == "xfs" { msg += " Reformat the filesystem with ftype=1 to enable d_type support." } - msg += " Running without d_type support will no longer be supported in Docker 17.12." - return errors.New(msg) + msg += " Backing filesystems without d_type support are not supported." + + return graphdriver.NotSupportedError(msg) } diff --git a/components/engine/daemon/info.go b/components/engine/daemon/info.go index b14e7ba809..bbb027ed54 100644 --- a/components/engine/daemon/info.go +++ b/components/engine/daemon/info.go @@ -154,24 +154,46 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { // SystemVersion returns version information about the daemon. func (daemon *Daemon) SystemVersion() types.Version { - v := types.Version{ - Version: dockerversion.Version, - GitCommit: dockerversion.GitCommit, - MinAPIVersion: api.MinVersion, - GoVersion: runtime.Version(), - Os: runtime.GOOS, - Arch: runtime.GOARCH, - BuildTime: dockerversion.BuildTime, - Experimental: daemon.configStore.Experimental, - } - kernelVersion := "" if kv, err := kernel.GetKernelVersion(); err != nil { logrus.Warnf("Could not get kernel version: %v", err) } else { kernelVersion = kv.String() } - v.KernelVersion = kernelVersion + + v := types.Version{ + Components: []types.ComponentVersion{ + { + Name: "Engine", + Version: dockerversion.Version, + Details: map[string]string{ + "GitCommit": dockerversion.GitCommit, + "ApiVersion": api.DefaultVersion, + "MinAPIVersion": api.MinVersion, + "GoVersion": runtime.Version(), + "Os": runtime.GOOS, + "Arch": runtime.GOARCH, + "BuildTime": dockerversion.BuildTime, + "KernelVersion": kernelVersion, + "Experimental": fmt.Sprintf("%t", daemon.configStore.Experimental), + }, + }, + }, + + // Populate deprecated fields for older clients + Version: dockerversion.Version, + GitCommit: dockerversion.GitCommit, + APIVersion: api.DefaultVersion, + MinAPIVersion: api.MinVersion, + GoVersion: runtime.Version(), + Os: runtime.GOOS, + Arch: runtime.GOARCH, + BuildTime: dockerversion.BuildTime, + KernelVersion: kernelVersion, + Experimental: daemon.configStore.Experimental, + } + + v.Platform.Name = dockerversion.PlatformName return v } diff --git a/components/engine/daemon/reload.go b/components/engine/daemon/reload.go index 0d16bc8d02..a20eb688f4 100644 --- a/components/engine/daemon/reload.go +++ b/components/engine/daemon/reload.go @@ -61,6 +61,9 @@ func (daemon *Daemon) Reload(conf *config.Config) (err error) { if err := daemon.reloadLiveRestore(conf, attributes); err != nil { return err } + if err := daemon.reloadNetworkDiagnosticPort(conf, attributes); err != nil { + return err + } return nil } @@ -308,3 +311,18 @@ func (daemon *Daemon) reloadLiveRestore(conf *config.Config, attributes map[stri attributes["live-restore"] = fmt.Sprintf("%t", daemon.configStore.LiveRestoreEnabled) return nil } + +// reloadNetworkDiagnosticPort updates the network controller starting the diagnose mode if the config is valid +func (daemon *Daemon) reloadNetworkDiagnosticPort(conf *config.Config, attributes map[string]string) error { + if conf == nil || daemon.netController == nil { + return nil + } + // Enable the network diagnose if the flag is set with a valid port withing the range + if conf.IsValueSet("network-diagnostic-port") && conf.NetworkDiagnosticPort > 0 && conf.NetworkDiagnosticPort < 65536 { + logrus.Warnf("Calling the diagnostic start with %d", conf.NetworkDiagnosticPort) + daemon.netController.StartDiagnose(conf.NetworkDiagnosticPort) + } else { + daemon.netController.StopDiagnose() + } + return nil +} diff --git a/components/engine/daemon/reload_test.go b/components/engine/daemon/reload_test.go index 96b1a2452d..03b249bada 100644 --- a/components/engine/daemon/reload_test.go +++ b/components/engine/daemon/reload_test.go @@ -10,6 +10,7 @@ import ( "github.com/docker/docker/pkg/discovery" _ "github.com/docker/docker/pkg/discovery/memory" "github.com/docker/docker/registry" + "github.com/docker/libnetwork" "github.com/stretchr/testify/assert" ) @@ -479,3 +480,71 @@ func TestDaemonDiscoveryReloadOnlyClusterAdvertise(t *testing.T) { t.Fatal(e) } } + +func TestDaemonReloadNetworkDiagnosticPort(t *testing.T) { + daemon := &Daemon{} + daemon.configStore = &config.Config{} + + valuesSet := make(map[string]interface{}) + valuesSet["network-diagnostic-port"] = 2000 + enableConfig := &config.Config{ + CommonConfig: config.CommonConfig{ + NetworkDiagnosticPort: 2000, + ValuesSet: valuesSet, + }, + } + disableConfig := &config.Config{ + CommonConfig: config.CommonConfig{}, + } + + netOptions, err := daemon.networkOptions(enableConfig, nil, nil) + if err != nil { + t.Fatal(err) + } + controller, err := libnetwork.New(netOptions...) + if err != nil { + t.Fatal(err) + } + daemon.netController = controller + + // Enable/Disable the server for some iterations + for i := 0; i < 10; i++ { + enableConfig.CommonConfig.NetworkDiagnosticPort++ + if err := daemon.Reload(enableConfig); err != nil { + t.Fatal(err) + } + // Check that the diagnose is enabled + if !daemon.netController.IsDiagnoseEnabled() { + t.Fatalf("diagnosed should be enable") + } + + // Reload + if err := daemon.Reload(disableConfig); err != nil { + t.Fatal(err) + } + // Check that the diagnose is disabled + if daemon.netController.IsDiagnoseEnabled() { + t.Fatalf("diagnosed should be disable") + } + } + + enableConfig.CommonConfig.NetworkDiagnosticPort++ + // 2 times the enable should not create problems + if err := daemon.Reload(enableConfig); err != nil { + t.Fatal(err) + } + // Check that the diagnose is enabled + if !daemon.netController.IsDiagnoseEnabled() { + t.Fatalf("diagnosed should be enable") + } + + // Check that another reload does not cause issues + if err := daemon.Reload(enableConfig); err != nil { + t.Fatal(err) + } + // Check that the diagnose is enable + if !daemon.netController.IsDiagnoseEnabled() { + t.Fatalf("diagnosed should be enable") + } + +} diff --git a/components/engine/dockerversion/version_lib.go b/components/engine/dockerversion/version_lib.go index 33f77d3ce6..72f48939d6 100644 --- a/components/engine/dockerversion/version_lib.go +++ b/components/engine/dockerversion/version_lib.go @@ -13,4 +13,5 @@ const ( ContainerdCommitID string = "library-import" RuncCommitID string = "library-import" InitCommitID string = "library-import" + PlatformName string = "" ) diff --git a/components/engine/docs/contributing/set-up-dev-env.md b/components/engine/docs/contributing/set-up-dev-env.md index b28b8fb677..28bea5b86f 100644 --- a/components/engine/docs/contributing/set-up-dev-env.md +++ b/components/engine/docs/contributing/set-up-dev-env.md @@ -10,8 +10,7 @@ Moby's development environment is itself, ultimately a Docker container. You use the `moby/moby` repository and its `Dockerfile` to create a Docker image, run a Docker container, and develop code in the container. -If you followed the procedures that -set up Git for contributing, you should have a fork of the `moby/moby` +If you followed the procedures that [set up Git for contributing](./set-up-git.md), you should have a fork of the `moby/moby` repository. You also created a branch called `dry-run-test`. In this section, you continue working with your fork on this branch. @@ -106,8 +105,7 @@ can take over 15 minutes to complete. ``` If you are following along with this guide, you created a `dry-run-test` - branch when you - set up Git for contributing. + branch when you [set up Git for contributing](./set-up-git.md). 3. Ensure you are on your `dry-run-test` branch. diff --git a/components/engine/hack/make.ps1 b/components/engine/hack/make.ps1 index 3380a5b693..42a2b319fa 100644 --- a/components/engine/hack/make.ps1 +++ b/components/engine/hack/make.ps1 @@ -365,7 +365,7 @@ Try { # Run autogen if building binaries or running unit tests. if ($Client -or $Daemon -or $TestUnit) { Write-Host "INFO: Invoking autogen..." - Try { .\hack\make\.go-autogen.ps1 -CommitString $gitCommit -DockerVersion $dockerVersion } + Try { .\hack\make\.go-autogen.ps1 -CommitString $gitCommit -DockerVersion $dockerVersion -Platform "$env:PLATFORM" } Catch [Exception] { Throw $_ } } diff --git a/components/engine/hack/make/.go-autogen b/components/engine/hack/make/.go-autogen index b68e3a7534..850c3ec9ac 100644 --- a/components/engine/hack/make/.go-autogen +++ b/components/engine/hack/make/.go-autogen @@ -18,6 +18,7 @@ const ( BuildTime string = "$BUILDTIME" IAmStatic string = "${IAMSTATIC:-true}" ContainerdCommitID string = "${CONTAINERD_COMMIT}" + PlatformName string = "${PLATFORM}" ) // AUTOGENERATED FILE; see /go/src/github.com/docker/docker/hack/make/.go-autogen diff --git a/components/engine/hack/make/.go-autogen.ps1 b/components/engine/hack/make/.go-autogen.ps1 index 768badb6a5..cc14e9ee9d 100644 --- a/components/engine/hack/make/.go-autogen.ps1 +++ b/components/engine/hack/make/.go-autogen.ps1 @@ -14,7 +14,8 @@ param( [Parameter(Mandatory=$true)][string]$CommitString, - [Parameter(Mandatory=$true)][string]$DockerVersion + [Parameter(Mandatory=$true)][string]$DockerVersion, + [Parameter(Mandatory=$false)][string]$Platform ) $ErrorActionPreference = "Stop" @@ -43,6 +44,7 @@ const ( GitCommit string = "'+$CommitString+'" Version string = "'+$DockerVersion+'" BuildTime string = "'+$buildDateTime+'" + PlatformName string = "'+$Platform+'" ) // AUTOGENERATED FILE; see hack\make\.go-autogen.ps1 diff --git a/components/engine/integration-cli/docker_cli_swarm_test.go b/components/engine/integration-cli/docker_cli_swarm_test.go index 9488fff45e..283f576f57 100644 --- a/components/engine/integration-cli/docker_cli_swarm_test.go +++ b/components/engine/integration-cli/docker_cli_swarm_test.go @@ -849,7 +849,7 @@ func (s *DockerSwarmSuite) TestSwarmServiceTTY(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) // We need to get the container id. - out, err = d.Cmd("ps", "-a", "-q", "--no-trunc") + out, err = d.Cmd("ps", "-q", "--no-trunc") c.Assert(err, checker.IsNil) id := strings.TrimSpace(out) @@ -872,7 +872,7 @@ func (s *DockerSwarmSuite) TestSwarmServiceTTY(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) // We need to get the container id. - out, err = d.Cmd("ps", "-a", "-q", "--no-trunc") + out, err = d.Cmd("ps", "-q", "--no-trunc") c.Assert(err, checker.IsNil) id = strings.TrimSpace(out) diff --git a/components/engine/pkg/chrootarchive/archive_unix.go b/components/engine/pkg/chrootarchive/archive_unix.go index f2325abd74..75c010e98f 100644 --- a/components/engine/pkg/chrootarchive/archive_unix.go +++ b/components/engine/pkg/chrootarchive/archive_unix.go @@ -66,10 +66,12 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T cmd.Stderr = output if err := cmd.Start(); err != nil { + w.Close() return fmt.Errorf("Untar error on re-exec cmd: %v", err) } //write the options to the pipe for the untar exec to read if err := json.NewEncoder(w).Encode(options); err != nil { + w.Close() return fmt.Errorf("Untar json encode to pipe failed: %v", err) } w.Close() diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 120dd9d378..1b80231566 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -30,7 +30,7 @@ github.com/moby/buildkit aaff9d591ef128560018433fe61beb802e149de8 github.com/tonistiigi/fsutil dea3a0da73aee887fc02142d995be764106ac5e2 #get libnetwork packages -github.com/docker/libnetwork 64ae58878fc8f95e4a167499d654e13fa36abdc7 +github.com/docker/libnetwork 9bca9a4a220b158cc94402e0f8c2c7714eb6f503 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec @@ -42,7 +42,7 @@ github.com/hashicorp/go-multierror fcdddc395df1ddf4247c69bd436e84cfa0733f7e github.com/hashicorp/serf 598c54895cc5a7b1a24a398d635e8c0ea0959870 github.com/docker/libkv 1d8431073ae03cdaedb198a89722f3aab6d418ef github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25 -github.com/vishvananda/netlink bd6d5de5ccef2d66b0a26177928d0d8895d7f969 +github.com/vishvananda/netlink b2de5d10e38ecce8607e6b438b6d174f389a004e github.com/BurntSushi/toml f706d00e3de6abe700c994cdd545a1a4915af060 github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374 github.com/deckarep/golang-set ef32fa3046d9f249d399f98ebaf9be944430fd1d @@ -114,7 +114,7 @@ github.com/dmcgowan/go-tar go1.10 github.com/stevvooe/ttrpc 76e68349ad9ab4d03d764c713826d31216715e4f # cluster -github.com/docker/swarmkit de950a7ed842c7b7e47e9451cde9bf8f96031894 +github.com/docker/swarmkit 4429c763170d9ca96929249353c3270c19e7d39e github.com/gogo/protobuf v0.4 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b6506826e diff --git a/components/engine/vendor/github.com/docker/libnetwork/agent.go b/components/engine/vendor/github.com/docker/libnetwork/agent.go index f12006583b..085b8c47ca 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/agent.go +++ b/components/engine/vendor/github.com/docker/libnetwork/agent.go @@ -293,11 +293,13 @@ func (c *controller) agentInit(listenAddr, bindAddrOrInterface, advertiseAddr, d c.Config().Daemon.NetworkControlPlaneMTU, netDBConf.PacketBufferSize) } nDB, err := networkdb.New(netDBConf) - if err != nil { return err } + // Register the diagnose handlers + c.DiagnoseServer.RegisterHandler(nDB, networkdb.NetDbPaths2Func) + var cancelList []func() ch, cancel := nDB.Watch(libnetworkEPTable, "", "") cancelList = append(cancelList, cancel) @@ -436,7 +438,7 @@ func (n *network) Services() map[string]ServiceInfo { for eid, value := range entries { var epRec EndpointRecord nid := n.ID() - if err := proto.Unmarshal(value.([]byte), &epRec); err != nil { + if err := proto.Unmarshal(value.Value, &epRec); err != nil { logrus.Errorf("Unmarshal of libnetworkEPTable failed for endpoint %s in network %s, %v", eid, nid, err) continue } @@ -461,7 +463,7 @@ func (n *network) Services() map[string]ServiceInfo { } entries := agent.networkDB.GetTableByNetwork(table.name, n.id) for key, value := range entries { - epID, info := d.DecodeTableEntry(table.name, key, value.([]byte)) + epID, info := d.DecodeTableEntry(table.name, key, value.Value) if ep, ok := eps[epID]; !ok { logrus.Errorf("Inconsistent driver and libnetwork state for endpoint %s", epID) } else { diff --git a/components/engine/vendor/github.com/docker/libnetwork/controller.go b/components/engine/vendor/github.com/docker/libnetwork/controller.go index 24494ae834..e9389487a3 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/controller.go +++ b/components/engine/vendor/github.com/docker/libnetwork/controller.go @@ -60,6 +60,7 @@ import ( "github.com/docker/libnetwork/cluster" "github.com/docker/libnetwork/config" "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/diagnose" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/drvregistry" @@ -133,6 +134,13 @@ type NetworkController interface { // SetKeys configures the encryption key for gossip and overlay data path SetKeys(keys []*types.EncryptionKey) error + + // StartDiagnose start the network diagnose mode + StartDiagnose(port int) + // StopDiagnose start the network diagnose mode + StopDiagnose() + // IsDiagnoseEnabled returns true if the diagnose is enabled + IsDiagnoseEnabled() bool } // NetworkWalker is a client provided function which will be used to walk the Networks. @@ -167,6 +175,7 @@ type controller struct { agentStopDone chan struct{} keys []*types.EncryptionKey clusterConfigAvailable bool + DiagnoseServer *diagnose.Server sync.Mutex } @@ -185,7 +194,9 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { serviceBindings: make(map[serviceKey]*service), agentInitDone: make(chan struct{}), networkLocker: locker.New(), + DiagnoseServer: diagnose.New(), } + c.DiagnoseServer.Init() if err := c.initStores(); err != nil { return nil, err @@ -1291,3 +1302,28 @@ func (c *controller) Stop() { c.stopExternalKeyListener() osl.GC() } + +// StartDiagnose start the network diagnose mode +func (c *controller) StartDiagnose(port int) { + c.Lock() + if !c.DiagnoseServer.IsDebugEnable() { + c.DiagnoseServer.EnableDebug("127.0.0.1", port) + } + c.Unlock() +} + +// StopDiagnose start the network diagnose mode +func (c *controller) StopDiagnose() { + c.Lock() + if c.DiagnoseServer.IsDebugEnable() { + c.DiagnoseServer.DisableDebug() + } + c.Unlock() +} + +// IsDiagnoseEnabled returns true if the diagnose is enabled +func (c *controller) IsDiagnoseEnabled() bool { + c.Lock() + defer c.Unlock() + return c.DiagnoseServer.IsDebugEnable() +} diff --git a/components/engine/vendor/github.com/docker/libnetwork/diagnose/diagnose.go b/components/engine/vendor/github.com/docker/libnetwork/diagnose/diagnose.go deleted file mode 100644 index 9682fff78b..0000000000 --- a/components/engine/vendor/github.com/docker/libnetwork/diagnose/diagnose.go +++ /dev/null @@ -1,133 +0,0 @@ -package diagnose - -import ( - "fmt" - "net" - "net/http" - "sync" - - "github.com/sirupsen/logrus" -) - -// HTTPHandlerFunc TODO -type HTTPHandlerFunc func(interface{}, http.ResponseWriter, *http.Request) - -type httpHandlerCustom struct { - ctx interface{} - F func(interface{}, http.ResponseWriter, *http.Request) -} - -// ServeHTTP TODO -func (h httpHandlerCustom) ServeHTTP(w http.ResponseWriter, r *http.Request) { - h.F(h.ctx, w, r) -} - -var diagPaths2Func = map[string]HTTPHandlerFunc{ - "/": notImplemented, - "/help": help, - "/ready": ready, -} - -// Server when the debug is enabled exposes a -// This data structure is protected by the Agent mutex so does not require and additional mutex here -type Server struct { - sk net.Listener - port int - mux *http.ServeMux - registeredHanders []string - sync.Mutex -} - -// Init TODO -func (n *Server) Init() { - n.mux = http.NewServeMux() - - // Register local handlers - n.RegisterHandler(n, diagPaths2Func) -} - -// RegisterHandler TODO -func (n *Server) RegisterHandler(ctx interface{}, hdlrs map[string]HTTPHandlerFunc) { - n.Lock() - defer n.Unlock() - for path, fun := range hdlrs { - n.mux.Handle(path, httpHandlerCustom{ctx, fun}) - n.registeredHanders = append(n.registeredHanders, path) - } -} - -// EnableDebug opens a TCP socket to debug the passed network DB -func (n *Server) EnableDebug(ip string, port int) { - n.Lock() - defer n.Unlock() - - n.port = port - logrus.SetLevel(logrus.DebugLevel) - - if n.sk != nil { - logrus.Infof("The server is already up and running") - return - } - - logrus.Infof("Starting the server listening on %d for commands", port) - - // // Create the socket - // var err error - // n.sk, err = net.Listen("tcp", listeningAddr) - // if err != nil { - // log.Fatal(err) - // } - // - // go func() { - // http.Serve(n.sk, n.mux) - // }() - http.ListenAndServe(fmt.Sprintf(":%d", port), n.mux) -} - -// DisableDebug stop the dubug and closes the tcp socket -func (n *Server) DisableDebug() { - n.Lock() - defer n.Unlock() - n.sk.Close() - n.sk = nil -} - -// IsDebugEnable returns true when the debug is enabled -func (n *Server) IsDebugEnable() bool { - n.Lock() - defer n.Unlock() - return n.sk != nil -} - -func notImplemented(ctx interface{}, w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "URL path: %s no method implemented check /help\n", r.URL.Path) -} - -func help(ctx interface{}, w http.ResponseWriter, r *http.Request) { - n, ok := ctx.(*Server) - if ok { - for _, path := range n.registeredHanders { - fmt.Fprintf(w, "%s\n", path) - } - } -} - -func ready(ctx interface{}, w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "OK\n") -} - -// DebugHTTPForm TODO -func DebugHTTPForm(r *http.Request) { - r.ParseForm() - for k, v := range r.Form { - logrus.Debugf("Form[%q] = %q\n", k, v) - } -} - -// HTTPReplyError TODO -func HTTPReplyError(w http.ResponseWriter, message, usage string) { - fmt.Fprintf(w, "%s\n", message) - if usage != "" { - fmt.Fprintf(w, "Usage: %s\n", usage) - } -} diff --git a/components/engine/vendor/github.com/docker/libnetwork/diagnose/server.go b/components/engine/vendor/github.com/docker/libnetwork/diagnose/server.go new file mode 100644 index 0000000000..c841e51966 --- /dev/null +++ b/components/engine/vendor/github.com/docker/libnetwork/diagnose/server.go @@ -0,0 +1,228 @@ +package diagnose + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sync" + "sync/atomic" + + stackdump "github.com/docker/docker/pkg/signal" + "github.com/docker/libnetwork/common" + "github.com/sirupsen/logrus" +) + +// HTTPHandlerFunc TODO +type HTTPHandlerFunc func(interface{}, http.ResponseWriter, *http.Request) + +type httpHandlerCustom struct { + ctx interface{} + F func(interface{}, http.ResponseWriter, *http.Request) +} + +// ServeHTTP TODO +func (h httpHandlerCustom) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.F(h.ctx, w, r) +} + +var diagPaths2Func = map[string]HTTPHandlerFunc{ + "/": notImplemented, + "/help": help, + "/ready": ready, + "/stackdump": stackTrace, +} + +// Server when the debug is enabled exposes a +// This data structure is protected by the Agent mutex so does not require and additional mutex here +type Server struct { + enable int32 + srv *http.Server + port int + mux *http.ServeMux + registeredHanders map[string]bool + sync.Mutex +} + +// New creates a new diagnose server +func New() *Server { + return &Server{ + registeredHanders: make(map[string]bool), + } +} + +// Init initialize the mux for the http handling and register the base hooks +func (s *Server) Init() { + s.mux = http.NewServeMux() + + // Register local handlers + s.RegisterHandler(s, diagPaths2Func) +} + +// RegisterHandler allows to register new handlers to the mux and to a specific path +func (s *Server) RegisterHandler(ctx interface{}, hdlrs map[string]HTTPHandlerFunc) { + s.Lock() + defer s.Unlock() + for path, fun := range hdlrs { + if _, ok := s.registeredHanders[path]; ok { + continue + } + s.mux.Handle(path, httpHandlerCustom{ctx, fun}) + s.registeredHanders[path] = true + } +} + +// ServeHTTP this is the method called bu the ListenAndServe, and is needed to allow us to +// use our custom mux +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mux.ServeHTTP(w, r) +} + +// EnableDebug opens a TCP socket to debug the passed network DB +func (s *Server) EnableDebug(ip string, port int) { + s.Lock() + defer s.Unlock() + + s.port = port + + if s.enable == 1 { + logrus.Info("The server is already up and running") + return + } + + logrus.Infof("Starting the diagnose server listening on %d for commands", port) + srv := &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: s} + s.srv = srv + s.enable = 1 + go func(n *Server) { + // Ingore ErrServerClosed that is returned on the Shutdown call + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + logrus.Errorf("ListenAndServe error: %s", err) + atomic.SwapInt32(&n.enable, 0) + } + }(s) + +} + +// DisableDebug stop the dubug and closes the tcp socket +func (s *Server) DisableDebug() { + s.Lock() + defer s.Unlock() + + s.srv.Shutdown(context.Background()) + s.srv = nil + s.enable = 0 + logrus.Info("Disabling the diagnose server") +} + +// IsDebugEnable returns true when the debug is enabled +func (s *Server) IsDebugEnable() bool { + s.Lock() + defer s.Unlock() + return s.enable == 1 +} + +func notImplemented(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + _, json := ParseHTTPFormOptions(r) + rsp := WrongCommand("not implemented", fmt.Sprintf("URL path: %s no method implemented check /help\n", r.URL.Path)) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("command not implemented done") + + HTTPReply(w, rsp, json) +} + +func help(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + _, json := ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("help done") + + n, ok := ctx.(*Server) + var result string + if ok { + for path := range n.registeredHanders { + result += fmt.Sprintf("%s\n", path) + } + HTTPReply(w, CommandSucceed(&StringCmd{Info: result}), json) + } +} + +func ready(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + _, json := ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("ready done") + HTTPReply(w, CommandSucceed(&StringCmd{Info: "OK"}), json) +} + +func stackTrace(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + _, json := ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("stack trace") + + path, err := stackdump.DumpStacks("/tmp/") + if err != nil { + log.WithError(err).Error("failed to write goroutines dump") + HTTPReply(w, FailCommand(err), json) + } else { + log.Info("stack trace done") + HTTPReply(w, CommandSucceed(&StringCmd{Info: fmt.Sprintf("goroutine stacks written to %s", path)}), json) + } +} + +// DebugHTTPForm helper to print the form url parameters +func DebugHTTPForm(r *http.Request) { + for k, v := range r.Form { + logrus.Debugf("Form[%q] = %q\n", k, v) + } +} + +// JSONOutput contains details on JSON output printing +type JSONOutput struct { + enable bool + prettyPrint bool +} + +// ParseHTTPFormOptions easily parse the JSON printing options +func ParseHTTPFormOptions(r *http.Request) (bool, *JSONOutput) { + _, unsafe := r.Form["unsafe"] + v, json := r.Form["json"] + var pretty bool + if len(v) > 0 { + pretty = v[0] == "pretty" + } + return unsafe, &JSONOutput{enable: json, prettyPrint: pretty} +} + +// HTTPReply helper function that takes care of sending the message out +func HTTPReply(w http.ResponseWriter, r *HTTPResult, j *JSONOutput) (int, error) { + var response []byte + if j.enable { + w.Header().Set("Content-Type", "application/json") + var err error + if j.prettyPrint { + response, err = json.MarshalIndent(r, "", " ") + if err != nil { + response, _ = json.MarshalIndent(FailCommand(err), "", " ") + } + } else { + response, err = json.Marshal(r) + if err != nil { + response, _ = json.Marshal(FailCommand(err)) + } + } + } else { + response = []byte(r.String()) + } + return fmt.Fprint(w, string(response)) +} diff --git a/components/engine/vendor/github.com/docker/libnetwork/diagnose/types.go b/components/engine/vendor/github.com/docker/libnetwork/diagnose/types.go new file mode 100644 index 0000000000..982c54a2b2 --- /dev/null +++ b/components/engine/vendor/github.com/docker/libnetwork/diagnose/types.go @@ -0,0 +1,122 @@ +package diagnose + +import "fmt" + +// StringInterface interface that has to be implemented by messages +type StringInterface interface { + String() string +} + +// CommandSucceed creates a success message +func CommandSucceed(result StringInterface) *HTTPResult { + return &HTTPResult{ + Message: "OK", + Details: result, + } +} + +// FailCommand creates a failure message with error +func FailCommand(err error) *HTTPResult { + return &HTTPResult{ + Message: "FAIL", + Details: &ErrorCmd{Error: err.Error()}, + } +} + +// WrongCommand creates a wrong command response +func WrongCommand(message, usage string) *HTTPResult { + return &HTTPResult{ + Message: message, + Details: &UsageCmd{Usage: usage}, + } +} + +// HTTPResult Diagnose Server HTTP result operation +type HTTPResult struct { + Message string `json:"message"` + Details StringInterface `json:"details"` +} + +func (h *HTTPResult) String() string { + rsp := h.Message + if h.Details != nil { + rsp += "\n" + h.Details.String() + } + return rsp +} + +// UsageCmd command with usage field +type UsageCmd struct { + Usage string `json:"usage"` +} + +func (u *UsageCmd) String() string { + return "Usage: " + u.Usage +} + +// StringCmd command with info string +type StringCmd struct { + Info string `json:"info"` +} + +func (s *StringCmd) String() string { + return s.Info +} + +// ErrorCmd command with error +type ErrorCmd struct { + Error string `json:"error"` +} + +func (e *ErrorCmd) String() string { + return "Error: " + e.Error +} + +// TableObj network db table object +type TableObj struct { + Length int `json:"size"` + Elements []StringInterface `json:"entries"` +} + +func (t *TableObj) String() string { + output := fmt.Sprintf("total entries: %d\n", t.Length) + for _, e := range t.Elements { + output += e.String() + } + return output +} + +// PeerEntryObj entry in the networkdb peer table +type PeerEntryObj struct { + Index int `json:"-"` + Name string `json:"-=name"` + IP string `json:"ip"` +} + +func (p *PeerEntryObj) String() string { + return fmt.Sprintf("%d) %s -> %s\n", p.Index, p.Name, p.IP) +} + +// TableEntryObj network db table entry object +type TableEntryObj struct { + Index int `json:"-"` + Key string `json:"key"` + Value string `json:"value"` + Owner string `json:"owner"` +} + +func (t *TableEntryObj) String() string { + return fmt.Sprintf("%d) k:`%s` -> v:`%s` owner:`%s`\n", t.Index, t.Key, t.Value, t.Owner) +} + +// TableEndpointsResult fully typed message for proper unmarshaling on the client side +type TableEndpointsResult struct { + TableObj + Elements []TableEntryObj `json:"entries"` +} + +// TablePeersResult fully typed message for proper unmarshaling on the client side +type TablePeersResult struct { + TableObj + Elements []PeerEntryObj `json:"entries"` +} diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go index 1742c8df31..1fa8f0e921 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go @@ -42,6 +42,14 @@ const ( DefaultGatewayV6AuxKey = "DefaultGatewayIPv6" ) +type defaultBridgeNetworkConflict struct { + ID string +} + +func (d defaultBridgeNetworkConflict) Error() string { + return fmt.Sprintf("Stale default bridge network %s", d.ID) +} + type iptableCleanFunc func() error type iptablesCleanFuncs []iptableCleanFunc @@ -137,6 +145,7 @@ type driver struct { networks map[string]*bridgeNetwork store datastore.DataStore nlh *netlink.Handle + configNetwork sync.Mutex sync.Mutex } @@ -322,41 +331,6 @@ func (n *bridgeNetwork) isolateNetwork(others []*bridgeNetwork, enable bool) err return nil } -// Checks whether this network's configuration for the network with this id conflicts with any of the passed networks -func (c *networkConfiguration) conflictsWithNetworks(id string, others []*bridgeNetwork) error { - for _, nw := range others { - - nw.Lock() - nwID := nw.id - nwConfig := nw.config - nwBridge := nw.bridge - nw.Unlock() - - if nwID == id { - continue - } - // Verify the name (which may have been set by newInterface()) does not conflict with - // existing bridge interfaces. Ironically the system chosen name gets stored in the config... - // Basically we are checking if the two original configs were both empty. - if nwConfig.BridgeName == c.BridgeName { - return types.ForbiddenErrorf("conflicts with network %s (%s) by bridge name", nwID, nwConfig.BridgeName) - } - // If this network config specifies the AddressIPv4, we need - // to make sure it does not conflict with any previously allocated - // bridges. This could not be completely caught by the config conflict - // check, because networks which config does not specify the AddressIPv4 - // get their address and subnet selected by the driver (see electBridgeIPv4()) - if c.AddressIPv4 != nil && nwBridge.bridgeIPv4 != nil { - if nwBridge.bridgeIPv4.Contains(c.AddressIPv4.IP) || - c.AddressIPv4.Contains(nwBridge.bridgeIPv4.IP) { - return types.ForbiddenErrorf("conflicts with network %s (%s) by ip network", nwID, nwConfig.BridgeName) - } - } - } - - return nil -} - func (d *driver) configure(option map[string]interface{}) error { var ( config *configuration @@ -602,11 +576,27 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d return err } - err = config.processIPAM(id, ipV4Data, ipV6Data) - if err != nil { + if err = config.processIPAM(id, ipV4Data, ipV6Data); err != nil { return err } + // start the critical section, from this point onward we are dealing with the list of networks + // so to be consistent we cannot allow that the list changes + d.configNetwork.Lock() + defer d.configNetwork.Unlock() + + // check network conflicts + if err = d.checkConflict(config); err != nil { + nerr, ok := err.(defaultBridgeNetworkConflict) + if !ok { + return err + } + // Got a conflict with a stale default network, clean that up and continue + logrus.Warn(nerr) + d.deleteNetwork(nerr.ID) + } + + // there is no conflict, now create the network if err = d.createNetwork(config); err != nil { return err } @@ -614,33 +604,47 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d return d.storeUpdate(config) } +func (d *driver) checkConflict(config *networkConfiguration) error { + networkList := d.getNetworks() + for _, nw := range networkList { + nw.Lock() + nwConfig := nw.config + nw.Unlock() + if err := nwConfig.Conflicts(config); err != nil { + if config.DefaultBridge { + // We encountered and identified a stale default network + // We must delete it as libnetwork is the source of truth + // The default network being created must be the only one + // This can happen only from docker 1.12 on ward + logrus.Infof("Found stale default bridge network %s (%s)", nwConfig.ID, nwConfig.BridgeName) + return defaultBridgeNetworkConflict{nwConfig.ID} + } + + return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s", + config.ID, config.BridgeName, nwConfig.ID, nwConfig.BridgeName, err.Error()) + } + } + return nil +} + func (d *driver) createNetwork(config *networkConfiguration) error { var err error defer osl.InitOSContext()() networkList := d.getNetworks() - for i, nw := range networkList { - nw.Lock() - nwConfig := nw.config - nw.Unlock() - if err := nwConfig.Conflicts(config); err != nil { - if config.DefaultBridge { - // We encountered and identified a stale default network - // We must delete it as libnetwork is the source of thruth - // The default network being created must be the only one - // This can happen only from docker 1.12 on ward - logrus.Infof("Removing stale default bridge network %s (%s)", nwConfig.ID, nwConfig.BridgeName) - if err := d.DeleteNetwork(nwConfig.ID); err != nil { - logrus.Warnf("Failed to remove stale default network: %s (%s): %v. Will remove from store.", nwConfig.ID, nwConfig.BridgeName, err) - d.storeDelete(nwConfig) - } - networkList = append(networkList[:i], networkList[i+1:]...) - } else { - return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s", - config.ID, config.BridgeName, nwConfig.ID, nwConfig.BridgeName, err.Error()) - } - } + + // Initialize handle when needed + d.Lock() + if d.nlh == nil { + d.nlh = ns.NlHandle() + } + d.Unlock() + + // Create or retrieve the bridge L3 interface + bridgeIface, err := newInterface(d.nlh, config) + if err != nil { + return err } // Create and set network handler in driver @@ -649,6 +653,7 @@ func (d *driver) createNetwork(config *networkConfiguration) error { endpoints: make(map[string]*bridgeEndpoint), config: config, portMapper: portmapper.New(d.config.UserlandProxyPath), + bridge: bridgeIface, driver: d, } @@ -665,35 +670,15 @@ func (d *driver) createNetwork(config *networkConfiguration) error { } }() - // Initialize handle when needed - d.Lock() - if d.nlh == nil { - d.nlh = ns.NlHandle() - } - d.Unlock() - - // Create or retrieve the bridge L3 interface - bridgeIface, err := newInterface(d.nlh, config) - if err != nil { - return err - } - network.bridge = bridgeIface - - // Verify the network configuration does not conflict with previously installed - // networks. This step is needed now because driver might have now set the bridge - // name on this config struct. And because we need to check for possible address - // conflicts, so we need to check against operationa lnetworks. - if err = config.conflictsWithNetworks(config.ID, networkList); err != nil { - return err - } - + // Add inter-network communication rules. setupNetworkIsolationRules := func(config *networkConfiguration, i *bridgeInterface) error { if err := network.isolateNetwork(networkList, true); err != nil { - if err := network.isolateNetwork(networkList, false); err != nil { + if err = network.isolateNetwork(networkList, false); err != nil { logrus.Warnf("Failed on removing the inter-network iptables rules on cleanup: %v", err) } return err } + // register the cleanup function network.registerIptCleanFunc(func() error { nwList := d.getNetworks() return network.isolateNetwork(nwList, false) @@ -767,10 +752,17 @@ func (d *driver) createNetwork(config *networkConfiguration) error { } func (d *driver) DeleteNetwork(nid string) error { + + d.configNetwork.Lock() + defer d.configNetwork.Unlock() + + return d.deleteNetwork(nid) +} + +func (d *driver) deleteNetwork(nid string) error { var err error defer osl.InitOSContext()() - // Get network handler and remove it from driver d.Lock() n, ok := d.networks[nid] @@ -814,12 +806,6 @@ func (d *driver) DeleteNetwork(nid string) error { } }() - // Sanity check - if n == nil { - err = driverapi.ErrNoNetwork(nid) - return err - } - switch config.BridgeIfaceCreator { case ifaceCreatedByLibnetwork, ifaceCreatorUnknown: // We only delete the bridge if it was created by the bridge driver and diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_firewalld.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_firewalld.go index fc45a7e983..50cbdb1ddc 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_firewalld.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_firewalld.go @@ -9,7 +9,7 @@ func (n *bridgeNetwork) setupFirewalld(config *networkConfiguration, i *bridgeIn d.Unlock() // Sanity check. - if driverConfig.EnableIPTables == false { + if !driverConfig.EnableIPTables { return IPTableCfgError(config.BridgeName) } diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go index 3fbfccf007..11314170b1 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go @@ -696,6 +696,12 @@ func (n *network) initSandbox(restore bool) error { var nlSock *nl.NetlinkSocket sbox.InvokeFunc(func() { nlSock, err = nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH) + if err != nil { + return + } + // set the receive timeout to not remain stuck on the RecvFrom if the fd gets closed + tv := syscall.NsecToTimeval(soTimeout.Nanoseconds()) + err = nlSock.SetReceiveTimeout(&tv) }) n.setNetlinkSocket(nlSock) @@ -721,6 +727,11 @@ func (n *network) watchMiss(nlSock *nl.NetlinkSocket) { // The netlink socket got closed, simply exit to not leak this goroutine return } + // When the receive timeout expires the receive will return EAGAIN + if err == syscall.EAGAIN { + // we continue here to avoid spam for timeouts + continue + } logrus.Errorf("Failed to receive from netlink: %v ", err) continue } diff --git a/components/engine/vendor/github.com/docker/libnetwork/ipvs/ipvs.go b/components/engine/vendor/github.com/docker/libnetwork/ipvs/ipvs.go index ebcdd808c3..effbb716eb 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ipvs/ipvs.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ipvs/ipvs.go @@ -5,12 +5,19 @@ package ipvs import ( "net" "syscall" + "time" "fmt" + "github.com/vishvananda/netlink/nl" "github.com/vishvananda/netns" ) +const ( + netlinkRecvSocketsTimeout = 3 * time.Second + netlinkSendSocketTimeout = 30 * time.Second +) + // Service defines an IPVS service in its entirety. type Service struct { // Virtual service address. @@ -82,6 +89,15 @@ func New(path string) (*Handle, error) { if err != nil { return nil, err } + // Add operation timeout to avoid deadlocks + tv := syscall.NsecToTimeval(netlinkSendSocketTimeout.Nanoseconds()) + if err := sock.SetSendTimeout(&tv); err != nil { + return nil, err + } + tv = syscall.NsecToTimeval(netlinkRecvSocketsTimeout.Nanoseconds()) + if err := sock.SetReceiveTimeout(&tv); err != nil { + return nil, err + } return &Handle{sock: sock}, nil } diff --git a/components/engine/vendor/github.com/docker/libnetwork/ipvs/netlink.go b/components/engine/vendor/github.com/docker/libnetwork/ipvs/netlink.go index 2089283d14..c062a1789d 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ipvs/netlink.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ipvs/netlink.go @@ -203,10 +203,6 @@ func newGenlRequest(familyID int, cmd uint8) *nl.NetlinkRequest { } func execute(s *nl.NetlinkSocket, req *nl.NetlinkRequest, resType uint16) ([][]byte, error) { - var ( - err error - ) - if err := s.Send(req); err != nil { return nil, err } @@ -222,6 +218,13 @@ done: for { msgs, err := s.Receive() if err != nil { + if s.GetFd() == -1 { + return nil, fmt.Errorf("Socket got closed on receive") + } + if err == syscall.EAGAIN { + // timeout fired + continue + } return nil, err } for _, m := range msgs { diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/event_delegate.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/event_delegate.go index 6075718a63..6e11805610 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/event_delegate.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/event_delegate.go @@ -72,7 +72,7 @@ func (e *eventDelegate) NotifyLeave(mn *memberlist.Node) { // If the node instead left because was going down, then it makes sense to just delete all its state e.nDB.Lock() defer e.nDB.Unlock() - e.nDB.deleteNetworkEntriesForNode(mn.Name) + e.nDB.deleteNodeFromNetworks(mn.Name) e.nDB.deleteNodeTableEntries(mn.Name) if n, ok := e.nDB.nodes[mn.Name]; ok { delete(e.nDB.nodes, mn.Name) diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go index 025e0ca843..9ec6beca35 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go @@ -401,17 +401,23 @@ func (nDB *NetworkDB) UpdateEntry(tname, nid, key string, value []byte) error { return nil } +// TableElem elem +type TableElem struct { + Value []byte + owner string +} + // GetTableByNetwork walks the networkdb by the give table and network id and // returns a map of keys and values -func (nDB *NetworkDB) GetTableByNetwork(tname, nid string) map[string]interface{} { - entries := make(map[string]interface{}) +func (nDB *NetworkDB) GetTableByNetwork(tname, nid string) map[string]*TableElem { + entries := make(map[string]*TableElem) nDB.indexes[byTable].WalkPrefix(fmt.Sprintf("/%s/%s", tname, nid), func(k string, v interface{}) bool { entry := v.(*entry) if entry.deleting { return false } key := k[strings.LastIndex(k, "/")+1:] - entries[key] = entry.value + entries[key] = &TableElem{Value: entry.value, owner: entry.node} return false }) return entries @@ -445,7 +451,7 @@ func (nDB *NetworkDB) DeleteEntry(tname, nid, key string) error { return nil } -func (nDB *NetworkDB) deleteNetworkEntriesForNode(deletedNode string) { +func (nDB *NetworkDB) deleteNodeFromNetworks(deletedNode string) { for nid, nodes := range nDB.networkNodes { updatedNodes := make([]string, 0, len(nodes)) for _, node := range nodes { @@ -547,7 +553,9 @@ func (nDB *NetworkDB) deleteNodeTableEntries(node string) { nDB.deleteEntry(nid, tname, key) - nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, oldEntry.value)) + if !oldEntry.deleting { + nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, oldEntry.value)) + } return false }) } diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdbdiagnose.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdbdiagnose.go index 01429a5381..3c6032b4ea 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdbdiagnose.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdbdiagnose.go @@ -1,17 +1,19 @@ package networkdb import ( + "encoding/base64" "fmt" "net/http" "strings" - stackdump "github.com/docker/docker/pkg/signal" + "github.com/docker/libnetwork/common" "github.com/docker/libnetwork/diagnose" "github.com/sirupsen/logrus" ) const ( missingParameter = "missing parameter" + dbNotAvailable = "database not available" ) // NetDbPaths2Func TODO @@ -26,14 +28,21 @@ var NetDbPaths2Func = map[string]diagnose.HTTPHandlerFunc{ "/deleteentry": dbDeleteEntry, "/getentry": dbGetEntry, "/gettable": dbGetTable, - "/dump": dbStackTrace, } func dbJoin(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + _, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("join cluster") + if len(r.Form["members"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?members=ip1,ip2,...", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?members=ip1,ip2,...", r.URL.Path)) + log.Error("join cluster failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -41,51 +50,88 @@ func dbJoin(ctx interface{}, w http.ResponseWriter, r *http.Request) { if ok { err := nDB.Join(strings.Split(r.Form["members"][0], ",")) if err != nil { - fmt.Fprintf(w, "%s error in the DB join %s\n", r.URL.Path, err) + rsp := diagnose.FailCommand(fmt.Errorf("%s error in the DB join %s", r.URL.Path, err)) + log.WithError(err).Error("join cluster failed") + diagnose.HTTPReply(w, rsp, json) return } - fmt.Fprintf(w, "OK\n") + log.Info("join cluster done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbPeers(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + _, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("network peers") + if len(r.Form["nid"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?nid=test", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?nid=test", r.URL.Path)) + log.Error("network peers failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } nDB, ok := ctx.(*NetworkDB) if ok { peers := nDB.Peers(r.Form["nid"][0]) - fmt.Fprintf(w, "Network:%s Total peers: %d\n", r.Form["nid"], len(peers)) + rsp := &diagnose.TableObj{Length: len(peers)} for i, peerInfo := range peers { - fmt.Fprintf(w, "%d) %s -> %s\n", i, peerInfo.Name, peerInfo.IP) + rsp.Elements = append(rsp.Elements, &diagnose.PeerEntryObj{Index: i, Name: peerInfo.Name, IP: peerInfo.IP}) } + log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("network peers done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(rsp), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbClusterPeers(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + diagnose.DebugHTTPForm(r) + _, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("cluster peers") + nDB, ok := ctx.(*NetworkDB) if ok { peers := nDB.ClusterPeers() - fmt.Fprintf(w, "Total peers: %d\n", len(peers)) + rsp := &diagnose.TableObj{Length: len(peers)} for i, peerInfo := range peers { - fmt.Fprintf(w, "%d) %s -> %s\n", i, peerInfo.Name, peerInfo.IP) + rsp.Elements = append(rsp.Elements, &diagnose.PeerEntryObj{Index: i, Name: peerInfo.Name, IP: peerInfo.IP}) } + log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("cluster peers done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(rsp), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbCreateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + unsafe, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("create entry") + if len(r.Form["tname"]) < 1 || len(r.Form["nid"]) < 1 || len(r.Form["key"]) < 1 || len(r.Form["value"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path)) + log.Error("create entry failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -93,25 +139,48 @@ func dbCreateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { nid := r.Form["nid"][0] key := r.Form["key"][0] value := r.Form["value"][0] + decodedValue := []byte(value) + if !unsafe { + var err error + decodedValue, err = base64.StdEncoding.DecodeString(value) + if err != nil { + log.WithError(err).Error("create entry failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) + return + } + } nDB, ok := ctx.(*NetworkDB) if ok { - if err := nDB.CreateEntry(tname, nid, key, []byte(value)); err != nil { - diagnose.HTTPReplyError(w, err.Error(), "") + if err := nDB.CreateEntry(tname, nid, key, decodedValue); err != nil { + rsp := diagnose.FailCommand(err) + diagnose.HTTPReply(w, rsp, json) + log.WithError(err).Error("create entry failed") return } - fmt.Fprintf(w, "OK\n") + log.Info("create entry done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbUpdateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + unsafe, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("update entry") + if len(r.Form["tname"]) < 1 || len(r.Form["nid"]) < 1 || len(r.Form["key"]) < 1 || len(r.Form["value"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path)) + log.Error("update entry failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -119,24 +188,46 @@ func dbUpdateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { nid := r.Form["nid"][0] key := r.Form["key"][0] value := r.Form["value"][0] + decodedValue := []byte(value) + if !unsafe { + var err error + decodedValue, err = base64.StdEncoding.DecodeString(value) + if err != nil { + log.WithError(err).Error("update entry failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) + return + } + } nDB, ok := ctx.(*NetworkDB) if ok { - if err := nDB.UpdateEntry(tname, nid, key, []byte(value)); err != nil { - diagnose.HTTPReplyError(w, err.Error(), "") + if err := nDB.UpdateEntry(tname, nid, key, decodedValue); err != nil { + log.WithError(err).Error("update entry failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) return } - fmt.Fprintf(w, "OK\n") + log.Info("update entry done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbDeleteEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + _, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("delete entry") + if len(r.Form["tname"]) < 1 || len(r.Form["nid"]) < 1 || len(r.Form["key"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path)) + log.Error("delete entry failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -148,20 +239,32 @@ func dbDeleteEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { if ok { err := nDB.DeleteEntry(tname, nid, key) if err != nil { - diagnose.HTTPReplyError(w, err.Error(), "") + log.WithError(err).Error("delete entry failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) return } - fmt.Fprintf(w, "OK\n") + log.Info("delete entry done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbGetEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + unsafe, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("get entry") + if len(r.Form["tname"]) < 1 || len(r.Form["nid"]) < 1 || len(r.Form["key"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path)) + log.Error("get entry failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -173,18 +276,39 @@ func dbGetEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { if ok { value, err := nDB.GetEntry(tname, nid, key) if err != nil { - diagnose.HTTPReplyError(w, err.Error(), "") + log.WithError(err).Error("get entry failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) return } - fmt.Fprintf(w, "key:`%s` value:`%s`\n", key, string(value)) + + var encodedValue string + if unsafe { + encodedValue = string(value) + } else { + encodedValue = base64.StdEncoding.EncodeToString(value) + } + + rsp := &diagnose.TableEntryObj{Key: key, Value: encodedValue} + log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("update entry done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(rsp), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbJoinNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + _, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("join network") + if len(r.Form["nid"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path)) + log.Error("join network failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -193,18 +317,30 @@ func dbJoinNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) { nDB, ok := ctx.(*NetworkDB) if ok { if err := nDB.JoinNetwork(nid); err != nil { - diagnose.HTTPReplyError(w, err.Error(), "") + log.WithError(err).Error("join network failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) return } - fmt.Fprintf(w, "OK\n") + log.Info("join network done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbLeaveNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + _, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("leave network") + if len(r.Form["nid"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path)) + log.Error("leave network failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -213,19 +349,31 @@ func dbLeaveNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) { nDB, ok := ctx.(*NetworkDB) if ok { if err := nDB.LeaveNetwork(nid); err != nil { - diagnose.HTTPReplyError(w, err.Error(), "") + log.WithError(err).Error("leave network failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) return } - fmt.Fprintf(w, "OK\n") + log.Info("leave network done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbGetTable(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + unsafe, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("get table") + if len(r.Form["tname"]) < 1 || len(r.Form["nid"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id", r.URL.Path)) + log.Error("get table failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -235,20 +383,26 @@ func dbGetTable(ctx interface{}, w http.ResponseWriter, r *http.Request) { nDB, ok := ctx.(*NetworkDB) if ok { table := nDB.GetTableByNetwork(tname, nid) - fmt.Fprintf(w, "total elements: %d\n", len(table)) - i := 0 + rsp := &diagnose.TableObj{Length: len(table)} + var i = 0 for k, v := range table { - fmt.Fprintf(w, "%d) k:`%s` -> v:`%s`\n", i, k, string(v.([]byte))) - i++ + var encodedValue string + if unsafe { + encodedValue = string(v.Value) + } else { + encodedValue = base64.StdEncoding.EncodeToString(v.Value) + } + rsp.Elements = append(rsp.Elements, + &diagnose.TableEntryObj{ + Index: i, + Key: k, + Value: encodedValue, + Owner: v.owner, + }) } + log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("get table done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(rsp), json) + return } -} - -func dbStackTrace(ctx interface{}, w http.ResponseWriter, r *http.Request) { - path, err := stackdump.DumpStacks("/tmp/") - if err != nil { - logrus.WithError(err).Error("failed to write goroutines dump") - } else { - fmt.Fprintf(w, "goroutine stacks written to %s", path) - } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } diff --git a/components/engine/vendor/github.com/docker/libnetwork/vendor.conf b/components/engine/vendor/github.com/docker/libnetwork/vendor.conf index f00d8ced44..bdcb5a1a54 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/vendor.conf +++ b/components/engine/vendor/github.com/docker/libnetwork/vendor.conf @@ -45,7 +45,7 @@ github.com/sirupsen/logrus v1.0.3 github.com/stretchr/testify dab07ac62d4905d3e48d17dc549c684ac3b7c15a github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16 github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065 -github.com/vishvananda/netlink bd6d5de5ccef2d66b0a26177928d0d8895d7f969 +github.com/vishvananda/netlink b2de5d10e38ecce8607e6b438b6d174f389a004e github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25 golang.org/x/crypto 558b6879de74bc843225cde5686419267ff707ca golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 diff --git a/components/engine/vendor/github.com/docker/swarmkit/agent/exec/controller.go b/components/engine/vendor/github.com/docker/swarmkit/agent/exec/controller.go index 9b4fc7bca1..c9e9343fd7 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/agent/exec/controller.go +++ b/components/engine/vendor/github.com/docker/swarmkit/agent/exec/controller.go @@ -288,7 +288,9 @@ func Do(ctx context.Context, task *api.Task, ctlr Controller) (*api.TaskStatus, status.PortStatus = portStatus }() - if task.DesiredState == api.TaskStateShutdown { + // this branch bounds the largest state achievable in the agent as SHUTDOWN, which + // is exactly the correct behavior for the agent. + if task.DesiredState >= api.TaskStateShutdown { if status.State >= api.TaskStateCompleted { return noop() } diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/ca.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/ca.pb.go index caaa06cc83..7d289126a1 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/ca.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/ca.pb.go @@ -131,6 +131,8 @@ LeaveResponse ProcessRaftMessageRequest ProcessRaftMessageResponse + StreamRaftMessageRequest + StreamRaftMessageResponse ResolveAddressRequest ResolveAddressResponse InternalRaftRequest @@ -235,6 +237,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -984,12 +987,12 @@ func NewRaftProxyCAServer(local CAServer, connSelector raftselector.ConnProvider redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) @@ -1126,12 +1129,12 @@ func NewRaftProxyNodeCAServer(local NodeCAServer, connSelector raftselector.Conn redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/control.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/control.pb.go index 13ef4822cb..fb28c5d330 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/control.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/control.pb.go @@ -19,6 +19,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -5859,12 +5860,12 @@ func NewRaftProxyControlServer(local ControlServer, connSelector raftselector.Co redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/dispatcher.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/dispatcher.pb.go index 8e9b038cad..120df8811f 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/dispatcher.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/dispatcher.pb.go @@ -24,6 +24,7 @@ import github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -1602,12 +1603,12 @@ func NewRaftProxyDispatcherServer(local DispatcherServer, connSelector raftselec redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/health.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/health.pb.go index 757db6acbf..ed7df73054 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/health.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/health.pb.go @@ -17,6 +17,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -286,12 +287,12 @@ func NewRaftProxyHealthServer(local HealthServer, connSelector raftselector.Conn redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/logbroker.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/logbroker.pb.go index 58515aa11e..1108088fba 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/logbroker.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/logbroker.pb.go @@ -20,6 +20,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -1273,12 +1274,12 @@ func NewRaftProxyLogsServer(local LogsServer, connSelector raftselector.ConnProv redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) @@ -1396,12 +1397,12 @@ func NewRaftProxyLogBrokerServer(local LogBrokerServer, connSelector raftselecto redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/raft.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/raft.pb.go index 4710ee6b01..ddee3e18a6 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/raft.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/raft.pb.go @@ -21,6 +21,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -133,6 +134,23 @@ func (m *ProcessRaftMessageResponse) Reset() { *m = ProcessRa func (*ProcessRaftMessageResponse) ProtoMessage() {} func (*ProcessRaftMessageResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{6} } +// Raft message streaming request. +type StreamRaftMessageRequest struct { + Message *raftpb.Message `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` +} + +func (m *StreamRaftMessageRequest) Reset() { *m = StreamRaftMessageRequest{} } +func (*StreamRaftMessageRequest) ProtoMessage() {} +func (*StreamRaftMessageRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{7} } + +// Raft message streaming response. +type StreamRaftMessageResponse struct { +} + +func (m *StreamRaftMessageResponse) Reset() { *m = StreamRaftMessageResponse{} } +func (*StreamRaftMessageResponse) ProtoMessage() {} +func (*StreamRaftMessageResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{8} } + type ResolveAddressRequest struct { // raft_id is the ID to resolve to an address. RaftID uint64 `protobuf:"varint,1,opt,name=raft_id,json=raftId,proto3" json:"raft_id,omitempty"` @@ -140,7 +158,7 @@ type ResolveAddressRequest struct { func (m *ResolveAddressRequest) Reset() { *m = ResolveAddressRequest{} } func (*ResolveAddressRequest) ProtoMessage() {} -func (*ResolveAddressRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{7} } +func (*ResolveAddressRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{9} } type ResolveAddressResponse struct { // Addr specifies the address of the member @@ -149,7 +167,7 @@ type ResolveAddressResponse struct { func (m *ResolveAddressResponse) Reset() { *m = ResolveAddressResponse{} } func (*ResolveAddressResponse) ProtoMessage() {} -func (*ResolveAddressResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{8} } +func (*ResolveAddressResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{10} } // Contains one of many protobuf encoded objects to replicate // over the raft backend with a request ID to track when the @@ -161,7 +179,7 @@ type InternalRaftRequest struct { func (m *InternalRaftRequest) Reset() { *m = InternalRaftRequest{} } func (*InternalRaftRequest) ProtoMessage() {} -func (*InternalRaftRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{9} } +func (*InternalRaftRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{11} } // StoreAction defines a target and operation to apply on the storage system. type StoreAction struct { @@ -181,7 +199,7 @@ type StoreAction struct { func (m *StoreAction) Reset() { *m = StoreAction{} } func (*StoreAction) ProtoMessage() {} -func (*StoreAction) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{10} } +func (*StoreAction) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{12} } type isStoreAction_Target interface { isStoreAction_Target() @@ -512,6 +530,8 @@ func init() { proto.RegisterType((*LeaveResponse)(nil), "docker.swarmkit.v1.LeaveResponse") proto.RegisterType((*ProcessRaftMessageRequest)(nil), "docker.swarmkit.v1.ProcessRaftMessageRequest") proto.RegisterType((*ProcessRaftMessageResponse)(nil), "docker.swarmkit.v1.ProcessRaftMessageResponse") + proto.RegisterType((*StreamRaftMessageRequest)(nil), "docker.swarmkit.v1.StreamRaftMessageRequest") + proto.RegisterType((*StreamRaftMessageResponse)(nil), "docker.swarmkit.v1.StreamRaftMessageResponse") proto.RegisterType((*ResolveAddressRequest)(nil), "docker.swarmkit.v1.ResolveAddressRequest") proto.RegisterType((*ResolveAddressResponse)(nil), "docker.swarmkit.v1.ResolveAddressResponse") proto.RegisterType((*InternalRaftRequest)(nil), "docker.swarmkit.v1.InternalRaftRequest") @@ -539,6 +559,14 @@ func (p *authenticatedWrapperRaftServer) ProcessRaftMessage(ctx context.Context, return p.local.ProcessRaftMessage(ctx, r) } +func (p *authenticatedWrapperRaftServer) StreamRaftMessage(stream Raft_StreamRaftMessageServer) error { + + if err := p.authorize(stream.Context(), []string{"swarm-manager"}); err != nil { + return err + } + return p.local.StreamRaftMessage(stream) +} + func (p *authenticatedWrapperRaftServer) ResolveAddress(ctx context.Context, r *ResolveAddressRequest) (*ResolveAddressResponse, error) { if err := p.authorize(ctx, []string{"swarm-manager"}); err != nil { @@ -673,6 +701,16 @@ func (m *ProcessRaftMessageResponse) Copy() *ProcessRaftMessageResponse { } func (m *ProcessRaftMessageResponse) CopyFrom(src interface{}) {} +func (m *StreamRaftMessageResponse) Copy() *StreamRaftMessageResponse { + if m == nil { + return nil + } + o := &StreamRaftMessageResponse{} + o.CopyFrom(m) + return o +} + +func (m *StreamRaftMessageResponse) CopyFrom(src interface{}) {} func (m *ResolveAddressRequest) Copy() *ResolveAddressRequest { if m == nil { return nil @@ -813,6 +851,12 @@ type RaftClient interface { // ProcessRaftMessage sends a raft message to be processed on a raft member, it is // called from the RaftMember willing to send a message to its destination ('To' field) ProcessRaftMessage(ctx context.Context, in *ProcessRaftMessageRequest, opts ...grpc.CallOption) (*ProcessRaftMessageResponse, error) + // StreamRaftMessage accepts a stream of raft messages of type StreamRaftMessageRequest + // to be processed on a raft member, returning a StreamRaftMessageResponse + // when processing of the streamed messages is complete. A single stream corresponds + // to a single raft message, which may be disassembled and streamed as individual messages. + // It is called from the Raft leader, which uses it to stream messages to a raft member. + StreamRaftMessage(ctx context.Context, opts ...grpc.CallOption) (Raft_StreamRaftMessageClient, error) // ResolveAddress returns the address where the node with the given ID can be reached. ResolveAddress(ctx context.Context, in *ResolveAddressRequest, opts ...grpc.CallOption) (*ResolveAddressResponse, error) } @@ -834,6 +878,40 @@ func (c *raftClient) ProcessRaftMessage(ctx context.Context, in *ProcessRaftMess return out, nil } +func (c *raftClient) StreamRaftMessage(ctx context.Context, opts ...grpc.CallOption) (Raft_StreamRaftMessageClient, error) { + stream, err := grpc.NewClientStream(ctx, &_Raft_serviceDesc.Streams[0], c.cc, "/docker.swarmkit.v1.Raft/StreamRaftMessage", opts...) + if err != nil { + return nil, err + } + x := &raftStreamRaftMessageClient{stream} + return x, nil +} + +type Raft_StreamRaftMessageClient interface { + Send(*StreamRaftMessageRequest) error + CloseAndRecv() (*StreamRaftMessageResponse, error) + grpc.ClientStream +} + +type raftStreamRaftMessageClient struct { + grpc.ClientStream +} + +func (x *raftStreamRaftMessageClient) Send(m *StreamRaftMessageRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *raftStreamRaftMessageClient) CloseAndRecv() (*StreamRaftMessageResponse, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(StreamRaftMessageResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + func (c *raftClient) ResolveAddress(ctx context.Context, in *ResolveAddressRequest, opts ...grpc.CallOption) (*ResolveAddressResponse, error) { out := new(ResolveAddressResponse) err := grpc.Invoke(ctx, "/docker.swarmkit.v1.Raft/ResolveAddress", in, out, c.cc, opts...) @@ -849,6 +927,12 @@ type RaftServer interface { // ProcessRaftMessage sends a raft message to be processed on a raft member, it is // called from the RaftMember willing to send a message to its destination ('To' field) ProcessRaftMessage(context.Context, *ProcessRaftMessageRequest) (*ProcessRaftMessageResponse, error) + // StreamRaftMessage accepts a stream of raft messages of type StreamRaftMessageRequest + // to be processed on a raft member, returning a StreamRaftMessageResponse + // when processing of the streamed messages is complete. A single stream corresponds + // to a single raft message, which may be disassembled and streamed as individual messages. + // It is called from the Raft leader, which uses it to stream messages to a raft member. + StreamRaftMessage(Raft_StreamRaftMessageServer) error // ResolveAddress returns the address where the node with the given ID can be reached. ResolveAddress(context.Context, *ResolveAddressRequest) (*ResolveAddressResponse, error) } @@ -875,6 +959,32 @@ func _Raft_ProcessRaftMessage_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _Raft_StreamRaftMessage_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(RaftServer).StreamRaftMessage(&raftStreamRaftMessageServer{stream}) +} + +type Raft_StreamRaftMessageServer interface { + SendAndClose(*StreamRaftMessageResponse) error + Recv() (*StreamRaftMessageRequest, error) + grpc.ServerStream +} + +type raftStreamRaftMessageServer struct { + grpc.ServerStream +} + +func (x *raftStreamRaftMessageServer) SendAndClose(m *StreamRaftMessageResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *raftStreamRaftMessageServer) Recv() (*StreamRaftMessageRequest, error) { + m := new(StreamRaftMessageRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + func _Raft_ResolveAddress_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ResolveAddressRequest) if err := dec(in); err != nil { @@ -906,7 +1016,13 @@ var _Raft_serviceDesc = grpc.ServiceDesc{ Handler: _Raft_ResolveAddress_Handler, }, }, - Streams: []grpc.StreamDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamRaftMessage", + Handler: _Raft_StreamRaftMessage_Handler, + ClientStreams: true, + }, + }, Metadata: "github.com/docker/swarmkit/api/raft.proto", } @@ -1212,6 +1328,52 @@ func (m *ProcessRaftMessageResponse) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *StreamRaftMessageRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StreamRaftMessageRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Message != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintRaft(dAtA, i, uint64(m.Message.Size())) + n4, err := m.Message.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n4 + } + return i, nil +} + +func (m *StreamRaftMessageResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StreamRaftMessageResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + func (m *ResolveAddressRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1315,11 +1477,11 @@ func (m *StoreAction) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintRaft(dAtA, i, uint64(m.Action)) } if m.Target != nil { - nn4, err := m.Target.MarshalTo(dAtA[i:]) + nn5, err := m.Target.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += nn4 + i += nn5 } return i, nil } @@ -1330,11 +1492,11 @@ func (m *StoreAction_Node) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Node.Size())) - n5, err := m.Node.MarshalTo(dAtA[i:]) + n6, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n5 + i += n6 } return i, nil } @@ -1344,11 +1506,11 @@ func (m *StoreAction_Service) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintRaft(dAtA, i, uint64(m.Service.Size())) - n6, err := m.Service.MarshalTo(dAtA[i:]) + n7, err := m.Service.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n6 + i += n7 } return i, nil } @@ -1358,11 +1520,11 @@ func (m *StoreAction_Task) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x22 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Task.Size())) - n7, err := m.Task.MarshalTo(dAtA[i:]) + n8, err := m.Task.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n7 + i += n8 } return i, nil } @@ -1372,11 +1534,11 @@ func (m *StoreAction_Network) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintRaft(dAtA, i, uint64(m.Network.Size())) - n8, err := m.Network.MarshalTo(dAtA[i:]) + n9, err := m.Network.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n8 + i += n9 } return i, nil } @@ -1386,11 +1548,11 @@ func (m *StoreAction_Cluster) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x32 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Cluster.Size())) - n9, err := m.Cluster.MarshalTo(dAtA[i:]) + n10, err := m.Cluster.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n9 + i += n10 } return i, nil } @@ -1400,11 +1562,11 @@ func (m *StoreAction_Secret) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x3a i++ i = encodeVarintRaft(dAtA, i, uint64(m.Secret.Size())) - n10, err := m.Secret.MarshalTo(dAtA[i:]) + n11, err := m.Secret.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n10 + i += n11 } return i, nil } @@ -1414,11 +1576,11 @@ func (m *StoreAction_Resource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x42 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Resource.Size())) - n11, err := m.Resource.MarshalTo(dAtA[i:]) + n12, err := m.Resource.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n11 + i += n12 } return i, nil } @@ -1428,11 +1590,11 @@ func (m *StoreAction_Extension) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x4a i++ i = encodeVarintRaft(dAtA, i, uint64(m.Extension.Size())) - n12, err := m.Extension.MarshalTo(dAtA[i:]) + n13, err := m.Extension.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n12 + i += n13 } return i, nil } @@ -1442,11 +1604,11 @@ func (m *StoreAction_Config) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x52 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Config.Size())) - n13, err := m.Config.MarshalTo(dAtA[i:]) + n14, err := m.Config.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n14 } return i, nil } @@ -1488,12 +1650,12 @@ func NewRaftProxyRaftServer(local RaftServer, connSelector raftselector.ConnProv redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) @@ -1585,6 +1747,63 @@ func (p *raftProxyRaftServer) ProcessRaftMessage(ctx context.Context, r *Process return resp, err } +type Raft_StreamRaftMessageServerWrapper struct { + Raft_StreamRaftMessageServer + ctx context.Context +} + +func (s Raft_StreamRaftMessageServerWrapper) Context() context.Context { + return s.ctx +} + +func (p *raftProxyRaftServer) StreamRaftMessage(stream Raft_StreamRaftMessageServer) error { + ctx := stream.Context() + conn, err := p.connSelector.LeaderConn(ctx) + if err != nil { + if err == raftselector.ErrIsLeader { + ctx, err = p.runCtxMods(ctx, p.localCtxMods) + if err != nil { + return err + } + streamWrapper := Raft_StreamRaftMessageServerWrapper{ + Raft_StreamRaftMessageServer: stream, + ctx: ctx, + } + return p.local.StreamRaftMessage(streamWrapper) + } + return err + } + ctx, err = p.runCtxMods(ctx, p.remoteCtxMods) + if err != nil { + return err + } + clientStream, err := NewRaftClient(conn).StreamRaftMessage(ctx) + + if err != nil { + return err + } + + for { + msg, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + return err + } + if err := clientStream.Send(msg); err != nil { + return err + } + } + + reply, err := clientStream.CloseAndRecv() + if err != nil { + return err + } + + return stream.SendAndClose(reply) +} + func (p *raftProxyRaftServer) ResolveAddress(ctx context.Context, r *ResolveAddressRequest) (*ResolveAddressResponse, error) { conn, err := p.connSelector.LeaderConn(ctx) @@ -1630,12 +1849,12 @@ func NewRaftProxyRaftMembershipServer(local RaftMembershipServer, connSelector r redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) @@ -1843,6 +2062,22 @@ func (m *ProcessRaftMessageResponse) Size() (n int) { return n } +func (m *StreamRaftMessageRequest) Size() (n int) { + var l int + _ = l + if m.Message != nil { + l = m.Message.Size() + n += 1 + l + sovRaft(uint64(l)) + } + return n +} + +func (m *StreamRaftMessageResponse) Size() (n int) { + var l int + _ = l + return n +} + func (m *ResolveAddressRequest) Size() (n int) { var l int _ = l @@ -2057,6 +2292,25 @@ func (this *ProcessRaftMessageResponse) String() string { }, "") return s } +func (this *StreamRaftMessageRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&StreamRaftMessageRequest{`, + `Message:` + strings.Replace(fmt.Sprintf("%v", this.Message), "Message", "raftpb.Message", 1) + `,`, + `}`, + }, "") + return s +} +func (this *StreamRaftMessageResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&StreamRaftMessageResponse{`, + `}`, + }, "") + return s +} func (this *ResolveAddressRequest) String() string { if this == nil { return "nil" @@ -2861,6 +3115,139 @@ func (m *ProcessRaftMessageResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *StreamRaftMessageRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StreamRaftMessageRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StreamRaftMessageRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRaft + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Message == nil { + m.Message = &raftpb.Message{} + } + if err := m.Message.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRaft(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRaft + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *StreamRaftMessageResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StreamRaftMessageResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StreamRaftMessageResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipRaft(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRaft + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *ResolveAddressRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -3574,66 +3961,69 @@ var ( func init() { proto.RegisterFile("github.com/docker/swarmkit/api/raft.proto", fileDescriptorRaft) } var fileDescriptorRaft = []byte{ - // 974 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x96, 0x4f, 0x6f, 0x1b, 0x45, - 0x18, 0xc6, 0x77, 0xd7, 0x5b, 0x27, 0x79, 0xd3, 0x26, 0xd1, 0x94, 0x84, 0xed, 0x52, 0x1c, 0x77, - 0x8b, 0x84, 0x13, 0x9a, 0xb5, 0x30, 0x48, 0x45, 0x85, 0x1e, 0x62, 0xc7, 0x92, 0x4d, 0x5b, 0xa7, - 0xda, 0x24, 0xd0, 0x5b, 0x58, 0xef, 0x4e, 0xdc, 0xc5, 0xf6, 0x8e, 0x99, 0x19, 0x3b, 0x70, 0x41, - 0x3d, 0xa2, 0x5c, 0x39, 0x80, 0x90, 0x7a, 0x82, 0x73, 0x3f, 0x00, 0x1f, 0x00, 0x45, 0x9c, 0xb8, - 0xc1, 0x29, 0xa2, 0xfe, 0x00, 0xf0, 0x15, 0xd0, 0xcc, 0xee, 0x3a, 0xc6, 0x59, 0x3b, 0xb9, 0x24, - 0xa3, 0x9d, 0xdf, 0xf3, 0x3e, 0xef, 0x3b, 0x7f, 0xde, 0x31, 0x6c, 0xb4, 0x02, 0xfe, 0xbc, 0xdf, - 0xb4, 0x3d, 0xd2, 0x2d, 0xfa, 0xc4, 0x6b, 0x63, 0x5a, 0x64, 0xc7, 0x2e, 0xed, 0xb6, 0x03, 0x5e, - 0x74, 0x7b, 0x41, 0x91, 0xba, 0x47, 0xdc, 0xee, 0x51, 0xc2, 0x09, 0x42, 0xd1, 0xbc, 0x9d, 0xcc, - 0xdb, 0x83, 0xf7, 0xcd, 0x7b, 0x97, 0xc8, 0x49, 0xf3, 0x4b, 0xec, 0x71, 0x16, 0x45, 0x30, 0x37, - 0x2f, 0xa1, 0xf9, 0x37, 0x3d, 0x9c, 0xb0, 0x5b, 0x63, 0xac, 0x47, 0x28, 0x26, 0xac, 0x88, 0xb9, - 0xe7, 0xcb, 0x84, 0xe4, 0x9f, 0x5e, 0x73, 0x2c, 0x39, 0xf3, 0x8d, 0x16, 0x69, 0x11, 0x39, 0x2c, - 0x8a, 0x51, 0xfc, 0xf5, 0xfe, 0x0c, 0x43, 0x49, 0x34, 0xfb, 0x47, 0xc5, 0x5e, 0xa7, 0xdf, 0x0a, - 0xc2, 0xf8, 0x5f, 0x24, 0xb4, 0x5e, 0xa9, 0x00, 0x8e, 0x7b, 0xc4, 0x9f, 0xe0, 0x6e, 0x13, 0x53, - 0x74, 0x17, 0xe6, 0x84, 0xd7, 0x61, 0xe0, 0x1b, 0x6a, 0x5e, 0x2d, 0xe8, 0x65, 0x18, 0x9e, 0xad, - 0x67, 0x05, 0x50, 0xdf, 0x71, 0xb2, 0x62, 0xaa, 0xee, 0x0b, 0x28, 0x24, 0x3e, 0x16, 0x90, 0x96, - 0x57, 0x0b, 0x0b, 0x11, 0xd4, 0x20, 0x3e, 0x16, 0x90, 0x98, 0xaa, 0xfb, 0x08, 0x81, 0xee, 0xfa, - 0x3e, 0x35, 0x32, 0x82, 0x70, 0xe4, 0x18, 0x95, 0x21, 0xcb, 0xb8, 0xcb, 0xfb, 0xcc, 0xd0, 0xf3, - 0x6a, 0x61, 0xb1, 0xf4, 0x8e, 0x7d, 0x71, 0xa5, 0xed, 0xf3, 0x6c, 0xf6, 0x24, 0x5b, 0xd6, 0x4f, - 0xcf, 0xd6, 0x15, 0x27, 0x56, 0x5a, 0x77, 0x60, 0xf1, 0x53, 0x12, 0x84, 0x0e, 0xfe, 0xaa, 0x8f, - 0x19, 0x1f, 0xd9, 0xa8, 0xe7, 0x36, 0xd6, 0x4f, 0x2a, 0x5c, 0x8f, 0x18, 0xd6, 0x23, 0x21, 0xc3, - 0x57, 0xab, 0xea, 0x23, 0x98, 0xeb, 0x4a, 0x5b, 0x66, 0x68, 0xf9, 0x4c, 0x61, 0xb1, 0x94, 0x9b, - 0x9d, 0x9d, 0x93, 0xe0, 0xe8, 0x3d, 0x58, 0xa6, 0xb8, 0x4b, 0x06, 0xd8, 0x3f, 0x4c, 0x22, 0x64, - 0xf2, 0x99, 0x82, 0x5e, 0xd6, 0x56, 0x14, 0x67, 0x29, 0x9e, 0x8a, 0x44, 0xcc, 0x2a, 0xc3, 0xf5, - 0xc7, 0xd8, 0x1d, 0xe0, 0xa4, 0x80, 0x12, 0xe8, 0x62, 0xc5, 0x64, 0x62, 0x97, 0x7b, 0x4a, 0xd6, - 0x5a, 0x86, 0x1b, 0x71, 0x8c, 0xa8, 0x40, 0xeb, 0x31, 0xdc, 0x7a, 0x4a, 0x89, 0x87, 0x19, 0x8b, - 0x58, 0xc6, 0xdc, 0xd6, 0xc8, 0x61, 0x43, 0x14, 0x26, 0xbf, 0xc4, 0x26, 0xcb, 0x76, 0x74, 0xac, - 0xec, 0x04, 0x4c, 0xe6, 0x1f, 0xe8, 0x2f, 0x7e, 0xb0, 0x14, 0xeb, 0x36, 0x98, 0x69, 0xd1, 0x62, - 0xaf, 0x4f, 0x60, 0xd5, 0xc1, 0x8c, 0x74, 0x06, 0x78, 0xdb, 0xf7, 0xa9, 0x80, 0x62, 0x9f, 0xab, - 0xac, 0xb2, 0x75, 0x0f, 0xd6, 0x26, 0xd5, 0xf1, 0x26, 0xa5, 0xed, 0x64, 0x07, 0x6e, 0xd6, 0x43, - 0x8e, 0x69, 0xe8, 0x76, 0x44, 0x9c, 0xc4, 0x69, 0x0d, 0xb4, 0x91, 0x49, 0x76, 0x78, 0xb6, 0xae, - 0xd5, 0x77, 0x1c, 0x2d, 0xf0, 0xd1, 0x43, 0xc8, 0xba, 0x1e, 0x0f, 0x48, 0x18, 0xef, 0xe0, 0x7a, - 0xda, 0x6a, 0xee, 0x71, 0x42, 0xf1, 0xb6, 0xc4, 0x92, 0xa3, 0x15, 0x89, 0xac, 0xdf, 0x74, 0x58, - 0x1c, 0x9b, 0x45, 0x1f, 0x8f, 0xc2, 0x09, 0xab, 0xa5, 0xd2, 0xdd, 0x4b, 0xc2, 0x3d, 0x0a, 0x42, - 0x3f, 0x09, 0x86, 0xec, 0x78, 0x5f, 0x35, 0xb9, 0xe4, 0x46, 0x9a, 0x54, 0xdc, 0x98, 0x9a, 0x12, - 0xed, 0x29, 0xba, 0x0f, 0x73, 0x0c, 0xd3, 0x41, 0xe0, 0x61, 0x79, 0x65, 0x16, 0x4b, 0x6f, 0xa5, - 0xba, 0x45, 0x48, 0x4d, 0x71, 0x12, 0x5a, 0x18, 0x71, 0x97, 0xb5, 0xe3, 0x2b, 0x95, 0x6a, 0xb4, - 0xef, 0xb2, 0xb6, 0x30, 0x12, 0x9c, 0x30, 0x0a, 0x31, 0x3f, 0x26, 0xb4, 0x6d, 0x5c, 0x9b, 0x6e, - 0xd4, 0x88, 0x10, 0x61, 0x14, 0xd3, 0x42, 0xe8, 0x75, 0xfa, 0x8c, 0x63, 0x6a, 0x64, 0xa7, 0x0b, - 0x2b, 0x11, 0x22, 0x84, 0x31, 0x8d, 0x3e, 0x84, 0x2c, 0xc3, 0x1e, 0xc5, 0xdc, 0x98, 0x93, 0x3a, - 0x33, 0xbd, 0x32, 0x41, 0xd4, 0xc4, 0x45, 0x97, 0x23, 0xf4, 0x00, 0xe6, 0x29, 0x66, 0xa4, 0x4f, - 0x3d, 0x6c, 0xcc, 0x4b, 0xdd, 0xed, 0xd4, 0xcb, 0x11, 0x33, 0x35, 0xc5, 0x19, 0xf1, 0xe8, 0x21, - 0x2c, 0xe0, 0xaf, 0x39, 0x0e, 0x99, 0xd8, 0xbc, 0x05, 0x29, 0x7e, 0x3b, 0x4d, 0x5c, 0x4d, 0xa0, - 0x9a, 0xe2, 0x9c, 0x2b, 0x44, 0xc2, 0x1e, 0x09, 0x8f, 0x82, 0x96, 0x01, 0xd3, 0x13, 0xae, 0x48, - 0x42, 0x24, 0x1c, 0xb1, 0xe5, 0x79, 0xc8, 0x72, 0x97, 0xb6, 0x30, 0xdf, 0xfc, 0x57, 0x85, 0xe5, - 0x89, 0x73, 0x81, 0xde, 0x85, 0xb9, 0x83, 0xc6, 0xa3, 0xc6, 0xee, 0xe7, 0x8d, 0x15, 0xc5, 0x34, - 0x4f, 0x5e, 0xe6, 0xd7, 0x26, 0x88, 0x83, 0xb0, 0x1d, 0x92, 0xe3, 0x10, 0x95, 0xe0, 0xe6, 0xde, - 0xfe, 0xae, 0x53, 0x3d, 0xdc, 0xae, 0xec, 0xd7, 0x77, 0x1b, 0x87, 0x15, 0xa7, 0xba, 0xbd, 0x5f, - 0x5d, 0x51, 0xcd, 0x5b, 0x27, 0x2f, 0xf3, 0xab, 0x13, 0xa2, 0x0a, 0xc5, 0x2e, 0xc7, 0x17, 0x34, - 0x07, 0x4f, 0x77, 0x84, 0x46, 0x4b, 0xd5, 0x1c, 0xf4, 0xfc, 0x34, 0x8d, 0x53, 0x7d, 0xb2, 0xfb, - 0x59, 0x75, 0x25, 0x93, 0xaa, 0x71, 0x64, 0x13, 0x33, 0xdf, 0xfc, 0xee, 0xe7, 0x9c, 0xf2, 0xeb, - 0x2f, 0xb9, 0xc9, 0xea, 0x4a, 0xdf, 0x6b, 0xa0, 0x8b, 0x1b, 0x8a, 0x4e, 0x54, 0x40, 0x17, 0x9b, - 0x07, 0xda, 0x4a, 0x5b, 0xc1, 0xa9, 0x2d, 0xcb, 0xb4, 0xaf, 0x8a, 0xc7, 0x3d, 0x69, 0xf5, 0xf7, - 0x57, 0xff, 0xfc, 0xa8, 0x2d, 0xc3, 0x0d, 0xc9, 0x6f, 0x75, 0xdd, 0xd0, 0x6d, 0x61, 0x8a, 0xbe, - 0x85, 0xa5, 0xff, 0x37, 0x1b, 0xb4, 0x31, 0xed, 0x08, 0x5d, 0x68, 0x67, 0xe6, 0xe6, 0x55, 0xd0, - 0x99, 0xfe, 0xa5, 0x3f, 0x55, 0x58, 0x3a, 0x6f, 0xde, 0xec, 0x79, 0xd0, 0x43, 0x5f, 0x80, 0x2e, - 0x9e, 0x26, 0x94, 0xda, 0x9a, 0xc6, 0x1e, 0x36, 0x33, 0x3f, 0x1d, 0x98, 0x5d, 0xb4, 0x07, 0xd7, - 0xe4, 0xe3, 0x80, 0x52, 0x23, 0x8c, 0xbf, 0x3d, 0xe6, 0x9d, 0x19, 0xc4, 0x4c, 0x93, 0xb2, 0x71, - 0xfa, 0x3a, 0xa7, 0xfc, 0xf5, 0x3a, 0xa7, 0xbc, 0x18, 0xe6, 0xd4, 0xd3, 0x61, 0x4e, 0xfd, 0x63, - 0x98, 0x53, 0xff, 0x1e, 0xe6, 0xd4, 0x67, 0x99, 0x67, 0x7a, 0x33, 0x2b, 0x7f, 0x5b, 0x7c, 0xf0, - 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x64, 0x01, 0xa3, 0x4f, 0x74, 0x09, 0x00, 0x00, + // 1015 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x96, 0xc1, 0x6e, 0x1b, 0x45, + 0x18, 0xc7, 0x77, 0xed, 0xad, 0xd3, 0x7c, 0x69, 0x93, 0x30, 0x25, 0x61, 0xb3, 0x2d, 0x8e, 0xbb, + 0x45, 0xc2, 0x09, 0xc9, 0x5a, 0x18, 0xa4, 0xa2, 0x42, 0x0f, 0x71, 0x62, 0x29, 0x26, 0xad, 0x53, + 0x6d, 0x12, 0xe8, 0x2d, 0xac, 0x77, 0x27, 0xee, 0x62, 0x7b, 0xc7, 0xcc, 0x8c, 0x1d, 0xb8, 0xa0, + 0x1e, 0x21, 0x2f, 0x00, 0x42, 0xaa, 0x38, 0xc0, 0xb9, 0x0f, 0xc0, 0x03, 0xa0, 0x88, 0x13, 0x37, + 0x38, 0x45, 0xd4, 0x0f, 0x00, 0xaf, 0x80, 0x66, 0x76, 0xd7, 0x31, 0xf6, 0xda, 0xf1, 0x81, 0x4b, + 0x32, 0xda, 0xf9, 0xfd, 0xbf, 0xff, 0x37, 0x33, 0xdf, 0x7c, 0x63, 0x58, 0xab, 0xfb, 0xfc, 0x59, + 0xa7, 0x66, 0xb9, 0xa4, 0x55, 0xf0, 0x88, 0xdb, 0xc0, 0xb4, 0xc0, 0x4e, 0x1d, 0xda, 0x6a, 0xf8, + 0xbc, 0xe0, 0xb4, 0xfd, 0x02, 0x75, 0x4e, 0xb8, 0xd5, 0xa6, 0x84, 0x13, 0x84, 0xc2, 0x79, 0x2b, + 0x9e, 0xb7, 0xba, 0xef, 0x1a, 0x1b, 0x57, 0xc8, 0x49, 0xed, 0x73, 0xec, 0x72, 0x16, 0x46, 0x30, + 0xd6, 0xaf, 0xa0, 0xf9, 0x57, 0x6d, 0x1c, 0xb3, 0x9b, 0x03, 0xac, 0x4b, 0x28, 0x26, 0xac, 0x80, + 0xb9, 0xeb, 0xc9, 0x84, 0xe4, 0x9f, 0x76, 0x6d, 0x20, 0x39, 0xe3, 0xf5, 0x3a, 0xa9, 0x13, 0x39, + 0x2c, 0x88, 0x51, 0xf4, 0xf5, 0xfe, 0x04, 0x43, 0x49, 0xd4, 0x3a, 0x27, 0x85, 0x76, 0xb3, 0x53, + 0xf7, 0x83, 0xe8, 0x5f, 0x28, 0x34, 0x5f, 0xaa, 0x00, 0xb6, 0x73, 0xc2, 0x1f, 0xe3, 0x56, 0x0d, + 0x53, 0x74, 0x0f, 0x66, 0x84, 0xd7, 0xb1, 0xef, 0xe9, 0x6a, 0x4e, 0xcd, 0x6b, 0x25, 0xe8, 0x5d, + 0xac, 0x66, 0x04, 0x50, 0xd9, 0xb1, 0x33, 0x62, 0xaa, 0xe2, 0x09, 0x28, 0x20, 0x1e, 0x16, 0x50, + 0x2a, 0xa7, 0xe6, 0x67, 0x43, 0xa8, 0x4a, 0x3c, 0x2c, 0x20, 0x31, 0x55, 0xf1, 0x10, 0x02, 0xcd, + 0xf1, 0x3c, 0xaa, 0xa7, 0x05, 0x61, 0xcb, 0x31, 0x2a, 0x41, 0x86, 0x71, 0x87, 0x77, 0x98, 0xae, + 0xe5, 0xd4, 0xfc, 0x5c, 0xf1, 0x2d, 0x6b, 0x74, 0xa7, 0xad, 0xcb, 0x6c, 0x0e, 0x24, 0x5b, 0xd2, + 0xce, 0x2f, 0x56, 0x15, 0x3b, 0x52, 0x9a, 0x77, 0x61, 0xee, 0x63, 0xe2, 0x07, 0x36, 0xfe, 0xa2, + 0x83, 0x19, 0xef, 0xdb, 0xa8, 0x97, 0x36, 0xe6, 0x0f, 0x2a, 0xdc, 0x08, 0x19, 0xd6, 0x26, 0x01, + 0xc3, 0xd3, 0xad, 0xea, 0x03, 0x98, 0x69, 0x49, 0x5b, 0xa6, 0xa7, 0x72, 0xe9, 0xfc, 0x5c, 0x31, + 0x3b, 0x39, 0x3b, 0x3b, 0xc6, 0xd1, 0x3b, 0xb0, 0x40, 0x71, 0x8b, 0x74, 0xb1, 0x77, 0x1c, 0x47, + 0x48, 0xe7, 0xd2, 0x79, 0xad, 0x94, 0x5a, 0x54, 0xec, 0xf9, 0x68, 0x2a, 0x14, 0x31, 0xb3, 0x04, + 0x37, 0x1e, 0x61, 0xa7, 0x8b, 0xe3, 0x05, 0x14, 0x41, 0x13, 0x3b, 0x26, 0x13, 0xbb, 0xda, 0x53, + 0xb2, 0xe6, 0x02, 0xdc, 0x8c, 0x62, 0x84, 0x0b, 0x34, 0x1f, 0xc1, 0xca, 0x13, 0x4a, 0x5c, 0xcc, + 0x58, 0xc8, 0x32, 0xe6, 0xd4, 0xfb, 0x0e, 0x6b, 0x62, 0x61, 0xf2, 0x4b, 0x64, 0xb2, 0x60, 0x85, + 0x65, 0x65, 0xc5, 0x60, 0x3c, 0xff, 0x40, 0x7b, 0xfe, 0x9d, 0xa9, 0x98, 0x77, 0xc0, 0x48, 0x8a, + 0x16, 0x79, 0xed, 0x81, 0x7e, 0xc0, 0x29, 0x76, 0x5a, 0xff, 0x87, 0xd5, 0x6d, 0x58, 0x49, 0x08, + 0x16, 0x39, 0x7d, 0x04, 0x4b, 0x36, 0x66, 0xa4, 0xd9, 0xc5, 0x5b, 0x9e, 0x47, 0x45, 0x3a, 0x91, + 0xcd, 0x34, 0xe7, 0x69, 0x6e, 0xc0, 0xf2, 0xb0, 0x3a, 0x2a, 0x87, 0xa4, 0x9a, 0x69, 0xc2, 0xad, + 0x4a, 0xc0, 0x31, 0x0d, 0x9c, 0xa6, 0x88, 0x13, 0x3b, 0x2d, 0x43, 0xaa, 0x6f, 0x92, 0xe9, 0x5d, + 0xac, 0xa6, 0x2a, 0x3b, 0x76, 0xca, 0xf7, 0xd0, 0x43, 0xc8, 0x38, 0x2e, 0xf7, 0x49, 0x10, 0xd5, + 0xca, 0x6a, 0xd2, 0xb9, 0x1d, 0x70, 0x42, 0xf1, 0x96, 0xc4, 0xe2, 0x22, 0x0e, 0x45, 0xe6, 0xaf, + 0x1a, 0xcc, 0x0d, 0xcc, 0xa2, 0x0f, 0xfb, 0xe1, 0x84, 0xd5, 0x7c, 0xf1, 0xde, 0x15, 0xe1, 0xf6, + 0xfc, 0xc0, 0x8b, 0x83, 0x21, 0x2b, 0xaa, 0xa0, 0x94, 0xdc, 0x71, 0x3d, 0x49, 0x2a, 0xee, 0xe6, + 0xae, 0x12, 0x56, 0x0f, 0xba, 0x0f, 0x33, 0x0c, 0xd3, 0xae, 0xef, 0x62, 0x79, 0x39, 0xe7, 0x8a, + 0xb7, 0x13, 0xdd, 0x42, 0x64, 0x57, 0xb1, 0x63, 0x5a, 0x18, 0x71, 0x87, 0x35, 0xa2, 0xcb, 0x9b, + 0x68, 0x74, 0xe8, 0xb0, 0x86, 0x30, 0x12, 0x9c, 0x30, 0x0a, 0x30, 0x3f, 0x25, 0xb4, 0xa1, 0x5f, + 0x1b, 0x6f, 0x54, 0x0d, 0x11, 0x61, 0x14, 0xd1, 0x42, 0xe8, 0x36, 0x3b, 0x8c, 0x63, 0xaa, 0x67, + 0xc6, 0x0b, 0xb7, 0x43, 0x44, 0x08, 0x23, 0x1a, 0xbd, 0x0f, 0x19, 0x86, 0x5d, 0x8a, 0xb9, 0x3e, + 0x23, 0x75, 0x46, 0xf2, 0xca, 0x04, 0xb1, 0x2b, 0x5a, 0x8a, 0x1c, 0xa1, 0x07, 0x70, 0x9d, 0x62, + 0x46, 0x3a, 0xd4, 0xc5, 0xfa, 0x75, 0xa9, 0xbb, 0x93, 0x78, 0x0d, 0x23, 0x66, 0x57, 0xb1, 0xfb, + 0x3c, 0x7a, 0x08, 0xb3, 0xf8, 0x4b, 0x8e, 0x03, 0x26, 0x0e, 0x6f, 0x56, 0x8a, 0xdf, 0x4c, 0x12, + 0x97, 0x63, 0x68, 0x57, 0xb1, 0x2f, 0x15, 0x22, 0x61, 0x97, 0x04, 0x27, 0x7e, 0x5d, 0x87, 0xf1, + 0x09, 0x6f, 0x4b, 0x42, 0x24, 0x1c, 0xb2, 0xa5, 0xeb, 0x90, 0xe1, 0x0e, 0xad, 0x63, 0xbe, 0xfe, + 0x8f, 0x0a, 0x0b, 0x43, 0x75, 0x81, 0xde, 0x86, 0x99, 0xa3, 0xea, 0x5e, 0x75, 0xff, 0xd3, 0xea, + 0xa2, 0x62, 0x18, 0x67, 0x2f, 0x72, 0xcb, 0x43, 0xc4, 0x51, 0xd0, 0x08, 0xc8, 0x69, 0x80, 0x8a, + 0x70, 0xeb, 0xe0, 0x70, 0xdf, 0x2e, 0x1f, 0x6f, 0x6d, 0x1f, 0x56, 0xf6, 0xab, 0xc7, 0xdb, 0x76, + 0x79, 0xeb, 0xb0, 0xbc, 0xa8, 0x1a, 0x2b, 0x67, 0x2f, 0x72, 0x4b, 0x43, 0xa2, 0x6d, 0x8a, 0x1d, + 0x8e, 0x47, 0x34, 0x47, 0x4f, 0x76, 0x84, 0x26, 0x95, 0xa8, 0x39, 0x6a, 0x7b, 0x49, 0x1a, 0xbb, + 0xfc, 0x78, 0xff, 0x93, 0xf2, 0x62, 0x3a, 0x51, 0x63, 0xcb, 0x76, 0x69, 0xbc, 0xf1, 0xcd, 0x4f, + 0x59, 0xe5, 0x97, 0x9f, 0xb3, 0xc3, 0xab, 0x2b, 0xfe, 0x98, 0x06, 0x4d, 0xdc, 0x50, 0x74, 0xa6, + 0x02, 0x1a, 0x6d, 0x53, 0x68, 0x33, 0x69, 0x07, 0xc7, 0x36, 0x47, 0xc3, 0x9a, 0x16, 0x8f, 0x7a, + 0xd2, 0xd2, 0x6f, 0x2f, 0xff, 0xfe, 0x3e, 0xb5, 0x00, 0x37, 0x25, 0xbf, 0xd9, 0x72, 0x02, 0xa7, + 0x8e, 0x29, 0xfa, 0x56, 0x85, 0xd7, 0x46, 0x1a, 0x19, 0xda, 0x48, 0xbe, 0xc6, 0xc9, 0xcd, 0xd3, + 0xd8, 0x9c, 0x92, 0x9e, 0x98, 0x49, 0x5e, 0x45, 0x5f, 0xc3, 0xfc, 0x7f, 0x1b, 0x1f, 0x5a, 0x1b, + 0x57, 0xce, 0x23, 0xad, 0xd5, 0x58, 0x9f, 0x06, 0x9d, 0x98, 0x41, 0xf1, 0x0f, 0x15, 0xe6, 0x2f, + 0x9f, 0x2c, 0xf6, 0xcc, 0x6f, 0xa3, 0xcf, 0x40, 0x13, 0x0f, 0x32, 0x4a, 0x6c, 0x93, 0x03, 0xcf, + 0xb9, 0x91, 0x1b, 0x0f, 0x4c, 0x3e, 0x00, 0x17, 0xae, 0xc9, 0x27, 0x11, 0x25, 0x46, 0x18, 0x7c, + 0x71, 0x8d, 0xbb, 0x13, 0x88, 0x89, 0x26, 0x25, 0xfd, 0xfc, 0x55, 0x56, 0xf9, 0xf3, 0x55, 0x56, + 0x79, 0xde, 0xcb, 0xaa, 0xe7, 0xbd, 0xac, 0xfa, 0x7b, 0x2f, 0xab, 0xfe, 0xd5, 0xcb, 0xaa, 0x4f, + 0xd3, 0x4f, 0xb5, 0x5a, 0x46, 0xfe, 0xa2, 0x7a, 0xef, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x3e, + 0x7a, 0x8b, 0xe7, 0x6a, 0x0a, 0x00, 0x00, } diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/raft.proto b/components/engine/vendor/github.com/docker/swarmkit/api/raft.proto index b39831584d..b351c15bde 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/raft.proto +++ b/components/engine/vendor/github.com/docker/swarmkit/api/raft.proto @@ -16,6 +16,15 @@ service Raft { option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" }; }; + // StreamRaftMessage accepts a stream of raft messages of type StreamRaftMessageRequest + // to be processed on a raft member, returning a StreamRaftMessageResponse + // when processing of the streamed messages is complete. A single stream corresponds + // to a single raft message, which may be disassembled and streamed as individual messages. + // It is called from the Raft leader, which uses it to stream messages to a raft member. + rpc StreamRaftMessage(stream StreamRaftMessageRequest) returns (StreamRaftMessageResponse) { + option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" }; + }; + // ResolveAddress returns the address where the node with the given ID can be reached. rpc ResolveAddress(ResolveAddressRequest) returns (ResolveAddressResponse) { option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" }; @@ -82,6 +91,15 @@ message ProcessRaftMessageRequest { message ProcessRaftMessageResponse {} +// Raft message streaming request. +message StreamRaftMessageRequest { + option (docker.protobuf.plugin.deepcopy) = false; + raftpb.Message message = 1; +} + +// Raft message streaming response. +message StreamRaftMessageResponse {} + message ResolveAddressRequest { // raft_id is the ID to resolve to an address. uint64 raft_id = 1; diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/resource.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/resource.pb.go index ead5d27ef4..d4e27f3362 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/resource.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/resource.pb.go @@ -19,6 +19,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -403,12 +404,12 @@ func NewRaftProxyResourceAllocatorServer(local ResourceAllocatorServer, connSele redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/types.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/types.pb.go index 33e2281b67..67243dc88e 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/types.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/types.pb.go @@ -72,7 +72,16 @@ const ( TaskStateShutdown TaskState = 640 TaskStateFailed TaskState = 704 TaskStateRejected TaskState = 768 - TaskStateOrphaned TaskState = 832 + // TaskStateRemove is used to correctly handle service deletions and scale + // downs. This allows us to keep track of tasks that have been marked for + // deletion, but can't yet be removed because the agent is in the process of + // shutting them down. Once the agent has shut down tasks with desired state + // REMOVE, the task reaper is responsible for removing them. + TaskStateRemove TaskState = 800 + // TaskStateOrphaned is used to free up resources associated with service + // tasks on unresponsive nodes without having to delete those tasks. This + // state is directly assigned to the task by the orchestrator. + TaskStateOrphaned TaskState = 832 ) var TaskState_name = map[int32]string{ @@ -88,6 +97,7 @@ var TaskState_name = map[int32]string{ 640: "SHUTDOWN", 704: "FAILED", 768: "REJECTED", + 800: "REMOVE", 832: "ORPHANED", } var TaskState_value = map[string]int32{ @@ -103,6 +113,7 @@ var TaskState_value = map[string]int32{ "SHUTDOWN": 640, "FAILED": 704, "REJECTED": 768, + "REMOVE": 800, "ORPHANED": 832, } @@ -17029,319 +17040,321 @@ var ( func init() { proto.RegisterFile("github.com/docker/swarmkit/api/types.proto", fileDescriptorTypes) } var fileDescriptorTypes = []byte{ - // 5020 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x5a, 0x4d, 0x6c, 0x24, 0x49, - 0x56, 0x76, 0xfd, 0xba, 0xea, 0x55, 0xd9, 0x4e, 0x47, 0x7b, 0x7b, 0xdc, 0xb5, 0xdd, 0x76, 0x4d, - 0xce, 0xf4, 0xce, 0x6c, 0x6f, 0x53, 0xfd, 0xb7, 0xbb, 0xea, 0x99, 0x61, 0x77, 0xa6, 0xfe, 0x6c, - 0xd7, 0xb6, 0x5d, 0x55, 0x8a, 0x2a, 0x77, 0xef, 0x22, 0x41, 0x2a, 0x9d, 0x19, 0x2e, 0xe7, 0x38, - 0x2b, 0xa3, 0xc8, 0xcc, 0xb2, 0xbb, 0x58, 0x10, 0x2d, 0x0e, 0x80, 0x7c, 0x82, 0xdb, 0x22, 0x64, - 0x2e, 0x70, 0x42, 0x48, 0x1c, 0x40, 0x42, 0x70, 0x1a, 0x24, 0x0e, 0x7b, 0x83, 0x05, 0x09, 0xad, - 0x40, 0x32, 0xac, 0x0f, 0xdc, 0x56, 0x70, 0x59, 0x71, 0x01, 0x09, 0xc5, 0x4f, 0x66, 0xa5, 0xab, - 0xd3, 0x76, 0x0f, 0xb3, 0x17, 0xbb, 0xe2, 0xbd, 0xef, 0xbd, 0x78, 0xf1, 0x22, 0xe2, 0xc5, 0x7b, - 0x11, 0x09, 0xf7, 0x06, 0x96, 0x7f, 0x30, 0xde, 0xab, 0x18, 0x74, 0xf8, 0xc0, 0xa4, 0xc6, 0x21, - 0x71, 0x1f, 0x78, 0xc7, 0xba, 0x3b, 0x3c, 0xb4, 0xfc, 0x07, 0xfa, 0xc8, 0x7a, 0xe0, 0x4f, 0x46, - 0xc4, 0xab, 0x8c, 0x5c, 0xea, 0x53, 0x84, 0x04, 0xa0, 0x12, 0x00, 0x2a, 0x47, 0x8f, 0x4a, 0xeb, - 0x03, 0x4a, 0x07, 0x36, 0x79, 0xc0, 0x11, 0x7b, 0xe3, 0xfd, 0x07, 0xbe, 0x35, 0x24, 0x9e, 0xaf, - 0x0f, 0x47, 0x42, 0xa8, 0xb4, 0x36, 0x0b, 0x30, 0xc7, 0xae, 0xee, 0x5b, 0xd4, 0x91, 0xfc, 0x95, - 0x01, 0x1d, 0x50, 0xfe, 0xf3, 0x01, 0xfb, 0x25, 0xa8, 0xea, 0x3a, 0xcc, 0x3f, 0x27, 0xae, 0x67, - 0x51, 0x07, 0xad, 0x40, 0xc6, 0x72, 0x4c, 0xf2, 0x72, 0x35, 0x51, 0x4e, 0xbc, 0x9f, 0xc6, 0xa2, - 0xa1, 0x3e, 0x04, 0x68, 0xb1, 0x1f, 0x4d, 0xc7, 0x77, 0x27, 0x48, 0x81, 0xd4, 0x21, 0x99, 0x70, - 0x44, 0x1e, 0xb3, 0x9f, 0x8c, 0x72, 0xa4, 0xdb, 0xab, 0x49, 0x41, 0x39, 0xd2, 0x6d, 0xf5, 0x27, - 0x09, 0x28, 0x54, 0x1d, 0x87, 0xfa, 0xbc, 0x77, 0x0f, 0x21, 0x48, 0x3b, 0xfa, 0x90, 0x48, 0x21, - 0xfe, 0x1b, 0xd5, 0x21, 0x6b, 0xeb, 0x7b, 0xc4, 0xf6, 0x56, 0x93, 0xe5, 0xd4, 0xfb, 0x85, 0xc7, - 0x5f, 0xab, 0xbc, 0x3e, 0xe4, 0x4a, 0x44, 0x49, 0x65, 0x9b, 0xa3, 0xb9, 0x11, 0x58, 0x8a, 0xa2, - 0x6f, 0xc3, 0xbc, 0xe5, 0x98, 0x96, 0x41, 0xbc, 0xd5, 0x34, 0xd7, 0xb2, 0x16, 0xa7, 0x65, 0x6a, - 0x7d, 0x2d, 0xfd, 0xc3, 0xb3, 0xf5, 0x39, 0x1c, 0x08, 0x95, 0x3e, 0x80, 0x42, 0x44, 0x6d, 0xcc, - 0xd8, 0x56, 0x20, 0x73, 0xa4, 0xdb, 0x63, 0x22, 0x47, 0x27, 0x1a, 0x1f, 0x26, 0x9f, 0x26, 0xd4, - 0x4f, 0x60, 0xa5, 0xad, 0x0f, 0x89, 0xb9, 0x49, 0x1c, 0xe2, 0x5a, 0x06, 0x26, 0x1e, 0x1d, 0xbb, - 0x06, 0x61, 0x63, 0x3d, 0xb4, 0x1c, 0x33, 0x18, 0x2b, 0xfb, 0x1d, 0xaf, 0x45, 0xad, 0xc3, 0x5b, - 0x0d, 0xcb, 0x33, 0x5c, 0xe2, 0x93, 0xcf, 0xad, 0x24, 0x15, 0x28, 0x39, 0x4b, 0xc0, 0xd2, 0xac, - 0xf4, 0x2f, 0xc1, 0x0d, 0xe6, 0x62, 0x53, 0x73, 0x25, 0x45, 0xf3, 0x46, 0xc4, 0xe0, 0xca, 0x0a, - 0x8f, 0xdf, 0x8f, 0xf3, 0x50, 0xdc, 0x48, 0xb6, 0xe6, 0xf0, 0x32, 0x57, 0x13, 0x10, 0x7a, 0x23, - 0x62, 0x20, 0x03, 0x6e, 0x9a, 0xd2, 0xe8, 0x19, 0xf5, 0x49, 0xae, 0x3e, 0x76, 0x1a, 0x2f, 0x19, - 0xe6, 0xd6, 0x1c, 0x5e, 0x09, 0x94, 0x45, 0x3b, 0xa9, 0x01, 0xe4, 0x02, 0xdd, 0xea, 0x0f, 0x12, - 0x90, 0x0f, 0x98, 0x1e, 0xfa, 0x2a, 0xe4, 0x1d, 0xdd, 0xa1, 0x9a, 0x31, 0x1a, 0x7b, 0x7c, 0x40, - 0xa9, 0x5a, 0xf1, 0xfc, 0x6c, 0x3d, 0xd7, 0xd6, 0x1d, 0x5a, 0xef, 0xee, 0x7a, 0x38, 0xc7, 0xd8, - 0xf5, 0xd1, 0xd8, 0x43, 0x6f, 0x43, 0x71, 0x48, 0x86, 0xd4, 0x9d, 0x68, 0x7b, 0x13, 0x9f, 0x78, - 0xd2, 0x6d, 0x05, 0x41, 0xab, 0x31, 0x12, 0xfa, 0x16, 0xcc, 0x0f, 0x84, 0x49, 0xab, 0x29, 0xbe, - 0x7c, 0xde, 0x89, 0xb3, 0x7e, 0xc6, 0x6a, 0x1c, 0xc8, 0xa8, 0xbf, 0x97, 0x80, 0x95, 0x90, 0x4a, - 0x7e, 0x75, 0x6c, 0xb9, 0x64, 0x48, 0x1c, 0xdf, 0x43, 0xdf, 0x80, 0xac, 0x6d, 0x0d, 0x2d, 0xdf, - 0x93, 0x3e, 0xbf, 0x13, 0xa7, 0x36, 0x1c, 0x14, 0x96, 0x60, 0x54, 0x85, 0xa2, 0x4b, 0x3c, 0xe2, - 0x1e, 0x89, 0x15, 0x2f, 0x3d, 0x7a, 0x8d, 0xf0, 0x05, 0x11, 0x75, 0x03, 0x72, 0x5d, 0x5b, 0xf7, - 0xf7, 0xa9, 0x3b, 0x44, 0x2a, 0x14, 0x75, 0xd7, 0x38, 0xb0, 0x7c, 0x62, 0xf8, 0x63, 0x37, 0xd8, - 0x7d, 0x17, 0x68, 0xe8, 0x26, 0x24, 0xa9, 0xe8, 0x28, 0x5f, 0xcb, 0x9e, 0x9f, 0xad, 0x27, 0x3b, - 0x3d, 0x9c, 0xa4, 0x9e, 0xfa, 0x11, 0x2c, 0x77, 0xed, 0xf1, 0xc0, 0x72, 0x1a, 0xc4, 0x33, 0x5c, - 0x6b, 0xc4, 0xb4, 0xb3, 0x55, 0xc9, 0x62, 0x54, 0xb0, 0x2a, 0xd9, 0xef, 0x70, 0x6b, 0x27, 0xa7, - 0x5b, 0x5b, 0xfd, 0x9d, 0x24, 0x2c, 0x37, 0x9d, 0x81, 0xe5, 0x90, 0xa8, 0xf4, 0x5d, 0x58, 0x24, - 0x9c, 0xa8, 0x1d, 0x89, 0x70, 0x23, 0xf5, 0x2c, 0x08, 0x6a, 0x10, 0x83, 0x5a, 0x33, 0x71, 0xe1, - 0x51, 0xdc, 0xf0, 0x5f, 0xd3, 0x1e, 0x1b, 0x1d, 0x9a, 0x30, 0x3f, 0xe2, 0x83, 0xf0, 0xe4, 0xf4, - 0xde, 0x8d, 0xd3, 0xf5, 0xda, 0x38, 0x83, 0x20, 0x21, 0x65, 0xbf, 0x48, 0x90, 0xf8, 0xb3, 0x24, - 0x2c, 0xb5, 0xa9, 0x79, 0xc1, 0x0f, 0x25, 0xc8, 0x1d, 0x50, 0xcf, 0x8f, 0x04, 0xc4, 0xb0, 0x8d, - 0x9e, 0x42, 0x6e, 0x24, 0xa7, 0x4f, 0xce, 0xfe, 0xed, 0x78, 0x93, 0x05, 0x06, 0x87, 0x68, 0xf4, - 0x11, 0xe4, 0x83, 0x2d, 0xc3, 0x46, 0xfb, 0x06, 0x0b, 0x67, 0x8a, 0x47, 0xdf, 0x82, 0xac, 0x98, - 0x84, 0xd5, 0x34, 0x97, 0xbc, 0xfb, 0x46, 0x3e, 0xc7, 0x52, 0x08, 0x6d, 0x42, 0xce, 0xb7, 0x3d, - 0xcd, 0x72, 0xf6, 0xe9, 0x6a, 0x86, 0x2b, 0x58, 0x8f, 0x0d, 0x32, 0xd4, 0x24, 0xfd, 0xed, 0x5e, - 0xcb, 0xd9, 0xa7, 0xb5, 0xc2, 0xf9, 0xd9, 0xfa, 0xbc, 0x6c, 0xe0, 0x79, 0xdf, 0xf6, 0xd8, 0x0f, - 0xf5, 0xf7, 0x13, 0x50, 0x88, 0xa0, 0xd0, 0x1d, 0x00, 0xdf, 0x1d, 0x7b, 0xbe, 0xe6, 0x52, 0xea, - 0x73, 0x67, 0x15, 0x71, 0x9e, 0x53, 0x30, 0xa5, 0x3e, 0xaa, 0xc0, 0x0d, 0x83, 0xb8, 0xbe, 0x66, - 0x79, 0xde, 0x98, 0xb8, 0x9a, 0x37, 0xde, 0xfb, 0x94, 0x18, 0x3e, 0x77, 0x5c, 0x11, 0x2f, 0x33, - 0x56, 0x8b, 0x73, 0x7a, 0x82, 0x81, 0x9e, 0xc0, 0xcd, 0x28, 0x7e, 0x34, 0xde, 0xb3, 0x2d, 0x43, - 0x63, 0x93, 0x99, 0xe2, 0x22, 0x37, 0xa6, 0x22, 0x5d, 0xce, 0x7b, 0x46, 0x26, 0xea, 0x8f, 0x13, - 0xa0, 0x60, 0x7d, 0xdf, 0xdf, 0x21, 0xc3, 0x3d, 0xe2, 0xf6, 0x7c, 0xdd, 0x1f, 0x7b, 0xe8, 0x26, - 0x64, 0x6d, 0xa2, 0x9b, 0xc4, 0xe5, 0x46, 0xe5, 0xb0, 0x6c, 0xa1, 0x5d, 0xb6, 0x83, 0x75, 0xe3, - 0x40, 0xdf, 0xb3, 0x6c, 0xcb, 0x9f, 0x70, 0x53, 0x16, 0xe3, 0x97, 0xf0, 0xac, 0xce, 0x0a, 0x8e, - 0x08, 0xe2, 0x0b, 0x6a, 0xd0, 0x2a, 0xcc, 0x0f, 0x89, 0xe7, 0xe9, 0x03, 0xc2, 0x2d, 0xcd, 0xe3, - 0xa0, 0xa9, 0x7e, 0x04, 0xc5, 0xa8, 0x1c, 0x2a, 0xc0, 0xfc, 0x6e, 0xfb, 0x59, 0xbb, 0xf3, 0xa2, - 0xad, 0xcc, 0xa1, 0x25, 0x28, 0xec, 0xb6, 0x71, 0xb3, 0x5a, 0xdf, 0xaa, 0xd6, 0xb6, 0x9b, 0x4a, - 0x02, 0x2d, 0x40, 0x7e, 0xda, 0x4c, 0xaa, 0x7f, 0x91, 0x00, 0x60, 0xee, 0x96, 0x83, 0xfa, 0x10, - 0x32, 0x9e, 0xaf, 0xfb, 0x62, 0x55, 0x2e, 0x3e, 0x7e, 0xf7, 0xb2, 0x39, 0x94, 0xf6, 0xb2, 0x7f, - 0x04, 0x0b, 0x91, 0xa8, 0x85, 0xc9, 0x0b, 0x16, 0xb2, 0x00, 0xa1, 0x9b, 0xa6, 0x2b, 0x0d, 0xe7, - 0xbf, 0xd5, 0x8f, 0x20, 0xc3, 0xa5, 0x2f, 0x9a, 0x9b, 0x83, 0x74, 0x83, 0xfd, 0x4a, 0xa0, 0x3c, - 0x64, 0x70, 0xb3, 0xda, 0xf8, 0x9e, 0x92, 0x44, 0x0a, 0x14, 0x1b, 0xad, 0x5e, 0xbd, 0xd3, 0x6e, - 0x37, 0xeb, 0xfd, 0x66, 0x43, 0x49, 0xa9, 0x77, 0x21, 0xd3, 0x1a, 0x32, 0xcd, 0xb7, 0xd9, 0x92, - 0xdf, 0x27, 0x2e, 0x71, 0x8c, 0x60, 0x27, 0x4d, 0x09, 0xea, 0x4f, 0x0b, 0x90, 0xd9, 0xa1, 0x63, - 0xc7, 0x47, 0x8f, 0x23, 0x61, 0x6b, 0x31, 0x3e, 0x43, 0xe0, 0xc0, 0x4a, 0x7f, 0x32, 0x22, 0x32, - 0xac, 0xdd, 0x84, 0xac, 0xd8, 0x1c, 0x72, 0x38, 0xb2, 0xc5, 0xe8, 0xbe, 0xee, 0x0e, 0x88, 0x2f, - 0xc7, 0x23, 0x5b, 0xe8, 0x7d, 0x76, 0x62, 0xe9, 0x26, 0x75, 0xec, 0x09, 0xdf, 0x43, 0x39, 0x71, - 0x2c, 0x61, 0xa2, 0x9b, 0x1d, 0xc7, 0x9e, 0xe0, 0x90, 0x8b, 0xb6, 0xa0, 0xb8, 0x67, 0x39, 0xa6, - 0x46, 0x47, 0x22, 0xc8, 0x67, 0x2e, 0xdf, 0x71, 0xc2, 0xaa, 0x9a, 0xe5, 0x98, 0x1d, 0x01, 0xc6, - 0x85, 0xbd, 0x69, 0x03, 0xb5, 0x61, 0xf1, 0x88, 0xda, 0xe3, 0x21, 0x09, 0x75, 0x65, 0xb9, 0xae, - 0xf7, 0x2e, 0xd7, 0xf5, 0x9c, 0xe3, 0x03, 0x6d, 0x0b, 0x47, 0xd1, 0x26, 0x7a, 0x06, 0x0b, 0xfe, - 0x70, 0xb4, 0xef, 0x85, 0xea, 0xe6, 0xb9, 0xba, 0xaf, 0x5c, 0xe1, 0x30, 0x06, 0x0f, 0xb4, 0x15, - 0xfd, 0x48, 0x0b, 0x6d, 0x42, 0xc1, 0xa0, 0x8e, 0x67, 0x79, 0x3e, 0x71, 0x8c, 0xc9, 0x6a, 0x8e, - 0xfb, 0xfe, 0x8a, 0x51, 0xd6, 0xa7, 0x60, 0x1c, 0x95, 0x2c, 0xfd, 0x56, 0x0a, 0x0a, 0x11, 0x17, - 0xa0, 0x1e, 0x14, 0x46, 0x2e, 0x1d, 0xe9, 0x03, 0x7e, 0xe2, 0xc9, 0x49, 0x7d, 0xf4, 0x46, 0xee, - 0xab, 0x74, 0xa7, 0x82, 0x38, 0xaa, 0x45, 0x3d, 0x4d, 0x42, 0x21, 0xc2, 0x44, 0xf7, 0x20, 0x87, - 0xbb, 0xb8, 0xf5, 0xbc, 0xda, 0x6f, 0x2a, 0x73, 0xa5, 0xdb, 0x27, 0xa7, 0xe5, 0x55, 0xae, 0x2d, - 0xaa, 0xa0, 0xeb, 0x5a, 0x47, 0x6c, 0x0d, 0xbf, 0x0f, 0xf3, 0x01, 0x34, 0x51, 0xfa, 0xf2, 0xc9, - 0x69, 0xf9, 0xad, 0x59, 0x68, 0x04, 0x89, 0x7b, 0x5b, 0x55, 0xdc, 0x6c, 0x28, 0xc9, 0x78, 0x24, - 0xee, 0x1d, 0xe8, 0x2e, 0x31, 0xd1, 0x57, 0x20, 0x2b, 0x81, 0xa9, 0x52, 0xe9, 0xe4, 0xb4, 0x7c, - 0x73, 0x16, 0x38, 0xc5, 0xe1, 0xde, 0x76, 0xf5, 0x79, 0x53, 0x49, 0xc7, 0xe3, 0x70, 0xcf, 0xd6, - 0x8f, 0x08, 0x7a, 0x17, 0x32, 0x02, 0x96, 0x29, 0xdd, 0x3a, 0x39, 0x2d, 0x7f, 0xe9, 0x35, 0x75, - 0x0c, 0x55, 0x5a, 0xfd, 0xdd, 0x3f, 0x5e, 0x9b, 0xfb, 0x9b, 0x3f, 0x59, 0x53, 0x66, 0xd9, 0xa5, - 0xff, 0x49, 0xc0, 0xc2, 0x85, 0xb5, 0x83, 0x54, 0xc8, 0x3a, 0xd4, 0xa0, 0x23, 0x71, 0x10, 0xe6, - 0x6a, 0x70, 0x7e, 0xb6, 0x9e, 0x6d, 0xd3, 0x3a, 0x1d, 0x4d, 0xb0, 0xe4, 0xa0, 0x67, 0x33, 0x47, - 0xf9, 0x93, 0x37, 0x5c, 0x98, 0xb1, 0x87, 0xf9, 0xc7, 0xb0, 0x60, 0xba, 0xd6, 0x11, 0x71, 0x35, - 0x83, 0x3a, 0xfb, 0xd6, 0x40, 0x1e, 0x72, 0xa5, 0xd8, 0x7c, 0x93, 0x03, 0x71, 0x51, 0x08, 0xd4, - 0x39, 0xfe, 0x0b, 0x1c, 0xe3, 0xa5, 0xe7, 0x50, 0x8c, 0x2e, 0x75, 0x76, 0x2e, 0x79, 0xd6, 0xaf, - 0x11, 0x99, 0x58, 0xf2, 0x34, 0x14, 0xe7, 0x19, 0x45, 0xa4, 0x95, 0xef, 0x41, 0x7a, 0x48, 0x4d, - 0xa1, 0x67, 0xa1, 0x76, 0x83, 0x65, 0x13, 0xff, 0x72, 0xb6, 0x5e, 0xa0, 0x5e, 0x65, 0xc3, 0xb2, - 0xc9, 0x0e, 0x35, 0x09, 0xe6, 0x00, 0xf5, 0x08, 0xd2, 0x2c, 0xe6, 0xa0, 0x2f, 0x43, 0xba, 0xd6, - 0x6a, 0x37, 0x94, 0xb9, 0xd2, 0xf2, 0xc9, 0x69, 0x79, 0x81, 0xbb, 0x84, 0x31, 0xd8, 0xda, 0x45, - 0xeb, 0x90, 0x7d, 0xde, 0xd9, 0xde, 0xdd, 0x61, 0xcb, 0xeb, 0xc6, 0xc9, 0x69, 0x79, 0x29, 0x64, - 0x0b, 0xa7, 0xa1, 0x3b, 0x90, 0xe9, 0xef, 0x74, 0x37, 0x7a, 0x4a, 0xb2, 0x84, 0x4e, 0x4e, 0xcb, - 0x8b, 0x21, 0x9f, 0xdb, 0x5c, 0x5a, 0x96, 0xb3, 0x9a, 0x0f, 0xe9, 0xea, 0x8f, 0x12, 0x50, 0x88, - 0x6c, 0x38, 0xb6, 0x30, 0x1b, 0xcd, 0x8d, 0xea, 0xee, 0x76, 0x5f, 0x99, 0x8b, 0x2c, 0xcc, 0x08, - 0xa4, 0x41, 0xf6, 0xf5, 0xb1, 0xcd, 0xe2, 0x1c, 0xd4, 0x3b, 0xed, 0x5e, 0xab, 0xd7, 0x6f, 0xb6, - 0xfb, 0x4a, 0xa2, 0xb4, 0x7a, 0x72, 0x5a, 0x5e, 0x99, 0x05, 0x6f, 0x8c, 0x6d, 0x9b, 0x2d, 0xcd, - 0x7a, 0xb5, 0xbe, 0xc5, 0xd7, 0xfa, 0x74, 0x69, 0x46, 0x50, 0x75, 0xdd, 0x38, 0x20, 0x26, 0xba, - 0x0f, 0xf9, 0x46, 0x73, 0xbb, 0xb9, 0x59, 0xe5, 0xd1, 0xbd, 0x74, 0xe7, 0xe4, 0xb4, 0x7c, 0xeb, - 0xf5, 0xde, 0x6d, 0x32, 0xd0, 0x7d, 0x62, 0xce, 0x2c, 0xd1, 0x08, 0x44, 0xfd, 0x59, 0x12, 0x16, - 0x30, 0x2b, 0x87, 0x5d, 0xbf, 0x4b, 0x6d, 0xcb, 0x98, 0xa0, 0x2e, 0xe4, 0x0d, 0xea, 0x98, 0x56, - 0x24, 0x4e, 0x3c, 0xbe, 0x24, 0x25, 0x9a, 0x4a, 0x05, 0xad, 0x7a, 0x20, 0x89, 0xa7, 0x4a, 0xd0, - 0x03, 0xc8, 0x98, 0xc4, 0xd6, 0x27, 0x32, 0x37, 0xbb, 0x55, 0x11, 0x05, 0x77, 0x25, 0x28, 0xb8, - 0x2b, 0x0d, 0x59, 0x70, 0x63, 0x81, 0xe3, 0x35, 0x88, 0xfe, 0x52, 0xd3, 0x7d, 0x9f, 0x0c, 0x47, - 0xbe, 0x48, 0xcc, 0xd2, 0xb8, 0x30, 0xd4, 0x5f, 0x56, 0x25, 0x09, 0x3d, 0x82, 0xec, 0xb1, 0xe5, - 0x98, 0xf4, 0x58, 0xe6, 0x5e, 0x57, 0x28, 0x95, 0x40, 0xf5, 0x84, 0xa5, 0x24, 0x33, 0x66, 0xb2, - 0x35, 0xd4, 0xee, 0xb4, 0x9b, 0xc1, 0x1a, 0x92, 0xfc, 0x8e, 0xd3, 0xa6, 0x0e, 0xdb, 0xff, 0xd0, - 0x69, 0x6b, 0x1b, 0xd5, 0xd6, 0xf6, 0x2e, 0x66, 0xeb, 0x68, 0xe5, 0xe4, 0xb4, 0xac, 0x84, 0x90, - 0x0d, 0xdd, 0xb2, 0x59, 0x31, 0x70, 0x0b, 0x52, 0xd5, 0xf6, 0xf7, 0x94, 0x64, 0x49, 0x39, 0x39, - 0x2d, 0x17, 0x43, 0x76, 0xd5, 0x99, 0x4c, 0xfd, 0x3e, 0xdb, 0xaf, 0xfa, 0xf7, 0x29, 0x28, 0xee, - 0x8e, 0x4c, 0xdd, 0x27, 0x62, 0x9f, 0xa1, 0x32, 0x14, 0x46, 0xba, 0xab, 0xdb, 0x36, 0xb1, 0x2d, - 0x6f, 0x28, 0xaf, 0x12, 0xa2, 0x24, 0xf4, 0xc1, 0x9b, 0xba, 0xb1, 0x96, 0x63, 0x7b, 0xe7, 0x07, - 0xff, 0xb6, 0x9e, 0x08, 0x1c, 0xba, 0x0b, 0x8b, 0xfb, 0xc2, 0x5a, 0x4d, 0x37, 0xf8, 0xc4, 0xa6, - 0xf8, 0xc4, 0x56, 0xe2, 0x26, 0x36, 0x6a, 0x56, 0x45, 0x0e, 0xb2, 0xca, 0xa5, 0xf0, 0xc2, 0x7e, - 0xb4, 0x89, 0x9e, 0xc0, 0xfc, 0x90, 0x3a, 0x96, 0x4f, 0xdd, 0xeb, 0x67, 0x21, 0x40, 0xa2, 0x7b, - 0xb0, 0xcc, 0x26, 0x37, 0xb0, 0x87, 0xb3, 0xf9, 0x71, 0x9e, 0xc4, 0x4b, 0x43, 0xfd, 0xa5, 0xec, - 0x10, 0x33, 0x32, 0xaa, 0x41, 0x86, 0xba, 0x2c, 0x5f, 0xcc, 0x72, 0x73, 0xef, 0x5f, 0x6b, 0xae, - 0x68, 0x74, 0x98, 0x0c, 0x16, 0xa2, 0xea, 0x37, 0x61, 0xe1, 0xc2, 0x20, 0x58, 0x9a, 0xd4, 0xad, - 0xee, 0xf6, 0x9a, 0xca, 0x1c, 0x2a, 0x42, 0xae, 0xde, 0x69, 0xf7, 0x5b, 0xed, 0x5d, 0x96, 0xe7, - 0x15, 0x21, 0x87, 0x3b, 0xdb, 0xdb, 0xb5, 0x6a, 0xfd, 0x99, 0x92, 0x54, 0x2b, 0x50, 0x88, 0x68, - 0x43, 0x8b, 0x00, 0xbd, 0x7e, 0xa7, 0xab, 0x6d, 0xb4, 0x70, 0xaf, 0x2f, 0xb2, 0xc4, 0x5e, 0xbf, - 0x8a, 0xfb, 0x92, 0x90, 0x50, 0xff, 0x33, 0x19, 0xcc, 0xa8, 0x4c, 0x0c, 0x6b, 0x17, 0x13, 0xc3, - 0x2b, 0x8c, 0x97, 0xa9, 0xe1, 0xb4, 0x11, 0x26, 0x88, 0x1f, 0x00, 0xf0, 0x85, 0x43, 0x4c, 0x4d, - 0xf7, 0xe5, 0xc4, 0x97, 0x5e, 0x73, 0x72, 0x3f, 0xb8, 0xd1, 0xc2, 0x79, 0x89, 0xae, 0xfa, 0xe8, - 0x5b, 0x50, 0x34, 0xe8, 0x70, 0x64, 0x13, 0x29, 0x9c, 0xba, 0x56, 0xb8, 0x10, 0xe2, 0xab, 0x7e, - 0x34, 0x35, 0x4d, 0x5f, 0x4c, 0x9e, 0x7f, 0x3b, 0x11, 0x78, 0x26, 0x26, 0x1b, 0x2d, 0x42, 0x6e, - 0xb7, 0xdb, 0xa8, 0xf6, 0x5b, 0xed, 0x4d, 0x25, 0x81, 0x00, 0xb2, 0xdc, 0xd5, 0x0d, 0x25, 0xc9, - 0xb2, 0xe8, 0x7a, 0x67, 0xa7, 0xbb, 0xdd, 0xe4, 0x11, 0x0b, 0xad, 0x80, 0x12, 0x38, 0x5b, 0xe3, - 0x8e, 0x6c, 0x36, 0x94, 0x34, 0xba, 0x01, 0x4b, 0x21, 0x55, 0x4a, 0x66, 0xd0, 0x4d, 0x40, 0x21, - 0x71, 0xaa, 0x22, 0xab, 0xfe, 0x06, 0x2c, 0xd5, 0xa9, 0xe3, 0xeb, 0x96, 0x13, 0x56, 0x18, 0x8f, - 0xd9, 0xa0, 0x25, 0x49, 0xb3, 0xe4, 0x4d, 0x50, 0x6d, 0xe9, 0xfc, 0x6c, 0xbd, 0x10, 0x42, 0x5b, - 0x0d, 0x9e, 0x2a, 0xc9, 0x86, 0xc9, 0xf6, 0xef, 0xc8, 0x32, 0xb9, 0x73, 0x33, 0xb5, 0xf9, 0xf3, - 0xb3, 0xf5, 0x54, 0xb7, 0xd5, 0xc0, 0x8c, 0x86, 0xbe, 0x0c, 0x79, 0xf2, 0xd2, 0xf2, 0x35, 0x83, - 0x9d, 0x4b, 0xcc, 0x81, 0x19, 0x9c, 0x63, 0x84, 0x3a, 0x3b, 0x86, 0x6a, 0x00, 0x5d, 0xea, 0xfa, - 0xb2, 0xe7, 0xaf, 0x43, 0x66, 0x44, 0x5d, 0x7e, 0x77, 0x71, 0xe9, 0x8d, 0x1a, 0x83, 0x8b, 0x85, - 0x8a, 0x05, 0x58, 0xfd, 0x83, 0x14, 0x40, 0x5f, 0xf7, 0x0e, 0xa5, 0x92, 0xa7, 0x90, 0x0f, 0x6f, - 0x27, 0xe5, 0x25, 0xc8, 0x95, 0xb3, 0x1d, 0x82, 0xd1, 0x93, 0x60, 0xb1, 0x89, 0xda, 0x29, 0xb6, - 0x88, 0x0d, 0x3a, 0x8a, 0x2b, 0x3f, 0x2e, 0x16, 0x48, 0xec, 0x98, 0x27, 0xae, 0x2b, 0x67, 0x9e, - 0xfd, 0x44, 0x75, 0x7e, 0x2c, 0x08, 0xa7, 0xc9, 0xec, 0x3b, 0xf6, 0xda, 0x67, 0x66, 0x46, 0xb6, - 0xe6, 0xf0, 0x54, 0x0e, 0x7d, 0x0c, 0x05, 0x36, 0x6e, 0xcd, 0xe3, 0x3c, 0x99, 0x78, 0x5f, 0xea, - 0x2a, 0xa1, 0x01, 0xc3, 0x68, 0xea, 0xe5, 0x3b, 0x00, 0xfa, 0x68, 0x64, 0x5b, 0xc4, 0xd4, 0xf6, - 0x26, 0x3c, 0xd3, 0xce, 0xe3, 0xbc, 0xa4, 0xd4, 0x26, 0x6c, 0xbb, 0x04, 0x6c, 0xdd, 0xe7, 0xd9, - 0xf3, 0x35, 0x0e, 0x94, 0xe8, 0xaa, 0x5f, 0x53, 0x60, 0xd1, 0x1d, 0x3b, 0xcc, 0xa1, 0xd2, 0x3a, - 0xf5, 0xcf, 0x93, 0xf0, 0x56, 0x9b, 0xf8, 0xc7, 0xd4, 0x3d, 0xac, 0xfa, 0xbe, 0x6e, 0x1c, 0x0c, - 0x89, 0x23, 0xa7, 0x2f, 0x52, 0xd0, 0x24, 0x2e, 0x14, 0x34, 0xab, 0x30, 0xaf, 0xdb, 0x96, 0xee, - 0x11, 0x91, 0xbc, 0xe5, 0x71, 0xd0, 0x64, 0x65, 0x17, 0x2b, 0xe2, 0x88, 0xe7, 0x11, 0x71, 0xaf, - 0xc2, 0x0c, 0x0f, 0x08, 0xe8, 0xfb, 0x70, 0x53, 0xa6, 0x69, 0x7a, 0xd8, 0x15, 0x2b, 0x28, 0x82, - 0x0b, 0xda, 0x66, 0x6c, 0x55, 0x19, 0x6f, 0x9c, 0xcc, 0xe3, 0xa6, 0xe4, 0xce, 0xc8, 0x97, 0x59, - 0xe1, 0x8a, 0x19, 0xc3, 0x2a, 0x6d, 0xc2, 0xad, 0x4b, 0x45, 0x3e, 0xd7, 0xbd, 0xcd, 0x3f, 0x25, - 0x01, 0x5a, 0xdd, 0xea, 0x8e, 0x74, 0x52, 0x03, 0xb2, 0xfb, 0xfa, 0xd0, 0xb2, 0x27, 0x57, 0x45, - 0xc0, 0x29, 0xbe, 0x52, 0x15, 0xee, 0xd8, 0xe0, 0x32, 0x58, 0xca, 0xf2, 0x9a, 0x72, 0xbc, 0xe7, - 0x10, 0x3f, 0xac, 0x29, 0x79, 0x8b, 0x99, 0xe1, 0xea, 0x4e, 0xb8, 0x74, 0x45, 0x83, 0x4d, 0x00, - 0x4b, 0x79, 0x8e, 0xf5, 0x49, 0x10, 0xb6, 0x64, 0x13, 0x6d, 0xf1, 0xdb, 0x51, 0xe2, 0x1e, 0x11, - 0x73, 0x35, 0xc3, 0x9d, 0x7a, 0x9d, 0x3d, 0x58, 0xc2, 0x85, 0xef, 0x42, 0xe9, 0xd2, 0x47, 0x3c, - 0x65, 0x9a, 0xb2, 0x3e, 0x97, 0x8f, 0x1e, 0xc2, 0xc2, 0x85, 0x71, 0xbe, 0x56, 0xcc, 0xb7, 0xba, - 0xcf, 0xbf, 0xae, 0xa4, 0xe5, 0xaf, 0x6f, 0x2a, 0x59, 0xf5, 0x4f, 0x53, 0x22, 0xd0, 0x48, 0xaf, - 0xc6, 0xbf, 0x0a, 0xe4, 0xf8, 0xea, 0x36, 0xa8, 0x2d, 0x03, 0xc0, 0x7b, 0x57, 0xc7, 0x1f, 0x56, - 0xd3, 0x71, 0x38, 0x0e, 0x05, 0xd1, 0x3a, 0x14, 0xc4, 0x2a, 0xd6, 0xd8, 0x86, 0xe3, 0x6e, 0x5d, - 0xc0, 0x20, 0x48, 0x4c, 0x12, 0xdd, 0x85, 0x45, 0x7e, 0xf9, 0xe3, 0x1d, 0x10, 0x53, 0x60, 0xd2, - 0x1c, 0xb3, 0x10, 0x52, 0x39, 0x6c, 0x07, 0x8a, 0x92, 0xa0, 0xf1, 0x7c, 0x3e, 0xc3, 0x0d, 0xba, - 0x77, 0x9d, 0x41, 0x42, 0x84, 0xa7, 0xf9, 0x85, 0xd1, 0xb4, 0xa1, 0x36, 0x20, 0x17, 0x18, 0x8b, - 0x56, 0x21, 0xd5, 0xaf, 0x77, 0x95, 0xb9, 0xd2, 0xd2, 0xc9, 0x69, 0xb9, 0x10, 0x90, 0xfb, 0xf5, - 0x2e, 0xe3, 0xec, 0x36, 0xba, 0x4a, 0xe2, 0x22, 0x67, 0xb7, 0xd1, 0x2d, 0xa5, 0x59, 0x0e, 0xa6, - 0xee, 0x43, 0x21, 0xd2, 0x03, 0x7a, 0x07, 0xe6, 0x5b, 0xed, 0x4d, 0xdc, 0xec, 0xf5, 0x94, 0xb9, - 0xd2, 0xcd, 0x93, 0xd3, 0x32, 0x8a, 0x70, 0x5b, 0xce, 0x80, 0xcd, 0x0f, 0xba, 0x03, 0xe9, 0xad, - 0x0e, 0x3b, 0xdb, 0x45, 0x01, 0x11, 0x41, 0x6c, 0x51, 0xcf, 0x2f, 0xdd, 0x90, 0xc9, 0x5d, 0x54, - 0xb1, 0xfa, 0x87, 0x09, 0xc8, 0x8a, 0xcd, 0x14, 0x3b, 0x51, 0x55, 0x98, 0x0f, 0xae, 0x09, 0x44, - 0x71, 0xf7, 0xde, 0xe5, 0x85, 0x58, 0x45, 0xd6, 0x4d, 0x62, 0xf9, 0x05, 0x72, 0xa5, 0x0f, 0xa1, - 0x18, 0x65, 0x7c, 0xae, 0xc5, 0xf7, 0x7d, 0x28, 0xb0, 0xf5, 0x1d, 0x14, 0x64, 0x8f, 0x21, 0x2b, - 0x02, 0x42, 0x78, 0xd6, 0x5c, 0x5e, 0x15, 0x4a, 0x24, 0x7a, 0x0a, 0xf3, 0xa2, 0x92, 0x0c, 0x6e, - 0x87, 0xd7, 0xae, 0xde, 0x45, 0x38, 0x80, 0xab, 0x1f, 0x43, 0xba, 0x4b, 0x88, 0xcb, 0x7c, 0xef, - 0x50, 0x93, 0x4c, 0x8f, 0x67, 0x59, 0x04, 0x9b, 0xa4, 0xd5, 0x60, 0x45, 0xb0, 0x49, 0x5a, 0x66, - 0x78, 0xff, 0x95, 0x8c, 0xdc, 0x7f, 0xf5, 0xa1, 0xf8, 0x82, 0x58, 0x83, 0x03, 0x9f, 0x98, 0x5c, - 0xd1, 0x7d, 0x48, 0x8f, 0x48, 0x68, 0xfc, 0x6a, 0xec, 0x02, 0x23, 0xc4, 0xc5, 0x1c, 0xc5, 0xe2, - 0xc8, 0x31, 0x97, 0x96, 0x4f, 0x1a, 0xb2, 0xa5, 0xfe, 0x63, 0x12, 0x16, 0x5b, 0x9e, 0x37, 0xd6, - 0x1d, 0x23, 0xc8, 0xdc, 0xbe, 0x7d, 0x31, 0x73, 0x8b, 0x7d, 0xfb, 0xb9, 0x28, 0x72, 0xf1, 0x5a, - 0x4f, 0x9e, 0x9e, 0xc9, 0xf0, 0xf4, 0x54, 0x7f, 0x9a, 0x08, 0xee, 0xee, 0xee, 0x46, 0xb6, 0xbb, - 0xa8, 0x03, 0xa3, 0x9a, 0xc8, 0xae, 0x73, 0xe8, 0xd0, 0x63, 0x07, 0xbd, 0x0d, 0x19, 0xdc, 0x6c, - 0x37, 0x5f, 0x28, 0x09, 0xb1, 0x3c, 0x2f, 0x80, 0x30, 0x71, 0xc8, 0x31, 0xd3, 0xd4, 0x6d, 0xb6, - 0x1b, 0x2c, 0xd3, 0x4a, 0xc6, 0x68, 0xea, 0x12, 0xc7, 0xb4, 0x9c, 0x01, 0x7a, 0x07, 0xb2, 0xad, - 0x5e, 0x6f, 0x97, 0x97, 0x89, 0x6f, 0x9d, 0x9c, 0x96, 0x6f, 0x5c, 0x40, 0xf1, 0x7b, 0x5b, 0x93, - 0x81, 0x58, 0x99, 0xc3, 0x72, 0xb0, 0x18, 0x10, 0xcb, 0x9f, 0x05, 0x08, 0x77, 0xfa, 0xd5, 0x7e, - 0x53, 0xc9, 0xc4, 0x80, 0x30, 0x65, 0x7f, 0xe5, 0x76, 0xfb, 0xd7, 0x24, 0x28, 0x55, 0xc3, 0x20, - 0x23, 0x9f, 0xf1, 0x65, 0x65, 0xd9, 0x87, 0xdc, 0x88, 0xfd, 0xb2, 0x48, 0x90, 0x25, 0x3d, 0x8d, - 0x7d, 0xbd, 0x9c, 0x91, 0xab, 0x60, 0x6a, 0x93, 0xaa, 0x39, 0xb4, 0x3c, 0xcf, 0xa2, 0x8e, 0xa0, - 0xe1, 0x50, 0x53, 0xe9, 0xbf, 0x12, 0x70, 0x23, 0x06, 0x81, 0x1e, 0x42, 0xda, 0xa5, 0x76, 0x30, - 0x87, 0xb7, 0x2f, 0xbb, 0x96, 0x65, 0xa2, 0x98, 0x23, 0xd1, 0x1a, 0x80, 0x3e, 0xf6, 0xa9, 0xce, - 0xfb, 0xe7, 0xb3, 0x97, 0xc3, 0x11, 0x0a, 0x7a, 0x01, 0x59, 0x8f, 0x18, 0x2e, 0x09, 0x72, 0xe9, - 0x8f, 0xff, 0xbf, 0xd6, 0x57, 0x7a, 0x5c, 0x0d, 0x96, 0xea, 0x4a, 0x15, 0xc8, 0x0a, 0x0a, 0x5b, - 0xf6, 0xa6, 0xee, 0xeb, 0xf2, 0xd2, 0x9e, 0xff, 0x66, 0xab, 0x49, 0xb7, 0x07, 0xc1, 0x6a, 0xd2, - 0xed, 0x81, 0xfa, 0x77, 0x49, 0x80, 0xe6, 0x4b, 0x9f, 0xb8, 0x8e, 0x6e, 0xd7, 0xab, 0xa8, 0x19, - 0x89, 0xfe, 0x62, 0xb4, 0x5f, 0x8d, 0x7d, 0x89, 0x08, 0x25, 0x2a, 0xf5, 0x6a, 0x4c, 0xfc, 0xbf, - 0x05, 0xa9, 0xb1, 0x2b, 0x1f, 0xa4, 0x45, 0x1e, 0xbc, 0x8b, 0xb7, 0x31, 0xa3, 0xa1, 0xe6, 0x34, - 0x6c, 0xa5, 0x2e, 0x7f, 0x76, 0x8e, 0x74, 0x10, 0x1b, 0xba, 0xd8, 0xce, 0x37, 0x74, 0xcd, 0x20, - 0xf2, 0xe4, 0x28, 0x8a, 0x9d, 0x5f, 0xaf, 0xd6, 0x89, 0xeb, 0xe3, 0xac, 0xa1, 0xb3, 0xff, 0x5f, - 0x28, 0xbe, 0xdd, 0x07, 0x98, 0x0e, 0x0d, 0xad, 0x41, 0xa6, 0xbe, 0xd1, 0xeb, 0x6d, 0x2b, 0x73, - 0x22, 0x80, 0x4f, 0x59, 0x9c, 0xac, 0xfe, 0x75, 0x12, 0x72, 0xf5, 0xaa, 0x3c, 0x56, 0xeb, 0xa0, - 0xf0, 0xa8, 0xc4, 0x9f, 0x3a, 0xc8, 0xcb, 0x91, 0xe5, 0x4e, 0x64, 0x60, 0xb9, 0xa2, 0xa8, 0x5d, - 0x64, 0x22, 0xcc, 0xea, 0x26, 0x17, 0x40, 0x18, 0x8a, 0x44, 0x3a, 0x41, 0x33, 0xf4, 0x20, 0xc6, - 0xaf, 0x5d, 0xed, 0x2c, 0x51, 0x9e, 0x4c, 0xdb, 0x1e, 0x2e, 0x04, 0x4a, 0xea, 0xba, 0x87, 0x3e, - 0x80, 0x25, 0xcf, 0x1a, 0x38, 0x96, 0x33, 0xd0, 0x02, 0xe7, 0xf1, 0x77, 0x97, 0xda, 0xf2, 0xf9, - 0xd9, 0xfa, 0x42, 0x4f, 0xb0, 0xa4, 0x0f, 0x17, 0x24, 0xb2, 0xce, 0x5d, 0x89, 0xbe, 0x09, 0x8b, - 0x11, 0x51, 0xe6, 0x45, 0xe1, 0x76, 0xe5, 0xfc, 0x6c, 0xbd, 0x18, 0x4a, 0x3e, 0x23, 0x13, 0x5c, - 0x0c, 0x05, 0x9f, 0x11, 0x7e, 0xff, 0xb2, 0x4f, 0x5d, 0x83, 0x68, 0x2e, 0xdf, 0xd3, 0xfc, 0x04, - 0x4f, 0xe3, 0x02, 0xa7, 0x89, 0x6d, 0xae, 0x3e, 0x87, 0x1b, 0x1d, 0xd7, 0x38, 0x20, 0x9e, 0x2f, - 0x5c, 0x21, 0xbd, 0xf8, 0x31, 0xdc, 0xf6, 0x75, 0xef, 0x50, 0x3b, 0xb0, 0x3c, 0x9f, 0xba, 0x13, - 0xcd, 0x25, 0x3e, 0x71, 0x18, 0x5f, 0xe3, 0x8f, 0xb5, 0xf2, 0xd2, 0xef, 0x16, 0xc3, 0x6c, 0x09, - 0x08, 0x0e, 0x10, 0xdb, 0x0c, 0xa0, 0xb6, 0xa0, 0xc8, 0xca, 0x14, 0x79, 0x71, 0xc6, 0x46, 0x0f, - 0x36, 0x1d, 0x68, 0x6f, 0x7c, 0x4c, 0xe5, 0x6d, 0x3a, 0x10, 0x3f, 0xd5, 0xef, 0x82, 0xd2, 0xb0, - 0xbc, 0x91, 0xee, 0x1b, 0x07, 0xc1, 0x6d, 0x26, 0x6a, 0x80, 0x72, 0x40, 0x74, 0xd7, 0xdf, 0x23, - 0xba, 0xaf, 0x8d, 0x88, 0x6b, 0x51, 0xf3, 0xfa, 0x59, 0x5e, 0x0a, 0x45, 0xba, 0x5c, 0x42, 0xfd, - 0xef, 0x04, 0x00, 0xd6, 0xf7, 0x83, 0x8c, 0xec, 0x6b, 0xb0, 0xec, 0x39, 0xfa, 0xc8, 0x3b, 0xa0, - 0xbe, 0x66, 0x39, 0x3e, 0x71, 0x8f, 0x74, 0x5b, 0x5e, 0xe0, 0x28, 0x01, 0xa3, 0x25, 0xe9, 0xe8, - 0x3e, 0xa0, 0x43, 0x42, 0x46, 0x1a, 0xb5, 0x4d, 0x2d, 0x60, 0x8a, 0xa7, 0xe4, 0x34, 0x56, 0x18, - 0xa7, 0x63, 0x9b, 0xbd, 0x80, 0x8e, 0x6a, 0xb0, 0xc6, 0x86, 0x4f, 0x1c, 0xdf, 0xb5, 0x88, 0xa7, - 0xed, 0x53, 0x57, 0xf3, 0x6c, 0x7a, 0xac, 0xed, 0x53, 0xdb, 0xa6, 0xc7, 0xc4, 0x0d, 0xee, 0xc6, - 0x4a, 0x36, 0x1d, 0x34, 0x05, 0x68, 0x83, 0xba, 0x3d, 0x9b, 0x1e, 0x6f, 0x04, 0x08, 0x96, 0xb6, - 0x4d, 0xc7, 0xec, 0x5b, 0xc6, 0x61, 0x90, 0xb6, 0x85, 0xd4, 0xbe, 0x65, 0x1c, 0xa2, 0x77, 0x60, - 0x81, 0xd8, 0x84, 0x5f, 0x91, 0x08, 0x54, 0x86, 0xa3, 0x8a, 0x01, 0x91, 0x81, 0xd4, 0x4f, 0x40, - 0x69, 0x3a, 0x86, 0x3b, 0x19, 0x45, 0xe6, 0xfc, 0x3e, 0x20, 0x16, 0x24, 0x35, 0x9b, 0x1a, 0x87, - 0xda, 0x50, 0x77, 0xf4, 0x01, 0xb3, 0x4b, 0xbc, 0xf0, 0x29, 0x8c, 0xb3, 0x4d, 0x8d, 0xc3, 0x1d, - 0x49, 0x57, 0x3f, 0x00, 0xe8, 0x8d, 0x5c, 0xa2, 0x9b, 0x1d, 0x96, 0x4d, 0x30, 0xd7, 0xf1, 0x96, - 0x66, 0xca, 0x17, 0x52, 0xea, 0xca, 0xad, 0xae, 0x08, 0x46, 0x23, 0xa4, 0xab, 0xbf, 0x0c, 0x37, - 0xba, 0xb6, 0x6e, 0xf0, 0xaf, 0x05, 0xba, 0xe1, 0x93, 0x15, 0x7a, 0x0a, 0x59, 0x01, 0x95, 0x33, - 0x19, 0xbb, 0xdd, 0xa6, 0x7d, 0x6e, 0xcd, 0x61, 0x89, 0xaf, 0x15, 0x01, 0xa6, 0x7a, 0xd4, 0xbf, - 0x4c, 0x40, 0x3e, 0xd4, 0x8f, 0xca, 0xe2, 0x25, 0xc6, 0x77, 0x75, 0xcb, 0x91, 0x55, 0x7d, 0x1e, - 0x47, 0x49, 0xa8, 0x05, 0x85, 0x51, 0x28, 0x7d, 0x65, 0x3e, 0x17, 0x63, 0x35, 0x8e, 0xca, 0xa2, - 0x0f, 0x21, 0x1f, 0x3c, 0x49, 0x07, 0x11, 0xf6, 0xea, 0x17, 0xec, 0x29, 0x5c, 0xfd, 0x36, 0xc0, - 0x77, 0xa8, 0xe5, 0xf4, 0xe9, 0x21, 0x71, 0xf8, 0x13, 0x2b, 0xab, 0x09, 0x49, 0xe0, 0x45, 0xd9, - 0xe2, 0xa5, 0xbe, 0x98, 0x82, 0xf0, 0xa5, 0x51, 0x34, 0xd5, 0xbf, 0x4d, 0x42, 0x16, 0x53, 0xea, - 0xd7, 0xab, 0xa8, 0x0c, 0x59, 0x19, 0x27, 0xf8, 0xf9, 0x53, 0xcb, 0x9f, 0x9f, 0xad, 0x67, 0x44, - 0x80, 0xc8, 0x18, 0x3c, 0x32, 0x44, 0x22, 0x78, 0xf2, 0xb2, 0x08, 0x8e, 0x1e, 0x42, 0x51, 0x82, - 0xb4, 0x03, 0xdd, 0x3b, 0x10, 0x05, 0x5a, 0x6d, 0xf1, 0xfc, 0x6c, 0x1d, 0x04, 0x72, 0x4b, 0xf7, - 0x0e, 0x30, 0x08, 0x34, 0xfb, 0x8d, 0x9a, 0x50, 0xf8, 0x94, 0x5a, 0x8e, 0xe6, 0xf3, 0x41, 0xc8, - 0xcb, 0xc4, 0xd8, 0x79, 0x9c, 0x0e, 0x55, 0x7e, 0x6f, 0x00, 0x9f, 0x4e, 0x07, 0xdf, 0x84, 0x05, - 0x97, 0x52, 0x5f, 0x84, 0x2d, 0x8b, 0x3a, 0xf2, 0x9e, 0xa2, 0x1c, 0x7b, 0x7d, 0x4d, 0xa9, 0x8f, - 0x25, 0x0e, 0x17, 0xdd, 0x48, 0x0b, 0x3d, 0x84, 0x15, 0x5b, 0xf7, 0x7c, 0x8d, 0xc7, 0x3b, 0x73, - 0xaa, 0x2d, 0xcb, 0xb7, 0x1a, 0x62, 0xbc, 0x0d, 0xce, 0x0a, 0x24, 0xd4, 0x7f, 0x4e, 0x40, 0x81, - 0x0d, 0xc6, 0xda, 0xb7, 0x0c, 0x96, 0xe4, 0x7d, 0xfe, 0xdc, 0xe3, 0x16, 0xa4, 0x0c, 0xcf, 0x95, - 0x4e, 0xe5, 0x87, 0x6f, 0xbd, 0x87, 0x31, 0xa3, 0xa1, 0x4f, 0x20, 0x2b, 0xef, 0x4b, 0x44, 0xda, - 0xa1, 0x5e, 0x9f, 0x8e, 0x4a, 0xdf, 0x48, 0x39, 0xbe, 0x96, 0xa7, 0xd6, 0x89, 0x43, 0x00, 0x47, - 0x49, 0xe8, 0x26, 0x24, 0x0d, 0xe1, 0x2e, 0xf9, 0x41, 0x4b, 0xbd, 0x8d, 0x93, 0x86, 0xa3, 0xfe, - 0x28, 0x01, 0x0b, 0xd3, 0x0d, 0xcf, 0x56, 0xc0, 0x6d, 0xc8, 0x7b, 0xe3, 0x3d, 0x6f, 0xe2, 0xf9, - 0x64, 0x18, 0x3c, 0x1f, 0x87, 0x04, 0xd4, 0x82, 0xbc, 0x6e, 0x0f, 0xa8, 0x6b, 0xf9, 0x07, 0x43, - 0x59, 0x89, 0xc6, 0xa7, 0x0a, 0x51, 0x9d, 0x95, 0x6a, 0x20, 0x82, 0xa7, 0xd2, 0xc1, 0xb9, 0x2f, - 0xbe, 0x31, 0xe0, 0xe7, 0xfe, 0xdb, 0x50, 0xb4, 0xf5, 0x21, 0xbf, 0x40, 0xf2, 0xad, 0xa1, 0x18, - 0x47, 0x1a, 0x17, 0x24, 0xad, 0x6f, 0x0d, 0x89, 0xaa, 0x42, 0x3e, 0x54, 0x86, 0x96, 0xa0, 0x50, - 0x6d, 0xf6, 0xb4, 0x47, 0x8f, 0x9f, 0x6a, 0x9b, 0xf5, 0x1d, 0x65, 0x4e, 0xe6, 0xa6, 0x7f, 0x95, - 0x80, 0x05, 0x19, 0x8e, 0x64, 0xbe, 0xff, 0x0e, 0xcc, 0xbb, 0xfa, 0xbe, 0x1f, 0x54, 0x24, 0x69, - 0xb1, 0xaa, 0x59, 0x84, 0x67, 0x15, 0x09, 0x63, 0xc5, 0x57, 0x24, 0x91, 0x0f, 0x1a, 0x52, 0x57, - 0x7e, 0xd0, 0x90, 0xfe, 0xb9, 0x7c, 0xd0, 0xa0, 0xfe, 0x26, 0xc0, 0x86, 0x65, 0x93, 0xbe, 0xb8, - 0x6b, 0x8a, 0xab, 0x2f, 0x59, 0x0e, 0x27, 0xef, 0x32, 0x83, 0x1c, 0xae, 0xd5, 0xc0, 0x8c, 0xc6, - 0x58, 0x03, 0xcb, 0x94, 0x9b, 0x91, 0xb3, 0x36, 0x19, 0x6b, 0x60, 0x99, 0xe1, 0xcb, 0x5b, 0xfa, - 0xba, 0x97, 0xb7, 0xd3, 0x04, 0x2c, 0xc9, 0xdc, 0x35, 0x0c, 0xbf, 0x5f, 0x85, 0xbc, 0x48, 0x63, - 0xa7, 0x05, 0x1d, 0x7f, 0xc4, 0x17, 0xb8, 0x56, 0x03, 0xe7, 0x04, 0xbb, 0x65, 0xa2, 0x75, 0x28, - 0x48, 0x68, 0xe4, 0xe3, 0x27, 0x10, 0xa4, 0x36, 0x33, 0xff, 0xeb, 0x90, 0xde, 0xb7, 0x6c, 0x22, - 0x17, 0x7a, 0x6c, 0x00, 0x98, 0x3a, 0x60, 0x6b, 0x0e, 0x73, 0x74, 0x2d, 0x17, 0x5c, 0xc6, 0x71, - 0xfb, 0x64, 0xd9, 0x19, 0xb5, 0x4f, 0x54, 0xa0, 0x33, 0xf6, 0x09, 0x1c, 0xb3, 0x4f, 0xb0, 0x85, - 0x7d, 0x12, 0x1a, 0xb5, 0x4f, 0x90, 0x7e, 0x2e, 0xf6, 0x6d, 0xc3, 0xcd, 0x9a, 0xad, 0x1b, 0x87, - 0xb6, 0xe5, 0xf9, 0xc4, 0x8c, 0x46, 0x8c, 0xc7, 0x90, 0xbd, 0x90, 0x74, 0x5e, 0x75, 0x6b, 0x29, - 0x91, 0xea, 0x7f, 0x24, 0xa0, 0xb8, 0x45, 0x74, 0xdb, 0x3f, 0x98, 0x5e, 0x0d, 0xf9, 0xc4, 0xf3, - 0xe5, 0x61, 0xc5, 0x7f, 0xa3, 0x6f, 0x40, 0x2e, 0xcc, 0x49, 0xae, 0x7d, 0x7f, 0x0b, 0xa1, 0xe8, - 0x09, 0xcc, 0xb3, 0x3d, 0x46, 0xc7, 0x41, 0xb1, 0x73, 0xd5, 0xd3, 0x8e, 0x44, 0xb2, 0x43, 0xc6, - 0x25, 0x3c, 0x09, 0xe1, 0x4b, 0x29, 0x83, 0x83, 0x26, 0xfa, 0x45, 0x28, 0xf2, 0x97, 0x89, 0x20, - 0xe7, 0xca, 0x5c, 0xa7, 0xb3, 0x20, 0x1e, 0x17, 0x45, 0xbe, 0xf5, 0xbf, 0x09, 0x58, 0xd9, 0xd1, - 0x27, 0x7b, 0x44, 0x86, 0x0d, 0x62, 0x62, 0x62, 0x50, 0xd7, 0x44, 0xdd, 0x68, 0xb8, 0xb9, 0xe2, - 0xad, 0x32, 0x4e, 0x38, 0x3e, 0xea, 0x04, 0x05, 0x58, 0x32, 0x52, 0x80, 0xad, 0x40, 0xc6, 0xa1, - 0x8e, 0x41, 0x64, 0x2c, 0x12, 0x0d, 0xd5, 0x8a, 0x86, 0x9a, 0x52, 0xf8, 0x8c, 0xc8, 0x1f, 0x01, - 0xdb, 0xd4, 0x0f, 0x7b, 0x43, 0x9f, 0x40, 0xa9, 0xd7, 0xac, 0xe3, 0x66, 0xbf, 0xd6, 0xf9, 0xae, - 0xd6, 0xab, 0x6e, 0xf7, 0xaa, 0x8f, 0x1f, 0x6a, 0xdd, 0xce, 0xf6, 0xf7, 0x1e, 0x3d, 0x79, 0xf8, - 0x0d, 0x25, 0x51, 0x2a, 0x9f, 0x9c, 0x96, 0x6f, 0xb7, 0xab, 0xf5, 0x6d, 0xb1, 0x63, 0xf6, 0xe8, - 0xcb, 0x9e, 0x6e, 0x7b, 0xfa, 0xe3, 0x87, 0x5d, 0x6a, 0x4f, 0x18, 0x86, 0x2d, 0xeb, 0x62, 0xf4, - 0xbc, 0x8a, 0x1e, 0xc3, 0x89, 0x4b, 0x8f, 0xe1, 0xe9, 0x69, 0x9e, 0xbc, 0xe4, 0x34, 0xdf, 0x80, - 0x15, 0xc3, 0xa5, 0x9e, 0xa7, 0xb1, 0xec, 0x9f, 0x98, 0x33, 0xf5, 0xc5, 0x97, 0xce, 0xcf, 0xd6, - 0x97, 0xeb, 0x8c, 0xdf, 0xe3, 0x6c, 0xa9, 0x7e, 0xd9, 0x88, 0x90, 0x78, 0x4f, 0xea, 0x1f, 0xa5, - 0x58, 0x22, 0x65, 0x1d, 0x59, 0x36, 0x19, 0x10, 0x0f, 0x3d, 0x87, 0x25, 0xc3, 0x25, 0x26, 0x4b, - 0xeb, 0x75, 0x3b, 0xfa, 0x11, 0xed, 0x2f, 0xc4, 0xe6, 0x34, 0xa1, 0x60, 0xa5, 0x1e, 0x4a, 0xf5, - 0x46, 0xc4, 0xc0, 0x8b, 0xc6, 0x85, 0x36, 0xfa, 0x14, 0x96, 0x3c, 0x62, 0x5b, 0xce, 0xf8, 0xa5, - 0x66, 0x50, 0xc7, 0x27, 0x2f, 0x83, 0x17, 0xb1, 0xeb, 0xf4, 0xf6, 0x9a, 0xdb, 0x4c, 0xaa, 0x2e, - 0x84, 0x6a, 0xe8, 0xfc, 0x6c, 0x7d, 0xf1, 0x22, 0x0d, 0x2f, 0x4a, 0xcd, 0xb2, 0x5d, 0x6a, 0xc3, - 0xe2, 0x45, 0x6b, 0xd0, 0x8a, 0xdc, 0xfb, 0x3c, 0x84, 0x04, 0x7b, 0x1b, 0xdd, 0x86, 0x9c, 0x4b, - 0x06, 0x96, 0xe7, 0xbb, 0xc2, 0xcd, 0x8c, 0x13, 0x52, 0xd8, 0xce, 0x17, 0x5f, 0x40, 0x95, 0x7e, - 0x1d, 0x66, 0x7a, 0x64, 0x9b, 0xc5, 0xb4, 0x3c, 0x7d, 0x4f, 0xaa, 0xcc, 0xe1, 0xa0, 0xc9, 0xd6, - 0xe0, 0xd8, 0x0b, 0x13, 0x35, 0xfe, 0x9b, 0xd1, 0x78, 0x46, 0x21, 0xbf, 0x07, 0xe3, 0x39, 0x43, - 0xf0, 0x61, 0x69, 0x3a, 0xf2, 0x61, 0xe9, 0x0a, 0x64, 0x6c, 0x72, 0x44, 0x6c, 0x71, 0x96, 0x63, - 0xd1, 0xb8, 0xf7, 0x10, 0x8a, 0xc1, 0x17, 0x8c, 0xfc, 0xcb, 0x89, 0x1c, 0xa4, 0xfb, 0xd5, 0xde, - 0x33, 0x65, 0x0e, 0x01, 0x64, 0xc5, 0xe2, 0x14, 0xaf, 0x75, 0xf5, 0x4e, 0x7b, 0xa3, 0xb5, 0xa9, - 0x24, 0xef, 0xfd, 0x2c, 0x05, 0xf9, 0xf0, 0xbd, 0x88, 0x9d, 0x1d, 0xed, 0xe6, 0x8b, 0x60, 0x75, - 0x87, 0xf4, 0x36, 0x39, 0x46, 0x6f, 0x4f, 0x6f, 0xa1, 0x3e, 0x11, 0x0f, 0xe4, 0x21, 0x3b, 0xb8, - 0x81, 0x7a, 0x17, 0x72, 0xd5, 0x5e, 0xaf, 0xb5, 0xd9, 0x6e, 0x36, 0x94, 0xcf, 0x12, 0xa5, 0x2f, - 0x9d, 0x9c, 0x96, 0x97, 0x43, 0x50, 0xd5, 0x13, 0x8b, 0x8f, 0xa3, 0xea, 0xf5, 0x66, 0xb7, 0xdf, - 0x6c, 0x28, 0xaf, 0x92, 0xb3, 0x28, 0x7e, 0xab, 0xc2, 0x3f, 0xdd, 0xc9, 0x77, 0x71, 0xb3, 0x5b, - 0xc5, 0xac, 0xc3, 0xcf, 0x92, 0xe2, 0x72, 0x6c, 0xda, 0xa3, 0x4b, 0x46, 0xba, 0xcb, 0xfa, 0x5c, - 0x0b, 0xbe, 0x85, 0x7b, 0x95, 0x12, 0x9f, 0x77, 0x4c, 0x1f, 0xbf, 0x88, 0x6e, 0x4e, 0x58, 0x6f, - 0xfc, 0xd5, 0x91, 0xab, 0x49, 0xcd, 0xf4, 0xd6, 0x63, 0xb1, 0x87, 0x69, 0x51, 0x61, 0x1e, 0xef, - 0xb6, 0xdb, 0x0c, 0xf4, 0x2a, 0x3d, 0x33, 0x3a, 0x3c, 0x76, 0x58, 0xc5, 0x8c, 0xee, 0x42, 0x2e, - 0x78, 0x94, 0x54, 0x3e, 0x4b, 0xcf, 0x18, 0x54, 0x0f, 0x5e, 0x54, 0x79, 0x87, 0x5b, 0xbb, 0x7d, - 0xfe, 0xa9, 0xde, 0xab, 0xcc, 0x6c, 0x87, 0x07, 0x63, 0xdf, 0xa4, 0xc7, 0x0e, 0xdb, 0xb3, 0xf2, - 0x1e, 0xee, 0xb3, 0x8c, 0xb8, 0xb4, 0x08, 0x31, 0xf2, 0x12, 0xee, 0x5d, 0xc8, 0xe1, 0xe6, 0x77, - 0xc4, 0x57, 0x7d, 0xaf, 0xb2, 0x33, 0x7a, 0x30, 0xf9, 0x94, 0x18, 0xb2, 0xb7, 0x0e, 0xee, 0x6e, - 0x55, 0xb9, 0xcb, 0x67, 0x51, 0x1d, 0x77, 0x74, 0xa0, 0x3b, 0xc4, 0x9c, 0x7e, 0xe3, 0x12, 0xb2, - 0xee, 0xfd, 0x0a, 0xe4, 0x82, 0xcc, 0x14, 0xad, 0x41, 0xf6, 0x45, 0x07, 0x3f, 0x6b, 0x62, 0x65, - 0x4e, 0xf8, 0x30, 0xe0, 0xbc, 0x10, 0x35, 0x45, 0x19, 0xe6, 0x77, 0xaa, 0xed, 0xea, 0x66, 0x13, - 0x07, 0x57, 0xe4, 0x01, 0x40, 0xa6, 0x57, 0x25, 0x45, 0x76, 0x10, 0xea, 0xac, 0xad, 0xfe, 0xf0, - 0x27, 0x6b, 0x73, 0x3f, 0xfe, 0xc9, 0xda, 0xdc, 0xab, 0xf3, 0xb5, 0xc4, 0x0f, 0xcf, 0xd7, 0x12, - 0xff, 0x70, 0xbe, 0x96, 0xf8, 0xf7, 0xf3, 0xb5, 0xc4, 0x5e, 0x96, 0x1f, 0x02, 0x4f, 0xfe, 0x2f, - 0x00, 0x00, 0xff, 0xff, 0x4b, 0xdb, 0xdc, 0xec, 0xf0, 0x31, 0x00, 0x00, + // 5043 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x7a, 0x4d, 0x6c, 0x24, 0x49, + 0x56, 0xbf, 0xeb, 0xd3, 0x55, 0xaf, 0xca, 0x76, 0x3a, 0xda, 0xdb, 0xe3, 0xae, 0xed, 0xb6, 0x6b, + 0x72, 0xa6, 0x77, 0x66, 0x7b, 0xfb, 0x5f, 0xfd, 0xb5, 0xbb, 0xea, 0x99, 0xf9, 0xef, 0xce, 0xd4, + 0x97, 0xed, 0xda, 0xb6, 0xab, 0x4a, 0x51, 0xe5, 0xee, 0x5d, 0x24, 0x48, 0xa5, 0x33, 0xc3, 0xe5, + 0x1c, 0x67, 0x65, 0x14, 0x99, 0x59, 0x76, 0x17, 0x0b, 0xa2, 0xc5, 0x01, 0x90, 0x4f, 0x70, 0x62, + 0x11, 0x32, 0x42, 0x82, 0x13, 0x42, 0xe2, 0x00, 0x12, 0x82, 0xd3, 0x20, 0x71, 0xd8, 0x1b, 0x2c, + 0x48, 0x68, 0x05, 0x92, 0x61, 0x7d, 0xe0, 0xb6, 0x82, 0x0b, 0xe2, 0x02, 0x12, 0x8a, 0x8f, 0xcc, + 0x4a, 0x57, 0xa7, 0xed, 0x1e, 0x66, 0x2f, 0x76, 0xc5, 0x7b, 0xbf, 0xf7, 0xe2, 0xc5, 0x8b, 0x88, + 0x17, 0xef, 0x45, 0x24, 0xdc, 0x1b, 0x58, 0xfe, 0xc1, 0x78, 0xaf, 0x62, 0xd0, 0xe1, 0x03, 0x93, + 0x1a, 0x87, 0xc4, 0x7d, 0xe0, 0x1d, 0xeb, 0xee, 0xf0, 0xd0, 0xf2, 0x1f, 0xe8, 0x23, 0xeb, 0x81, + 0x3f, 0x19, 0x11, 0xaf, 0x32, 0x72, 0xa9, 0x4f, 0x11, 0x12, 0x80, 0x4a, 0x00, 0xa8, 0x1c, 0x3d, + 0x2a, 0xad, 0x0f, 0x28, 0x1d, 0xd8, 0xe4, 0x01, 0x47, 0xec, 0x8d, 0xf7, 0x1f, 0xf8, 0xd6, 0x90, + 0x78, 0xbe, 0x3e, 0x1c, 0x09, 0xa1, 0xd2, 0xda, 0x2c, 0xc0, 0x1c, 0xbb, 0xba, 0x6f, 0x51, 0x47, + 0xf2, 0x57, 0x06, 0x74, 0x40, 0xf9, 0xcf, 0x07, 0xec, 0x97, 0xa0, 0xaa, 0xeb, 0x30, 0xff, 0x9c, + 0xb8, 0x9e, 0x45, 0x1d, 0xb4, 0x02, 0x19, 0xcb, 0x31, 0xc9, 0xcb, 0xd5, 0x44, 0x39, 0xf1, 0x7e, + 0x1a, 0x8b, 0x86, 0xfa, 0x10, 0xa0, 0xc5, 0x7e, 0x34, 0x1d, 0xdf, 0x9d, 0x20, 0x05, 0x52, 0x87, + 0x64, 0xc2, 0x11, 0x79, 0xcc, 0x7e, 0x32, 0xca, 0x91, 0x6e, 0xaf, 0x26, 0x05, 0xe5, 0x48, 0xb7, + 0xd5, 0x9f, 0x24, 0xa0, 0x50, 0x75, 0x1c, 0xea, 0xf3, 0xde, 0x3d, 0x84, 0x20, 0xed, 0xe8, 0x43, + 0x22, 0x85, 0xf8, 0x6f, 0x54, 0x87, 0xac, 0xad, 0xef, 0x11, 0xdb, 0x5b, 0x4d, 0x96, 0x53, 0xef, + 0x17, 0x1e, 0x7f, 0xad, 0xf2, 0xfa, 0x90, 0x2b, 0x11, 0x25, 0x95, 0x6d, 0x8e, 0xe6, 0x46, 0x60, + 0x29, 0x8a, 0xbe, 0x0d, 0xf3, 0x96, 0x63, 0x5a, 0x06, 0xf1, 0x56, 0xd3, 0x5c, 0xcb, 0x5a, 0x9c, + 0x96, 0xa9, 0xf5, 0xb5, 0xf4, 0x0f, 0xcf, 0xd6, 0xe7, 0x70, 0x20, 0x54, 0xfa, 0x00, 0x0a, 0x11, + 0xb5, 0x31, 0x63, 0x5b, 0x81, 0xcc, 0x91, 0x6e, 0x8f, 0x89, 0x1c, 0x9d, 0x68, 0x7c, 0x98, 0x7c, + 0x9a, 0x50, 0x3f, 0x81, 0x95, 0xb6, 0x3e, 0x24, 0xe6, 0x26, 0x71, 0x88, 0x6b, 0x19, 0x98, 0x78, + 0x74, 0xec, 0x1a, 0x84, 0x8d, 0xf5, 0xd0, 0x72, 0xcc, 0x60, 0xac, 0xec, 0x77, 0xbc, 0x16, 0xb5, + 0x0e, 0x6f, 0x35, 0x2c, 0xcf, 0x70, 0x89, 0x4f, 0x3e, 0xb7, 0x92, 0x54, 0xa0, 0xe4, 0x2c, 0x01, + 0x4b, 0xb3, 0xd2, 0x3f, 0x07, 0x37, 0x98, 0x8b, 0x4d, 0xcd, 0x95, 0x14, 0xcd, 0x1b, 0x11, 0x83, + 0x2b, 0x2b, 0x3c, 0x7e, 0x3f, 0xce, 0x43, 0x71, 0x23, 0xd9, 0x9a, 0xc3, 0xcb, 0x5c, 0x4d, 0x40, + 0xe8, 0x8d, 0x88, 0x81, 0x0c, 0xb8, 0x69, 0x4a, 0xa3, 0x67, 0xd4, 0x27, 0xb9, 0xfa, 0xd8, 0x69, + 0xbc, 0x64, 0x98, 0x5b, 0x73, 0x78, 0x25, 0x50, 0x16, 0xed, 0xa4, 0x06, 0x90, 0x0b, 0x74, 0xab, + 0x3f, 0x48, 0x40, 0x3e, 0x60, 0x7a, 0xe8, 0xab, 0x90, 0x77, 0x74, 0x87, 0x6a, 0xc6, 0x68, 0xec, + 0xf1, 0x01, 0xa5, 0x6a, 0xc5, 0xf3, 0xb3, 0xf5, 0x5c, 0x5b, 0x77, 0x68, 0xbd, 0xbb, 0xeb, 0xe1, + 0x1c, 0x63, 0xd7, 0x47, 0x63, 0x0f, 0xbd, 0x0d, 0xc5, 0x21, 0x19, 0x52, 0x77, 0xa2, 0xed, 0x4d, + 0x7c, 0xe2, 0x49, 0xb7, 0x15, 0x04, 0xad, 0xc6, 0x48, 0xe8, 0x5b, 0x30, 0x3f, 0x10, 0x26, 0xad, + 0xa6, 0xf8, 0xf2, 0x79, 0x27, 0xce, 0xfa, 0x19, 0xab, 0x71, 0x20, 0xa3, 0xfe, 0x56, 0x02, 0x56, + 0x42, 0x2a, 0xf9, 0xc5, 0xb1, 0xe5, 0x92, 0x21, 0x71, 0x7c, 0x0f, 0x7d, 0x03, 0xb2, 0xb6, 0x35, + 0xb4, 0x7c, 0x4f, 0xfa, 0xfc, 0x4e, 0x9c, 0xda, 0x70, 0x50, 0x58, 0x82, 0x51, 0x15, 0x8a, 0x2e, + 0xf1, 0x88, 0x7b, 0x24, 0x56, 0xbc, 0xf4, 0xe8, 0x35, 0xc2, 0x17, 0x44, 0xd4, 0x0d, 0xc8, 0x75, + 0x6d, 0xdd, 0xdf, 0xa7, 0xee, 0x10, 0xa9, 0x50, 0xd4, 0x5d, 0xe3, 0xc0, 0xf2, 0x89, 0xe1, 0x8f, + 0xdd, 0x60, 0xf7, 0x5d, 0xa0, 0xa1, 0x9b, 0x90, 0xa4, 0xa2, 0xa3, 0x7c, 0x2d, 0x7b, 0x7e, 0xb6, + 0x9e, 0xec, 0xf4, 0x70, 0x92, 0x7a, 0xea, 0x47, 0xb0, 0xdc, 0xb5, 0xc7, 0x03, 0xcb, 0x69, 0x10, + 0xcf, 0x70, 0xad, 0x11, 0xd3, 0xce, 0x56, 0x25, 0x8b, 0x51, 0xc1, 0xaa, 0x64, 0xbf, 0xc3, 0xad, + 0x9d, 0x9c, 0x6e, 0x6d, 0xf5, 0x37, 0x92, 0xb0, 0xdc, 0x74, 0x06, 0x96, 0x43, 0xa2, 0xd2, 0x77, + 0x61, 0x91, 0x70, 0xa2, 0x76, 0x24, 0xc2, 0x8d, 0xd4, 0xb3, 0x20, 0xa8, 0x41, 0x0c, 0x6a, 0xcd, + 0xc4, 0x85, 0x47, 0x71, 0xc3, 0x7f, 0x4d, 0x7b, 0x6c, 0x74, 0x68, 0xc2, 0xfc, 0x88, 0x0f, 0xc2, + 0x93, 0xd3, 0x7b, 0x37, 0x4e, 0xd7, 0x6b, 0xe3, 0x0c, 0x82, 0x84, 0x94, 0xfd, 0x22, 0x41, 0xe2, + 0x4f, 0x92, 0xb0, 0xd4, 0xa6, 0xe6, 0x05, 0x3f, 0x94, 0x20, 0x77, 0x40, 0x3d, 0x3f, 0x12, 0x10, + 0xc3, 0x36, 0x7a, 0x0a, 0xb9, 0x91, 0x9c, 0x3e, 0x39, 0xfb, 0xb7, 0xe3, 0x4d, 0x16, 0x18, 0x1c, + 0xa2, 0xd1, 0x47, 0x90, 0x0f, 0xb6, 0x0c, 0x1b, 0xed, 0x1b, 0x2c, 0x9c, 0x29, 0x1e, 0x7d, 0x0b, + 0xb2, 0x62, 0x12, 0x56, 0xd3, 0x5c, 0xf2, 0xee, 0x1b, 0xf9, 0x1c, 0x4b, 0x21, 0xb4, 0x09, 0x39, + 0xdf, 0xf6, 0x34, 0xcb, 0xd9, 0xa7, 0xab, 0x19, 0xae, 0x60, 0x3d, 0x36, 0xc8, 0x50, 0x93, 0xf4, + 0xb7, 0x7b, 0x2d, 0x67, 0x9f, 0xd6, 0x0a, 0xe7, 0x67, 0xeb, 0xf3, 0xb2, 0x81, 0xe7, 0x7d, 0xdb, + 0x63, 0x3f, 0xd4, 0xdf, 0x4e, 0x40, 0x21, 0x82, 0x42, 0x77, 0x00, 0x7c, 0x77, 0xec, 0xf9, 0x9a, + 0x4b, 0xa9, 0xcf, 0x9d, 0x55, 0xc4, 0x79, 0x4e, 0xc1, 0x94, 0xfa, 0xa8, 0x02, 0x37, 0x0c, 0xe2, + 0xfa, 0x9a, 0xe5, 0x79, 0x63, 0xe2, 0x6a, 0xde, 0x78, 0xef, 0x53, 0x62, 0xf8, 0xdc, 0x71, 0x45, + 0xbc, 0xcc, 0x58, 0x2d, 0xce, 0xe9, 0x09, 0x06, 0x7a, 0x02, 0x37, 0xa3, 0xf8, 0xd1, 0x78, 0xcf, + 0xb6, 0x0c, 0x8d, 0x4d, 0x66, 0x8a, 0x8b, 0xdc, 0x98, 0x8a, 0x74, 0x39, 0xef, 0x19, 0x99, 0xa8, + 0x3f, 0x4e, 0x80, 0x82, 0xf5, 0x7d, 0x7f, 0x87, 0x0c, 0xf7, 0x88, 0xdb, 0xf3, 0x75, 0x7f, 0xec, + 0xa1, 0x9b, 0x90, 0xb5, 0x89, 0x6e, 0x12, 0x97, 0x1b, 0x95, 0xc3, 0xb2, 0x85, 0x76, 0xd9, 0x0e, + 0xd6, 0x8d, 0x03, 0x7d, 0xcf, 0xb2, 0x2d, 0x7f, 0xc2, 0x4d, 0x59, 0x8c, 0x5f, 0xc2, 0xb3, 0x3a, + 0x2b, 0x38, 0x22, 0x88, 0x2f, 0xa8, 0x41, 0xab, 0x30, 0x3f, 0x24, 0x9e, 0xa7, 0x0f, 0x08, 0xb7, + 0x34, 0x8f, 0x83, 0xa6, 0xfa, 0x11, 0x14, 0xa3, 0x72, 0xa8, 0x00, 0xf3, 0xbb, 0xed, 0x67, 0xed, + 0xce, 0x8b, 0xb6, 0x32, 0x87, 0x96, 0xa0, 0xb0, 0xdb, 0xc6, 0xcd, 0x6a, 0x7d, 0xab, 0x5a, 0xdb, + 0x6e, 0x2a, 0x09, 0xb4, 0x00, 0xf9, 0x69, 0x33, 0xa9, 0xfe, 0x59, 0x02, 0x80, 0xb9, 0x5b, 0x0e, + 0xea, 0x43, 0xc8, 0x78, 0xbe, 0xee, 0x8b, 0x55, 0xb9, 0xf8, 0xf8, 0xdd, 0xcb, 0xe6, 0x50, 0xda, + 0xcb, 0xfe, 0x11, 0x2c, 0x44, 0xa2, 0x16, 0x26, 0x2f, 0x58, 0xc8, 0x02, 0x84, 0x6e, 0x9a, 0xae, + 0x34, 0x9c, 0xff, 0x56, 0x3f, 0x82, 0x0c, 0x97, 0xbe, 0x68, 0x6e, 0x0e, 0xd2, 0x0d, 0xf6, 0x2b, + 0x81, 0xf2, 0x90, 0xc1, 0xcd, 0x6a, 0xe3, 0x7b, 0x4a, 0x12, 0x29, 0x50, 0x6c, 0xb4, 0x7a, 0xf5, + 0x4e, 0xbb, 0xdd, 0xac, 0xf7, 0x9b, 0x0d, 0x25, 0xa5, 0xde, 0x85, 0x4c, 0x6b, 0xc8, 0x34, 0xdf, + 0x66, 0x4b, 0x7e, 0x9f, 0xb8, 0xc4, 0x31, 0x82, 0x9d, 0x34, 0x25, 0xa8, 0x3f, 0x2d, 0x40, 0x66, + 0x87, 0x8e, 0x1d, 0x1f, 0x3d, 0x8e, 0x84, 0xad, 0xc5, 0xf8, 0x0c, 0x81, 0x03, 0x2b, 0xfd, 0xc9, + 0x88, 0xc8, 0xb0, 0x76, 0x13, 0xb2, 0x62, 0x73, 0xc8, 0xe1, 0xc8, 0x16, 0xa3, 0xfb, 0xba, 0x3b, + 0x20, 0xbe, 0x1c, 0x8f, 0x6c, 0xa1, 0xf7, 0xd9, 0x89, 0xa5, 0x9b, 0xd4, 0xb1, 0x27, 0x7c, 0x0f, + 0xe5, 0xc4, 0xb1, 0x84, 0x89, 0x6e, 0x76, 0x1c, 0x7b, 0x82, 0x43, 0x2e, 0xda, 0x82, 0xe2, 0x9e, + 0xe5, 0x98, 0x1a, 0x1d, 0x89, 0x20, 0x9f, 0xb9, 0x7c, 0xc7, 0x09, 0xab, 0x6a, 0x96, 0x63, 0x76, + 0x04, 0x18, 0x17, 0xf6, 0xa6, 0x0d, 0xd4, 0x86, 0xc5, 0x23, 0x6a, 0x8f, 0x87, 0x24, 0xd4, 0x95, + 0xe5, 0xba, 0xde, 0xbb, 0x5c, 0xd7, 0x73, 0x8e, 0x0f, 0xb4, 0x2d, 0x1c, 0x45, 0x9b, 0xe8, 0x19, + 0x2c, 0xf8, 0xc3, 0xd1, 0xbe, 0x17, 0xaa, 0x9b, 0xe7, 0xea, 0xbe, 0x72, 0x85, 0xc3, 0x18, 0x3c, + 0xd0, 0x56, 0xf4, 0x23, 0x2d, 0xb4, 0x09, 0x05, 0x83, 0x3a, 0x9e, 0xe5, 0xf9, 0xc4, 0x31, 0x26, + 0xab, 0x39, 0xee, 0xfb, 0x2b, 0x46, 0x59, 0x9f, 0x82, 0x71, 0x54, 0xb2, 0xf4, 0x6b, 0x29, 0x28, + 0x44, 0x5c, 0x80, 0x7a, 0x50, 0x18, 0xb9, 0x74, 0xa4, 0x0f, 0xf8, 0x89, 0x27, 0x27, 0xf5, 0xd1, + 0x1b, 0xb9, 0xaf, 0xd2, 0x9d, 0x0a, 0xe2, 0xa8, 0x16, 0xf5, 0x34, 0x09, 0x85, 0x08, 0x13, 0xdd, + 0x83, 0x1c, 0xee, 0xe2, 0xd6, 0xf3, 0x6a, 0xbf, 0xa9, 0xcc, 0x95, 0x6e, 0x9f, 0x9c, 0x96, 0x57, + 0xb9, 0xb6, 0xa8, 0x82, 0xae, 0x6b, 0x1d, 0xb1, 0x35, 0xfc, 0x3e, 0xcc, 0x07, 0xd0, 0x44, 0xe9, + 0xcb, 0x27, 0xa7, 0xe5, 0xb7, 0x66, 0xa1, 0x11, 0x24, 0xee, 0x6d, 0x55, 0x71, 0xb3, 0xa1, 0x24, + 0xe3, 0x91, 0xb8, 0x77, 0xa0, 0xbb, 0xc4, 0x44, 0x5f, 0x81, 0xac, 0x04, 0xa6, 0x4a, 0xa5, 0x93, + 0xd3, 0xf2, 0xcd, 0x59, 0xe0, 0x14, 0x87, 0x7b, 0xdb, 0xd5, 0xe7, 0x4d, 0x25, 0x1d, 0x8f, 0xc3, + 0x3d, 0x5b, 0x3f, 0x22, 0xe8, 0x5d, 0xc8, 0x08, 0x58, 0xa6, 0x74, 0xeb, 0xe4, 0xb4, 0xfc, 0xa5, + 0xd7, 0xd4, 0x31, 0x54, 0x69, 0xf5, 0x37, 0xff, 0x70, 0x6d, 0xee, 0xaf, 0xfe, 0x68, 0x4d, 0x99, + 0x65, 0x97, 0xfe, 0x3b, 0x01, 0x0b, 0x17, 0xd6, 0x0e, 0x52, 0x21, 0xeb, 0x50, 0x83, 0x8e, 0xc4, + 0x41, 0x98, 0xab, 0xc1, 0xf9, 0xd9, 0x7a, 0xb6, 0x4d, 0xeb, 0x74, 0x34, 0xc1, 0x92, 0x83, 0x9e, + 0xcd, 0x1c, 0xe5, 0x4f, 0xde, 0x70, 0x61, 0xc6, 0x1e, 0xe6, 0x1f, 0xc3, 0x82, 0xe9, 0x5a, 0x47, + 0xc4, 0xd5, 0x0c, 0xea, 0xec, 0x5b, 0x03, 0x79, 0xc8, 0x95, 0x62, 0xf3, 0x4d, 0x0e, 0xc4, 0x45, + 0x21, 0x50, 0xe7, 0xf8, 0x2f, 0x70, 0x8c, 0x97, 0x9e, 0x43, 0x31, 0xba, 0xd4, 0xd9, 0xb9, 0xe4, + 0x59, 0xbf, 0x44, 0x64, 0x62, 0xc9, 0xd3, 0x50, 0x9c, 0x67, 0x14, 0x91, 0x56, 0xbe, 0x07, 0xe9, + 0x21, 0x35, 0x85, 0x9e, 0x85, 0xda, 0x0d, 0x96, 0x4d, 0xfc, 0xd3, 0xd9, 0x7a, 0x81, 0x7a, 0x95, + 0x0d, 0xcb, 0x26, 0x3b, 0xd4, 0x24, 0x98, 0x03, 0xd4, 0x23, 0x48, 0xb3, 0x98, 0x83, 0xbe, 0x0c, + 0xe9, 0x5a, 0xab, 0xdd, 0x50, 0xe6, 0x4a, 0xcb, 0x27, 0xa7, 0xe5, 0x05, 0xee, 0x12, 0xc6, 0x60, + 0x6b, 0x17, 0xad, 0x43, 0xf6, 0x79, 0x67, 0x7b, 0x77, 0x87, 0x2d, 0xaf, 0x1b, 0x27, 0xa7, 0xe5, + 0xa5, 0x90, 0x2d, 0x9c, 0x86, 0xee, 0x40, 0xa6, 0xbf, 0xd3, 0xdd, 0xe8, 0x29, 0xc9, 0x12, 0x3a, + 0x39, 0x2d, 0x2f, 0x86, 0x7c, 0x6e, 0x73, 0x69, 0x59, 0xce, 0x6a, 0x3e, 0xa4, 0xab, 0x3f, 0x4a, + 0x40, 0x21, 0xb2, 0xe1, 0xd8, 0xc2, 0x6c, 0x34, 0x37, 0xaa, 0xbb, 0xdb, 0x7d, 0x65, 0x2e, 0xb2, + 0x30, 0x23, 0x90, 0x06, 0xd9, 0xd7, 0xc7, 0x36, 0x8b, 0x73, 0x50, 0xef, 0xb4, 0x7b, 0xad, 0x5e, + 0xbf, 0xd9, 0xee, 0x2b, 0x89, 0xd2, 0xea, 0xc9, 0x69, 0x79, 0x65, 0x16, 0xbc, 0x31, 0xb6, 0x6d, + 0xb6, 0x34, 0xeb, 0xd5, 0xfa, 0x16, 0x5f, 0xeb, 0xd3, 0xa5, 0x19, 0x41, 0xd5, 0x75, 0xe3, 0x80, + 0x98, 0xe8, 0x3e, 0xe4, 0x1b, 0xcd, 0xed, 0xe6, 0x66, 0x95, 0x47, 0xf7, 0xd2, 0x9d, 0x93, 0xd3, + 0xf2, 0xad, 0xd7, 0x7b, 0xb7, 0xc9, 0x40, 0xf7, 0x89, 0x39, 0xb3, 0x44, 0x23, 0x10, 0xf5, 0x3f, + 0x93, 0xb0, 0x80, 0x59, 0x39, 0xec, 0xfa, 0x5d, 0x6a, 0x5b, 0xc6, 0x04, 0x75, 0x21, 0x6f, 0x50, + 0xc7, 0xb4, 0x22, 0x71, 0xe2, 0xf1, 0x25, 0x29, 0xd1, 0x54, 0x2a, 0x68, 0xd5, 0x03, 0x49, 0x3c, + 0x55, 0x82, 0x1e, 0x40, 0xc6, 0x24, 0xb6, 0x3e, 0x91, 0xb9, 0xd9, 0xad, 0x8a, 0x28, 0xb8, 0x2b, + 0x41, 0xc1, 0x5d, 0x69, 0xc8, 0x82, 0x1b, 0x0b, 0x1c, 0xaf, 0x41, 0xf4, 0x97, 0x9a, 0xee, 0xfb, + 0x64, 0x38, 0xf2, 0x45, 0x62, 0x96, 0xc6, 0x85, 0xa1, 0xfe, 0xb2, 0x2a, 0x49, 0xe8, 0x11, 0x64, + 0x8f, 0x2d, 0xc7, 0xa4, 0xc7, 0x32, 0xf7, 0xba, 0x42, 0xa9, 0x04, 0xaa, 0x27, 0x2c, 0x25, 0x99, + 0x31, 0x93, 0xad, 0xa1, 0x76, 0xa7, 0xdd, 0x0c, 0xd6, 0x90, 0xe4, 0x77, 0x9c, 0x36, 0x75, 0xd8, + 0xfe, 0x87, 0x4e, 0x5b, 0xdb, 0xa8, 0xb6, 0xb6, 0x77, 0x31, 0x5b, 0x47, 0x2b, 0x27, 0xa7, 0x65, + 0x25, 0x84, 0x6c, 0xe8, 0x96, 0xcd, 0x8a, 0x81, 0x5b, 0x90, 0xaa, 0xb6, 0xbf, 0xa7, 0x24, 0x4b, + 0xca, 0xc9, 0x69, 0xb9, 0x18, 0xb2, 0xab, 0xce, 0x64, 0xea, 0xf7, 0xd9, 0x7e, 0xd5, 0xbf, 0x4d, + 0x41, 0x71, 0x77, 0x64, 0xea, 0x3e, 0x11, 0xfb, 0x0c, 0x95, 0xa1, 0x30, 0xd2, 0x5d, 0xdd, 0xb6, + 0x89, 0x6d, 0x79, 0x43, 0x79, 0x95, 0x10, 0x25, 0xa1, 0x0f, 0xde, 0xd4, 0x8d, 0xb5, 0x1c, 0xdb, + 0x3b, 0x3f, 0xf8, 0x97, 0xf5, 0x44, 0xe0, 0xd0, 0x5d, 0x58, 0xdc, 0x17, 0xd6, 0x6a, 0xba, 0xc1, + 0x27, 0x36, 0xc5, 0x27, 0xb6, 0x12, 0x37, 0xb1, 0x51, 0xb3, 0x2a, 0x72, 0x90, 0x55, 0x2e, 0x85, + 0x17, 0xf6, 0xa3, 0x4d, 0xf4, 0x04, 0xe6, 0x87, 0xd4, 0xb1, 0x7c, 0xea, 0x5e, 0x3f, 0x0b, 0x01, + 0x12, 0xdd, 0x83, 0x65, 0x36, 0xb9, 0x81, 0x3d, 0x9c, 0xcd, 0x8f, 0xf3, 0x24, 0x5e, 0x1a, 0xea, + 0x2f, 0x65, 0x87, 0x98, 0x91, 0x51, 0x0d, 0x32, 0xd4, 0x65, 0xf9, 0x62, 0x96, 0x9b, 0x7b, 0xff, + 0x5a, 0x73, 0x45, 0xa3, 0xc3, 0x64, 0xb0, 0x10, 0x55, 0xbf, 0x09, 0x0b, 0x17, 0x06, 0xc1, 0xd2, + 0xa4, 0x6e, 0x75, 0xb7, 0xd7, 0x54, 0xe6, 0x50, 0x11, 0x72, 0xf5, 0x4e, 0xbb, 0xdf, 0x6a, 0xef, + 0xb2, 0x3c, 0xaf, 0x08, 0x39, 0xdc, 0xd9, 0xde, 0xae, 0x55, 0xeb, 0xcf, 0x94, 0xa4, 0x5a, 0x81, + 0x42, 0x44, 0x1b, 0x5a, 0x04, 0xe8, 0xf5, 0x3b, 0x5d, 0x6d, 0xa3, 0x85, 0x7b, 0x7d, 0x91, 0x25, + 0xf6, 0xfa, 0x55, 0xdc, 0x97, 0x84, 0x84, 0xfa, 0xef, 0xc9, 0x60, 0x46, 0x65, 0x62, 0x58, 0xbb, + 0x98, 0x18, 0x5e, 0x61, 0xbc, 0x4c, 0x0d, 0xa7, 0x8d, 0x30, 0x41, 0xfc, 0x00, 0x80, 0x2f, 0x1c, + 0x62, 0x6a, 0xba, 0x2f, 0x27, 0xbe, 0xf4, 0x9a, 0x93, 0xfb, 0xc1, 0x8d, 0x16, 0xce, 0x4b, 0x74, + 0xd5, 0x47, 0xdf, 0x82, 0xa2, 0x41, 0x87, 0x23, 0x9b, 0x48, 0xe1, 0xd4, 0xb5, 0xc2, 0x85, 0x10, + 0x5f, 0xf5, 0xa3, 0xa9, 0x69, 0xfa, 0x62, 0xf2, 0xfc, 0xeb, 0x89, 0xc0, 0x33, 0x31, 0xd9, 0x68, + 0x11, 0x72, 0xbb, 0xdd, 0x46, 0xb5, 0xdf, 0x6a, 0x6f, 0x2a, 0x09, 0x04, 0x90, 0xe5, 0xae, 0x6e, + 0x28, 0x49, 0x96, 0x45, 0xd7, 0x3b, 0x3b, 0xdd, 0xed, 0x26, 0x8f, 0x58, 0x68, 0x05, 0x94, 0xc0, + 0xd9, 0x1a, 0x77, 0x64, 0xb3, 0xa1, 0xa4, 0xd1, 0x0d, 0x58, 0x0a, 0xa9, 0x52, 0x32, 0x83, 0x6e, + 0x02, 0x0a, 0x89, 0x53, 0x15, 0x59, 0xf5, 0x57, 0x60, 0xa9, 0x4e, 0x1d, 0x5f, 0xb7, 0x9c, 0xb0, + 0xc2, 0x78, 0xcc, 0x06, 0x2d, 0x49, 0x9a, 0x25, 0x6f, 0x82, 0x6a, 0x4b, 0xe7, 0x67, 0xeb, 0x85, + 0x10, 0xda, 0x6a, 0xf0, 0x54, 0x49, 0x36, 0x4c, 0xb6, 0x7f, 0x47, 0x96, 0xc9, 0x9d, 0x9b, 0xa9, + 0xcd, 0x9f, 0x9f, 0xad, 0xa7, 0xba, 0xad, 0x06, 0x66, 0x34, 0xf4, 0x65, 0xc8, 0x93, 0x97, 0x96, + 0xaf, 0x19, 0xec, 0x5c, 0x62, 0x0e, 0xcc, 0xe0, 0x1c, 0x23, 0xd4, 0xd9, 0x31, 0x54, 0x03, 0xe8, + 0x52, 0xd7, 0x97, 0x3d, 0x7f, 0x1d, 0x32, 0x23, 0xea, 0xf2, 0xbb, 0x8b, 0x4b, 0x6f, 0xd4, 0x18, + 0x5c, 0x2c, 0x54, 0x2c, 0xc0, 0xea, 0xef, 0xa6, 0x00, 0xfa, 0xba, 0x77, 0x28, 0x95, 0x3c, 0x85, + 0x7c, 0x78, 0x3b, 0x29, 0x2f, 0x41, 0xae, 0x9c, 0xed, 0x10, 0x8c, 0x9e, 0x04, 0x8b, 0x4d, 0xd4, + 0x4e, 0xb1, 0x45, 0x6c, 0xd0, 0x51, 0x5c, 0xf9, 0x71, 0xb1, 0x40, 0x62, 0xc7, 0x3c, 0x71, 0x5d, + 0x39, 0xf3, 0xec, 0x27, 0xaa, 0xf3, 0x63, 0x41, 0x38, 0x4d, 0x66, 0xdf, 0xb1, 0xd7, 0x3e, 0x33, + 0x33, 0xb2, 0x35, 0x87, 0xa7, 0x72, 0xe8, 0x63, 0x28, 0xb0, 0x71, 0x6b, 0x1e, 0xe7, 0xc9, 0xc4, + 0xfb, 0x52, 0x57, 0x09, 0x0d, 0x18, 0x46, 0x53, 0x2f, 0xdf, 0x01, 0xd0, 0x47, 0x23, 0xdb, 0x22, + 0xa6, 0xb6, 0x37, 0xe1, 0x99, 0x76, 0x1e, 0xe7, 0x25, 0xa5, 0x36, 0x61, 0xdb, 0x25, 0x60, 0xeb, + 0x3e, 0xcf, 0x9e, 0xaf, 0x71, 0xa0, 0x44, 0x57, 0xfd, 0x9a, 0x02, 0x8b, 0xee, 0xd8, 0x61, 0x0e, + 0x95, 0xd6, 0xa9, 0x7f, 0x9a, 0x84, 0xb7, 0xda, 0xc4, 0x3f, 0xa6, 0xee, 0x61, 0xd5, 0xf7, 0x75, + 0xe3, 0x60, 0x48, 0x1c, 0x39, 0x7d, 0x91, 0x82, 0x26, 0x71, 0xa1, 0xa0, 0x59, 0x85, 0x79, 0xdd, + 0xb6, 0x74, 0x8f, 0x88, 0xe4, 0x2d, 0x8f, 0x83, 0x26, 0x2b, 0xbb, 0x58, 0x11, 0x47, 0x3c, 0x8f, + 0x88, 0x7b, 0x15, 0x66, 0x78, 0x40, 0x40, 0xdf, 0x87, 0x9b, 0x32, 0x4d, 0xd3, 0xc3, 0xae, 0x58, + 0x41, 0x11, 0x5c, 0xd0, 0x36, 0x63, 0xab, 0xca, 0x78, 0xe3, 0x64, 0x1e, 0x37, 0x25, 0x77, 0x46, + 0xbe, 0xcc, 0x0a, 0x57, 0xcc, 0x18, 0x56, 0x69, 0x13, 0x6e, 0x5d, 0x2a, 0xf2, 0xb9, 0xee, 0x6d, + 0xfe, 0x21, 0x09, 0xd0, 0xea, 0x56, 0x77, 0xa4, 0x93, 0x1a, 0x90, 0xdd, 0xd7, 0x87, 0x96, 0x3d, + 0xb9, 0x2a, 0x02, 0x4e, 0xf1, 0x95, 0xaa, 0x70, 0xc7, 0x06, 0x97, 0xc1, 0x52, 0x96, 0xd7, 0x94, + 0xe3, 0x3d, 0x87, 0xf8, 0x61, 0x4d, 0xc9, 0x5b, 0xcc, 0x0c, 0x57, 0x77, 0xc2, 0xa5, 0x2b, 0x1a, + 0x6c, 0x02, 0x58, 0xca, 0x73, 0xac, 0x4f, 0x82, 0xb0, 0x25, 0x9b, 0x68, 0x8b, 0xdf, 0x8e, 0x12, + 0xf7, 0x88, 0x98, 0xab, 0x19, 0xee, 0xd4, 0xeb, 0xec, 0xc1, 0x12, 0x2e, 0x7c, 0x17, 0x4a, 0x97, + 0x3e, 0xe2, 0x29, 0xd3, 0x94, 0xf5, 0xb9, 0x7c, 0xf4, 0x10, 0x16, 0x2e, 0x8c, 0xf3, 0xb5, 0x62, + 0xbe, 0xd5, 0x7d, 0xfe, 0x75, 0x25, 0x2d, 0x7f, 0x7d, 0x53, 0xc9, 0xaa, 0x7f, 0x9c, 0x12, 0x81, + 0x46, 0x7a, 0x35, 0xfe, 0x55, 0x20, 0xc7, 0x57, 0xb7, 0x41, 0x6d, 0x19, 0x00, 0xde, 0xbb, 0x3a, + 0xfe, 0xb0, 0x9a, 0x8e, 0xc3, 0x71, 0x28, 0x88, 0xd6, 0xa1, 0x20, 0x56, 0xb1, 0xc6, 0x36, 0x1c, + 0x77, 0xeb, 0x02, 0x06, 0x41, 0x62, 0x92, 0xe8, 0x2e, 0x2c, 0xf2, 0xcb, 0x1f, 0xef, 0x80, 0x98, + 0x02, 0x93, 0xe6, 0x98, 0x85, 0x90, 0xca, 0x61, 0x3b, 0x50, 0x94, 0x04, 0x8d, 0xe7, 0xf3, 0x19, + 0x6e, 0xd0, 0xbd, 0xeb, 0x0c, 0x12, 0x22, 0x3c, 0xcd, 0x2f, 0x8c, 0xa6, 0x0d, 0xb5, 0x01, 0xb9, + 0xc0, 0x58, 0xb4, 0x0a, 0xa9, 0x7e, 0xbd, 0xab, 0xcc, 0x95, 0x96, 0x4e, 0x4e, 0xcb, 0x85, 0x80, + 0xdc, 0xaf, 0x77, 0x19, 0x67, 0xb7, 0xd1, 0x55, 0x12, 0x17, 0x39, 0xbb, 0x8d, 0x6e, 0x29, 0xcd, + 0x72, 0x30, 0x75, 0x1f, 0x0a, 0x91, 0x1e, 0xd0, 0x3b, 0x30, 0xdf, 0x6a, 0x6f, 0xe2, 0x66, 0xaf, + 0xa7, 0xcc, 0x95, 0x6e, 0x9e, 0x9c, 0x96, 0x51, 0x84, 0xdb, 0x72, 0x06, 0x6c, 0x7e, 0xd0, 0x1d, + 0x48, 0x6f, 0x75, 0xd8, 0xd9, 0x2e, 0x0a, 0x88, 0x08, 0x62, 0x8b, 0x7a, 0x7e, 0xe9, 0x86, 0x4c, + 0xee, 0xa2, 0x8a, 0xd5, 0xdf, 0x4b, 0x40, 0x56, 0x6c, 0xa6, 0xd8, 0x89, 0xaa, 0xc2, 0x7c, 0x70, + 0x4d, 0x20, 0x8a, 0xbb, 0xf7, 0x2e, 0x2f, 0xc4, 0x2a, 0xb2, 0x6e, 0x12, 0xcb, 0x2f, 0x90, 0x2b, + 0x7d, 0x08, 0xc5, 0x28, 0xe3, 0x73, 0x2d, 0xbe, 0xef, 0x43, 0x81, 0xad, 0xef, 0xa0, 0x20, 0x7b, + 0x0c, 0x59, 0x11, 0x10, 0xc2, 0xb3, 0xe6, 0xf2, 0xaa, 0x50, 0x22, 0xd1, 0x53, 0x98, 0x17, 0x95, + 0x64, 0x70, 0x3b, 0xbc, 0x76, 0xf5, 0x2e, 0xc2, 0x01, 0x5c, 0xfd, 0x18, 0xd2, 0x5d, 0x42, 0x5c, + 0xe6, 0x7b, 0x87, 0x9a, 0x64, 0x7a, 0x3c, 0xcb, 0x22, 0xd8, 0x24, 0xad, 0x06, 0x2b, 0x82, 0x4d, + 0xd2, 0x32, 0xc3, 0xfb, 0xaf, 0x64, 0xe4, 0xfe, 0xab, 0x0f, 0xc5, 0x17, 0xc4, 0x1a, 0x1c, 0xf8, + 0xc4, 0xe4, 0x8a, 0xee, 0x43, 0x7a, 0x44, 0x42, 0xe3, 0x57, 0x63, 0x17, 0x18, 0x21, 0x2e, 0xe6, + 0x28, 0x16, 0x47, 0x8e, 0xb9, 0xb4, 0x7c, 0xd2, 0x90, 0x2d, 0xf5, 0xef, 0x93, 0xb0, 0xd8, 0xf2, + 0xbc, 0xb1, 0xee, 0x18, 0x41, 0xe6, 0xf6, 0xed, 0x8b, 0x99, 0x5b, 0xec, 0xdb, 0xcf, 0x45, 0x91, + 0x8b, 0xd7, 0x7a, 0xf2, 0xf4, 0x4c, 0x86, 0xa7, 0xa7, 0xfa, 0xd3, 0x44, 0x70, 0x77, 0x77, 0x37, + 0xb2, 0xdd, 0x45, 0x1d, 0x18, 0xd5, 0x44, 0x76, 0x9d, 0x43, 0x87, 0x1e, 0x3b, 0xe8, 0x6d, 0xc8, + 0xe0, 0x66, 0xbb, 0xf9, 0x42, 0x49, 0x88, 0xe5, 0x79, 0x01, 0x84, 0x89, 0x43, 0x8e, 0x99, 0xa6, + 0x6e, 0xb3, 0xdd, 0x60, 0x99, 0x56, 0x32, 0x46, 0x53, 0x97, 0x38, 0xa6, 0xe5, 0x0c, 0xd0, 0x3b, + 0x90, 0x6d, 0xf5, 0x7a, 0xbb, 0xbc, 0x4c, 0x7c, 0xeb, 0xe4, 0xb4, 0x7c, 0xe3, 0x02, 0x8a, 0xdf, + 0xdb, 0x9a, 0x0c, 0xc4, 0xca, 0x1c, 0x96, 0x83, 0xc5, 0x80, 0x58, 0xfe, 0x2c, 0x40, 0xb8, 0xd3, + 0xaf, 0xf6, 0x9b, 0x4a, 0x26, 0x06, 0x84, 0x29, 0xfb, 0x2b, 0xb7, 0xdb, 0x3f, 0x27, 0x41, 0xa9, + 0x1a, 0x06, 0x19, 0xf9, 0x8c, 0x2f, 0x2b, 0xcb, 0x3e, 0xe4, 0x46, 0xec, 0x97, 0x45, 0x82, 0x2c, + 0xe9, 0x69, 0xec, 0xeb, 0xe5, 0x8c, 0x5c, 0x05, 0x53, 0x9b, 0x54, 0xcd, 0xa1, 0xe5, 0x79, 0x16, + 0x75, 0x04, 0x0d, 0x87, 0x9a, 0x4a, 0xff, 0x91, 0x80, 0x1b, 0x31, 0x08, 0xf4, 0x10, 0xd2, 0x2e, + 0xb5, 0x83, 0x39, 0xbc, 0x7d, 0xd9, 0xb5, 0x2c, 0x13, 0xc5, 0x1c, 0x89, 0xd6, 0x00, 0xf4, 0xb1, + 0x4f, 0x75, 0xde, 0x3f, 0x9f, 0xbd, 0x1c, 0x8e, 0x50, 0xd0, 0x0b, 0xc8, 0x7a, 0xc4, 0x70, 0x49, + 0x90, 0x4b, 0x7f, 0xfc, 0x7f, 0xb5, 0xbe, 0xd2, 0xe3, 0x6a, 0xb0, 0x54, 0x57, 0xaa, 0x40, 0x56, + 0x50, 0xd8, 0xb2, 0x37, 0x75, 0x5f, 0x97, 0x97, 0xf6, 0xfc, 0x37, 0x5b, 0x4d, 0xba, 0x3d, 0x08, + 0x56, 0x93, 0x6e, 0x0f, 0xd4, 0xbf, 0x49, 0x02, 0x34, 0x5f, 0xfa, 0xc4, 0x75, 0x74, 0xbb, 0x5e, + 0x45, 0xcd, 0x48, 0xf4, 0x17, 0xa3, 0xfd, 0x6a, 0xec, 0x4b, 0x44, 0x28, 0x51, 0xa9, 0x57, 0x63, + 0xe2, 0xff, 0x2d, 0x48, 0x8d, 0x5d, 0xf9, 0x20, 0x2d, 0xf2, 0xe0, 0x5d, 0xbc, 0x8d, 0x19, 0x0d, + 0x35, 0xa7, 0x61, 0x2b, 0x75, 0xf9, 0xb3, 0x73, 0xa4, 0x83, 0xd8, 0xd0, 0xc5, 0x76, 0xbe, 0xa1, + 0x6b, 0x06, 0x91, 0x27, 0x47, 0x51, 0xec, 0xfc, 0x7a, 0xb5, 0x4e, 0x5c, 0x1f, 0x67, 0x0d, 0x9d, + 0xfd, 0xff, 0x42, 0xf1, 0xed, 0x3e, 0xc0, 0x74, 0x68, 0x68, 0x0d, 0x32, 0xf5, 0x8d, 0x5e, 0x6f, + 0x5b, 0x99, 0x13, 0x01, 0x7c, 0xca, 0xe2, 0x64, 0xf5, 0x2f, 0x93, 0x90, 0xab, 0x57, 0xe5, 0xb1, + 0x5a, 0x07, 0x85, 0x47, 0x25, 0xfe, 0xd4, 0x41, 0x5e, 0x8e, 0x2c, 0x77, 0x22, 0x03, 0xcb, 0x15, + 0x45, 0xed, 0x22, 0x13, 0x61, 0x56, 0x37, 0xb9, 0x00, 0xc2, 0x50, 0x24, 0xd2, 0x09, 0x9a, 0xa1, + 0x07, 0x31, 0x7e, 0xed, 0x6a, 0x67, 0x89, 0xf2, 0x64, 0xda, 0xf6, 0x70, 0x21, 0x50, 0x52, 0xd7, + 0x3d, 0xf4, 0x01, 0x2c, 0x79, 0xd6, 0xc0, 0xb1, 0x9c, 0x81, 0x16, 0x38, 0x8f, 0xbf, 0xbb, 0xd4, + 0x96, 0xcf, 0xcf, 0xd6, 0x17, 0x7a, 0x82, 0x25, 0x7d, 0xb8, 0x20, 0x91, 0x75, 0xee, 0x4a, 0xf4, + 0x4d, 0x58, 0x8c, 0x88, 0x32, 0x2f, 0x0a, 0xb7, 0x2b, 0xe7, 0x67, 0xeb, 0xc5, 0x50, 0xf2, 0x19, + 0x99, 0xe0, 0x62, 0x28, 0xf8, 0x8c, 0xf0, 0xfb, 0x97, 0x7d, 0xea, 0x1a, 0x44, 0x73, 0xf9, 0x9e, + 0xe6, 0x27, 0x78, 0x1a, 0x17, 0x38, 0x4d, 0x6c, 0x73, 0xf5, 0x39, 0xdc, 0xe8, 0xb8, 0xc6, 0x01, + 0xf1, 0x7c, 0xe1, 0x0a, 0xe9, 0xc5, 0x8f, 0xe1, 0xb6, 0xaf, 0x7b, 0x87, 0xda, 0x81, 0xe5, 0xf9, + 0xd4, 0x9d, 0x68, 0x2e, 0xf1, 0x89, 0xc3, 0xf8, 0x1a, 0x7f, 0xac, 0x95, 0x97, 0x7e, 0xb7, 0x18, + 0x66, 0x4b, 0x40, 0x70, 0x80, 0xd8, 0x66, 0x00, 0xb5, 0x05, 0x45, 0x56, 0xa6, 0xc8, 0x8b, 0x33, + 0x36, 0x7a, 0xb0, 0xe9, 0x40, 0x7b, 0xe3, 0x63, 0x2a, 0x6f, 0xd3, 0x81, 0xf8, 0xa9, 0x7e, 0x17, + 0x94, 0x86, 0xe5, 0x8d, 0x74, 0xdf, 0x38, 0x08, 0x6e, 0x33, 0x51, 0x03, 0x94, 0x03, 0xa2, 0xbb, + 0xfe, 0x1e, 0xd1, 0x7d, 0x6d, 0x44, 0x5c, 0x8b, 0x9a, 0xd7, 0xcf, 0xf2, 0x52, 0x28, 0xd2, 0xe5, + 0x12, 0xea, 0x7f, 0x25, 0x00, 0xb0, 0xbe, 0x1f, 0x64, 0x64, 0x5f, 0x83, 0x65, 0xcf, 0xd1, 0x47, + 0xde, 0x01, 0xf5, 0x35, 0xcb, 0xf1, 0x89, 0x7b, 0xa4, 0xdb, 0xf2, 0x02, 0x47, 0x09, 0x18, 0x2d, + 0x49, 0x47, 0xf7, 0x01, 0x1d, 0x12, 0x32, 0xd2, 0xa8, 0x6d, 0x6a, 0x01, 0x53, 0x3c, 0x25, 0xa7, + 0xb1, 0xc2, 0x38, 0x1d, 0xdb, 0xec, 0x05, 0x74, 0x54, 0x83, 0x35, 0x36, 0x7c, 0xe2, 0xf8, 0xae, + 0x45, 0x3c, 0x6d, 0x9f, 0xba, 0x9a, 0x67, 0xd3, 0x63, 0x6d, 0x9f, 0xda, 0x36, 0x3d, 0x26, 0x6e, + 0x70, 0x37, 0x56, 0xb2, 0xe9, 0xa0, 0x29, 0x40, 0x1b, 0xd4, 0xed, 0xd9, 0xf4, 0x78, 0x23, 0x40, + 0xb0, 0xb4, 0x6d, 0x3a, 0x66, 0xdf, 0x32, 0x0e, 0x83, 0xb4, 0x2d, 0xa4, 0xf6, 0x2d, 0xe3, 0x10, + 0xbd, 0x03, 0x0b, 0xc4, 0x26, 0xfc, 0x8a, 0x44, 0xa0, 0x32, 0x1c, 0x55, 0x0c, 0x88, 0x0c, 0xa4, + 0x7e, 0x02, 0x4a, 0xd3, 0x31, 0xdc, 0xc9, 0x28, 0x32, 0xe7, 0xf7, 0x01, 0xb1, 0x20, 0xa9, 0xd9, + 0xd4, 0x38, 0xd4, 0x86, 0xba, 0xa3, 0x0f, 0x98, 0x5d, 0xe2, 0x85, 0x4f, 0x61, 0x9c, 0x6d, 0x6a, + 0x1c, 0xee, 0x48, 0xba, 0xfa, 0x01, 0x40, 0x6f, 0xe4, 0x12, 0xdd, 0xec, 0xb0, 0x6c, 0x82, 0xb9, + 0x8e, 0xb7, 0x34, 0x53, 0xbe, 0x90, 0x52, 0x57, 0x6e, 0x75, 0x45, 0x30, 0x1a, 0x21, 0x5d, 0xfd, + 0x79, 0xb8, 0xd1, 0xb5, 0x75, 0x83, 0x7f, 0x2d, 0xd0, 0x0d, 0x9f, 0xac, 0xd0, 0x53, 0xc8, 0x0a, + 0xa8, 0x9c, 0xc9, 0xd8, 0xed, 0x36, 0xed, 0x73, 0x6b, 0x0e, 0x4b, 0x7c, 0xad, 0x08, 0x30, 0xd5, + 0xa3, 0xfe, 0x79, 0x02, 0xf2, 0xa1, 0x7e, 0x54, 0x16, 0x2f, 0x31, 0xbe, 0xab, 0x5b, 0x8e, 0xac, + 0xea, 0xf3, 0x38, 0x4a, 0x42, 0x2d, 0x28, 0x8c, 0x42, 0xe9, 0x2b, 0xf3, 0xb9, 0x18, 0xab, 0x71, + 0x54, 0x16, 0x7d, 0x08, 0xf9, 0xe0, 0x49, 0x3a, 0x88, 0xb0, 0x57, 0xbf, 0x60, 0x4f, 0xe1, 0xea, + 0xb7, 0x01, 0xbe, 0x43, 0x2d, 0xa7, 0x4f, 0x0f, 0x89, 0xc3, 0x9f, 0x58, 0x59, 0x4d, 0x48, 0x02, + 0x2f, 0xca, 0x16, 0x2f, 0xf5, 0xc5, 0x14, 0x84, 0x2f, 0x8d, 0xa2, 0xa9, 0xfe, 0x75, 0x12, 0xb2, + 0x98, 0x52, 0xbf, 0x5e, 0x45, 0x65, 0xc8, 0xca, 0x38, 0xc1, 0xcf, 0x9f, 0x5a, 0xfe, 0xfc, 0x6c, + 0x3d, 0x23, 0x02, 0x44, 0xc6, 0xe0, 0x91, 0x21, 0x12, 0xc1, 0x93, 0x97, 0x45, 0x70, 0xf4, 0x10, + 0x8a, 0x12, 0xa4, 0x1d, 0xe8, 0xde, 0x81, 0x28, 0xd0, 0x6a, 0x8b, 0xe7, 0x67, 0xeb, 0x20, 0x90, + 0x5b, 0xba, 0x77, 0x80, 0x41, 0xa0, 0xd9, 0x6f, 0xd4, 0x84, 0xc2, 0xa7, 0xd4, 0x72, 0x34, 0x9f, + 0x0f, 0x42, 0x5e, 0x26, 0xc6, 0xce, 0xe3, 0x74, 0xa8, 0xf2, 0x7b, 0x03, 0xf8, 0x74, 0x3a, 0xf8, + 0x26, 0x2c, 0xb8, 0x94, 0xfa, 0x22, 0x6c, 0x59, 0xd4, 0x91, 0xf7, 0x14, 0xe5, 0xd8, 0xeb, 0x6b, + 0x4a, 0x7d, 0x2c, 0x71, 0xb8, 0xe8, 0x46, 0x5a, 0xe8, 0x21, 0xac, 0xd8, 0xba, 0xe7, 0x6b, 0x3c, + 0xde, 0x99, 0x53, 0x6d, 0x59, 0xbe, 0xd5, 0x10, 0xe3, 0x6d, 0x70, 0x56, 0x20, 0xa1, 0xfe, 0x63, + 0x02, 0x0a, 0x6c, 0x30, 0xd6, 0xbe, 0x65, 0xb0, 0x24, 0xef, 0xf3, 0xe7, 0x1e, 0xb7, 0x20, 0x65, + 0x78, 0xae, 0x74, 0x2a, 0x3f, 0x7c, 0xeb, 0x3d, 0x8c, 0x19, 0x0d, 0x7d, 0x02, 0x59, 0x79, 0x5f, + 0x22, 0xd2, 0x0e, 0xf5, 0xfa, 0x74, 0x54, 0xfa, 0x46, 0xca, 0xf1, 0xb5, 0x3c, 0xb5, 0x4e, 0x1c, + 0x02, 0x38, 0x4a, 0x42, 0x37, 0x21, 0x69, 0x08, 0x77, 0xc9, 0x0f, 0x5a, 0xea, 0x6d, 0x9c, 0x34, + 0x1c, 0xf5, 0x47, 0x09, 0x58, 0x98, 0x6e, 0x78, 0xb6, 0x02, 0x6e, 0x43, 0xde, 0x1b, 0xef, 0x79, + 0x13, 0xcf, 0x27, 0xc3, 0xe0, 0xf9, 0x38, 0x24, 0xa0, 0x16, 0xe4, 0x75, 0x7b, 0x40, 0x5d, 0xcb, + 0x3f, 0x18, 0xca, 0x4a, 0x34, 0x3e, 0x55, 0x88, 0xea, 0xac, 0x54, 0x03, 0x11, 0x3c, 0x95, 0x0e, + 0xce, 0x7d, 0xf1, 0x8d, 0x01, 0x3f, 0xf7, 0xdf, 0x86, 0xa2, 0xad, 0x0f, 0xf9, 0x05, 0x92, 0x6f, + 0x0d, 0xc5, 0x38, 0xd2, 0xb8, 0x20, 0x69, 0x7d, 0x6b, 0x48, 0x54, 0x15, 0xf2, 0xa1, 0x32, 0xb4, + 0x04, 0x85, 0x6a, 0xb3, 0xa7, 0x3d, 0x7a, 0xfc, 0x54, 0xdb, 0xac, 0xef, 0x28, 0x73, 0x32, 0x37, + 0xfd, 0x8b, 0x04, 0x2c, 0xc8, 0x70, 0x24, 0xf3, 0xfd, 0x77, 0x60, 0xde, 0xd5, 0xf7, 0xfd, 0xa0, + 0x22, 0x49, 0x8b, 0x55, 0xcd, 0x22, 0x3c, 0xab, 0x48, 0x18, 0x2b, 0xbe, 0x22, 0x89, 0x7c, 0xd0, + 0x90, 0xba, 0xf2, 0x83, 0x86, 0xf4, 0xcf, 0xe4, 0x83, 0x06, 0xf5, 0x57, 0x01, 0x36, 0x2c, 0x9b, + 0xf4, 0xc5, 0x5d, 0x53, 0x5c, 0x7d, 0xc9, 0x72, 0x38, 0x79, 0x97, 0x19, 0xe4, 0x70, 0xad, 0x06, + 0x66, 0x34, 0xc6, 0x1a, 0x58, 0xa6, 0xdc, 0x8c, 0x9c, 0xb5, 0xc9, 0x58, 0x03, 0xcb, 0x0c, 0x5f, + 0xde, 0xd2, 0xd7, 0xbd, 0xbc, 0x9d, 0x26, 0x60, 0x49, 0xe6, 0xae, 0x61, 0xf8, 0xfd, 0x2a, 0xe4, + 0x45, 0x1a, 0x3b, 0x2d, 0xe8, 0xf8, 0x23, 0xbe, 0xc0, 0xb5, 0x1a, 0x38, 0x27, 0xd8, 0x2d, 0x13, + 0xad, 0x43, 0x41, 0x42, 0x23, 0x1f, 0x3f, 0x81, 0x20, 0xb5, 0x99, 0xf9, 0x5f, 0x87, 0xf4, 0xbe, + 0x65, 0x13, 0xb9, 0xd0, 0x63, 0x03, 0xc0, 0xd4, 0x01, 0x5b, 0x73, 0x98, 0xa3, 0x6b, 0xb9, 0xe0, + 0x32, 0x8e, 0xdb, 0x27, 0xcb, 0xce, 0xa8, 0x7d, 0xa2, 0x02, 0x9d, 0xb1, 0x4f, 0xe0, 0x98, 0x7d, + 0x82, 0x2d, 0xec, 0x93, 0xd0, 0xa8, 0x7d, 0x82, 0xf4, 0x33, 0xb1, 0x6f, 0x1b, 0x6e, 0xd6, 0x6c, + 0xdd, 0x38, 0xb4, 0x2d, 0xcf, 0x27, 0x66, 0x34, 0x62, 0x3c, 0x86, 0xec, 0x85, 0xa4, 0xf3, 0xaa, + 0x5b, 0x4b, 0x89, 0x54, 0xff, 0x2d, 0x01, 0xc5, 0x2d, 0xa2, 0xdb, 0xfe, 0xc1, 0xf4, 0x6a, 0xc8, + 0x27, 0x9e, 0x2f, 0x0f, 0x2b, 0xfe, 0x1b, 0x7d, 0x03, 0x72, 0x61, 0x4e, 0x72, 0xed, 0xfb, 0x5b, + 0x08, 0x45, 0x4f, 0x60, 0x9e, 0xed, 0x31, 0x3a, 0x0e, 0x8a, 0x9d, 0xab, 0x9e, 0x76, 0x24, 0x92, + 0x1d, 0x32, 0x2e, 0xe1, 0x49, 0x08, 0x5f, 0x4a, 0x19, 0x1c, 0x34, 0xd1, 0xff, 0x87, 0x22, 0x7f, + 0x99, 0x08, 0x72, 0xae, 0xcc, 0x75, 0x3a, 0x0b, 0xe2, 0x71, 0x51, 0xe4, 0x5b, 0xff, 0x93, 0x80, + 0x95, 0x1d, 0x7d, 0xb2, 0x47, 0x64, 0xd8, 0x20, 0x26, 0x26, 0x06, 0x75, 0x4d, 0xd4, 0x8d, 0x86, + 0x9b, 0x2b, 0xde, 0x2a, 0xe3, 0x84, 0xe3, 0xa3, 0x4e, 0x50, 0x80, 0x25, 0x23, 0x05, 0xd8, 0x0a, + 0x64, 0x1c, 0xea, 0x18, 0x44, 0xc6, 0x22, 0xd1, 0x50, 0xad, 0x68, 0xa8, 0x29, 0x85, 0xcf, 0x88, + 0xfc, 0x11, 0xb0, 0x4d, 0xfd, 0xb0, 0x37, 0xf4, 0x09, 0x94, 0x7a, 0xcd, 0x3a, 0x6e, 0xf6, 0x6b, + 0x9d, 0xef, 0x6a, 0xbd, 0xea, 0x76, 0xaf, 0xfa, 0xf8, 0xa1, 0xd6, 0xed, 0x6c, 0x7f, 0xef, 0xd1, + 0x93, 0x87, 0xdf, 0x50, 0x12, 0xa5, 0xf2, 0xc9, 0x69, 0xf9, 0x76, 0xbb, 0x5a, 0xdf, 0x16, 0x3b, + 0x66, 0x8f, 0xbe, 0xec, 0xe9, 0xb6, 0xa7, 0x3f, 0x7e, 0xd8, 0xa5, 0xf6, 0x84, 0x61, 0xd8, 0xb2, + 0x2e, 0x46, 0xcf, 0xab, 0xe8, 0x31, 0x9c, 0xb8, 0xf4, 0x18, 0x9e, 0x9e, 0xe6, 0xc9, 0x4b, 0x4e, + 0xf3, 0x0d, 0x58, 0x31, 0x5c, 0xea, 0x79, 0x1a, 0xcb, 0xfe, 0x89, 0x39, 0x53, 0x5f, 0x7c, 0xe9, + 0xfc, 0x6c, 0x7d, 0xb9, 0xce, 0xf8, 0x3d, 0xce, 0x96, 0xea, 0x97, 0x8d, 0x08, 0x89, 0xf7, 0xa4, + 0xfe, 0x7e, 0x8a, 0x25, 0x52, 0xd6, 0x91, 0x65, 0x93, 0x01, 0xf1, 0xd0, 0x73, 0x58, 0x32, 0x5c, + 0x62, 0xb2, 0xb4, 0x5e, 0xb7, 0xa3, 0x1f, 0xd1, 0xfe, 0xbf, 0xd8, 0x9c, 0x26, 0x14, 0xac, 0xd4, + 0x43, 0xa9, 0xde, 0x88, 0x18, 0x78, 0xd1, 0xb8, 0xd0, 0x46, 0x9f, 0xc2, 0x92, 0x47, 0x6c, 0xcb, + 0x19, 0xbf, 0xd4, 0x0c, 0xea, 0xf8, 0xe4, 0x65, 0xf0, 0x22, 0x76, 0x9d, 0xde, 0x5e, 0x73, 0x9b, + 0x49, 0xd5, 0x85, 0x50, 0x0d, 0x9d, 0x9f, 0xad, 0x2f, 0x5e, 0xa4, 0xe1, 0x45, 0xa9, 0x59, 0xb6, + 0x4b, 0x6d, 0x58, 0xbc, 0x68, 0x0d, 0x5a, 0x91, 0x7b, 0x9f, 0x87, 0x90, 0x60, 0x6f, 0xa3, 0xdb, + 0x90, 0x73, 0xc9, 0xc0, 0xf2, 0x7c, 0x57, 0xb8, 0x99, 0x71, 0x42, 0x0a, 0xdb, 0xf9, 0xe2, 0x0b, + 0xa8, 0xd2, 0x2f, 0xc3, 0x4c, 0x8f, 0x6c, 0xb3, 0x98, 0x96, 0xa7, 0xef, 0x49, 0x95, 0x39, 0x1c, + 0x34, 0xd9, 0x1a, 0x1c, 0x7b, 0x61, 0xa2, 0xc6, 0x7f, 0x33, 0x1a, 0xcf, 0x28, 0xe4, 0xf7, 0x60, + 0x3c, 0x67, 0x08, 0x3e, 0x2c, 0x4d, 0x47, 0x3e, 0x2c, 0x5d, 0x81, 0x8c, 0x4d, 0x8e, 0x88, 0x2d, + 0xce, 0x72, 0x2c, 0x1a, 0xf7, 0x1e, 0x42, 0x31, 0xf8, 0x82, 0x91, 0x7f, 0x39, 0x91, 0x83, 0x74, + 0xbf, 0xda, 0x7b, 0xa6, 0xcc, 0x21, 0x80, 0xac, 0x58, 0x9c, 0xe2, 0xb5, 0xae, 0xde, 0x69, 0x6f, + 0xb4, 0x36, 0x95, 0xe4, 0xbd, 0xdf, 0x49, 0x43, 0x3e, 0x7c, 0x2f, 0x62, 0x67, 0x47, 0xbb, 0xf9, + 0x22, 0x58, 0xdd, 0x21, 0xbd, 0x4d, 0x8e, 0xd1, 0xdb, 0xd3, 0x5b, 0xa8, 0x4f, 0xc4, 0x03, 0x79, + 0xc8, 0x0e, 0x6e, 0xa0, 0xde, 0x85, 0x5c, 0xb5, 0xd7, 0x6b, 0x6d, 0xb6, 0x9b, 0x0d, 0xe5, 0xb3, + 0x44, 0xe9, 0x4b, 0x27, 0xa7, 0xe5, 0xe5, 0x10, 0x54, 0xf5, 0xc4, 0xe2, 0xe3, 0xa8, 0x7a, 0xbd, + 0xd9, 0xed, 0x37, 0x1b, 0xca, 0xab, 0xe4, 0x2c, 0x8a, 0xdf, 0xaa, 0xf0, 0x4f, 0x77, 0xf2, 0x5d, + 0xdc, 0xec, 0x56, 0x31, 0xeb, 0xf0, 0xb3, 0xa4, 0xb8, 0x1c, 0x9b, 0xf6, 0xe8, 0x92, 0x91, 0xee, + 0xb2, 0x3e, 0xd7, 0x82, 0x6f, 0xe1, 0x5e, 0xa5, 0xc4, 0xe7, 0x1d, 0xd3, 0xc7, 0x2f, 0xa2, 0x9b, + 0x13, 0xd6, 0x1b, 0x7f, 0x75, 0xe4, 0x6a, 0x52, 0x33, 0xbd, 0xf5, 0x58, 0xec, 0x61, 0x5a, 0x54, + 0x98, 0xc7, 0xbb, 0xed, 0x36, 0x03, 0xbd, 0x4a, 0xcf, 0x8c, 0x0e, 0x8f, 0x1d, 0x56, 0x31, 0xa3, + 0xbb, 0x90, 0x0b, 0x1e, 0x25, 0x95, 0xcf, 0xd2, 0x33, 0x06, 0xd5, 0x83, 0x17, 0x55, 0xde, 0xe1, + 0xd6, 0x6e, 0x9f, 0x7f, 0xaa, 0xf7, 0x2a, 0x33, 0xdb, 0xe1, 0xc1, 0xd8, 0x37, 0xe9, 0xb1, 0xc3, + 0xf6, 0xac, 0xbc, 0x87, 0xfb, 0x2c, 0x23, 0x2e, 0x2d, 0x42, 0x8c, 0xbc, 0x84, 0x7b, 0x17, 0x72, + 0xb8, 0xf9, 0x1d, 0xf1, 0x55, 0xdf, 0xab, 0xec, 0x8c, 0x1e, 0x4c, 0x3e, 0x25, 0x06, 0xeb, 0xad, + 0x0c, 0x59, 0xdc, 0xdc, 0xe9, 0x3c, 0x6f, 0x2a, 0x7f, 0x90, 0x9d, 0xd1, 0x83, 0xc9, 0x90, 0xf2, + 0x6f, 0x9b, 0x72, 0x1d, 0xdc, 0xdd, 0xaa, 0xf2, 0x49, 0x99, 0xd5, 0xd3, 0x71, 0x47, 0x07, 0xba, + 0x43, 0xcc, 0xe9, 0x57, 0x30, 0x21, 0xeb, 0xde, 0x2f, 0x40, 0x2e, 0xc8, 0x5d, 0xd1, 0x1a, 0x64, + 0x5f, 0x74, 0xf0, 0xb3, 0x26, 0x56, 0xe6, 0x84, 0x97, 0x03, 0xce, 0x0b, 0x51, 0x75, 0x94, 0x61, + 0x7e, 0xa7, 0xda, 0xae, 0x6e, 0x36, 0x71, 0x70, 0x89, 0x1e, 0x00, 0x64, 0x02, 0x56, 0x52, 0x64, + 0x07, 0xa1, 0xce, 0xda, 0xea, 0x0f, 0x7f, 0xb2, 0x36, 0xf7, 0xe3, 0x9f, 0xac, 0xcd, 0xbd, 0x3a, + 0x5f, 0x4b, 0xfc, 0xf0, 0x7c, 0x2d, 0xf1, 0x77, 0xe7, 0x6b, 0x89, 0x7f, 0x3d, 0x5f, 0x4b, 0xec, + 0x65, 0xf9, 0x31, 0xf1, 0xe4, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x08, 0x6c, 0xc9, 0x88, 0x12, + 0x32, 0x00, 0x00, } diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/types.proto b/components/engine/vendor/github.com/docker/swarmkit/api/types.proto index 635d12b200..3b60c544c3 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/types.proto +++ b/components/engine/vendor/github.com/docker/swarmkit/api/types.proto @@ -458,7 +458,16 @@ enum TaskState { SHUTDOWN = 640 [(gogoproto.enumvalue_customname)="TaskStateShutdown"]; // orchestrator requested shutdown FAILED = 704 [(gogoproto.enumvalue_customname)="TaskStateFailed"]; // task execution failed with error REJECTED = 768 [(gogoproto.enumvalue_customname)="TaskStateRejected"]; // task could not be executed here. - ORPHANED = 832 [(gogoproto.enumvalue_customname)="TaskStateOrphaned"]; // The node on which this task is scheduled is Down for too long + // TaskStateRemove is used to correctly handle service deletions and scale + // downs. This allows us to keep track of tasks that have been marked for + // deletion, but can't yet be removed because the agent is in the process of + // shutting them down. Once the agent has shut down tasks with desired state + // REMOVE, the task reaper is responsible for removing them. + REMOVE = 800 [(gogoproto.enumvalue_customname)="TaskStateRemove"]; + // TaskStateOrphaned is used to free up resources associated with service + // tasks on unresponsive nodes without having to delete those tasks. This + // state is directly assigned to the task by the orchestrator. + ORPHANED = 832 [(gogoproto.enumvalue_customname)="TaskStateOrphaned"]; // NOTE(stevvooe): The state of a task is actually a lamport clock, in that // given two observations, the greater of the two can be considered diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/watch.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/watch.pb.go index 58fa8917e5..78f0014c7a 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/watch.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/watch.pb.go @@ -19,6 +19,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -2043,12 +2044,12 @@ func NewRaftProxyWatchServer(local WatchServer, connSelector raftselector.ConnPr redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/ca/auth.go b/components/engine/vendor/github.com/docker/swarmkit/ca/auth.go index 488d34dd33..df4547fb13 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/ca/auth.go +++ b/components/engine/vendor/github.com/docker/swarmkit/ca/auth.go @@ -10,10 +10,10 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" ) type localRequestKeyType struct{} @@ -52,13 +52,13 @@ func LogTLSState(ctx context.Context, tlsState *tls.ConnectionState) { // getCertificateSubject extracts the subject from a verified client certificate func getCertificateSubject(tlsState *tls.ConnectionState) (pkix.Name, error) { if tlsState == nil { - return pkix.Name{}, grpc.Errorf(codes.PermissionDenied, "request is not using TLS") + return pkix.Name{}, status.Errorf(codes.PermissionDenied, "request is not using TLS") } if len(tlsState.PeerCertificates) == 0 { - return pkix.Name{}, grpc.Errorf(codes.PermissionDenied, "no client certificates in request") + return pkix.Name{}, status.Errorf(codes.PermissionDenied, "no client certificates in request") } if len(tlsState.VerifiedChains) == 0 { - return pkix.Name{}, grpc.Errorf(codes.PermissionDenied, "no verified chains for remote certificate") + return pkix.Name{}, status.Errorf(codes.PermissionDenied, "no verified chains for remote certificate") } return tlsState.VerifiedChains[0][0].Subject, nil @@ -67,11 +67,11 @@ func getCertificateSubject(tlsState *tls.ConnectionState) (pkix.Name, error) { func tlsConnStateFromContext(ctx context.Context) (*tls.ConnectionState, error) { peer, ok := peer.FromContext(ctx) if !ok { - return nil, grpc.Errorf(codes.PermissionDenied, "Permission denied: no peer info") + return nil, status.Errorf(codes.PermissionDenied, "Permission denied: no peer info") } tlsInfo, ok := peer.AuthInfo.(credentials.TLSInfo) if !ok { - return nil, grpc.Errorf(codes.PermissionDenied, "Permission denied: peer didn't not present valid peer certificate") + return nil, status.Errorf(codes.PermissionDenied, "Permission denied: peer didn't not present valid peer certificate") } return &tlsInfo.State, nil } @@ -98,21 +98,21 @@ func AuthorizeOrgAndRole(ctx context.Context, org string, blacklistedCerts map[s return authorizeOrg(certSubj, org, blacklistedCerts) } - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of OUs: %v", ou) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of OUs: %v", ou) } // authorizeOrg takes in a certificate subject and an organization, and returns // the Node ID of the node. func authorizeOrg(certSubj pkix.Name, org string, blacklistedCerts map[string]*api.BlacklistedCertificate) (string, error) { if _, ok := blacklistedCerts[certSubj.CommonName]; ok { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: node %s was removed from swarm", certSubj.CommonName) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: node %s was removed from swarm", certSubj.CommonName) } if len(certSubj.Organization) > 0 && certSubj.Organization[0] == org { return certSubj.CommonName, nil } - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of organization: %s", org) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of organization: %s", org) } // AuthorizeForwardedRoleAndOrg checks for proper roles and organization of caller. The RPC may have @@ -123,7 +123,7 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde if isForwardedRequest(ctx) { _, err := AuthorizeOrgAndRole(ctx, org, blacklistedCerts, forwarderRoles...) if err != nil { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarder role: %v", err) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarder role: %v", err) } // This was a forwarded request. Authorize the forwarder, and @@ -132,15 +132,15 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde _, forwardedID, forwardedOrg, forwardedOUs := forwardedTLSInfoFromContext(ctx) if len(forwardedOUs) == 0 || forwardedID == "" || forwardedOrg == "" { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request") + return "", status.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request") } if !intersectArrays(forwardedOUs, authorizedRoles) { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarded role, expecting: %v", authorizedRoles) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarded role, expecting: %v", authorizedRoles) } if forwardedOrg != org { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: organization mismatch, expecting: %s", org) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: organization mismatch, expecting: %s", org) } return forwardedID, nil @@ -152,7 +152,7 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde return nodeID, nil } - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized peer role: %v", err) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: unauthorized peer role: %v", err) } // intersectArrays returns true when there is at least one element in common @@ -219,7 +219,7 @@ func RemoteNode(ctx context.Context) (RemoteNodeInfo, error) { peer, ok := peer.FromContext(ctx) if !ok { - return RemoteNodeInfo{}, grpc.Errorf(codes.PermissionDenied, "Permission denied: no peer info") + return RemoteNodeInfo{}, status.Errorf(codes.PermissionDenied, "Permission denied: no peer info") } directInfo := RemoteNodeInfo{ @@ -232,7 +232,7 @@ func RemoteNode(ctx context.Context) (RemoteNodeInfo, error) { if isForwardedRequest(ctx) { remoteAddr, cn, org, ous := forwardedTLSInfoFromContext(ctx) if len(ous) == 0 || cn == "" || org == "" { - return RemoteNodeInfo{}, grpc.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request") + return RemoteNodeInfo{}, status.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request") } return RemoteNodeInfo{ Roles: ous, diff --git a/components/engine/vendor/github.com/docker/swarmkit/ca/server.go b/components/engine/vendor/github.com/docker/swarmkit/ca/server.go index 16dbedc8da..a456df7900 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/ca/server.go +++ b/components/engine/vendor/github.com/docker/swarmkit/ca/server.go @@ -16,8 +16,8 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -149,7 +149,7 @@ func (s *Server) GetUnlockKey(ctx context.Context, request *api.GetUnlockKeyRequ // NodeCertificateStatus returns the current issuance status of an issuance request identified by the nodeID func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCertificateStatusRequest) (*api.NodeCertificateStatusResponse, error) { if request.NodeID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, codes.InvalidArgument.String()) + return nil, status.Errorf(codes.InvalidArgument, codes.InvalidArgument.String()) } serverCtx, err := s.isRunningLocked() @@ -180,7 +180,7 @@ func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCer // This node ID doesn't exist if node == nil { - return nil, grpc.Errorf(codes.NotFound, codes.NotFound.String()) + return nil, status.Errorf(codes.NotFound, codes.NotFound.String()) } log.G(ctx).WithFields(logrus.Fields{ @@ -236,7 +236,7 @@ func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCer func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNodeCertificateRequest) (*api.IssueNodeCertificateResponse, error) { // First, let's see if the remote node is presenting a non-empty CSR if len(request.CSR) == 0 { - return nil, grpc.Errorf(codes.InvalidArgument, codes.InvalidArgument.String()) + return nil, status.Errorf(codes.InvalidArgument, codes.InvalidArgument.String()) } if err := s.isReadyLocked(); err != nil { @@ -295,7 +295,7 @@ func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNod s.mu.Unlock() if role < 0 { - return nil, grpc.Errorf(codes.InvalidArgument, "A valid join token is necessary to join this cluster") + return nil, status.Errorf(codes.InvalidArgument, "A valid join token is necessary to join this cluster") } // Max number of collisions of ID or CN to tolerate before giving up @@ -369,7 +369,7 @@ func (s *Server) issueRenewCertificate(ctx context.Context, nodeID string, csr [ "method": "issueRenewCertificate", }).Warnf("node does not exist") // If this node doesn't exist, we shouldn't be renewing a certificate for it - return grpc.Errorf(codes.NotFound, "node %s not found when attempting to renew certificate", nodeID) + return status.Errorf(codes.NotFound, "node %s not found when attempting to renew certificate", nodeID) } // Create a new Certificate entry for this node with the new CSR and a RENEW state @@ -594,7 +594,7 @@ func (s *Server) isRunningLocked() (context.Context, error) { s.mu.Lock() if !s.isRunning() { s.mu.Unlock() - return nil, grpc.Errorf(codes.Aborted, "CA signer is stopped") + return nil, status.Errorf(codes.Aborted, "CA signer is stopped") } ctx := s.ctx s.mu.Unlock() @@ -605,10 +605,10 @@ func (s *Server) isReadyLocked() error { s.mu.Lock() defer s.mu.Unlock() if !s.isRunning() { - return grpc.Errorf(codes.Aborted, "CA signer is stopped") + return status.Errorf(codes.Aborted, "CA signer is stopped") } if s.joinTokens == nil { - return grpc.Errorf(codes.Aborted, "CA signer is still starting") + return status.Errorf(codes.Aborted, "CA signer is still starting") } return nil } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/ca_rotation.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/ca_rotation.go index 5e8fa43ce3..d39c7d2b69 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/ca_rotation.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/ca_rotation.go @@ -10,13 +10,12 @@ import ( "net/url" "time" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "github.com/cloudflare/cfssl/helpers" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/log" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var minRootExpiration = 1 * helpers.OneYear @@ -60,7 +59,7 @@ func newRootRotationObject(ctx context.Context, securityConfig *ca.SecurityConfi crossSignedCert, err = oldRootCA.CrossSignCACertificate(rootCert) } case !newRootHasSigner: // the original CA and the new CA both require external CAs - return nil, grpc.Errorf(codes.InvalidArgument, "rotating from one external CA to a different external CA is not supported") + return nil, status.Errorf(codes.InvalidArgument, "rotating from one external CA to a different external CA is not supported") default: // We need the same credentials but to connect to the original URLs (in case we are in the middle of a root rotation already) var urls []string @@ -70,7 +69,7 @@ func newRootRotationObject(ctx context.Context, securityConfig *ca.SecurityConfi } } if len(urls) == 0 { - return nil, grpc.Errorf(codes.InvalidArgument, + return nil, status.Errorf(codes.InvalidArgument, "must provide an external CA for the current external root CA to generate a cross-signed certificate") } rootPool := x509.NewCertPool() @@ -83,7 +82,7 @@ func newRootRotationObject(ctx context.Context, securityConfig *ca.SecurityConfi if err != nil { log.G(ctx).WithError(err).Error("unable to generate a cross-signed certificate for root rotation") - return nil, grpc.Errorf(codes.Internal, "unable to generate a cross-signed certificate for root rotation") + return nil, status.Errorf(codes.Internal, "unable to generate a cross-signed certificate for root rotation") } copied := apiRootCA.Copy() @@ -146,7 +145,7 @@ func validateHasAtLeastOneExternalCA(ctx context.Context, externalCAs map[string } } } - return nil, grpc.Errorf(codes.InvalidArgument, "there must be at least one valid, reachable external CA corresponding to the %s CA certificate", desc) + return nil, status.Errorf(codes.InvalidArgument, "there must be at least one valid, reachable external CA corresponding to the %s CA certificate", desc) } // validates that the list of external CAs have valid certs associated with them, and produce a mapping of subject/pubkey:external @@ -193,7 +192,7 @@ func validateCAConfig(ctx context.Context, securityConfig *ca.SecurityConfig, cl newConfig.SigningCACert = ca.NormalizePEMs(newConfig.SigningCACert) // ensure this is normalized before we use it if len(newConfig.SigningCAKey) > 0 && len(newConfig.SigningCACert) == 0 { - return nil, grpc.Errorf(codes.InvalidArgument, "if a signing CA key is provided, the signing CA cert must also be provided") + return nil, status.Errorf(codes.InvalidArgument, "if a signing CA key is provided, the signing CA cert must also be provided") } normalizedRootCA := ca.NormalizePEMs(cluster.RootCA.CACert) @@ -216,7 +215,7 @@ func validateCAConfig(ctx context.Context, securityConfig *ca.SecurityConfig, cl if cluster.RootCA.LastForcedRotation != newConfig.ForceRotate { newRootCA, err := ca.CreateRootCA(ca.DefaultRootCN) if err != nil { - return nil, grpc.Errorf(codes.Internal, err.Error()) + return nil, status.Errorf(codes.Internal, err.Error()) } return newRootRotationObject(ctx, securityConfig, &cluster.RootCA, newRootCA, oldCertExtCAs, newConfig.ForceRotate) } @@ -240,21 +239,21 @@ func validateCAConfig(ctx context.Context, securityConfig *ca.SecurityConfig, cl } newRootCA, err := ca.NewRootCA(newConfig.SigningCACert, signingCert, newConfig.SigningCAKey, ca.DefaultNodeCertExpiration, nil) if err != nil { - return nil, grpc.Errorf(codes.InvalidArgument, err.Error()) + return nil, status.Errorf(codes.InvalidArgument, err.Error()) } if len(newRootCA.Pool.Subjects()) != 1 { - return nil, grpc.Errorf(codes.InvalidArgument, "the desired CA certificate cannot contain multiple certificates") + return nil, status.Errorf(codes.InvalidArgument, "the desired CA certificate cannot contain multiple certificates") } parsedCert, err := helpers.ParseCertificatePEM(newConfig.SigningCACert) if err != nil { - return nil, grpc.Errorf(codes.InvalidArgument, "could not parse the desired CA certificate") + return nil, status.Errorf(codes.InvalidArgument, "could not parse the desired CA certificate") } // The new certificate's expiry must be at least one year away if parsedCert.NotAfter.Before(time.Now().Add(minRootExpiration)) { - return nil, grpc.Errorf(codes.InvalidArgument, "CA certificate expires too soon") + return nil, status.Errorf(codes.InvalidArgument, "CA certificate expires too soon") } if !hasSigningKey(newConfig) { diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go index 329313a950..0876113d70 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go @@ -11,8 +11,8 @@ import ( "github.com/docker/swarmkit/manager/state/store" gogotypes "github.com/gogo/protobuf/types" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -23,17 +23,17 @@ const ( func validateClusterSpec(spec *api.ClusterSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } // Validate that expiry time being provided is valid, and over our minimum if spec.CAConfig.NodeCertExpiry != nil { expiry, err := gogotypes.DurationFromProto(spec.CAConfig.NodeCertExpiry) if err != nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if expiry < ca.MinNodeCertExpiration { - return grpc.Errorf(codes.InvalidArgument, "minimum certificate expiry time is: %s", ca.MinNodeCertExpiration) + return status.Errorf(codes.InvalidArgument, "minimum certificate expiry time is: %s", ca.MinNodeCertExpiration) } } @@ -42,7 +42,7 @@ func validateClusterSpec(spec *api.ClusterSpec) error { if len(spec.AcceptancePolicy.Policies) > 0 { for _, policy := range spec.AcceptancePolicy.Policies { if policy.Secret != nil && strings.ToLower(policy.Secret.Alg) != "bcrypt" { - return grpc.Errorf(codes.InvalidArgument, "hashing algorithm is not supported: %s", policy.Secret.Alg) + return status.Errorf(codes.InvalidArgument, "hashing algorithm is not supported: %s", policy.Secret.Alg) } } } @@ -51,13 +51,17 @@ func validateClusterSpec(spec *api.ClusterSpec) error { if spec.Dispatcher.HeartbeatPeriod != nil { heartbeatPeriod, err := gogotypes.DurationFromProto(spec.Dispatcher.HeartbeatPeriod) if err != nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if heartbeatPeriod < 0 { - return grpc.Errorf(codes.InvalidArgument, "heartbeat time period cannot be a negative duration") + return status.Errorf(codes.InvalidArgument, "heartbeat time period cannot be a negative duration") } } + if spec.Annotations.Name != store.DefaultClusterName { + return status.Errorf(codes.InvalidArgument, "modification of cluster name is not allowed") + } + return nil } @@ -66,7 +70,7 @@ func validateClusterSpec(spec *api.ClusterSpec) error { // - Returns `NotFound` if the Cluster is not found. func (s *Server) GetCluster(ctx context.Context, request *api.GetClusterRequest) (*api.GetClusterResponse, error) { if request.ClusterID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var cluster *api.Cluster @@ -74,7 +78,7 @@ func (s *Server) GetCluster(ctx context.Context, request *api.GetClusterRequest) cluster = store.GetCluster(tx, request.ClusterID) }) if cluster == nil { - return nil, grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID) + return nil, status.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID) } redactedClusters := redactClusters([]*api.Cluster{cluster}) @@ -92,7 +96,7 @@ func (s *Server) GetCluster(ctx context.Context, request *api.GetClusterRequest) // - Returns an error if the update fails. func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRequest) (*api.UpdateClusterResponse, error) { if request.ClusterID == "" || request.ClusterVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateClusterSpec(request.Spec); err != nil { return nil, err @@ -102,7 +106,7 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe err := s.store.Update(func(tx store.Tx) error { cluster = store.GetCluster(tx, request.ClusterID) if cluster == nil { - return grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID) + return status.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID) } // This ensures that we have the current rootCA with which to generate tokens (expiration doesn't matter // for generating the tokens) @@ -110,7 +114,7 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe if err != nil { log.G(ctx).WithField( "method", "(*controlapi.Server).UpdateCluster").WithError(err).Error("invalid cluster root CA") - return grpc.Errorf(codes.Internal, "error loading cluster rootCA for update") + return status.Errorf(codes.Internal, "error loading cluster rootCA for update") } cluster.Meta.Version = *request.ClusterVersion diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/common.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/common.go index c01662379e..9e52179464 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/common.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/common.go @@ -10,8 +10,8 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/allocator" "github.com/docker/swarmkit/manager/state/store" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var isValidDNSName = regexp.MustCompile(`^[a-zA-Z0-9](?:[-_]*[A-Za-z0-9]+)*$`) @@ -70,25 +70,25 @@ func filterMatchLabels(match map[string]string, candidates map[string]string) bo func validateAnnotations(m api.Annotations) error { if m.Name == "" { - return grpc.Errorf(codes.InvalidArgument, "meta: name must be provided") + return status.Errorf(codes.InvalidArgument, "meta: name must be provided") } if !isValidDNSName.MatchString(m.Name) { // if the name doesn't match the regex - return grpc.Errorf(codes.InvalidArgument, "name must be valid as a DNS name component") + return status.Errorf(codes.InvalidArgument, "name must be valid as a DNS name component") } if len(m.Name) > 63 { // DNS labels are limited to 63 characters - return grpc.Errorf(codes.InvalidArgument, "name must be 63 characters or fewer") + return status.Errorf(codes.InvalidArgument, "name must be 63 characters or fewer") } return nil } func validateConfigOrSecretAnnotations(m api.Annotations) error { if m.Name == "" { - return grpc.Errorf(codes.InvalidArgument, "name must be provided") + return status.Errorf(codes.InvalidArgument, "name must be provided") } else if len(m.Name) > 64 || !isValidConfigOrSecretName.MatchString(m.Name) { // if the name doesn't match the regex - return grpc.Errorf(codes.InvalidArgument, + return status.Errorf(codes.InvalidArgument, "invalid name, only 64 [a-zA-Z0-9-_.] characters allowed, and the start and end character must be [a-zA-Z0-9]") } return nil @@ -102,7 +102,7 @@ func validateDriver(driver *api.Driver, pg plugingetter.PluginGetter, pluginType } if driver.Name == "" { - return grpc.Errorf(codes.InvalidArgument, "driver name: if driver is specified name is required") + return status.Errorf(codes.InvalidArgument, "driver name: if driver is specified name is required") } // First check against the known drivers @@ -119,16 +119,16 @@ func validateDriver(driver *api.Driver, pg plugingetter.PluginGetter, pluginType } if pg == nil { - return grpc.Errorf(codes.InvalidArgument, "plugin %s not supported", driver.Name) + return status.Errorf(codes.InvalidArgument, "plugin %s not supported", driver.Name) } p, err := pg.Get(driver.Name, pluginType, plugingetter.Lookup) if err != nil { - return grpc.Errorf(codes.InvalidArgument, "error during lookup of plugin %s", driver.Name) + return status.Errorf(codes.InvalidArgument, "error during lookup of plugin %s", driver.Name) } if p.IsV1() { - return grpc.Errorf(codes.InvalidArgument, "legacy plugin %s of type %s is not supported in swarm mode", driver.Name, pluginType) + return status.Errorf(codes.InvalidArgument, "legacy plugin %s of type %s is not supported in swarm mode", driver.Name, pluginType) } return nil diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/config.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/config.go index d0fe8a56c1..ae08885b00 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/config.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/config.go @@ -10,8 +10,8 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/sirupsen/logrus" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // MaxConfigSize is the maximum byte length of the `Config.Spec.Data` field. @@ -32,7 +32,7 @@ func configFromConfigSpec(spec *api.ConfigSpec) *api.Config { // - Returns an error if getting fails. func (s *Server) GetConfig(ctx context.Context, request *api.GetConfigRequest) (*api.GetConfigResponse, error) { if request.ConfigID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, "config ID must be provided") + return nil, status.Errorf(codes.InvalidArgument, "config ID must be provided") } var config *api.Config @@ -41,7 +41,7 @@ func (s *Server) GetConfig(ctx context.Context, request *api.GetConfigRequest) ( }) if config == nil { - return nil, grpc.Errorf(codes.NotFound, "config %s not found", request.ConfigID) + return nil, status.Errorf(codes.NotFound, "config %s not found", request.ConfigID) } return &api.GetConfigResponse{Config: config}, nil @@ -53,21 +53,21 @@ func (s *Server) GetConfig(ctx context.Context, request *api.GetConfigRequest) ( // - Returns an error if the update fails. func (s *Server) UpdateConfig(ctx context.Context, request *api.UpdateConfigRequest) (*api.UpdateConfigResponse, error) { if request.ConfigID == "" || request.ConfigVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var config *api.Config err := s.store.Update(func(tx store.Tx) error { config = store.GetConfig(tx, request.ConfigID) if config == nil { - return grpc.Errorf(codes.NotFound, "config %s not found", request.ConfigID) + return status.Errorf(codes.NotFound, "config %s not found", request.ConfigID) } // Check if the Name is different than the current name, or the config is non-nil and different // than the current config if config.Spec.Annotations.Name != request.Spec.Annotations.Name || (request.Spec.Data != nil && !bytes.Equal(request.Spec.Data, config.Spec.Data)) { - return grpc.Errorf(codes.InvalidArgument, "only updates to Labels are allowed") + return status.Errorf(codes.InvalidArgument, "only updates to Labels are allowed") } // We only allow updating Labels @@ -164,7 +164,7 @@ func (s *Server) CreateConfig(ctx context.Context, request *api.CreateConfigRequ switch err { case store.ErrNameConflict: - return nil, grpc.Errorf(codes.AlreadyExists, "config %s already exists", request.Spec.Annotations.Name) + return nil, status.Errorf(codes.AlreadyExists, "config %s already exists", request.Spec.Annotations.Name) case nil: log.G(ctx).WithFields(logrus.Fields{ "config.Name": request.Spec.Annotations.Name, @@ -184,20 +184,20 @@ func (s *Server) CreateConfig(ctx context.Context, request *api.CreateConfigRequ // - Returns an error if the deletion fails. func (s *Server) RemoveConfig(ctx context.Context, request *api.RemoveConfigRequest) (*api.RemoveConfigResponse, error) { if request.ConfigID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, "config ID must be provided") + return nil, status.Errorf(codes.InvalidArgument, "config ID must be provided") } err := s.store.Update(func(tx store.Tx) error { // Check if the config exists config := store.GetConfig(tx, request.ConfigID) if config == nil { - return grpc.Errorf(codes.NotFound, "could not find config %s", request.ConfigID) + return status.Errorf(codes.NotFound, "could not find config %s", request.ConfigID) } // Check if any services currently reference this config, return error if so services, err := store.FindServices(tx, store.ByReferencedConfigID(request.ConfigID)) if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using config %s: %v", request.ConfigID, err) + return status.Errorf(codes.Internal, "could not find services using config %s: %v", request.ConfigID, err) } if len(services) != 0 { @@ -213,14 +213,14 @@ func (s *Server) RemoveConfig(ctx context.Context, request *api.RemoveConfigRequ serviceStr = "service" } - return grpc.Errorf(codes.InvalidArgument, "config '%s' is in use by the following %s: %v", configName, serviceStr, serviceNameStr) + return status.Errorf(codes.InvalidArgument, "config '%s' is in use by the following %s: %v", configName, serviceStr, serviceNameStr) } return store.DeleteConfig(tx, request.ConfigID) }) switch err { case store.ErrNotExist: - return nil, grpc.Errorf(codes.NotFound, "config %s not found", request.ConfigID) + return nil, status.Errorf(codes.NotFound, "config %s not found", request.ConfigID) case nil: log.G(ctx).WithFields(logrus.Fields{ "config.ID": request.ConfigID, @@ -235,14 +235,14 @@ func (s *Server) RemoveConfig(ctx context.Context, request *api.RemoveConfigRequ func validateConfigSpec(spec *api.ConfigSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateConfigOrSecretAnnotations(spec.Annotations); err != nil { return err } if len(spec.Data) >= MaxConfigSize || len(spec.Data) < 1 { - return grpc.Errorf(codes.InvalidArgument, "config data must be larger than 0 and less than %d bytes", MaxConfigSize) + return status.Errorf(codes.InvalidArgument, "config data must be larger than 0 and less than %d bytes", MaxConfigSize) } return nil } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/network.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/network.go index b150de0e78..481b0cfe4b 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/network.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/network.go @@ -12,39 +12,39 @@ import ( "github.com/docker/swarmkit/manager/allocator/networkallocator" "github.com/docker/swarmkit/manager/state/store" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func validateIPAMConfiguration(ipamConf *api.IPAMConfig) error { if ipamConf == nil { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: cannot be empty") + return status.Errorf(codes.InvalidArgument, "ipam configuration: cannot be empty") } _, subnet, err := net.ParseCIDR(ipamConf.Subnet) if err != nil { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: invalid subnet %s", ipamConf.Subnet) + return status.Errorf(codes.InvalidArgument, "ipam configuration: invalid subnet %s", ipamConf.Subnet) } if ipamConf.Range != "" { ip, _, err := net.ParseCIDR(ipamConf.Range) if err != nil { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: invalid range %s", ipamConf.Range) + return status.Errorf(codes.InvalidArgument, "ipam configuration: invalid range %s", ipamConf.Range) } if !subnet.Contains(ip) { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain range %s", ipamConf.Subnet, ipamConf.Range) + return status.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain range %s", ipamConf.Subnet, ipamConf.Range) } } if ipamConf.Gateway != "" { ip := net.ParseIP(ipamConf.Gateway) if ip == nil { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: invalid gateway %s", ipamConf.Gateway) + return status.Errorf(codes.InvalidArgument, "ipam configuration: invalid gateway %s", ipamConf.Gateway) } if !subnet.Contains(ip) { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain gateway %s", ipamConf.Subnet, ipamConf.Gateway) + return status.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain gateway %s", ipamConf.Subnet, ipamConf.Gateway) } } @@ -73,15 +73,15 @@ func validateIPAM(ipam *api.IPAMOptions, pg plugingetter.PluginGetter) error { func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if spec.Ingress && spec.DriverConfig != nil && spec.DriverConfig.Name != "overlay" { - return grpc.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network") + return status.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network") } if spec.Attachable && spec.Ingress { - return grpc.Errorf(codes.InvalidArgument, "ingress network cannot be attachable") + return status.Errorf(codes.InvalidArgument, "ingress network cannot be attachable") } if err := validateAnnotations(spec.Annotations); err != nil { @@ -89,7 +89,7 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er } if _, ok := spec.Annotations.Labels[networkallocator.PredefinedLabel]; ok { - return grpc.Errorf(codes.PermissionDenied, "label %s is for internally created predefined networks and cannot be applied by users", + return status.Errorf(codes.PermissionDenied, "label %s is for internally created predefined networks and cannot be applied by users", networkallocator.PredefinedLabel) } if err := validateDriver(spec.DriverConfig, pg, driverapi.NetworkPluginEndpointType); err != nil { @@ -117,9 +117,9 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe err := s.store.Update(func(tx store.Tx) error { if request.Spec.Ingress { if n, err := allocator.GetIngressNetwork(s.store); err == nil { - return grpc.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID) + return status.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID) } else if err != allocator.ErrNoIngress { - return grpc.Errorf(codes.Internal, "failed ingress network presence check: %v", err) + return status.Errorf(codes.Internal, "failed ingress network presence check: %v", err) } } return store.CreateNetwork(tx, n) @@ -138,7 +138,7 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe // - Returns `NotFound` if the Network is not found. func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest) (*api.GetNetworkResponse, error) { if request.NetworkID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var n *api.Network @@ -146,7 +146,7 @@ func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest) n = store.GetNetwork(tx, request.NetworkID) }) if n == nil { - return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) + return nil, status.Errorf(codes.NotFound, "network %s not found", request.NetworkID) } return &api.GetNetworkResponse{ Network: n, @@ -159,7 +159,7 @@ func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest) // - Returns an error if the deletion fails. func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRequest) (*api.RemoveNetworkResponse, error) { if request.NetworkID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var ( @@ -171,7 +171,7 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe n = store.GetNetwork(tx, request.NetworkID) }) if n == nil { - return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) + return nil, status.Errorf(codes.NotFound, "network %s not found", request.NetworkID) } if allocator.IsIngressNetwork(n) { @@ -179,13 +179,13 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe } if v, ok := n.Spec.Annotations.Labels[networkallocator.PredefinedLabel]; ok && v == "true" { - return nil, grpc.Errorf(codes.FailedPrecondition, "network %s (%s) is a swarm predefined network and cannot be removed", + return nil, status.Errorf(codes.FailedPrecondition, "network %s (%s) is a swarm predefined network and cannot be removed", request.NetworkID, n.Spec.Annotations.Name) } if err := rm(n.ID); err != nil { if err == store.ErrNotExist { - return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) + return nil, status.Errorf(codes.NotFound, "network %s not found", request.NetworkID) } return nil, err } @@ -196,21 +196,21 @@ func (s *Server) removeNetwork(id string) error { return s.store.Update(func(tx store.Tx) error { services, err := store.FindServices(tx, store.ByReferencedNetworkID(id)) if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) + return status.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) } if len(services) != 0 { - return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID) + return status.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID) } tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(id)) if err != nil { - return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err) + return status.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err) } for _, t := range tasks { if t.DesiredState <= api.TaskStateRunning && t.Status.State <= api.TaskStateRunning { - return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID) + return status.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID) } } @@ -222,11 +222,11 @@ func (s *Server) removeIngressNetwork(id string) error { return s.store.Update(func(tx store.Tx) error { services, err := store.FindServices(tx, store.All) if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) + return status.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) } for _, srv := range services { if allocator.IsIngressNetworkNeeded(srv) { - return grpc.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID) + return status.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID) } } return store.DeleteNetwork(tx, id) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/node.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/node.go index bac6b8073d..e1fe3dec1d 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/node.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/node.go @@ -9,13 +9,13 @@ import ( "github.com/docker/swarmkit/manager/state/store" gogotypes "github.com/gogo/protobuf/types" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func validateNodeSpec(spec *api.NodeSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } return nil } @@ -25,7 +25,7 @@ func validateNodeSpec(spec *api.NodeSpec) error { // - Returns `NotFound` if the Node is not found. func (s *Server) GetNode(ctx context.Context, request *api.GetNodeRequest) (*api.GetNodeResponse, error) { if request.NodeID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var node *api.Node @@ -33,7 +33,7 @@ func (s *Server) GetNode(ctx context.Context, request *api.GetNodeRequest) (*api node = store.GetNode(tx, request.NodeID) }) if node == nil { - return nil, grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID) + return nil, status.Errorf(codes.NotFound, "node %s not found", request.NodeID) } if s.raft != nil { @@ -196,7 +196,7 @@ func (s *Server) ListNodes(ctx context.Context, request *api.ListNodesRequest) ( // - Returns an error if the update fails. func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) (*api.UpdateNodeResponse, error) { if request.NodeID == "" || request.NodeVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateNodeSpec(request.Spec); err != nil { return nil, err @@ -210,7 +210,7 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) err := s.store.Update(func(tx store.Tx) error { node = store.GetNode(tx, request.NodeID) if node == nil { - return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID) + return status.Errorf(codes.NotFound, "node %s not found", request.NodeID) } // Demotion sanity checks. @@ -218,20 +218,20 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) // Check for manager entries in Store. managers, err := store.FindNodes(tx, store.ByRole(api.NodeRoleManager)) if err != nil { - return grpc.Errorf(codes.Internal, "internal store error: %v", err) + return status.Errorf(codes.Internal, "internal store error: %v", err) } if len(managers) == 1 && managers[0].ID == node.ID { - return grpc.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm") + return status.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm") } // Check for node in memberlist if member = s.raft.GetMemberByNodeID(request.NodeID); member == nil { - return grpc.Errorf(codes.NotFound, "can't find manager in raft memberlist") + return status.Errorf(codes.NotFound, "can't find manager in raft memberlist") } // Quorum safeguard if !s.raft.CanRemoveMember(member.RaftID) { - return grpc.Errorf(codes.FailedPrecondition, "can't remove member from the raft: this would result in a loss of quorum") + return status.Errorf(codes.FailedPrecondition, "can't remove member from the raft: this would result in a loss of quorum") } } @@ -278,33 +278,33 @@ func removeNodeAttachments(tx store.Tx, nodeID string) error { // - Returns an error if the delete fails. func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest) (*api.RemoveNodeResponse, error) { if request.NodeID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } err := s.store.Update(func(tx store.Tx) error { node := store.GetNode(tx, request.NodeID) if node == nil { - return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID) + return status.Errorf(codes.NotFound, "node %s not found", request.NodeID) } if node.Spec.DesiredRole == api.NodeRoleManager { if s.raft == nil { - return grpc.Errorf(codes.FailedPrecondition, "node %s is a manager but cannot access node information from the raft memberlist", request.NodeID) + return status.Errorf(codes.FailedPrecondition, "node %s is a manager but cannot access node information from the raft memberlist", request.NodeID) } if member := s.raft.GetMemberByNodeID(request.NodeID); member != nil { - return grpc.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID) + return status.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID) } } if !request.Force && node.Status.State == api.NodeStatus_READY { - return grpc.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID) + return status.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID) } // lookup the cluster - clusters, err := store.FindClusters(tx, store.ByName("default")) + clusters, err := store.FindClusters(tx, store.ByName(store.DefaultClusterName)) if err != nil { return err } if len(clusters) != 1 { - return grpc.Errorf(codes.Internal, "could not fetch cluster object") + return status.Errorf(codes.Internal, "could not fetch cluster object") } cluster := clusters[0] diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go index bac4c102ef..fdcd2c412c 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go @@ -11,8 +11,8 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/sirupsen/logrus" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // assumes spec is not nil @@ -30,7 +30,7 @@ func secretFromSecretSpec(spec *api.SecretSpec) *api.Secret { // - Returns an error if getting fails. func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) (*api.GetSecretResponse, error) { if request.SecretID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, "secret ID must be provided") + return nil, status.Errorf(codes.InvalidArgument, "secret ID must be provided") } var secret *api.Secret @@ -39,7 +39,7 @@ func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) ( }) if secret == nil { - return nil, grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID) + return nil, status.Errorf(codes.NotFound, "secret %s not found", request.SecretID) } secret.Spec.Data = nil // clean the actual secret data so it's never returned @@ -52,20 +52,20 @@ func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) ( // - Returns an error if the update fails. func (s *Server) UpdateSecret(ctx context.Context, request *api.UpdateSecretRequest) (*api.UpdateSecretResponse, error) { if request.SecretID == "" || request.SecretVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var secret *api.Secret err := s.store.Update(func(tx store.Tx) error { secret = store.GetSecret(tx, request.SecretID) if secret == nil { - return grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID) + return status.Errorf(codes.NotFound, "secret %s not found", request.SecretID) } // Check if the Name is different than the current name, or the secret is non-nil and different // than the current secret if secret.Spec.Annotations.Name != request.Spec.Annotations.Name || (request.Spec.Data != nil && subtle.ConstantTimeCompare(request.Spec.Data, secret.Spec.Data) == 0) { - return grpc.Errorf(codes.InvalidArgument, "only updates to Labels are allowed") + return status.Errorf(codes.InvalidArgument, "only updates to Labels are allowed") } // We only allow updating Labels @@ -171,7 +171,7 @@ func (s *Server) CreateSecret(ctx context.Context, request *api.CreateSecretRequ switch err { case store.ErrNameConflict: - return nil, grpc.Errorf(codes.AlreadyExists, "secret %s already exists", request.Spec.Annotations.Name) + return nil, status.Errorf(codes.AlreadyExists, "secret %s already exists", request.Spec.Annotations.Name) case nil: secret.Spec.Data = nil // clean the actual secret data so it's never returned log.G(ctx).WithFields(logrus.Fields{ @@ -192,20 +192,20 @@ func (s *Server) CreateSecret(ctx context.Context, request *api.CreateSecretRequ // - Returns an error if the deletion fails. func (s *Server) RemoveSecret(ctx context.Context, request *api.RemoveSecretRequest) (*api.RemoveSecretResponse, error) { if request.SecretID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, "secret ID must be provided") + return nil, status.Errorf(codes.InvalidArgument, "secret ID must be provided") } err := s.store.Update(func(tx store.Tx) error { // Check if the secret exists secret := store.GetSecret(tx, request.SecretID) if secret == nil { - return grpc.Errorf(codes.NotFound, "could not find secret %s", request.SecretID) + return status.Errorf(codes.NotFound, "could not find secret %s", request.SecretID) } // Check if any services currently reference this secret, return error if so services, err := store.FindServices(tx, store.ByReferencedSecretID(request.SecretID)) if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using secret %s: %v", request.SecretID, err) + return status.Errorf(codes.Internal, "could not find services using secret %s: %v", request.SecretID, err) } if len(services) != 0 { @@ -221,14 +221,14 @@ func (s *Server) RemoveSecret(ctx context.Context, request *api.RemoveSecretRequ serviceStr = "service" } - return grpc.Errorf(codes.InvalidArgument, "secret '%s' is in use by the following %s: %v", secretName, serviceStr, serviceNameStr) + return status.Errorf(codes.InvalidArgument, "secret '%s' is in use by the following %s: %v", secretName, serviceStr, serviceNameStr) } return store.DeleteSecret(tx, request.SecretID) }) switch err { case store.ErrNotExist: - return nil, grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID) + return nil, status.Errorf(codes.NotFound, "secret %s not found", request.SecretID) case nil: log.G(ctx).WithFields(logrus.Fields{ "secret.ID": request.SecretID, @@ -243,7 +243,7 @@ func (s *Server) RemoveSecret(ctx context.Context, request *api.RemoveSecretRequ func validateSecretSpec(spec *api.SecretSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateConfigOrSecretAnnotations(spec.Annotations); err != nil { return err @@ -252,12 +252,12 @@ func validateSecretSpec(spec *api.SecretSpec) error { if spec.Driver != nil { // Ensure secret driver has a name if spec.Driver.Name == "" { - return grpc.Errorf(codes.InvalidArgument, "secret driver must have a name") + return status.Errorf(codes.InvalidArgument, "secret driver must have a name") } return nil } if err := validation.ValidateSecretPayload(spec.Data); err != nil { - return grpc.Errorf(codes.InvalidArgument, "%s", err.Error()) + return status.Errorf(codes.InvalidArgument, "%s", err.Error()) } return nil } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/service.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/service.go index 951c27b01e..e4c27df977 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/service.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/service.go @@ -19,8 +19,8 @@ import ( "github.com/docker/swarmkit/template" gogotypes "github.com/gogo/protobuf/types" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( @@ -37,11 +37,11 @@ func validateResources(r *api.Resources) error { } if r.NanoCPUs != 0 && r.NanoCPUs < 1e6 { - return grpc.Errorf(codes.InvalidArgument, "invalid cpu value %g: Must be at least %g", float64(r.NanoCPUs)/1e9, 1e6/1e9) + return status.Errorf(codes.InvalidArgument, "invalid cpu value %g: Must be at least %g", float64(r.NanoCPUs)/1e9, 1e6/1e9) } if r.MemoryBytes != 0 && r.MemoryBytes < 4*1024*1024 { - return grpc.Errorf(codes.InvalidArgument, "invalid memory value %d: Must be at least 4MiB", r.MemoryBytes) + return status.Errorf(codes.InvalidArgument, "invalid memory value %d: Must be at least 4MiB", r.MemoryBytes) } if err := genericresource.ValidateTask(r); err != nil { return nil @@ -70,7 +70,7 @@ func validateRestartPolicy(rp *api.RestartPolicy) error { return err } if delay < 0 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: restart-delay cannot be negative") + return status.Errorf(codes.InvalidArgument, "TaskSpec: restart-delay cannot be negative") } } @@ -80,7 +80,7 @@ func validateRestartPolicy(rp *api.RestartPolicy) error { return err } if win < 0 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: restart-window cannot be negative") + return status.Errorf(codes.InvalidArgument, "TaskSpec: restart-window cannot be negative") } } @@ -101,7 +101,7 @@ func validateUpdate(uc *api.UpdateConfig) error { } if uc.Delay < 0 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-delay cannot be negative") + return status.Errorf(codes.InvalidArgument, "TaskSpec: update-delay cannot be negative") } if uc.Monitor != nil { @@ -110,12 +110,12 @@ func validateUpdate(uc *api.UpdateConfig) error { return err } if monitor < 0 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-monitor cannot be negative") + return status.Errorf(codes.InvalidArgument, "TaskSpec: update-monitor cannot be negative") } } if uc.MaxFailureRatio < 0 || uc.MaxFailureRatio > 1 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-maxfailureratio cannot be less than 0 or bigger than 1") + return status.Errorf(codes.InvalidArgument, "TaskSpec: update-maxfailureratio cannot be less than 0 or bigger than 1") } return nil @@ -147,7 +147,7 @@ func validateContainerSpec(taskSpec api.TaskSpec) error { LogDriver: taskSpec.LogDriver, }) if err != nil { - return grpc.Errorf(codes.InvalidArgument, err.Error()) + return status.Errorf(codes.InvalidArgument, err.Error()) } if err := validateImage(container.Image); err != nil { @@ -164,11 +164,11 @@ func validateContainerSpec(taskSpec api.TaskSpec) error { // validateImage validates image name in containerSpec func validateImage(image string) error { if image == "" { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: image reference must be provided") + return status.Errorf(codes.InvalidArgument, "ContainerSpec: image reference must be provided") } if _, err := reference.ParseNormalizedNamed(image); err != nil { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: %q is not a valid repository/tag", image) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: %q is not a valid repository/tag", image) } return nil } @@ -178,7 +178,7 @@ func validateMounts(mounts []api.Mount) error { mountMap := make(map[string]bool) for _, mount := range mounts { if _, exists := mountMap[mount.Target]; exists { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: duplicate mount point: %s", mount.Target) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: duplicate mount point: %s", mount.Target) } mountMap[mount.Target] = true } @@ -198,7 +198,7 @@ func validateHealthCheck(hc *api.HealthConfig) error { return err } if interval != 0 && interval < time.Duration(minimumDuration) { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: Interval in HealthConfig cannot be less than %s", minimumDuration) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: Interval in HealthConfig cannot be less than %s", minimumDuration) } } @@ -208,7 +208,7 @@ func validateHealthCheck(hc *api.HealthConfig) error { return err } if timeout != 0 && timeout < time.Duration(minimumDuration) { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: Timeout in HealthConfig cannot be less than %s", minimumDuration) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: Timeout in HealthConfig cannot be less than %s", minimumDuration) } } @@ -218,12 +218,12 @@ func validateHealthCheck(hc *api.HealthConfig) error { return err } if sp != 0 && sp < time.Duration(minimumDuration) { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: StartPeriod in HealthConfig cannot be less than %s", minimumDuration) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: StartPeriod in HealthConfig cannot be less than %s", minimumDuration) } } if hc.Retries < 0 { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: Retries in HealthConfig cannot be negative") + return status.Errorf(codes.InvalidArgument, "ContainerSpec: Retries in HealthConfig cannot be negative") } return nil @@ -233,28 +233,28 @@ func validateGenericRuntimeSpec(taskSpec api.TaskSpec) error { generic := taskSpec.GetGeneric() if len(generic.Kind) < 3 { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime: Invalid name %q", generic.Kind) + return status.Errorf(codes.InvalidArgument, "Generic runtime: Invalid name %q", generic.Kind) } reservedNames := []string{"container", "attachment"} for _, n := range reservedNames { if strings.ToLower(generic.Kind) == n { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime: %q is a reserved name", generic.Kind) + return status.Errorf(codes.InvalidArgument, "Generic runtime: %q is a reserved name", generic.Kind) } } payload := generic.Payload if payload == nil { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime is missing payload") + return status.Errorf(codes.InvalidArgument, "Generic runtime is missing payload") } if payload.TypeUrl == "" { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime is missing payload type") + return status.Errorf(codes.InvalidArgument, "Generic runtime is missing payload type") } if len(payload.Value) == 0 { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime has an empty payload") + return status.Errorf(codes.InvalidArgument, "Generic runtime has an empty payload") } return nil @@ -284,7 +284,7 @@ func validateTaskSpec(taskSpec api.TaskSpec) error { } if taskSpec.GetRuntime() == nil { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime") + return status.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime") } switch taskSpec.GetRuntime().(type) { @@ -297,7 +297,7 @@ func validateTaskSpec(taskSpec api.TaskSpec) error { return err } default: - return grpc.Errorf(codes.Unimplemented, "RuntimeSpec: unimplemented runtime in service spec") + return status.Errorf(codes.Unimplemented, "RuntimeSpec: unimplemented runtime in service spec") } return nil @@ -324,7 +324,7 @@ func validateEndpointSpec(epSpec *api.EndpointSpec) error { // for the backend network and hence we accept that configuration. if epSpec.Mode == api.ResolutionModeDNSRoundRobin && port.PublishMode == api.PublishModeIngress { - return grpc.Errorf(codes.InvalidArgument, "EndpointSpec: port published with ingress mode can't be used with dnsrr mode") + return status.Errorf(codes.InvalidArgument, "EndpointSpec: port published with ingress mode can't be used with dnsrr mode") } // If published port is not specified, it does not conflict @@ -335,7 +335,7 @@ func validateEndpointSpec(epSpec *api.EndpointSpec) error { portSpec := portSpec{publishedPort: port.PublishedPort, protocol: port.Protocol} if _, ok := portSet[portSpec]; ok { - return grpc.Errorf(codes.InvalidArgument, "EndpointSpec: duplicate published ports provided") + return status.Errorf(codes.InvalidArgument, "EndpointSpec: duplicate published ports provided") } portSet[portSpec] = struct{}{} @@ -358,23 +358,23 @@ func validateSecretRefsSpec(spec api.TaskSpec) error { for _, secretRef := range container.Secrets { // SecretID and SecretName are mandatory, we have invalid references without them if secretRef.SecretID == "" || secretRef.SecretName == "" { - return grpc.Errorf(codes.InvalidArgument, "malformed secret reference") + return status.Errorf(codes.InvalidArgument, "malformed secret reference") } // Every secret reference requires a Target if secretRef.GetTarget() == nil { - return grpc.Errorf(codes.InvalidArgument, "malformed secret reference, no target provided") + return status.Errorf(codes.InvalidArgument, "malformed secret reference, no target provided") } // If this is a file target, we will ensure filename uniqueness if secretRef.GetFile() != nil { fileName := secretRef.GetFile().Name if fileName == "" { - return grpc.Errorf(codes.InvalidArgument, "malformed file secret reference, invalid target file name provided") + return status.Errorf(codes.InvalidArgument, "malformed file secret reference, invalid target file name provided") } // If this target is already in use, we have conflicting targets if prevSecretName, ok := existingTargets[fileName]; ok { - return grpc.Errorf(codes.InvalidArgument, "secret references '%s' and '%s' have a conflicting target: '%s'", prevSecretName, secretRef.SecretName, fileName) + return status.Errorf(codes.InvalidArgument, "secret references '%s' and '%s' have a conflicting target: '%s'", prevSecretName, secretRef.SecretName, fileName) } existingTargets[fileName] = secretRef.SecretName @@ -398,12 +398,12 @@ func validateConfigRefsSpec(spec api.TaskSpec) error { for _, configRef := range container.Configs { // ConfigID and ConfigName are mandatory, we have invalid references without them if configRef.ConfigID == "" || configRef.ConfigName == "" { - return grpc.Errorf(codes.InvalidArgument, "malformed config reference") + return status.Errorf(codes.InvalidArgument, "malformed config reference") } // Every config reference requires a Target if configRef.GetTarget() == nil { - return grpc.Errorf(codes.InvalidArgument, "malformed config reference, no target provided") + return status.Errorf(codes.InvalidArgument, "malformed config reference, no target provided") } // If this is a file target, we will ensure filename uniqueness @@ -411,12 +411,12 @@ func validateConfigRefsSpec(spec api.TaskSpec) error { fileName := configRef.GetFile().Name // Validate the file name if fileName == "" { - return grpc.Errorf(codes.InvalidArgument, "malformed file config reference, invalid target file name provided") + return status.Errorf(codes.InvalidArgument, "malformed file config reference, invalid target file name provided") } // If this target is already in use, we have conflicting targets if prevConfigName, ok := existingTargets[fileName]; ok { - return grpc.Errorf(codes.InvalidArgument, "config references '%s' and '%s' have a conflicting target: '%s'", prevConfigName, configRef.ConfigName, fileName) + return status.Errorf(codes.InvalidArgument, "config references '%s' and '%s' have a conflicting target: '%s'", prevConfigName, configRef.ConfigName, fileName) } existingTargets[fileName] = configRef.ConfigName @@ -436,7 +436,7 @@ func (s *Server) validateNetworks(networks []*api.NetworkAttachmentConfig) error continue } if allocator.IsIngressNetwork(network) { - return grpc.Errorf(codes.InvalidArgument, + return status.Errorf(codes.InvalidArgument, "Service cannot be explicitly attached to the ingress network %q", network.Spec.Annotations.Name) } } @@ -448,11 +448,11 @@ func validateMode(s *api.ServiceSpec) error { switch m.(type) { case *api.ServiceSpec_Replicated: if int64(m.(*api.ServiceSpec_Replicated).Replicated.Replicas) < 0 { - return grpc.Errorf(codes.InvalidArgument, "Number of replicas must be non-negative") + return status.Errorf(codes.InvalidArgument, "Number of replicas must be non-negative") } case *api.ServiceSpec_Global: default: - return grpc.Errorf(codes.InvalidArgument, "Unrecognized service mode") + return status.Errorf(codes.InvalidArgument, "Unrecognized service mode") } return nil @@ -460,7 +460,7 @@ func validateMode(s *api.ServiceSpec) error { func validateServiceSpec(spec *api.ServiceSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateAnnotations(spec.Annotations); err != nil { return err @@ -536,7 +536,7 @@ func (s *Server) checkPortConflicts(spec *api.ServiceSpec, serviceID string) err switch pc.PublishMode { case api.PublishModeHost: if _, ok := ingressPorts[pcToStruct(pc)]; ok { - return grpc.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as a host-published port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID) + return status.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as a host-published port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID) } // Multiple services with same port in host publish mode can @@ -546,7 +546,7 @@ func (s *Server) checkPortConflicts(spec *api.ServiceSpec, serviceID string) err _, ingressConflict := ingressPorts[pcToStruct(pc)] _, hostModeConflict := hostModePorts[pcToStruct(pc)] if ingressConflict || hostModeConflict { - return grpc.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as an ingress port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID) + return status.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as an ingress port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID) } } @@ -598,7 +598,7 @@ func (s *Server) checkSecretExistence(tx store.Tx, spec *api.ServiceSpec) error secretStr = "secret" } - return grpc.Errorf(codes.InvalidArgument, "%s not found: %v", secretStr, strings.Join(failedSecrets, ", ")) + return status.Errorf(codes.InvalidArgument, "%s not found: %v", secretStr, strings.Join(failedSecrets, ", ")) } @@ -627,7 +627,7 @@ func (s *Server) checkConfigExistence(tx store.Tx, spec *api.ServiceSpec) error configStr = "config" } - return grpc.Errorf(codes.InvalidArgument, "%s not found: %v", configStr, strings.Join(failedConfigs, ", ")) + return status.Errorf(codes.InvalidArgument, "%s not found: %v", configStr, strings.Join(failedConfigs, ", ")) } @@ -662,7 +662,7 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe if allocator.IsIngressNetworkNeeded(service) { if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress { - return nil, grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") + return nil, status.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") } } @@ -694,7 +694,7 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe // - Returns `NotFound` if the Service is not found. func (s *Server) GetService(ctx context.Context, request *api.GetServiceRequest) (*api.GetServiceResponse, error) { if request.ServiceID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var service *api.Service @@ -702,7 +702,7 @@ func (s *Server) GetService(ctx context.Context, request *api.GetServiceRequest) service = store.GetService(tx, request.ServiceID) }) if service == nil { - return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID) + return nil, status.Errorf(codes.NotFound, "service %s not found", request.ServiceID) } if request.InsertDefaults { @@ -721,7 +721,7 @@ func (s *Server) GetService(ctx context.Context, request *api.GetServiceRequest) // - Returns an error if the update fails. func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRequest) (*api.UpdateServiceResponse, error) { if request.ServiceID == "" || request.ServiceVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateServiceSpec(request.Spec); err != nil { return nil, err @@ -732,7 +732,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe service = store.GetService(tx, request.ServiceID) }) if service == nil { - return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID) + return nil, status.Errorf(codes.NotFound, "service %s not found", request.ServiceID) } if request.Spec.Endpoint != nil && !reflect.DeepEqual(request.Spec.Endpoint, service.Spec.Endpoint) { @@ -744,7 +744,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe err := s.store.Update(func(tx store.Tx) error { service = store.GetService(tx, request.ServiceID) if service == nil { - return grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID) + return status.Errorf(codes.NotFound, "service %s not found", request.ServiceID) } // It's not okay to update Service.Spec.Networks on its own. @@ -754,7 +754,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe if (len(request.Spec.Networks) != 0 || len(service.Spec.Networks) != 0) && !reflect.DeepEqual(request.Spec.Networks, service.Spec.Networks) && reflect.DeepEqual(request.Spec.Task.Networks, service.Spec.Task.Networks) { - return grpc.Errorf(codes.Unimplemented, errNetworkUpdateNotSupported.Error()) + return status.Errorf(codes.Unimplemented, errNetworkUpdateNotSupported.Error()) } // Check to see if all the secrets being added exist as objects @@ -773,18 +773,18 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe // with service mode change (comparing current config with previous config). // proper way to change service mode is to delete and re-add. if reflect.TypeOf(service.Spec.Mode) != reflect.TypeOf(request.Spec.Mode) { - return grpc.Errorf(codes.Unimplemented, errModeChangeNotAllowed.Error()) + return status.Errorf(codes.Unimplemented, errModeChangeNotAllowed.Error()) } if service.Spec.Annotations.Name != request.Spec.Annotations.Name { - return grpc.Errorf(codes.Unimplemented, errRenameNotSupported.Error()) + return status.Errorf(codes.Unimplemented, errRenameNotSupported.Error()) } service.Meta.Version = *request.ServiceVersion if request.Rollback == api.UpdateServiceRequest_PREVIOUS { if service.PreviousSpec == nil { - return grpc.Errorf(codes.FailedPrecondition, "service %s does not have a previous spec", request.ServiceID) + return status.Errorf(codes.FailedPrecondition, "service %s does not have a previous spec", request.ServiceID) } curSpec := service.Spec.Copy() @@ -815,7 +815,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe if allocator.IsIngressNetworkNeeded(service) { if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress { - return grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") + return status.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") } } @@ -836,7 +836,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe // - Returns an error if the deletion fails. func (s *Server) RemoveService(ctx context.Context, request *api.RemoveServiceRequest) (*api.RemoveServiceResponse, error) { if request.ServiceID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } err := s.store.Update(func(tx store.Tx) error { @@ -844,7 +844,7 @@ func (s *Server) RemoveService(ctx context.Context, request *api.RemoveServiceRe }) if err != nil { if err == store.ErrNotExist { - return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID) + return nil, status.Errorf(codes.NotFound, "service %s not found", request.ServiceID) } return nil, err } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/task.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/task.go index d2ae21522f..51b7bf82d8 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/task.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/task.go @@ -6,8 +6,8 @@ import ( "github.com/docker/swarmkit/manager/orchestrator" "github.com/docker/swarmkit/manager/state/store" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // GetTask returns a Task given a TaskID. @@ -15,7 +15,7 @@ import ( // - Returns `NotFound` if the Task is not found. func (s *Server) GetTask(ctx context.Context, request *api.GetTaskRequest) (*api.GetTaskResponse, error) { if request.TaskID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var task *api.Task @@ -23,7 +23,7 @@ func (s *Server) GetTask(ctx context.Context, request *api.GetTaskRequest) (*api task = store.GetTask(tx, request.TaskID) }) if task == nil { - return nil, grpc.Errorf(codes.NotFound, "task %s not found", request.TaskID) + return nil, status.Errorf(codes.NotFound, "task %s not found", request.TaskID) } return &api.GetTaskResponse{ Task: task, @@ -36,7 +36,7 @@ func (s *Server) GetTask(ctx context.Context, request *api.GetTaskRequest) (*api // - Returns an error if the deletion fails. func (s *Server) RemoveTask(ctx context.Context, request *api.RemoveTaskRequest) (*api.RemoveTaskResponse, error) { if request.TaskID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } err := s.store.Update(func(tx store.Tx) error { @@ -44,7 +44,7 @@ func (s *Server) RemoveTask(ctx context.Context, request *api.RemoveTaskRequest) }) if err != nil { if err == store.ErrNotExist { - return nil, grpc.Errorf(codes.NotFound, "task %s not found", request.TaskID) + return nil, status.Errorf(codes.NotFound, "task %s not found", request.TaskID) } return nil, err } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go index 4de6a30c4d..13d68293ae 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go @@ -7,11 +7,8 @@ import ( "sync" "time" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/transport" - "github.com/docker/go-events" + "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/api/equality" "github.com/docker/swarmkit/ca" @@ -25,6 +22,9 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/grpc/transport" ) const ( @@ -66,8 +66,18 @@ var ( ErrSessionInvalid = errors.New("session invalid") // ErrNodeNotFound returned when the Node doesn't exist in raft. ErrNodeNotFound = errors.New("node not found") + + // Scheduling delay timer. + schedulingDelayTimer metrics.Timer ) +func init() { + ns := metrics.NewNamespace("swarm", "dispatcher", nil) + schedulingDelayTimer = ns.NewTimer("scheduling_delay", + "Scheduling delay is the time a task takes to go from NEW to RUNNING state.") + metrics.Register(ns) +} + // Config is configuration for Dispatcher. For default you should use // DefaultConfig. type Config struct { @@ -322,7 +332,7 @@ func (d *Dispatcher) isRunningLocked() (context.Context, error) { d.mu.Lock() if !d.isRunning() { d.mu.Unlock() - return nil, grpc.Errorf(codes.Aborted, "dispatcher is stopped") + return nil, status.Errorf(codes.Aborted, "dispatcher is stopped") } ctx := d.ctx d.mu.Unlock() @@ -556,7 +566,7 @@ func (d *Dispatcher) UpdateTaskStatus(ctx context.Context, r *api.UpdateTaskStat } if t.NodeID != nodeID { - err := grpc.Errorf(codes.PermissionDenied, "cannot update a task not assigned this node") + err := status.Errorf(codes.PermissionDenied, "cannot update a task not assigned this node") log.WithField("task.id", u.TaskID).Error(err) return nil, err } @@ -632,6 +642,17 @@ func (d *Dispatcher) processUpdates(ctx context.Context) { return nil } + // Update scheduling delay metric for running tasks. + // We use the status update time on the leader to calculate the scheduling delay. + // Because of this, the recorded scheduling delay will be an overestimate and include + // the network delay between the worker and the leader. + // This is not ideal, but its a known overestimation, rather than using the status update time + // from the worker node, which may cause unknown incorrect results due to possible clock skew. + if status.State == api.TaskStateRunning { + start := time.Unix(status.AppliedAt.GetSeconds(), int64(status.AppliedAt.GetNanos())) + schedulingDelayTimer.UpdateSince(start) + } + task.Status = *status task.Status.AppliedBy = d.securityConfig.ClientTLSCreds.NodeID() task.Status.AppliedAt = ptypes.MustTimestampProto(time.Now()) @@ -1189,7 +1210,7 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio log.WithError(err).Error("failed to remove node") } // still return an abort if the transport closure was ineffective. - return grpc.Errorf(codes.Aborted, "node must disconnect") + return status.Errorf(codes.Aborted, "node must disconnect") } for { diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/nodes.go b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/nodes.go index 8a0de558d6..cf35bb869a 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/nodes.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/nodes.go @@ -4,12 +4,11 @@ import ( "sync" "time" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/manager/dispatcher/heartbeat" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const rateLimitCount = 3 @@ -36,7 +35,7 @@ func (rn *registeredNode) checkSessionID(sessionID string) error { // changed. If it has, we will the stream and make the node // re-register. if sessionID == "" || rn.SessionID != sessionID { - return grpc.Errorf(codes.InvalidArgument, ErrSessionInvalid.Error()) + return status.Errorf(codes.InvalidArgument, ErrSessionInvalid.Error()) } return nil @@ -97,7 +96,7 @@ func (s *nodeStore) CheckRateLimit(id string) error { } existRn.Attempts++ if existRn.Attempts > rateLimitCount { - return grpc.Errorf(codes.Unavailable, "node %s exceeded rate limit count of registrations", id) + return status.Errorf(codes.Unavailable, "node %s exceeded rate limit count of registrations", id) } existRn.Registered = time.Now() } @@ -136,7 +135,7 @@ func (s *nodeStore) Get(id string) (*registeredNode, error) { rn, ok := s.nodes[id] s.mu.RUnlock() if !ok { - return nil, grpc.Errorf(codes.NotFound, ErrNodeNotRegistered.Error()) + return nil, status.Errorf(codes.NotFound, ErrNodeNotRegistered.Error()) } return rn, nil } @@ -146,7 +145,7 @@ func (s *nodeStore) GetWithSession(id, sid string) (*registeredNode, error) { rn, ok := s.nodes[id] s.mu.RUnlock() if !ok { - return nil, grpc.Errorf(codes.NotFound, ErrNodeNotRegistered.Error()) + return nil, status.Errorf(codes.NotFound, ErrNodeNotRegistered.Error()) } return rn, rn.checkSessionID(sid) } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/health/health.go b/components/engine/vendor/github.com/docker/swarmkit/manager/health/health.go index bf220bdc18..ef6658b09d 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/health/health.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/health/health.go @@ -12,8 +12,8 @@ import ( "github.com/docker/swarmkit/api" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // Server represents a Health Check server to check @@ -46,7 +46,7 @@ func (s *Server) Check(ctx context.Context, in *api.HealthCheckRequest) (*api.He Status: status, }, nil } - return nil, grpc.Errorf(codes.NotFound, "unknown service") + return nil, status.Errorf(codes.NotFound, "unknown service") } // SetServingStatus is called when need to reset the serving status of a service diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go b/components/engine/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go index 860b55c30e..de2b93633f 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go @@ -6,9 +6,6 @@ import ( "io" "sync" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "github.com/docker/go-events" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" @@ -18,6 +15,8 @@ import ( "github.com/docker/swarmkit/watch" "github.com/sirupsen/logrus" "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( @@ -93,11 +92,11 @@ func (lb *LogBroker) Stop() error { func validateSelector(selector *api.LogSelector) error { if selector == nil { - return grpc.Errorf(codes.InvalidArgument, "log selector must be provided") + return status.Errorf(codes.InvalidArgument, "log selector must be provided") } if len(selector.ServiceIDs) == 0 && len(selector.TaskIDs) == 0 && len(selector.NodeIDs) == 0 { - return grpc.Errorf(codes.InvalidArgument, "log selector must not be empty") + return status.Errorf(codes.InvalidArgument, "log selector must not be empty") } return nil @@ -401,17 +400,17 @@ func (lb *LogBroker) PublishLogs(stream api.LogBroker_PublishLogsServer) (err er } if logMsg.SubscriptionID == "" { - return grpc.Errorf(codes.InvalidArgument, "missing subscription ID") + return status.Errorf(codes.InvalidArgument, "missing subscription ID") } if currentSubscription == nil { currentSubscription = lb.getSubscription(logMsg.SubscriptionID) if currentSubscription == nil { - return grpc.Errorf(codes.NotFound, "unknown subscription ID") + return status.Errorf(codes.NotFound, "unknown subscription ID") } } else { if logMsg.SubscriptionID != currentSubscription.message.ID { - return grpc.Errorf(codes.InvalidArgument, "different subscription IDs in the same session") + return status.Errorf(codes.InvalidArgument, "different subscription IDs in the same session") } } @@ -427,7 +426,7 @@ func (lb *LogBroker) PublishLogs(stream api.LogBroker_PublishLogsServer) (err er // Make sure logs are emitted using the right Node ID to avoid impersonation. for _, msg := range logMsg.Messages { if msg.Context.NodeID != remote.NodeID { - return grpc.Errorf(codes.PermissionDenied, "invalid NodeID: expected=%s;received=%s", remote.NodeID, msg.Context.NodeID) + return status.Errorf(codes.PermissionDenied, "invalid NodeID: expected=%s;received=%s", remote.NodeID, msg.Context.NodeID) } } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go b/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go index 39db22b09a..1ce727dcac 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go @@ -38,6 +38,7 @@ import ( "github.com/docker/swarmkit/manager/resourceapi" "github.com/docker/swarmkit/manager/scheduler" "github.com/docker/swarmkit/manager/state/raft" + "github.com/docker/swarmkit/manager/state/raft/transport" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/manager/watchapi" "github.com/docker/swarmkit/remotes" @@ -54,9 +55,6 @@ import ( const ( // defaultTaskHistoryRetentionLimit is the number of tasks to keep. defaultTaskHistoryRetentionLimit = 5 - - // Default value for grpc max message size. - grpcMaxMessageSize = 128 << 20 ) // RemoteAddrs provides a listening address and an optional advertise address @@ -234,7 +232,7 @@ func New(config *Config) (*Manager, error) { grpc.Creds(config.SecurityConfig.ServerTLSCreds), grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor), - grpc.MaxMsgSize(grpcMaxMessageSize), + grpc.MaxMsgSize(transport.GRPCMaxMsgSize), } m := &Manager{ @@ -404,7 +402,7 @@ func (m *Manager) Run(parent context.Context) error { ) m.raftNode.MemoryStore().View(func(readTx store.ReadTx) { - clusters, err = store.FindClusters(readTx, store.ByName("default")) + clusters, err = store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) }) @@ -954,13 +952,18 @@ func (m *Manager) becomeLeader(ctx context.Context) { // store. Don't check the error because // we expect this to fail unless this // is a brand new cluster. - store.CreateCluster(tx, defaultClusterObject( + err := store.CreateCluster(tx, defaultClusterObject( clusterID, initialCAConfig, raftCfg, api.EncryptionConfig{AutoLockManagers: m.config.AutoLockManagers}, unlockKeys, rootCA)) + + if err != nil && err != store.ErrExist { + log.G(ctx).WithError(err).Errorf("error creating cluster object") + } + // Add Node entry for ourself, if one // doesn't exist already. freshCluster := nil == store.CreateNode(tx, managerNode(nodeID, m.config.Availability)) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/global/global.go b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/global/global.go index a1d2873e80..f6217ff7b4 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/global/global.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/global/global.go @@ -73,7 +73,7 @@ func (g *Orchestrator) Run(ctx context.Context) error { var err error g.store.View(func(readTx store.ReadTx) { var clusters []*api.Cluster - clusters, err = store.FindClusters(readTx, store.ByName("default")) + clusters, err = store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) if len(clusters) != 1 { return // just pick up the cluster when it is created. @@ -147,7 +147,7 @@ func (g *Orchestrator) Run(ctx context.Context) error { if !orchestrator.IsGlobalService(v.Service) { continue } - orchestrator.DeleteServiceTasks(ctx, g.store, v.Service) + orchestrator.SetServiceTasksRemove(ctx, g.store, v.Service) // delete the service from service map delete(g.globalServices, v.Service.ID) g.restarts.ClearServiceHistory(v.Service.ID) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/replicated/services.go b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/replicated/services.go index f4d0511f80..04aea8795a 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/replicated/services.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/replicated/services.go @@ -17,7 +17,7 @@ import ( // responds to changes in individual tasks (or nodes which run them). func (r *Orchestrator) initCluster(readTx store.ReadTx) error { - clusters, err := store.FindClusters(readTx, store.ByName("default")) + clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) if err != nil { return err } @@ -50,7 +50,7 @@ func (r *Orchestrator) handleServiceEvent(ctx context.Context, event events.Even if !orchestrator.IsReplicatedService(v.Service) { return } - orchestrator.DeleteServiceTasks(ctx, r.store, v.Service) + orchestrator.SetServiceTasksRemove(ctx, r.store, v.Service) r.restarts.ClearServiceHistory(v.Service.ID) delete(r.reconcileServices, v.Service.ID) case api.EventCreateService: @@ -86,6 +86,12 @@ func (r *Orchestrator) resolveService(ctx context.Context, task *api.Task) *api. return service } +// reconcile decides what actions must be taken depending on the number of +// specificed slots and actual running slots. If the actual running slots are +// fewer than what is requested, it creates new tasks. If the actual running +// slots are more than requested, then it decides which slots must be removed +// and sets desired state of those tasks to REMOVE (the actual removal is handled +// by the task reaper, after the agent shuts the tasks down). func (r *Orchestrator) reconcile(ctx context.Context, service *api.Service) { runningSlots, deadSlots, err := r.updatableAndDeadSlots(ctx, service) if err != nil { @@ -157,7 +163,11 @@ func (r *Orchestrator) reconcile(ctx context.Context, service *api.Service) { r.updater.Update(ctx, r.cluster, service, sortedSlots[:specifiedSlots]) err = r.store.Batch(func(batch *store.Batch) error { r.deleteTasksMap(ctx, batch, deadSlots) - r.deleteTasks(ctx, batch, sortedSlots[specifiedSlots:]) + // for all slots that we are removing, we set the desired state of those tasks + // to REMOVE. Then, the agent is responsible for shutting them down, and the + // task reaper is responsible for actually removing them from the store after + // shutdown. + r.setTasksDesiredState(ctx, batch, sortedSlots[specifiedSlots:], api.TaskStateRemove) return nil }) if err != nil { @@ -198,10 +208,34 @@ func (r *Orchestrator) addTasks(ctx context.Context, batch *store.Batch, service } } -func (r *Orchestrator) deleteTasks(ctx context.Context, batch *store.Batch, slots []orchestrator.Slot) { +// setTasksDesiredState sets the desired state for all tasks for the given slots to the +// requested state +func (r *Orchestrator) setTasksDesiredState(ctx context.Context, batch *store.Batch, slots []orchestrator.Slot, newDesiredState api.TaskState) { for _, slot := range slots { for _, t := range slot { - r.deleteTask(ctx, batch, t) + err := batch.Update(func(tx store.Tx) error { + // time travel is not allowed. if the current desired state is + // above the one we're trying to go to we can't go backwards. + // we have nothing to do and we should skip to the next task + if t.DesiredState > newDesiredState { + // log a warning, though. we shouln't be trying to rewrite + // a state to an earlier state + log.G(ctx).Warnf( + "cannot update task %v in desired state %v to an earlier desired state %v", + t.ID, t.DesiredState, newDesiredState, + ) + return nil + } + // update desired state + t.DesiredState = newDesiredState + + return store.UpdateTask(tx, t) + }) + + // log an error if we get one + if err != nil { + log.G(ctx).WithError(err).Errorf("failed to update task to %v", newDesiredState.String()) + } } } } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/service.go b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/service.go index 4e52c83abf..7356c38cd5 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/service.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/service.go @@ -27,8 +27,10 @@ func IsGlobalService(service *api.Service) bool { return ok } -// DeleteServiceTasks deletes the tasks associated with a service. -func DeleteServiceTasks(ctx context.Context, s *store.MemoryStore, service *api.Service) { +// SetServiceTasksRemove sets the desired state of tasks associated with a service +// to REMOVE, so that they can be properly shut down by the agent and later removed +// by the task reaper. +func SetServiceTasksRemove(ctx context.Context, s *store.MemoryStore, service *api.Service) { var ( tasks []*api.Task err error @@ -44,8 +46,23 @@ func DeleteServiceTasks(ctx context.Context, s *store.MemoryStore, service *api. err = s.Batch(func(batch *store.Batch) error { for _, t := range tasks { err := batch.Update(func(tx store.Tx) error { - if err := store.DeleteTask(tx, t.ID); err != nil { - log.G(ctx).WithError(err).Errorf("failed to delete task") + // time travel is not allowed. if the current desired state is + // above the one we're trying to go to we can't go backwards. + // we have nothing to do and we should skip to the next task + if t.DesiredState > api.TaskStateRemove { + // log a warning, though. we shouln't be trying to rewrite + // a state to an earlier state + log.G(ctx).Warnf( + "cannot update task %v in desired state %v to an earlier desired state %v", + t.ID, t.DesiredState, api.TaskStateRemove, + ) + return nil + } + // update desired state to REMOVE + t.DesiredState = api.TaskStateRemove + + if err := store.UpdateTask(tx, t); err != nil { + log.G(ctx).WithError(err).Errorf("failed transaction: update task desired state to REMOVE") } return nil }) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go index 577319c8e8..bcef801f63 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go @@ -23,12 +23,19 @@ const ( // exist for the same service/instance or service/nodeid combination. type TaskReaper struct { store *store.MemoryStore + // taskHistory is the number of tasks to keep taskHistory int64 - dirty map[orchestrator.SlotTuple]struct{} - orphaned []string - stopChan chan struct{} - doneChan chan struct{} + + // List of slot tubles to be inspected for task history cleanup. + dirty map[orchestrator.SlotTuple]struct{} + + // List of tasks collected for cleanup, which includes two kinds of tasks + // - serviceless orphaned tasks + // - tasks with desired state REMOVE that have already been shut down + cleanup []string + stopChan chan struct{} + doneChan chan struct{} } // New creates a new TaskReaper. @@ -41,7 +48,13 @@ func New(store *store.MemoryStore) *TaskReaper { } } -// Run is the TaskReaper's main loop. +// Run is the TaskReaper's watch loop which collects candidates for cleanup. +// Task history is mainly used in task restarts but is also available for administrative purposes. +// Note that the task history is stored per-slot-per-service for replicated services +// and per-node-per-service for global services. History does not apply to serviceless +// since they are not attached to a service. In addition, the TaskReaper watch loop is also +// responsible for cleaning up tasks associated with slots that were removed as part of +// service scale down or service removal. func (tr *TaskReaper) Run(ctx context.Context) { watcher, watchCancel := state.Watch(tr.store.WatchQueue(), api.EventCreateTask{}, api.EventUpdateTask{}, api.EventUpdateCluster{}) @@ -50,7 +63,8 @@ func (tr *TaskReaper) Run(ctx context.Context) { watchCancel() }() - var tasks []*api.Task + var orphanedTasks []*api.Task + var removeTasks []*api.Task tr.store.View(func(readTx store.ReadTx) { var err error @@ -59,29 +73,54 @@ func (tr *TaskReaper) Run(ctx context.Context) { tr.taskHistory = clusters[0].Spec.Orchestration.TaskHistoryRetentionLimit } - tasks, err = store.FindTasks(readTx, store.ByTaskState(api.TaskStateOrphaned)) + // On startup, scan the entire store and inspect orphaned tasks from previous life. + orphanedTasks, err = store.FindTasks(readTx, store.ByTaskState(api.TaskStateOrphaned)) if err != nil { log.G(ctx).WithError(err).Error("failed to find Orphaned tasks in task reaper init") } + removeTasks, err = store.FindTasks(readTx, store.ByDesiredState(api.TaskStateRemove)) + if err != nil { + log.G(ctx).WithError(err).Error("failed to find tasks with desired state REMOVE in task reaper init") + } }) - if len(tasks) > 0 { - for _, t := range tasks { - // Do not reap service tasks immediately + if len(orphanedTasks)+len(removeTasks) > 0 { + for _, t := range orphanedTasks { + // Do not reap service tasks immediately. + // Let them go through the regular history cleanup process + // of checking TaskHistoryRetentionLimit. if t.ServiceID != "" { continue } - tr.orphaned = append(tr.orphaned, t.ID) + // Serviceless tasks can be cleaned up right away since they are not attached to a service. + tr.cleanup = append(tr.cleanup, t.ID) } - - if len(tr.orphaned) > 0 { + // tasks with desired state REMOVE that have progressed beyond SHUTDOWN can be cleaned up + // right away + for _, t := range removeTasks { + if t.Status.State >= api.TaskStateShutdown { + tr.cleanup = append(tr.cleanup, t.ID) + } + } + // Clean up tasks in 'cleanup' right away + if len(tr.cleanup) > 0 { tr.tick() } } + // Clean up when we hit TaskHistoryRetentionLimit or when the timer expires, + // whichever happens first. timer := time.NewTimer(reaperBatchingInterval) + // Watch for: + // 1. EventCreateTask for cleaning slots, which is the best time to cleanup that node/slot. + // 2. EventUpdateTask for cleaning + // - serviceless orphaned tasks (when orchestrator updates the task status to ORPHANED) + // - tasks which have desired state REMOVE and have been shut down by the agent + // (these are tasks which are associated with slots removed as part of service + // remove or scale down) + // 3. EventUpdateCluster for TaskHistoryRetentionLimit update. for { select { case event := <-watcher: @@ -95,14 +134,21 @@ func (tr *TaskReaper) Run(ctx context.Context) { }] = struct{}{} case api.EventUpdateTask: t := v.Task + // add serviceless orphaned tasks if t.Status.State >= api.TaskStateOrphaned && t.ServiceID == "" { - tr.orphaned = append(tr.orphaned, t.ID) + tr.cleanup = append(tr.cleanup, t.ID) + } + // add tasks that have progressed beyond SHUTDOWN and have desired state REMOVE. These + // tasks are associated with slots that were removed as part of a service scale down + // or service removal. + if t.DesiredState == api.TaskStateRemove && t.Status.State >= api.TaskStateShutdown { + tr.cleanup = append(tr.cleanup, t.ID) } case api.EventUpdateCluster: tr.taskHistory = v.Cluster.Spec.Orchestration.TaskHistoryRetentionLimit } - if len(tr.dirty)+len(tr.orphaned) > maxDirty { + if len(tr.dirty)+len(tr.cleanup) > maxDirty { timer.Stop() tr.tick() } else { @@ -118,19 +164,22 @@ func (tr *TaskReaper) Run(ctx context.Context) { } } +// tick performs task history cleanup. func (tr *TaskReaper) tick() { - if len(tr.dirty) == 0 && len(tr.orphaned) == 0 { + if len(tr.dirty) == 0 && len(tr.cleanup) == 0 { return } defer func() { - tr.orphaned = nil + tr.cleanup = nil }() deleteTasks := make(map[string]struct{}) - for _, tID := range tr.orphaned { + for _, tID := range tr.cleanup { deleteTasks[tID] = struct{}{} } + + // Check history of dirty tasks for cleanup. tr.store.View(func(tx store.ReadTx) { for dirty := range tr.dirty { service := store.GetService(tx, dirty.ServiceID) @@ -141,8 +190,8 @@ func (tr *TaskReaper) tick() { taskHistory := tr.taskHistory // If MaxAttempts is set, keep at least one more than - // that number of tasks. This is necessary reconstruct - // restart history when the orchestrator starts up. + // that number of tasks (this overrides TaskHistoryRetentionLimit). + // This is necessary to reconstruct restart history when the orchestrator starts up. // TODO(aaronl): Consider hiding tasks beyond the normal // retention limit in the UI. // TODO(aaronl): There are some ways to cut down the @@ -156,6 +205,7 @@ func (tr *TaskReaper) tick() { taskHistory = int64(service.Spec.Task.Restart.MaxAttempts) + 1 } + // Negative value for TaskHistoryRetentionLimit is an indication to never clean up task history. if taskHistory < 0 { continue } @@ -164,6 +214,7 @@ func (tr *TaskReaper) tick() { switch service.Spec.GetMode().(type) { case *api.ServiceSpec_Replicated: + // Clean out the slot for which we received EventCreateTask. var err error historicTasks, err = store.FindTasks(tx, store.BySlot(dirty.ServiceID, dirty.Slot)) if err != nil { @@ -171,6 +222,7 @@ func (tr *TaskReaper) tick() { } case *api.ServiceSpec_Global: + // Clean out the node history in case of global services. tasksByNode, err := store.FindTasks(tx, store.ByNodeID(dirty.NodeID)) if err != nil { continue @@ -215,6 +267,7 @@ func (tr *TaskReaper) tick() { } }) + // Perform cleanup. if len(deleteTasks) > 0 { tr.store.Batch(func(batch *store.Batch) error { for taskID := range deleteTasks { diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/resourceapi/allocator.go b/components/engine/vendor/github.com/docker/swarmkit/manager/resourceapi/allocator.go index 87b01ebd3b..ec19fba850 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/resourceapi/allocator.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/resourceapi/allocator.go @@ -10,8 +10,8 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/protobuf/ptypes" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( @@ -50,11 +50,11 @@ func (ra *ResourceAllocator) AttachNetwork(ctx context.Context, request *api.Att } }) if network == nil { - return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.Config.Target) + return nil, status.Errorf(codes.NotFound, "network %s not found", request.Config.Target) } if !network.Spec.Attachable { - return nil, grpc.Errorf(codes.PermissionDenied, "network %s not manually attachable", request.Config.Target) + return nil, status.Errorf(codes.PermissionDenied, "network %s not manually attachable", request.Config.Target) } t := &api.Task{ @@ -98,7 +98,7 @@ func (ra *ResourceAllocator) AttachNetwork(ctx context.Context, request *api.Att // - Returns an error if the deletion fails. func (ra *ResourceAllocator) DetachNetwork(ctx context.Context, request *api.DetachNetworkRequest) (*api.DetachNetworkResponse, error) { if request.AttachmentID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } nodeInfo, err := ca.RemoteNode(ctx) @@ -109,10 +109,10 @@ func (ra *ResourceAllocator) DetachNetwork(ctx context.Context, request *api.Det if err := ra.store.Update(func(tx store.Tx) error { t := store.GetTask(tx, request.AttachmentID) if t == nil { - return grpc.Errorf(codes.NotFound, "attachment %s not found", request.AttachmentID) + return status.Errorf(codes.NotFound, "attachment %s not found", request.AttachmentID) } if t.NodeID != nodeInfo.NodeID { - return grpc.Errorf(codes.PermissionDenied, "attachment %s doesn't belong to this node", request.AttachmentID) + return status.Errorf(codes.PermissionDenied, "attachment %s doesn't belong to this node", request.AttachmentID) } return store.DeleteTask(tx, request.AttachmentID) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go b/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go index 3b1c73fe2d..dab3c6695d 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go @@ -169,7 +169,10 @@ func (f *PluginFilter) Check(n *NodeInfo) bool { } } - if f.t.Spec.LogDriver != nil && f.t.Spec.LogDriver.Name != "none" { + // It's possible that the LogDriver object does not carry a name, just some + // configuration options. In that case, the plugin filter shouldn't fail to + // schedule the task + if f.t.Spec.LogDriver != nil && f.t.Spec.LogDriver.Name != "none" && f.t.Spec.LogDriver.Name != "" { // If there are no log driver types in the list at all, most likely this is // an older daemon that did not report this information. In this case don't filter if typeFound, exists := f.pluginExistsOnNode("Log", f.t.Spec.LogDriver.Name, nodePlugins); !exists && typeFound { @@ -294,6 +297,14 @@ func (f *PlatformFilter) platformEqual(imgPlatform, nodePlatform api.Platform) b nodePlatform.Architecture = "amd64" } + // normalize "aarch64" architectures to "arm64" + if imgPlatform.Architecture == "aarch64" { + imgPlatform.Architecture = "arm64" + } + if nodePlatform.Architecture == "aarch64" { + nodePlatform.Architecture = "arm64" + } + if (imgPlatform.Architecture == "" || imgPlatform.Architecture == nodePlatform.Architecture) && (imgPlatform.OS == "" || imgPlatform.OS == nodePlatform.OS) { return true } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go index 28c7cfa47e..56b7c7c966 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go @@ -2,6 +2,7 @@ package raft import ( "fmt" + "io" "math" "math/rand" "net" @@ -14,6 +15,7 @@ import ( "github.com/coreos/etcd/raft/raftpb" "github.com/docker/docker/pkg/signal" "github.com/docker/go-events" + "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/log" @@ -34,6 +36,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" ) var ( @@ -62,6 +65,9 @@ var ( // work around lint lostQuorumMessage = "The swarm does not have a leader. It's possible that too few managers are online. Make sure more than half of the managers are online." errLostQuorum = errors.New(lostQuorumMessage) + + // Timer to capture ProposeValue() latency. + proposeLatencyTimer metrics.Timer ) // LeadershipState indicates whether the node is a leader or follower. @@ -180,12 +186,9 @@ type NodeOptions struct { ClockSource clock.Clock // SendTimeout is the timeout on the sending messages to other raft // nodes. Leave this as 0 to get the default value. - SendTimeout time.Duration - // LargeSendTimeout is the timeout on the sending snapshots to other raft - // nodes. Leave this as 0 to get the default value. - LargeSendTimeout time.Duration - TLSCredentials credentials.TransportCredentials - KeyRotator EncryptionKeyRotator + SendTimeout time.Duration + TLSCredentials credentials.TransportCredentials + KeyRotator EncryptionKeyRotator // DisableStackDump prevents Run from dumping goroutine stacks when the // store becomes stuck. DisableStackDump bool @@ -193,6 +196,9 @@ type NodeOptions struct { func init() { rand.Seed(time.Now().UnixNano()) + ns := metrics.NewNamespace("swarm", "raft", nil) + proposeLatencyTimer = ns.NewTimer("transaction_latency", "Raft transaction latency.") + metrics.Register(ns) } // NewNode generates a new Raft node @@ -207,11 +213,6 @@ func NewNode(opts NodeOptions) *Node { if opts.SendTimeout == 0 { opts.SendTimeout = 2 * time.Second } - if opts.LargeSendTimeout == 0 { - // a "slow" 100Mbps connection can send over 240MB data in 20 seconds - // which is well over the gRPC message limit of 128MB allowed by SwarmKit - opts.LargeSendTimeout = 20 * time.Second - } raftStore := raft.NewMemoryStorage() @@ -357,7 +358,6 @@ func (n *Node) initTransport() { transportConfig := &transport.Config{ HeartbeatInterval: time.Duration(n.Config.ElectionTick) * n.opts.TickInterval, SendTimeout: n.opts.SendTimeout, - LargeSendTimeout: n.opts.LargeSendTimeout, Credentials: n.opts.TLSCredentials, Raft: n, } @@ -664,7 +664,7 @@ func (n *Node) Run(ctx context.Context) error { if n.snapshotInProgress == nil && (n.needsSnapshot(ctx) || raftConfig.SnapshotInterval > 0 && n.appliedIndex-n.snapshotMeta.Index >= raftConfig.SnapshotInterval) { - n.doSnapshot(ctx, raftConfig) + n.triggerSnapshot(ctx, raftConfig) } if wasLeader && atomic.LoadUint32(&n.signalledLeadership) != 1 { @@ -706,7 +706,7 @@ func (n *Node) Run(ctx context.Context) error { // there was a key rotation that took place before while the snapshot // was in progress - we have to take another snapshot and encrypt with the new key n.rotationQueued = false - n.doSnapshot(ctx, raftConfig) + n.triggerSnapshot(ctx, raftConfig) } case <-n.keyRotator.RotationNotify(): // There are 2 separate checks: rotationQueued, and n.needsSnapshot(). @@ -719,7 +719,7 @@ func (n *Node) Run(ctx context.Context) error { case n.snapshotInProgress != nil: n.rotationQueued = true case n.needsSnapshot(ctx): - n.doSnapshot(ctx, n.getCurrentRaftConfig()) + n.triggerSnapshot(ctx, n.getCurrentRaftConfig()) } case <-ctx.Done(): return nil @@ -929,11 +929,11 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons defer n.membershipLock.Unlock() if !n.IsMember() { - return nil, grpc.Errorf(codes.FailedPrecondition, "%s", ErrNoRaftMember.Error()) + return nil, status.Errorf(codes.FailedPrecondition, "%s", ErrNoRaftMember.Error()) } if !n.isLeader() { - return nil, grpc.Errorf(codes.FailedPrecondition, "%s", ErrLostLeadership.Error()) + return nil, status.Errorf(codes.FailedPrecondition, "%s", ErrLostLeadership.Error()) } remoteAddr := req.Addr @@ -944,7 +944,7 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons requestHost, requestPort, err := net.SplitHostPort(remoteAddr) if err != nil { - return nil, grpc.Errorf(codes.InvalidArgument, "invalid address %s in raft join request", remoteAddr) + return nil, status.Errorf(codes.InvalidArgument, "invalid address %s in raft join request", remoteAddr) } requestIP := net.ParseIP(requestHost) @@ -1118,7 +1118,7 @@ func (n *Node) UpdateNode(id uint64, addr string) { // membership to an active member of the raft func (n *Node) Leave(ctx context.Context, req *api.LeaveRequest) (*api.LeaveResponse, error) { if req.Node == nil { - return nil, grpc.Errorf(codes.InvalidArgument, "no node information provided") + return nil, status.Errorf(codes.InvalidArgument, "no node information provided") } nodeInfo, err := ca.RemoteNode(ctx) @@ -1302,6 +1302,82 @@ func (n *Node) reportNewAddress(ctx context.Context, id uint64) error { return n.transport.UpdatePeerAddr(id, newAddr) } +// StreamRaftMessage is the server endpoint for streaming Raft messages. +// It accepts a stream of raft messages to be processed on this raft member, +// returning a StreamRaftMessageResponse when processing of the streamed +// messages is complete. +// It is called from the Raft leader, which uses it to stream messages +// to this raft member. +// A single stream corresponds to a single raft message, +// which may be disassembled and streamed by the sender +// as individual messages. Therefore, each of the messages +// received by the stream will have the same raft message type and index. +// Currently, only messages of type raftpb.MsgSnap can be disassembled, sent +// and received on the stream. +func (n *Node) StreamRaftMessage(stream api.Raft_StreamRaftMessageServer) error { + // recvdMsg is the current messasge received from the stream. + // assembledMessage is where the data from recvdMsg is appended to. + var recvdMsg, assembledMessage *api.StreamRaftMessageRequest + var err error + + // First message index. + var raftMsgIndex uint64 + + for { + recvdMsg, err = stream.Recv() + if err == io.EOF { + break + } else if err != nil { + log.G(stream.Context()).WithError(err).Error("error while reading from stream") + return err + } + + // Initialized the message to be used for assembling + // the raft message. + if assembledMessage == nil { + // For all message types except raftpb.MsgSnap, + // we don't expect more than a single message + // on the stream so we'll get an EOF on the next Recv() + // and go on to process the received message. + assembledMessage = recvdMsg + raftMsgIndex = recvdMsg.Message.Index + continue + } + + // Verify raft message index. + if recvdMsg.Message.Index != raftMsgIndex { + errMsg := fmt.Sprintf("Raft message chunk with index %d is different from the previously received raft message index %d", + recvdMsg.Message.Index, raftMsgIndex) + log.G(stream.Context()).Errorf(errMsg) + return status.Errorf(codes.InvalidArgument, "%s", errMsg) + } + + // Verify that multiple message received on a stream + // can only be of type raftpb.MsgSnap. + if recvdMsg.Message.Type != raftpb.MsgSnap { + errMsg := fmt.Sprintf("Raft message chunk is not of type %d", + raftpb.MsgSnap) + log.G(stream.Context()).Errorf(errMsg) + return status.Errorf(codes.InvalidArgument, "%s", errMsg) + } + + // Append the received snapshot data. + assembledMessage.Message.Snapshot.Data = append(assembledMessage.Message.Snapshot.Data, recvdMsg.Message.Snapshot.Data...) + } + + // We should have the complete snapshot. Verify and process. + if err == io.EOF { + _, err = n.ProcessRaftMessage(stream.Context(), &api.ProcessRaftMessageRequest{Message: assembledMessage.Message}) + if err == nil { + // Translate the response of ProcessRaftMessage() from + // ProcessRaftMessageResponse to StreamRaftMessageResponse if needed. + return stream.SendAndClose(&api.StreamRaftMessageResponse{}) + } + } + + return err +} + // ProcessRaftMessage calls 'Step' which advances the // raft state machine with the provided message on the // receiving node @@ -1315,7 +1391,7 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa // a node in the remove set if n.cluster.IsIDRemoved(msg.Message.From) { n.processRaftMessageLogger(ctx, msg).Debug("received message from removed member") - return nil, grpc.Errorf(codes.NotFound, "%s", membership.ErrMemberRemoved.Error()) + return nil, status.Errorf(codes.NotFound, "%s", membership.ErrMemberRemoved.Error()) } ctx, cancel := n.WithContext(ctx) @@ -1393,7 +1469,7 @@ func (n *Node) ResolveAddress(ctx context.Context, msg *api.ResolveAddressReques member := n.cluster.GetMember(msg.RaftID) if member == nil { - return nil, grpc.Errorf(codes.NotFound, "member %x not found", msg.RaftID) + return nil, status.Errorf(codes.NotFound, "member %x not found", msg.RaftID) } return &api.ResolveAddressResponse{Addr: member.Addr}, nil } @@ -1497,9 +1573,11 @@ func (n *Node) registerNode(node *api.RaftMember) error { // ProposeValue calls Propose on the underlying raft library(etcd/raft) and waits // on the commit log action before returning a result func (n *Node) ProposeValue(ctx context.Context, storeAction []api.StoreAction, cb func()) error { + defer metrics.StartTimer(proposeLatencyTimer)() ctx, cancel := n.WithContext(ctx) defer cancel() _, err := n.processInternalRaftRequest(ctx, &api.InternalRaftRequest{Action: storeAction}, cb) + if err != nil { return err } @@ -1808,12 +1886,13 @@ func (n *Node) processEntry(ctx context.Context, entry raftpb.Entry) error { } if !n.wait.trigger(r.ID, r) { - log.G(ctx).Errorf("wait not found for raft request id %x", r.ID) - // There was no wait on this ID, meaning we don't have a // transaction in progress that would be committed to the - // memory store by the "trigger" call. Either a different node - // wrote this to raft, or we wrote it before losing the leader + // memory store by the "trigger" call. This could mean that: + // 1. Startup is in progress, and the raft WAL is being parsed, + // processed and applied to the store, or + // 2. Either a different node wrote this to raft, + // or we wrote it before losing the leader // position and cancelling the transaction. This entry still needs // to be committed since other nodes have already committed it. // Create a new transaction to commit this entry. @@ -1827,7 +1906,6 @@ func (n *Node) processEntry(ctx context.Context, entry raftpb.Entry) error { err := n.memoryStore.ApplyStoreActions(r.Action) if err != nil { log.G(ctx).WithError(err).Error("failed to apply actions from raft") - // TODO(anshul) return err here ? } } return nil diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/storage.go b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/storage.go index d12c43af2c..f538317c06 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/storage.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/storage.go @@ -5,6 +5,7 @@ import ( "github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft/raftpb" + "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/encryption" @@ -15,6 +16,18 @@ import ( "golang.org/x/net/context" ) +var ( + // Snapshot create latency timer. + snapshotLatencyTimer metrics.Timer +) + +func init() { + ns := metrics.NewNamespace("swarm", "raft", nil) + snapshotLatencyTimer = ns.NewTimer("snapshot_latency", + "Raft snapshot create latency.") + metrics.Register(ns) +} + func (n *Node) readFromDisk(ctx context.Context) (*raftpb.Snapshot, storage.WALData, error) { keys := n.keyRotator.GetKeys() @@ -169,7 +182,7 @@ func (n *Node) newRaftLogs(nodeID string) (raft.Peer, error) { return raft.Peer{ID: n.Config.ID, Context: metadata}, nil } -func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { +func (n *Node) triggerSnapshot(ctx context.Context, raftConfig api.RaftConfig) { snapshot := api.Snapshot{Version: api.Snapshot_V0} for _, member := range n.cluster.Members() { snapshot.Membership.Members = append(snapshot.Membership.Members, @@ -185,6 +198,9 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { n.asyncTasks.Add(1) n.snapshotInProgress = make(chan raftpb.SnapshotMetadata, 1) // buffered in case Shutdown is called during the snapshot go func(appliedIndex uint64, snapshotMeta raftpb.SnapshotMetadata) { + // Deferred latency capture. + defer metrics.StartTimer(snapshotLatencyTimer)() + defer func() { n.asyncTasks.Done() n.snapshotInProgress <- snapshotMeta diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go index 8c7ca75458..eb849c0803 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go @@ -18,6 +18,11 @@ import ( "github.com/pkg/errors" ) +const ( + // GRPCMaxMsgSize is the max allowed gRPC message size for raft messages. + GRPCMaxMsgSize = 4 << 20 +) + type peer struct { id uint64 @@ -132,17 +137,112 @@ func (p *peer) resolveAddr(ctx context.Context, id uint64) (string, error) { return resp.Addr, nil } -func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error { - timeout := p.tr.config.SendTimeout - // if a snapshot is being sent, set timeout to LargeSendTimeout because - // sending snapshots can take more time than other messages sent between peers. - // The same applies to AppendEntries as well, where messages can get large. - if m.Type == raftpb.MsgSnap || m.Type == raftpb.MsgApp { - timeout = p.tr.config.LargeSendTimeout +// Returns the raft message struct size (not including the payload size) for the given raftpb.Message. +// The payload is typically the snapshot or append entries. +func raftMessageStructSize(m *raftpb.Message) int { + return (&api.ProcessRaftMessageRequest{Message: m}).Size() - len(m.Snapshot.Data) +} + +// Returns the max allowable payload based on MaxRaftMsgSize and +// the struct size for the given raftpb.Message. +func raftMessagePayloadSize(m *raftpb.Message) int { + return GRPCMaxMsgSize - raftMessageStructSize(m) +} + +// Split a large raft message into smaller messages. +// Currently this means splitting the []Snapshot.Data into chunks whose size +// is dictacted by MaxRaftMsgSize. +func splitSnapshotData(ctx context.Context, m *raftpb.Message) []api.StreamRaftMessageRequest { + var messages []api.StreamRaftMessageRequest + if m.Type != raftpb.MsgSnap { + return messages } - ctx, cancel := context.WithTimeout(ctx, timeout) + + // get the size of the data to be split. + size := len(m.Snapshot.Data) + + // Get the max payload size. + payloadSize := raftMessagePayloadSize(m) + + // split the snapshot into smaller messages. + for snapDataIndex := 0; snapDataIndex < size; { + chunkSize := size - snapDataIndex + if chunkSize > payloadSize { + chunkSize = payloadSize + } + + raftMsg := *m + + // sub-slice for this snapshot chunk. + raftMsg.Snapshot.Data = m.Snapshot.Data[snapDataIndex : snapDataIndex+chunkSize] + + snapDataIndex += chunkSize + + // add message to the list of messages to be sent. + msg := api.StreamRaftMessageRequest{Message: &raftMsg} + messages = append(messages, msg) + } + + return messages +} + +// Function to check if this message needs to be split to be streamed +// (because it is larger than GRPCMaxMsgSize). +// Returns true if the message type is MsgSnap +// and size larger than MaxRaftMsgSize. +func needsSplitting(m *raftpb.Message) bool { + raftMsg := api.ProcessRaftMessageRequest{Message: m} + return m.Type == raftpb.MsgSnap && raftMsg.Size() > GRPCMaxMsgSize +} + +func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error { + ctx, cancel := context.WithTimeout(ctx, p.tr.config.SendTimeout) defer cancel() - _, err := api.NewRaftClient(p.conn()).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m}) + + var err error + var stream api.Raft_StreamRaftMessageClient + stream, err = api.NewRaftClient(p.conn()).StreamRaftMessage(ctx) + + if err == nil { + // Split the message if needed. + // Currently only supported for MsgSnap. + var msgs []api.StreamRaftMessageRequest + if needsSplitting(&m) { + msgs = splitSnapshotData(ctx, &m) + } else { + raftMsg := api.StreamRaftMessageRequest{Message: &m} + msgs = append(msgs, raftMsg) + } + + // Stream + for _, msg := range msgs { + err = stream.Send(&msg) + if err != nil { + log.G(ctx).WithError(err).Error("error streaming message to peer") + stream.CloseAndRecv() + break + } + } + + // Finished sending all the messages. + // Close and receive response. + if err == nil { + _, err = stream.CloseAndRecv() + + if err != nil { + log.G(ctx).WithError(err).Error("error receiving response") + } + } + } else { + log.G(ctx).WithError(err).Error("error sending message to peer") + } + + // Try doing a regular rpc if the receiver doesn't support streaming. + if grpc.Code(err) == codes.Unimplemented { + _, err = api.NewRaftClient(p.conn()).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m}) + } + + // Handle errors. if grpc.Code(err) == codes.NotFound && grpc.ErrorDesc(err) == membership.ErrMemberRemoved.Error() { p.tr.config.NodeRemoved() } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go index 6f096ef9b2..b741c4aa67 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go @@ -36,7 +36,6 @@ type Raft interface { type Config struct { HeartbeatInterval time.Duration SendTimeout time.Duration - LargeSendTimeout time.Duration Credentials credentials.TransportCredentials RaftID string diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/state/store/memory.go b/components/engine/vendor/github.com/docker/swarmkit/manager/state/store/memory.go index 01245a6966..e64565fae8 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/state/store/memory.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/state/store/memory.go @@ -11,6 +11,7 @@ import ( "time" "github.com/docker/go-events" + "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" pb "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/state" @@ -76,8 +77,38 @@ var ( // WedgeTimeout is the maximum amount of time the store lock may be // held before declaring a suspected deadlock. WedgeTimeout = 30 * time.Second + + // update()/write tx latency timer. + updateLatencyTimer metrics.Timer + + // view()/read tx latency timer. + viewLatencyTimer metrics.Timer + + // lookup() latency timer. + lookupLatencyTimer metrics.Timer + + // Batch() latency timer. + batchLatencyTimer metrics.Timer + + // timer to capture the duration for which the memory store mutex is locked. + storeLockDurationTimer metrics.Timer ) +func init() { + ns := metrics.NewNamespace("swarm", "store", nil) + updateLatencyTimer = ns.NewTimer("write_tx_latency", + "Raft store write tx latency.") + viewLatencyTimer = ns.NewTimer("read_tx_latency", + "Raft store read tx latency.") + lookupLatencyTimer = ns.NewTimer("lookup_latency", + "Raft store read latency.") + batchLatencyTimer = ns.NewTimer("batch_latency", + "Raft store batch latency.") + storeLockDurationTimer = ns.NewTimer("memory_store_lock_duration", + "Duration for which the raft memory store lock was held.") + metrics.Register(ns) +} + func register(os ObjectStoreConfig) { objectStorers = append(objectStorers, os) schema.Tables[os.Table.Name] = os.Table @@ -94,8 +125,13 @@ func (m *timedMutex) Lock() { m.lockedAt.Store(time.Now()) } +// Unlocks the timedMutex and captures the duration +// for which it was locked in a metric. func (m *timedMutex) Unlock() { + unlockedTimestamp := m.lockedAt.Load() m.Mutex.Unlock() + lockedFor := time.Since(unlockedTimestamp.(time.Time)) + storeLockDurationTimer.Update(lockedFor) m.lockedAt.Store(time.Time{}) } @@ -184,6 +220,7 @@ type readTx struct { // View executes a read transaction. func (s *MemoryStore) View(cb func(ReadTx)) { + defer metrics.StartTimer(viewLatencyTimer)() memDBTx := s.memDB.Txn(false) readTx := readTx{ @@ -280,6 +317,7 @@ func applyStoreAction(tx Tx, sa api.StoreAction) error { } func (s *MemoryStore) update(proposer state.Proposer, cb func(Tx) error) error { + defer metrics.StartTimer(updateLatencyTimer)() s.updateLock.Lock() memDBTx := s.memDB.Txn(true) @@ -329,7 +367,6 @@ func (s *MemoryStore) update(proposer state.Proposer, cb func(Tx) error) error { } s.updateLock.Unlock() return err - } func (s *MemoryStore) updateLocal(cb func(Tx) error) error { @@ -458,6 +495,7 @@ func (batch *Batch) commit() error { // If Batch returns an error, no guarantees are made about how many updates // were committed successfully. func (s *MemoryStore) Batch(cb func(*Batch) error) error { + defer metrics.StartTimer(batchLatencyTimer)() s.updateLock.Lock() batch := Batch{ @@ -498,6 +536,7 @@ func (tx tx) changelistStoreActions() ([]api.StoreAction, error) { // lookup is an internal typed wrapper around memdb. func (tx readTx) lookup(table, index, id string) api.StoreObject { + defer metrics.StartTimer(lookupLatencyTimer)() j, err := tx.memDBTx.First(table, index, id) if err != nil { return nil diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/watchapi/watch.go b/components/engine/vendor/github.com/docker/swarmkit/manager/watchapi/watch.go index 53bed49f1c..223dcb55d6 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/watchapi/watch.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/watchapi/watch.go @@ -1,12 +1,11 @@ package watchapi import ( - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/state" "github.com/docker/swarmkit/manager/state/store" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // Watch starts a stream that returns any changes to objects that match @@ -26,7 +25,7 @@ func (s *Server) Watch(request *api.WatchRequest, stream api.Watch_WatchServer) watchArgs, err := api.ConvertWatchArgs(request.Entries) if err != nil { - return grpc.Errorf(codes.InvalidArgument, "%s", err.Error()) + return status.Errorf(codes.InvalidArgument, "%s", err.Error()) } watchArgs = append(watchArgs, state.EventCommit{}) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/README.md b/components/engine/vendor/github.com/vishvananda/netlink/README.md index 2367fae097..0b61be217e 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/README.md +++ b/components/engine/vendor/github.com/vishvananda/netlink/README.md @@ -38,15 +38,18 @@ Add a new bridge and add eth1 into it: package main import ( - "net" + "fmt" "github.com/vishvananda/netlink" ) func main() { la := netlink.NewLinkAttrs() la.Name = "foo" - mybridge := &netlink.Bridge{la}} - _ := netlink.LinkAdd(mybridge) + mybridge := &netlink.Bridge{LinkAttrs: la} + err := netlink.LinkAdd(mybridge) + if err != nil { + fmt.Printf("could not add %s: %v\n", la.Name, err) + } eth1, _ := netlink.LinkByName("eth1") netlink.LinkSetMaster(eth1, mybridge) } @@ -63,7 +66,6 @@ Add a new ip address to loopback: package main import ( - "net" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/addr_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/addr_linux.go index f33242a7c2..8808b42d9b 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/addr_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/addr_linux.go @@ -2,7 +2,6 @@ package netlink import ( "fmt" - "log" "net" "strings" "syscall" @@ -65,7 +64,7 @@ func (h *Handle) addrHandle(link Link, addr *Addr, req *nl.NetlinkRequest) error msg := nl.NewIfAddrmsg(family) msg.Index = uint32(base.Index) msg.Scope = uint8(addr.Scope) - prefixlen, _ := addr.Mask.Size() + prefixlen, masklen := addr.Mask.Size() msg.Prefixlen = uint8(prefixlen) req.AddData(msg) @@ -103,9 +102,14 @@ func (h *Handle) addrHandle(link Link, addr *Addr, req *nl.NetlinkRequest) error } } - if addr.Broadcast != nil { - req.AddData(nl.NewRtAttr(syscall.IFA_BROADCAST, addr.Broadcast)) + if addr.Broadcast == nil { + calcBroadcast := make(net.IP, masklen/8) + for i := range localAddrData { + calcBroadcast[i] = localAddrData[i] | ^addr.Mask[i] + } + addr.Broadcast = calcBroadcast } + req.AddData(nl.NewRtAttr(syscall.IFA_BROADCAST, addr.Broadcast)) if addr.Label != "" { labelData := nl.NewRtAttr(syscall.IFA_LABEL, nl.ZeroTerminated(addr.Label)) @@ -232,16 +236,34 @@ type AddrUpdate struct { // AddrSubscribe takes a chan down which notifications will be sent // when addresses change. Close the 'done' chan to stop subscription. func AddrSubscribe(ch chan<- AddrUpdate, done <-chan struct{}) error { - return addrSubscribe(netns.None(), netns.None(), ch, done) + return addrSubscribeAt(netns.None(), netns.None(), ch, done, nil) } // AddrSubscribeAt works like AddrSubscribe plus it allows the caller // to choose the network namespace in which to subscribe (ns). func AddrSubscribeAt(ns netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}) error { - return addrSubscribe(ns, netns.None(), ch, done) + return addrSubscribeAt(ns, netns.None(), ch, done, nil) } -func addrSubscribe(newNs, curNs netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}) error { +// AddrSubscribeOptions contains a set of options to use with +// AddrSubscribeWithOptions. +type AddrSubscribeOptions struct { + Namespace *netns.NsHandle + ErrorCallback func(error) +} + +// AddrSubscribeWithOptions work like AddrSubscribe but enable to +// provide additional options to modify the behavior. Currently, the +// namespace can be provided as well as an error callback. +func AddrSubscribeWithOptions(ch chan<- AddrUpdate, done <-chan struct{}, options AddrSubscribeOptions) error { + if options.Namespace == nil { + none := netns.None() + options.Namespace = &none + } + return addrSubscribeAt(*options.Namespace, netns.None(), ch, done, options.ErrorCallback) +} + +func addrSubscribeAt(newNs, curNs netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}, cberr func(error)) error { s, err := nl.SubscribeAt(newNs, curNs, syscall.NETLINK_ROUTE, syscall.RTNLGRP_IPV4_IFADDR, syscall.RTNLGRP_IPV6_IFADDR) if err != nil { return err @@ -257,20 +279,26 @@ func addrSubscribe(newNs, curNs netns.NsHandle, ch chan<- AddrUpdate, done <-cha for { msgs, err := s.Receive() if err != nil { - log.Printf("netlink.AddrSubscribe: Receive() error: %v", err) + if cberr != nil { + cberr(err) + } return } for _, m := range msgs { msgType := m.Header.Type if msgType != syscall.RTM_NEWADDR && msgType != syscall.RTM_DELADDR { - log.Printf("netlink.AddrSubscribe: bad message type: %d", msgType) - continue + if cberr != nil { + cberr(fmt.Errorf("bad message type: %d", msgType)) + } + return } addr, _, ifindex, err := parseAddr(m.Data) if err != nil { - log.Printf("netlink.AddrSubscribe: could not parse address: %v", err) - continue + if cberr != nil { + cberr(fmt.Errorf("could not parse address: %v", err)) + } + return } ch <- AddrUpdate{LinkAddress: *addr.IPNet, diff --git a/components/engine/vendor/github.com/vishvananda/netlink/bridge_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/bridge_linux.go new file mode 100644 index 0000000000..a65d6a1319 --- /dev/null +++ b/components/engine/vendor/github.com/vishvananda/netlink/bridge_linux.go @@ -0,0 +1,115 @@ +package netlink + +import ( + "fmt" + "syscall" + + "github.com/vishvananda/netlink/nl" +) + +// BridgeVlanList gets a map of device id to bridge vlan infos. +// Equivalent to: `bridge vlan show` +func BridgeVlanList() (map[int32][]*nl.BridgeVlanInfo, error) { + return pkgHandle.BridgeVlanList() +} + +// BridgeVlanList gets a map of device id to bridge vlan infos. +// Equivalent to: `bridge vlan show` +func (h *Handle) BridgeVlanList() (map[int32][]*nl.BridgeVlanInfo, error) { + req := h.newNetlinkRequest(syscall.RTM_GETLINK, syscall.NLM_F_DUMP) + msg := nl.NewIfInfomsg(syscall.AF_BRIDGE) + req.AddData(msg) + req.AddData(nl.NewRtAttr(nl.IFLA_EXT_MASK, nl.Uint32Attr(uint32(nl.RTEXT_FILTER_BRVLAN)))) + + msgs, err := req.Execute(syscall.NETLINK_ROUTE, syscall.RTM_NEWLINK) + if err != nil { + return nil, err + } + ret := make(map[int32][]*nl.BridgeVlanInfo) + for _, m := range msgs { + msg := nl.DeserializeIfInfomsg(m) + + attrs, err := nl.ParseRouteAttr(m[msg.Len():]) + if err != nil { + return nil, err + } + for _, attr := range attrs { + switch attr.Attr.Type { + case nl.IFLA_AF_SPEC: + //nested attr + nestAttrs, err := nl.ParseRouteAttr(attr.Value) + if err != nil { + return nil, fmt.Errorf("failed to parse nested attr %v", err) + } + for _, nestAttr := range nestAttrs { + switch nestAttr.Attr.Type { + case nl.IFLA_BRIDGE_VLAN_INFO: + vlanInfo := nl.DeserializeBridgeVlanInfo(nestAttr.Value) + ret[msg.Index] = append(ret[msg.Index], vlanInfo) + } + } + } + } + } + return ret, nil +} + +// BridgeVlanAdd adds a new vlan filter entry +// Equivalent to: `bridge vlan add dev DEV vid VID [ pvid ] [ untagged ] [ self ] [ master ]` +func BridgeVlanAdd(link Link, vid uint16, pvid, untagged, self, master bool) error { + return pkgHandle.BridgeVlanAdd(link, vid, pvid, untagged, self, master) +} + +// BridgeVlanAdd adds a new vlan filter entry +// Equivalent to: `bridge vlan add dev DEV vid VID [ pvid ] [ untagged ] [ self ] [ master ]` +func (h *Handle) BridgeVlanAdd(link Link, vid uint16, pvid, untagged, self, master bool) error { + return h.bridgeVlanModify(syscall.RTM_SETLINK, link, vid, pvid, untagged, self, master) +} + +// BridgeVlanDel adds a new vlan filter entry +// Equivalent to: `bridge vlan del dev DEV vid VID [ pvid ] [ untagged ] [ self ] [ master ]` +func BridgeVlanDel(link Link, vid uint16, pvid, untagged, self, master bool) error { + return pkgHandle.BridgeVlanDel(link, vid, pvid, untagged, self, master) +} + +// BridgeVlanDel adds a new vlan filter entry +// Equivalent to: `bridge vlan del dev DEV vid VID [ pvid ] [ untagged ] [ self ] [ master ]` +func (h *Handle) BridgeVlanDel(link Link, vid uint16, pvid, untagged, self, master bool) error { + return h.bridgeVlanModify(syscall.RTM_DELLINK, link, vid, pvid, untagged, self, master) +} + +func (h *Handle) bridgeVlanModify(cmd int, link Link, vid uint16, pvid, untagged, self, master bool) error { + base := link.Attrs() + h.ensureIndex(base) + req := h.newNetlinkRequest(cmd, syscall.NLM_F_ACK) + + msg := nl.NewIfInfomsg(syscall.AF_BRIDGE) + msg.Index = int32(base.Index) + req.AddData(msg) + + br := nl.NewRtAttr(nl.IFLA_AF_SPEC, nil) + var flags uint16 + if self { + flags |= nl.BRIDGE_FLAGS_SELF + } + if master { + flags |= nl.BRIDGE_FLAGS_MASTER + } + if flags > 0 { + nl.NewRtAttrChild(br, nl.IFLA_BRIDGE_FLAGS, nl.Uint16Attr(flags)) + } + vlanInfo := &nl.BridgeVlanInfo{Vid: vid} + if pvid { + vlanInfo.Flags |= nl.BRIDGE_VLAN_INFO_PVID + } + if untagged { + vlanInfo.Flags |= nl.BRIDGE_VLAN_INFO_UNTAGGED + } + nl.NewRtAttrChild(br, nl.IFLA_BRIDGE_VLAN_INFO, vlanInfo.Serialize()) + req.AddData(br) + _, err := req.Execute(syscall.NETLINK_ROUTE, 0) + if err != nil { + return err + } + return nil +} diff --git a/components/engine/vendor/github.com/vishvananda/netlink/conntrack_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/conntrack_linux.go index 20df903006..ecf0445659 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/conntrack_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/conntrack_linux.go @@ -22,7 +22,11 @@ const ( // https://github.com/torvalds/linux/blob/master/include/uapi/linux/netfilter/nfnetlink.h -> #define NFNL_SUBSYS_CTNETLINK_EXP 2 ConntrackExpectTable = 2 ) - +const ( + // For Parsing Mark + TCP_PROTO = 6 + UDP_PROTO = 17 +) const ( // backward compatibility with golang 1.6 which does not have io.SeekCurrent seekCurrent = 1 @@ -56,7 +60,7 @@ func ConntrackTableFlush(table ConntrackTableType) error { // ConntrackDeleteFilter deletes entries on the specified table on the base of the filter // conntrack -D [table] parameters Delete conntrack or expectation -func ConntrackDeleteFilter(table ConntrackTableType, family InetFamily, filter *ConntrackFilter) (uint, error) { +func ConntrackDeleteFilter(table ConntrackTableType, family InetFamily, filter CustomConntrackFilter) (uint, error) { return pkgHandle.ConntrackDeleteFilter(table, family, filter) } @@ -88,7 +92,7 @@ func (h *Handle) ConntrackTableFlush(table ConntrackTableType) error { // ConntrackDeleteFilter deletes entries on the specified table on the base of the filter using the netlink handle passed // conntrack -D [table] parameters Delete conntrack or expectation -func (h *Handle) ConntrackDeleteFilter(table ConntrackTableType, family InetFamily, filter *ConntrackFilter) (uint, error) { +func (h *Handle) ConntrackDeleteFilter(table ConntrackTableType, family InetFamily, filter CustomConntrackFilter) (uint, error) { res, err := h.dumpConntrackTable(table, family) if err != nil { return 0, err @@ -142,15 +146,16 @@ type ConntrackFlow struct { FamilyType uint8 Forward ipTuple Reverse ipTuple + Mark uint32 } func (s *ConntrackFlow) String() string { // conntrack cmd output: - // udp 17 src=127.0.0.1 dst=127.0.0.1 sport=4001 dport=1234 [UNREPLIED] src=127.0.0.1 dst=127.0.0.1 sport=1234 dport=4001 - return fmt.Sprintf("%s\t%d src=%s dst=%s sport=%d dport=%d\tsrc=%s dst=%s sport=%d dport=%d", + // udp 17 src=127.0.0.1 dst=127.0.0.1 sport=4001 dport=1234 [UNREPLIED] src=127.0.0.1 dst=127.0.0.1 sport=1234 dport=4001 mark=0 + return fmt.Sprintf("%s\t%d src=%s dst=%s sport=%d dport=%d\tsrc=%s dst=%s sport=%d dport=%d mark=%d", nl.L4ProtoMap[s.Forward.Protocol], s.Forward.Protocol, s.Forward.SrcIP.String(), s.Forward.DstIP.String(), s.Forward.SrcPort, s.Forward.DstPort, - s.Reverse.SrcIP.String(), s.Reverse.DstIP.String(), s.Reverse.SrcPort, s.Reverse.DstPort) + s.Reverse.SrcIP.String(), s.Reverse.DstIP.String(), s.Reverse.SrcPort, s.Reverse.DstPort, s.Mark) } // This method parse the ip tuple structure @@ -160,7 +165,7 @@ func (s *ConntrackFlow) String() string { // // // -func parseIpTuple(reader *bytes.Reader, tpl *ipTuple) { +func parseIpTuple(reader *bytes.Reader, tpl *ipTuple) uint8 { for i := 0; i < 2; i++ { _, t, _, v := parseNfAttrTLV(reader) switch t { @@ -189,6 +194,7 @@ func parseIpTuple(reader *bytes.Reader, tpl *ipTuple) { // Skip some padding 2 byte reader.Seek(2, seekCurrent) } + return tpl.Protocol } func parseNfAttrTLV(r *bytes.Reader) (isNested bool, attrType, len uint16, value []byte) { @@ -216,6 +222,7 @@ func parseBERaw16(r *bytes.Reader, v *uint16) { func parseRawData(data []byte) *ConntrackFlow { s := &ConntrackFlow{} + var proto uint8 // First there is the Nfgenmsg header // consume only the family field reader := bytes.NewReader(data) @@ -234,7 +241,7 @@ func parseRawData(data []byte) *ConntrackFlow { nested, t, l := parseNfAttrTL(reader) if nested && t == nl.CTA_TUPLE_ORIG { if nested, t, _ = parseNfAttrTL(reader); nested && t == nl.CTA_TUPLE_IP { - parseIpTuple(reader, &s.Forward) + proto = parseIpTuple(reader, &s.Forward) } } else if nested && t == nl.CTA_TUPLE_REPLY { if nested, t, _ = parseNfAttrTL(reader); nested && t == nl.CTA_TUPLE_IP { @@ -248,7 +255,19 @@ func parseRawData(data []byte) *ConntrackFlow { } } } - + if proto == TCP_PROTO { + reader.Seek(64, seekCurrent) + _, t, _, v := parseNfAttrTLV(reader) + if t == nl.CTA_MARK { + s.Mark = uint32(v[3]) + } + } else if proto == UDP_PROTO { + reader.Seek(16, seekCurrent) + _, t, _, v := parseNfAttrTLV(reader) + if t == nl.CTA_MARK { + s.Mark = uint32(v[3]) + } + } return s } @@ -290,6 +309,12 @@ const ( ConntrackNatAnyIP // -any-nat ip Source or destination NAT ip ) +type CustomConntrackFilter interface { + // MatchConntrackFlow applies the filter to the flow and returns true if the flow matches + // the filter or false otherwise + MatchConntrackFlow(flow *ConntrackFlow) bool +} + type ConntrackFilter struct { ipFilter map[ConntrackFilterType]net.IP } @@ -342,3 +367,5 @@ func (f *ConntrackFilter) MatchConntrackFlow(flow *ConntrackFlow) bool { return match } + +var _ CustomConntrackFilter = (*ConntrackFilter)(nil) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/filter.go b/components/engine/vendor/github.com/vishvananda/netlink/filter.go index 938b28b0b0..1120c79d6a 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/filter.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/filter.go @@ -2,8 +2,6 @@ package netlink import ( "fmt" - - "github.com/vishvananda/netlink/nl" ) type Filter interface { @@ -184,14 +182,6 @@ func NewMirredAction(redirIndex int) *MirredAction { } } -// Constants used in TcU32Sel.Flags. -const ( - TC_U32_TERMINAL = nl.TC_U32_TERMINAL - TC_U32_OFFSET = nl.TC_U32_OFFSET - TC_U32_VAROFFSET = nl.TC_U32_VAROFFSET - TC_U32_EAT = nl.TC_U32_EAT -) - // Sel of the U32 filters that contains multiple TcU32Key. This is the copy // and the frontend representation of nl.TcU32Sel. It is serialized into canonical // nl.TcU32Sel with the appropriate endianness. diff --git a/components/engine/vendor/github.com/vishvananda/netlink/filter_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/filter_linux.go index dc0f90af88..5025bd56c1 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/filter_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/filter_linux.go @@ -11,6 +11,14 @@ import ( "github.com/vishvananda/netlink/nl" ) +// Constants used in TcU32Sel.Flags. +const ( + TC_U32_TERMINAL = nl.TC_U32_TERMINAL + TC_U32_OFFSET = nl.TC_U32_OFFSET + TC_U32_VAROFFSET = nl.TC_U32_VAROFFSET + TC_U32_EAT = nl.TC_U32_EAT +) + // Fw filter filters on firewall marks // NOTE: this is in filter_linux because it refers to nl.TcPolice which // is defined in nl/tc_linux.go @@ -128,9 +136,11 @@ func (h *Handle) FilterAdd(filter Filter) error { req.AddData(nl.NewRtAttr(nl.TCA_KIND, nl.ZeroTerminated(filter.Type()))) options := nl.NewRtAttr(nl.TCA_OPTIONS, nil) - if u32, ok := filter.(*U32); ok { + + switch filter := filter.(type) { + case *U32: // Convert TcU32Sel into nl.TcU32Sel as it is without copy. - sel := (*nl.TcU32Sel)(unsafe.Pointer(u32.Sel)) + sel := (*nl.TcU32Sel)(unsafe.Pointer(filter.Sel)) if sel == nil { // match all sel = &nl.TcU32Sel{ @@ -158,56 +168,56 @@ func (h *Handle) FilterAdd(filter Filter) error { } sel.Nkeys = uint8(len(sel.Keys)) nl.NewRtAttrChild(options, nl.TCA_U32_SEL, sel.Serialize()) - if u32.ClassId != 0 { - nl.NewRtAttrChild(options, nl.TCA_U32_CLASSID, nl.Uint32Attr(u32.ClassId)) + if filter.ClassId != 0 { + nl.NewRtAttrChild(options, nl.TCA_U32_CLASSID, nl.Uint32Attr(filter.ClassId)) } actionsAttr := nl.NewRtAttrChild(options, nl.TCA_U32_ACT, nil) // backwards compatibility - if u32.RedirIndex != 0 { - u32.Actions = append([]Action{NewMirredAction(u32.RedirIndex)}, u32.Actions...) + if filter.RedirIndex != 0 { + filter.Actions = append([]Action{NewMirredAction(filter.RedirIndex)}, filter.Actions...) } - if err := EncodeActions(actionsAttr, u32.Actions); err != nil { + if err := EncodeActions(actionsAttr, filter.Actions); err != nil { return err } - } else if fw, ok := filter.(*Fw); ok { - if fw.Mask != 0 { + case *Fw: + if filter.Mask != 0 { b := make([]byte, 4) - native.PutUint32(b, fw.Mask) + native.PutUint32(b, filter.Mask) nl.NewRtAttrChild(options, nl.TCA_FW_MASK, b) } - if fw.InDev != "" { - nl.NewRtAttrChild(options, nl.TCA_FW_INDEV, nl.ZeroTerminated(fw.InDev)) + if filter.InDev != "" { + nl.NewRtAttrChild(options, nl.TCA_FW_INDEV, nl.ZeroTerminated(filter.InDev)) } - if (fw.Police != nl.TcPolice{}) { + if (filter.Police != nl.TcPolice{}) { police := nl.NewRtAttrChild(options, nl.TCA_FW_POLICE, nil) - nl.NewRtAttrChild(police, nl.TCA_POLICE_TBF, fw.Police.Serialize()) - if (fw.Police.Rate != nl.TcRateSpec{}) { - payload := SerializeRtab(fw.Rtab) + nl.NewRtAttrChild(police, nl.TCA_POLICE_TBF, filter.Police.Serialize()) + if (filter.Police.Rate != nl.TcRateSpec{}) { + payload := SerializeRtab(filter.Rtab) nl.NewRtAttrChild(police, nl.TCA_POLICE_RATE, payload) } - if (fw.Police.PeakRate != nl.TcRateSpec{}) { - payload := SerializeRtab(fw.Ptab) + if (filter.Police.PeakRate != nl.TcRateSpec{}) { + payload := SerializeRtab(filter.Ptab) nl.NewRtAttrChild(police, nl.TCA_POLICE_PEAKRATE, payload) } } - if fw.ClassId != 0 { + if filter.ClassId != 0 { b := make([]byte, 4) - native.PutUint32(b, fw.ClassId) + native.PutUint32(b, filter.ClassId) nl.NewRtAttrChild(options, nl.TCA_FW_CLASSID, b) } - } else if bpf, ok := filter.(*BpfFilter); ok { + case *BpfFilter: var bpfFlags uint32 - if bpf.ClassId != 0 { - nl.NewRtAttrChild(options, nl.TCA_BPF_CLASSID, nl.Uint32Attr(bpf.ClassId)) + if filter.ClassId != 0 { + nl.NewRtAttrChild(options, nl.TCA_BPF_CLASSID, nl.Uint32Attr(filter.ClassId)) } - if bpf.Fd >= 0 { - nl.NewRtAttrChild(options, nl.TCA_BPF_FD, nl.Uint32Attr((uint32(bpf.Fd)))) + if filter.Fd >= 0 { + nl.NewRtAttrChild(options, nl.TCA_BPF_FD, nl.Uint32Attr((uint32(filter.Fd)))) } - if bpf.Name != "" { - nl.NewRtAttrChild(options, nl.TCA_BPF_NAME, nl.ZeroTerminated(bpf.Name)) + if filter.Name != "" { + nl.NewRtAttrChild(options, nl.TCA_BPF_NAME, nl.ZeroTerminated(filter.Name)) } - if bpf.DirectAction { + if filter.DirectAction { bpfFlags |= nl.TCA_BPF_FLAG_ACT_DIRECT } nl.NewRtAttrChild(options, nl.TCA_BPF_FLAGS, nl.Uint32Attr(bpfFlags)) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/handle_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/handle_linux.go index a04ceae6b6..d37b087c33 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/handle_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/handle_linux.go @@ -45,12 +45,27 @@ func (h *Handle) SetSocketTimeout(to time.Duration) error { } tv := syscall.NsecToTimeval(to.Nanoseconds()) for _, sh := range h.sockets { - fd := sh.Socket.GetFd() - err := syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, &tv) - if err != nil { + if err := sh.Socket.SetSendTimeout(&tv); err != nil { return err } - err = syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, syscall.SO_SNDTIMEO, &tv) + if err := sh.Socket.SetReceiveTimeout(&tv); err != nil { + return err + } + } + return nil +} + +// SetSocketReceiveBufferSize sets the receive buffer size for each +// socket in the netlink handle. The maximum value is capped by +// /proc/sys/net/core/rmem_max. +func (h *Handle) SetSocketReceiveBufferSize(size int, force bool) error { + opt := syscall.SO_RCVBUF + if force { + opt = syscall.SO_RCVBUFFORCE + } + for _, sh := range h.sockets { + fd := sh.Socket.GetFd() + err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, opt, size) if err != nil { return err } @@ -58,6 +73,24 @@ func (h *Handle) SetSocketTimeout(to time.Duration) error { return nil } +// GetSocketReceiveBufferSize gets the receiver buffer size for each +// socket in the netlink handle. The retrieved value should be the +// double to the one set for SetSocketReceiveBufferSize. +func (h *Handle) GetSocketReceiveBufferSize() ([]int, error) { + results := make([]int, len(h.sockets)) + i := 0 + for _, sh := range h.sockets { + fd := sh.Socket.GetFd() + size, err := syscall.GetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF) + if err != nil { + return nil, err + } + results[i] = size + i++ + } + return results, nil +} + // NewHandle returns a netlink handle on the network namespace // specified by ns. If ns=netns.None(), current network namespace // will be assumed diff --git a/components/engine/vendor/github.com/vishvananda/netlink/handle_unspecified.go b/components/engine/vendor/github.com/vishvananda/netlink/handle_unspecified.go index 32cf022732..7da21a6a18 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/handle_unspecified.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/handle_unspecified.go @@ -145,6 +145,10 @@ func (h *Handle) LinkSetFlood(link Link, mode bool) error { return ErrNotImplemented } +func (h *Handle) LinkSetTxQLen(link Link, qlen int) error { + return ErrNotImplemented +} + func (h *Handle) setProtinfoAttr(link Link, mode bool, attr int) error { return ErrNotImplemented } diff --git a/components/engine/vendor/github.com/vishvananda/netlink/link.go b/components/engine/vendor/github.com/vishvananda/netlink/link.go index 547e92ec12..5aa3a1790a 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/link.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/link.go @@ -37,6 +37,7 @@ type LinkAttrs struct { EncapType string Protinfo *Protinfo OperState LinkOperState + NetNsID int } // LinkOperState represents the values of the IFLA_OPERSTATE link @@ -171,6 +172,7 @@ type LinkXdp struct { Fd int Attached bool Flags uint32 + ProgId uint32 } // Device links cannot be created via netlink. These links @@ -339,6 +341,7 @@ type Vxlan struct { UDPCSum bool NoAge bool GBP bool + FlowBased bool Age int Limit int Port int @@ -684,6 +687,7 @@ type Gretap struct { EncapType uint16 EncapFlags uint16 Link uint32 + FlowBased bool } func (gretap *Gretap) Attrs() *LinkAttrs { @@ -729,6 +733,28 @@ func (iptun *Vti) Type() string { return "vti" } +type Gretun struct { + LinkAttrs + Link uint32 + IFlags uint16 + OFlags uint16 + IKey uint32 + OKey uint32 + Local net.IP + Remote net.IP + Ttl uint8 + Tos uint8 + PMtuDisc uint8 +} + +func (gretun *Gretun) Attrs() *LinkAttrs { + return &gretun.LinkAttrs +} + +func (gretun *Gretun) Type() string { + return "gre" +} + type Vrf struct { LinkAttrs Table uint32 diff --git a/components/engine/vendor/github.com/vishvananda/netlink/link_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/link_linux.go index 1c1bc52c4d..e94fd9766c 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/link_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/link_linux.go @@ -379,6 +379,74 @@ func (h *Handle) LinkSetVfTxRate(link Link, vf, rate int) error { return err } +// LinkSetVfSpoofchk enables/disables spoof check on a vf for the link. +// Equivalent to: `ip link set $link vf $vf spoofchk $check` +func LinkSetVfSpoofchk(link Link, vf int, check bool) error { + return pkgHandle.LinkSetVfSpoofchk(link, vf, check) +} + +// LinkSetVfSpookfchk enables/disables spoof check on a vf for the link. +// Equivalent to: `ip link set $link vf $vf spoofchk $check` +func (h *Handle) LinkSetVfSpoofchk(link Link, vf int, check bool) error { + var setting uint32 + base := link.Attrs() + h.ensureIndex(base) + req := h.newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) + + msg := nl.NewIfInfomsg(syscall.AF_UNSPEC) + msg.Index = int32(base.Index) + req.AddData(msg) + + data := nl.NewRtAttr(nl.IFLA_VFINFO_LIST, nil) + info := nl.NewRtAttrChild(data, nl.IFLA_VF_INFO, nil) + if check { + setting = 1 + } + vfmsg := nl.VfSpoofchk{ + Vf: uint32(vf), + Setting: setting, + } + nl.NewRtAttrChild(info, nl.IFLA_VF_SPOOFCHK, vfmsg.Serialize()) + req.AddData(data) + + _, err := req.Execute(syscall.NETLINK_ROUTE, 0) + return err +} + +// LinkSetVfTrust enables/disables trust state on a vf for the link. +// Equivalent to: `ip link set $link vf $vf trust $state` +func LinkSetVfTrust(link Link, vf int, state bool) error { + return pkgHandle.LinkSetVfTrust(link, vf, state) +} + +// LinkSetVfTrust enables/disables trust state on a vf for the link. +// Equivalent to: `ip link set $link vf $vf trust $state` +func (h *Handle) LinkSetVfTrust(link Link, vf int, state bool) error { + var setting uint32 + base := link.Attrs() + h.ensureIndex(base) + req := h.newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) + + msg := nl.NewIfInfomsg(syscall.AF_UNSPEC) + msg.Index = int32(base.Index) + req.AddData(msg) + + data := nl.NewRtAttr(nl.IFLA_VFINFO_LIST, nil) + info := nl.NewRtAttrChild(data, nl.IFLA_VF_INFO, nil) + if state { + setting = 1 + } + vfmsg := nl.VfTrust{ + Vf: uint32(vf), + Setting: setting, + } + nl.NewRtAttrChild(info, nl.IFLA_VF_TRUST, vfmsg.Serialize()) + req.AddData(data) + + _, err := req.Execute(syscall.NETLINK_ROUTE, 0) + return err +} + // LinkSetMaster sets the master of the link device. // Equivalent to: `ip link set $link master $master` func LinkSetMaster(link Link, master *Bridge) error { @@ -500,6 +568,12 @@ func (h *Handle) LinkSetNsFd(link Link, fd int) error { // LinkSetXdpFd adds a bpf function to the driver. The fd must be a bpf // program loaded with bpf(type=BPF_PROG_TYPE_XDP) func LinkSetXdpFd(link Link, fd int) error { + return LinkSetXdpFdWithFlags(link, fd, 0) +} + +// LinkSetXdpFdWithFlags adds a bpf function to the driver with the given +// options. The fd must be a bpf program loaded with bpf(type=BPF_PROG_TYPE_XDP) +func LinkSetXdpFdWithFlags(link Link, fd, flags int) error { base := link.Attrs() ensureIndex(base) req := nl.NewNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) @@ -508,7 +582,7 @@ func LinkSetXdpFd(link Link, fd int) error { msg.Index = int32(base.Index) req.AddData(msg) - addXdpAttrs(&LinkXdp{Fd: fd}, req) + addXdpAttrs(&LinkXdp{Fd: fd, Flags: uint32(flags)}, req) _, err := req.Execute(syscall.NETLINK_ROUTE, 0) return err @@ -528,7 +602,13 @@ type vxlanPortRange struct { func addVxlanAttrs(vxlan *Vxlan, linkInfo *nl.RtAttr) { data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) + + if vxlan.FlowBased { + vxlan.VxlanId = 0 + } + nl.NewRtAttrChild(data, nl.IFLA_VXLAN_ID, nl.Uint32Attr(uint32(vxlan.VxlanId))) + if vxlan.VtepDevIndex != 0 { nl.NewRtAttrChild(data, nl.IFLA_VXLAN_LINK, nl.Uint32Attr(uint32(vxlan.VtepDevIndex))) } @@ -569,6 +649,9 @@ func addVxlanAttrs(vxlan *Vxlan, linkInfo *nl.RtAttr) { if vxlan.GBP { nl.NewRtAttrChild(data, nl.IFLA_VXLAN_GBP, []byte{}) } + if vxlan.FlowBased { + nl.NewRtAttrChild(data, nl.IFLA_VXLAN_FLOWBASED, boolAttr(vxlan.FlowBased)) + } if vxlan.NoAge { nl.NewRtAttrChild(data, nl.IFLA_VXLAN_AGEING, nl.Uint32Attr(0)) } else if vxlan.Age > 0 { @@ -818,16 +901,17 @@ func (h *Handle) linkModify(link Link, flags int) error { linkInfo := nl.NewRtAttr(syscall.IFLA_LINKINFO, nil) nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_KIND, nl.NonZeroTerminated(link.Type())) - if vlan, ok := link.(*Vlan); ok { + switch link := link.(type) { + case *Vlan: b := make([]byte, 2) - native.PutUint16(b, uint16(vlan.VlanId)) + native.PutUint16(b, uint16(link.VlanId)) data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) nl.NewRtAttrChild(data, nl.IFLA_VLAN_ID, b) - } else if veth, ok := link.(*Veth); ok { + case *Veth: data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) peer := nl.NewRtAttrChild(data, nl.VETH_INFO_PEER, nil) nl.NewIfInfomsgChild(peer, syscall.AF_UNSPEC) - nl.NewRtAttrChild(peer, syscall.IFLA_IFNAME, nl.ZeroTerminated(veth.PeerName)) + nl.NewRtAttrChild(peer, syscall.IFLA_IFNAME, nl.ZeroTerminated(link.PeerName)) if base.TxQLen >= 0 { nl.NewRtAttrChild(peer, syscall.IFLA_TXQLEN, nl.Uint32Attr(uint32(base.TxQLen))) } @@ -835,35 +919,37 @@ func (h *Handle) linkModify(link Link, flags int) error { nl.NewRtAttrChild(peer, syscall.IFLA_MTU, nl.Uint32Attr(uint32(base.MTU))) } - } else if vxlan, ok := link.(*Vxlan); ok { - addVxlanAttrs(vxlan, linkInfo) - } else if bond, ok := link.(*Bond); ok { - addBondAttrs(bond, linkInfo) - } else if ipv, ok := link.(*IPVlan); ok { + case *Vxlan: + addVxlanAttrs(link, linkInfo) + case *Bond: + addBondAttrs(link, linkInfo) + case *IPVlan: data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) - nl.NewRtAttrChild(data, nl.IFLA_IPVLAN_MODE, nl.Uint16Attr(uint16(ipv.Mode))) - } else if macv, ok := link.(*Macvlan); ok { - if macv.Mode != MACVLAN_MODE_DEFAULT { + nl.NewRtAttrChild(data, nl.IFLA_IPVLAN_MODE, nl.Uint16Attr(uint16(link.Mode))) + case *Macvlan: + if link.Mode != MACVLAN_MODE_DEFAULT { data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) - nl.NewRtAttrChild(data, nl.IFLA_MACVLAN_MODE, nl.Uint32Attr(macvlanModes[macv.Mode])) + nl.NewRtAttrChild(data, nl.IFLA_MACVLAN_MODE, nl.Uint32Attr(macvlanModes[link.Mode])) } - } else if macv, ok := link.(*Macvtap); ok { - if macv.Mode != MACVLAN_MODE_DEFAULT { + case *Macvtap: + if link.Mode != MACVLAN_MODE_DEFAULT { data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) - nl.NewRtAttrChild(data, nl.IFLA_MACVLAN_MODE, nl.Uint32Attr(macvlanModes[macv.Mode])) + nl.NewRtAttrChild(data, nl.IFLA_MACVLAN_MODE, nl.Uint32Attr(macvlanModes[link.Mode])) } - } else if gretap, ok := link.(*Gretap); ok { - addGretapAttrs(gretap, linkInfo) - } else if iptun, ok := link.(*Iptun); ok { - addIptunAttrs(iptun, linkInfo) - } else if vti, ok := link.(*Vti); ok { - addVtiAttrs(vti, linkInfo) - } else if vrf, ok := link.(*Vrf); ok { - addVrfAttrs(vrf, linkInfo) - } else if bridge, ok := link.(*Bridge); ok { - addBridgeAttrs(bridge, linkInfo) - } else if gtp, ok := link.(*GTP); ok { - addGTPAttrs(gtp, linkInfo) + case *Gretap: + addGretapAttrs(link, linkInfo) + case *Iptun: + addIptunAttrs(link, linkInfo) + case *Gretun: + addGretunAttrs(link, linkInfo) + case *Vti: + addVtiAttrs(link, linkInfo) + case *Vrf: + addVrfAttrs(link, linkInfo) + case *Bridge: + addBridgeAttrs(link, linkInfo) + case *GTP: + addGTPAttrs(link, linkInfo) } req.AddData(linkInfo) @@ -1093,6 +1179,8 @@ func LinkDeserialize(hdr *syscall.NlMsghdr, m []byte) (Link, error) { link = &Gretap{} case "ipip": link = &Iptun{} + case "gre": + link = &Gretun{} case "vti": link = &Vti{} case "vrf": @@ -1124,6 +1212,8 @@ func LinkDeserialize(hdr *syscall.NlMsghdr, m []byte) (Link, error) { parseGretapData(link, data) case "ipip": parseIptunData(link, data) + case "gre": + parseGretunData(link, data) case "vti": parseVtiData(link, data) case "vrf": @@ -1178,6 +1268,8 @@ func LinkDeserialize(hdr *syscall.NlMsghdr, m []byte) (Link, error) { } case syscall.IFLA_OPERSTATE: base.OperState = LinkOperState(uint8(attr.Value[0])) + case nl.IFLA_LINK_NETNSID: + base.NetNsID = int(native.Uint32(attr.Value[0:4])) } } @@ -1239,16 +1331,34 @@ type LinkUpdate struct { // LinkSubscribe takes a chan down which notifications will be sent // when links change. Close the 'done' chan to stop subscription. func LinkSubscribe(ch chan<- LinkUpdate, done <-chan struct{}) error { - return linkSubscribe(netns.None(), netns.None(), ch, done) + return linkSubscribeAt(netns.None(), netns.None(), ch, done, nil) } // LinkSubscribeAt works like LinkSubscribe plus it allows the caller // to choose the network namespace in which to subscribe (ns). func LinkSubscribeAt(ns netns.NsHandle, ch chan<- LinkUpdate, done <-chan struct{}) error { - return linkSubscribe(ns, netns.None(), ch, done) + return linkSubscribeAt(ns, netns.None(), ch, done, nil) } -func linkSubscribe(newNs, curNs netns.NsHandle, ch chan<- LinkUpdate, done <-chan struct{}) error { +// LinkSubscribeOptions contains a set of options to use with +// LinkSubscribeWithOptions. +type LinkSubscribeOptions struct { + Namespace *netns.NsHandle + ErrorCallback func(error) +} + +// LinkSubscribeWithOptions work like LinkSubscribe but enable to +// provide additional options to modify the behavior. Currently, the +// namespace can be provided as well as an error callback. +func LinkSubscribeWithOptions(ch chan<- LinkUpdate, done <-chan struct{}, options LinkSubscribeOptions) error { + if options.Namespace == nil { + none := netns.None() + options.Namespace = &none + } + return linkSubscribeAt(*options.Namespace, netns.None(), ch, done, options.ErrorCallback) +} + +func linkSubscribeAt(newNs, curNs netns.NsHandle, ch chan<- LinkUpdate, done <-chan struct{}, cberr func(error)) error { s, err := nl.SubscribeAt(newNs, curNs, syscall.NETLINK_ROUTE, syscall.RTNLGRP_LINK) if err != nil { return err @@ -1264,12 +1374,18 @@ func linkSubscribe(newNs, curNs netns.NsHandle, ch chan<- LinkUpdate, done <-cha for { msgs, err := s.Receive() if err != nil { + if cberr != nil { + cberr(err) + } return } for _, m := range msgs { ifmsg := nl.DeserializeIfInfomsg(m.Data) link, err := LinkDeserialize(&m.Header, m.Data) if err != nil { + if cberr != nil { + cberr(err) + } return } ch <- LinkUpdate{IfInfomsg: *ifmsg, Header: m.Header, Link: link} @@ -1363,6 +1479,33 @@ func (h *Handle) setProtinfoAttr(link Link, mode bool, attr int) error { return nil } +// LinkSetTxQLen sets the transaction queue length for the link. +// Equivalent to: `ip link set $link txqlen $qlen` +func LinkSetTxQLen(link Link, qlen int) error { + return pkgHandle.LinkSetTxQLen(link, qlen) +} + +// LinkSetTxQLen sets the transaction queue length for the link. +// Equivalent to: `ip link set $link txqlen $qlen` +func (h *Handle) LinkSetTxQLen(link Link, qlen int) error { + base := link.Attrs() + h.ensureIndex(base) + req := h.newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) + + msg := nl.NewIfInfomsg(syscall.AF_UNSPEC) + msg.Index = int32(base.Index) + req.AddData(msg) + + b := make([]byte, 4) + native.PutUint32(b, uint32(qlen)) + + data := nl.NewRtAttr(syscall.IFLA_TXQLEN, b) + req.AddData(data) + + _, err := req.Execute(syscall.NETLINK_ROUTE, 0) + return err +} + func parseVlanData(link Link, data []syscall.NetlinkRouteAttr) { vlan := link.(*Vlan) for _, datum := range data { @@ -1407,6 +1550,8 @@ func parseVxlanData(link Link, data []syscall.NetlinkRouteAttr) { vxlan.UDPCSum = int8(datum.Value[0]) != 0 case nl.IFLA_VXLAN_GBP: vxlan.GBP = true + case nl.IFLA_VXLAN_FLOWBASED: + vxlan.FlowBased = int8(datum.Value[0]) != 0 case nl.IFLA_VXLAN_AGEING: vxlan.Age = int(native.Uint32(datum.Value[0:4])) vxlan.NoAge = vxlan.Age == 0 @@ -1547,6 +1692,12 @@ func linkFlags(rawFlags uint32) net.Flags { func addGretapAttrs(gretap *Gretap, linkInfo *nl.RtAttr) { data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) + if gretap.FlowBased { + // In flow based mode, no other attributes need to be configured + nl.NewRtAttrChild(data, nl.IFLA_GRE_COLLECT_METADATA, boolAttr(gretap.FlowBased)) + return + } + ip := gretap.Local.To4() if ip != nil { nl.NewRtAttrChild(data, nl.IFLA_GRE_LOCAL, []byte(ip)) @@ -1613,6 +1764,69 @@ func parseGretapData(link Link, data []syscall.NetlinkRouteAttr) { gre.EncapType = native.Uint16(datum.Value[0:2]) case nl.IFLA_GRE_ENCAP_FLAGS: gre.EncapFlags = native.Uint16(datum.Value[0:2]) + case nl.IFLA_GRE_COLLECT_METADATA: + gre.FlowBased = int8(datum.Value[0]) != 0 + } + } +} + +func addGretunAttrs(gre *Gretun, linkInfo *nl.RtAttr) { + data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) + + ip := gre.Local.To4() + if ip != nil { + nl.NewRtAttrChild(data, nl.IFLA_GRE_LOCAL, []byte(ip)) + } + ip = gre.Remote.To4() + if ip != nil { + nl.NewRtAttrChild(data, nl.IFLA_GRE_REMOTE, []byte(ip)) + } + + if gre.IKey != 0 { + nl.NewRtAttrChild(data, nl.IFLA_GRE_IKEY, htonl(gre.IKey)) + gre.IFlags |= uint16(nl.GRE_KEY) + } + + if gre.OKey != 0 { + nl.NewRtAttrChild(data, nl.IFLA_GRE_OKEY, htonl(gre.OKey)) + gre.OFlags |= uint16(nl.GRE_KEY) + } + + nl.NewRtAttrChild(data, nl.IFLA_GRE_IFLAGS, htons(gre.IFlags)) + nl.NewRtAttrChild(data, nl.IFLA_GRE_OFLAGS, htons(gre.OFlags)) + + if gre.Link != 0 { + nl.NewRtAttrChild(data, nl.IFLA_GRE_LINK, nl.Uint32Attr(gre.Link)) + } + + nl.NewRtAttrChild(data, nl.IFLA_GRE_PMTUDISC, nl.Uint8Attr(gre.PMtuDisc)) + nl.NewRtAttrChild(data, nl.IFLA_GRE_TTL, nl.Uint8Attr(gre.Ttl)) + nl.NewRtAttrChild(data, nl.IFLA_GRE_TOS, nl.Uint8Attr(gre.Tos)) +} + +func parseGretunData(link Link, data []syscall.NetlinkRouteAttr) { + gre := link.(*Gretun) + for _, datum := range data { + switch datum.Attr.Type { + case nl.IFLA_GRE_OKEY: + gre.IKey = ntohl(datum.Value[0:4]) + case nl.IFLA_GRE_IKEY: + gre.OKey = ntohl(datum.Value[0:4]) + case nl.IFLA_GRE_LOCAL: + gre.Local = net.IP(datum.Value[0:4]) + case nl.IFLA_GRE_REMOTE: + gre.Remote = net.IP(datum.Value[0:4]) + case nl.IFLA_GRE_IFLAGS: + gre.IFlags = ntohs(datum.Value[0:2]) + case nl.IFLA_GRE_OFLAGS: + gre.OFlags = ntohs(datum.Value[0:2]) + + case nl.IFLA_GRE_TTL: + gre.Ttl = uint8(datum.Value[0]) + case nl.IFLA_GRE_TOS: + gre.Tos = uint8(datum.Value[0]) + case nl.IFLA_GRE_PMTUDISC: + gre.PMtuDisc = uint8(datum.Value[0]) } } } @@ -1630,8 +1844,10 @@ func addXdpAttrs(xdp *LinkXdp, req *nl.NetlinkRequest) { b := make([]byte, 4) native.PutUint32(b, uint32(xdp.Fd)) nl.NewRtAttrChild(attrs, nl.IFLA_XDP_FD, b) - native.PutUint32(b, xdp.Flags) - nl.NewRtAttrChild(attrs, nl.IFLA_XDP_FLAGS, b) + if xdp.Flags != 0 { + native.PutUint32(b, xdp.Flags) + nl.NewRtAttrChild(attrs, nl.IFLA_XDP_FLAGS, b) + } req.AddData(attrs) } @@ -1649,6 +1865,8 @@ func parseLinkXdp(data []byte) (*LinkXdp, error) { xdp.Attached = attr.Value[0] != 0 case nl.IFLA_XDP_FLAGS: xdp.Flags = native.Uint32(attr.Value[0:4]) + case nl.IFLA_XDP_PROG_ID: + xdp.ProgId = native.Uint32(attr.Value[0:4]) } } return xdp, nil diff --git a/components/engine/vendor/github.com/vishvananda/netlink/neigh.go b/components/engine/vendor/github.com/vishvananda/netlink/neigh.go index 0e5eb90c9e..6a6f71ce86 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/neigh.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/neigh.go @@ -14,6 +14,7 @@ type Neigh struct { Flags int IP net.IP HardwareAddr net.HardwareAddr + LLIPAddr net.IP //Used in the case of NHRP } // String returns $ip/$hwaddr $label diff --git a/components/engine/vendor/github.com/vishvananda/netlink/neigh_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/neigh_linux.go index f069db2573..5edc8b4125 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/neigh_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/neigh_linux.go @@ -128,6 +128,7 @@ func (h *Handle) NeighDel(neigh *Neigh) error { func neighHandle(neigh *Neigh, req *nl.NetlinkRequest) error { var family int + if neigh.Family > 0 { family = neigh.Family } else { @@ -151,7 +152,10 @@ func neighHandle(neigh *Neigh, req *nl.NetlinkRequest) error { dstData := nl.NewRtAttr(NDA_DST, ipData) req.AddData(dstData) - if neigh.Flags != NTF_PROXY || neigh.HardwareAddr != nil { + if neigh.LLIPAddr != nil { + llIPData := nl.NewRtAttr(NDA_LLADDR, neigh.LLIPAddr.To4()) + req.AddData(llIPData) + } else if neigh.Flags != NTF_PROXY || neigh.HardwareAddr != nil { hwData := nl.NewRtAttr(NDA_LLADDR, []byte(neigh.HardwareAddr)) req.AddData(hwData) } @@ -237,12 +241,33 @@ func NeighDeserialize(m []byte) (*Neigh, error) { return nil, err } + // This should be cached for perfomance + // once per table dump + link, err := LinkByIndex(neigh.LinkIndex) + if err != nil { + return nil, err + } + encapType := link.Attrs().EncapType + for _, attr := range attrs { switch attr.Attr.Type { case NDA_DST: neigh.IP = net.IP(attr.Value) case NDA_LLADDR: - neigh.HardwareAddr = net.HardwareAddr(attr.Value) + // BUG: Is this a bug in the netlink library? + // #define RTA_LENGTH(len) (RTA_ALIGN(sizeof(struct rtattr)) + (len)) + // #define RTA_PAYLOAD(rta) ((int)((rta)->rta_len) - RTA_LENGTH(0)) + attrLen := attr.Attr.Len - syscall.SizeofRtAttr + if attrLen == 4 && (encapType == "ipip" || + encapType == "sit" || + encapType == "gre") { + neigh.LLIPAddr = net.IP(attr.Value) + } else if attrLen == 16 && + encapType == "tunnel6" { + neigh.IP = net.IP(attr.Value) + } else { + neigh.HardwareAddr = net.HardwareAddr(attr.Value) + } } } diff --git a/components/engine/vendor/github.com/vishvananda/netlink/netlink_unspecified.go b/components/engine/vendor/github.com/vishvananda/netlink/netlink_unspecified.go index 2d57c16d74..86111b92ce 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/netlink_unspecified.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/netlink_unspecified.go @@ -108,6 +108,10 @@ func LinkSetFlood(link Link, mode bool) error { return ErrNotImplemented } +func LinkSetTxQLen(link Link, qlen int) error { + return ErrNotImplemented +} + func LinkAdd(link Link) error { return ErrNotImplemented } diff --git a/components/engine/vendor/github.com/vishvananda/netlink/nl/bridge_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/nl/bridge_linux.go new file mode 100644 index 0000000000..6c0d333387 --- /dev/null +++ b/components/engine/vendor/github.com/vishvananda/netlink/nl/bridge_linux.go @@ -0,0 +1,74 @@ +package nl + +import ( + "fmt" + "unsafe" +) + +const ( + SizeofBridgeVlanInfo = 0x04 +) + +/* Bridge Flags */ +const ( + BRIDGE_FLAGS_MASTER = iota /* Bridge command to/from master */ + BRIDGE_FLAGS_SELF /* Bridge command to/from lowerdev */ +) + +/* Bridge management nested attributes + * [IFLA_AF_SPEC] = { + * [IFLA_BRIDGE_FLAGS] + * [IFLA_BRIDGE_MODE] + * [IFLA_BRIDGE_VLAN_INFO] + * } + */ +const ( + IFLA_BRIDGE_FLAGS = iota + IFLA_BRIDGE_MODE + IFLA_BRIDGE_VLAN_INFO +) + +const ( + BRIDGE_VLAN_INFO_MASTER = 1 << iota + BRIDGE_VLAN_INFO_PVID + BRIDGE_VLAN_INFO_UNTAGGED + BRIDGE_VLAN_INFO_RANGE_BEGIN + BRIDGE_VLAN_INFO_RANGE_END +) + +// struct bridge_vlan_info { +// __u16 flags; +// __u16 vid; +// }; + +type BridgeVlanInfo struct { + Flags uint16 + Vid uint16 +} + +func (b *BridgeVlanInfo) Serialize() []byte { + return (*(*[SizeofBridgeVlanInfo]byte)(unsafe.Pointer(b)))[:] +} + +func DeserializeBridgeVlanInfo(b []byte) *BridgeVlanInfo { + return (*BridgeVlanInfo)(unsafe.Pointer(&b[0:SizeofBridgeVlanInfo][0])) +} + +func (b *BridgeVlanInfo) PortVID() bool { + return b.Flags&BRIDGE_VLAN_INFO_PVID > 0 +} + +func (b *BridgeVlanInfo) EngressUntag() bool { + return b.Flags&BRIDGE_VLAN_INFO_UNTAGGED > 0 +} + +func (b *BridgeVlanInfo) String() string { + return fmt.Sprintf("%+v", *b) +} + +/* New extended info filters for IFLA_EXT_MASK */ +const ( + RTEXT_FILTER_VF = 1 << iota + RTEXT_FILTER_BRVLAN + RTEXT_FILTER_BRVLAN_COMPRESSED +) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/nl/conntrack_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/nl/conntrack_linux.go index 6692b53ed9..380cc5967b 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/nl/conntrack_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/nl/conntrack_linux.go @@ -79,8 +79,8 @@ const ( CTA_TUPLE_ORIG = 1 CTA_TUPLE_REPLY = 2 CTA_STATUS = 3 - CTA_TIMEOUT = 8 - CTA_MARK = 9 + CTA_TIMEOUT = 7 + CTA_MARK = 8 CTA_PROTOINFO = 4 ) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/nl/link_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/nl/link_linux.go index f7b9575919..9ae65a12c2 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/nl/link_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/nl/link_linux.go @@ -231,7 +231,8 @@ const ( * on/off switch */ IFLA_VF_STATS /* network device statistics */ - IFLA_VF_MAX = IFLA_VF_STATS + IFLA_VF_TRUST /* Trust state of VF */ + IFLA_VF_MAX = IFLA_VF_TRUST ) const ( @@ -259,6 +260,7 @@ const ( SizeofVfSpoofchk = 0x08 SizeofVfLinkState = 0x08 SizeofVfRssQueryEn = 0x08 + SizeofVfTrust = 0x08 ) // struct ifla_vf_mac { @@ -419,12 +421,42 @@ func (msg *VfRssQueryEn) Serialize() []byte { return (*(*[SizeofVfRssQueryEn]byte)(unsafe.Pointer(msg)))[:] } +// struct ifla_vf_trust { +// __u32 vf; +// __u32 setting; +// }; + +type VfTrust struct { + Vf uint32 + Setting uint32 +} + +func (msg *VfTrust) Len() int { + return SizeofVfTrust +} + +func DeserializeVfTrust(b []byte) *VfTrust { + return (*VfTrust)(unsafe.Pointer(&b[0:SizeofVfTrust][0])) +} + +func (msg *VfTrust) Serialize() []byte { + return (*(*[SizeofVfTrust]byte)(unsafe.Pointer(msg)))[:] +} + +const ( + XDP_FLAGS_UPDATE_IF_NOEXIST = 1 << iota + XDP_FLAGS_SKB_MODE + XDP_FLAGS_DRV_MODE + XDP_FLAGS_MASK = XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_SKB_MODE | XDP_FLAGS_DRV_MODE +) + const ( IFLA_XDP_UNSPEC = iota IFLA_XDP_FD /* fd of xdp program to attach, or -1 to remove */ IFLA_XDP_ATTACHED /* read-only bool indicating if prog is attached */ IFLA_XDP_FLAGS /* xdp prog related flags */ - IFLA_XDP_MAX = IFLA_XDP_FLAGS + IFLA_XDP_PROG_ID /* xdp prog id */ + IFLA_XDP_MAX = IFLA_XDP_PROG_ID ) const ( diff --git a/components/engine/vendor/github.com/vishvananda/netlink/nl/nl_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/nl/nl_linux.go index 1329acd864..72f7f6af3c 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/nl/nl_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/nl/nl_linux.go @@ -621,6 +621,20 @@ func (s *NetlinkSocket) Receive() ([]syscall.NetlinkMessage, error) { return syscall.ParseNetlinkMessage(rb) } +// SetSendTimeout allows to set a send timeout on the socket +func (s *NetlinkSocket) SetSendTimeout(timeout *syscall.Timeval) error { + // Set a send timeout of SOCKET_SEND_TIMEOUT, this will allow the Send to periodically unblock and avoid that a routine + // remains stuck on a send on a closed fd + return syscall.SetsockoptTimeval(int(s.fd), syscall.SOL_SOCKET, syscall.SO_SNDTIMEO, timeout) +} + +// SetReceiveTimeout allows to set a receive timeout on the socket +func (s *NetlinkSocket) SetReceiveTimeout(timeout *syscall.Timeval) error { + // Set a read timeout of SOCKET_READ_TIMEOUT, this will allow the Read to periodically unblock and avoid that a routine + // remains stuck on a recvmsg on a closed fd + return syscall.SetsockoptTimeval(int(s.fd), syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, timeout) +} + func (s *NetlinkSocket) GetPid() (uint32, error) { fd := int(atomic.LoadInt32(&s.fd)) lsa, err := syscall.Getsockname(fd) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/qdisc_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/qdisc_linux.go index 2c0deddb32..1123396e47 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/qdisc_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/qdisc_linux.go @@ -160,71 +160,73 @@ func qdiscPayload(req *nl.NetlinkRequest, qdisc Qdisc) error { req.AddData(nl.NewRtAttr(nl.TCA_KIND, nl.ZeroTerminated(qdisc.Type()))) options := nl.NewRtAttr(nl.TCA_OPTIONS, nil) - if prio, ok := qdisc.(*Prio); ok { + + switch qdisc := qdisc.(type) { + case *Prio: tcmap := nl.TcPrioMap{ - Bands: int32(prio.Bands), - Priomap: prio.PriorityMap, + Bands: int32(qdisc.Bands), + Priomap: qdisc.PriorityMap, } options = nl.NewRtAttr(nl.TCA_OPTIONS, tcmap.Serialize()) - } else if tbf, ok := qdisc.(*Tbf); ok { + case *Tbf: opt := nl.TcTbfQopt{} - opt.Rate.Rate = uint32(tbf.Rate) - opt.Peakrate.Rate = uint32(tbf.Peakrate) - opt.Limit = tbf.Limit - opt.Buffer = tbf.Buffer + opt.Rate.Rate = uint32(qdisc.Rate) + opt.Peakrate.Rate = uint32(qdisc.Peakrate) + opt.Limit = qdisc.Limit + opt.Buffer = qdisc.Buffer nl.NewRtAttrChild(options, nl.TCA_TBF_PARMS, opt.Serialize()) - if tbf.Rate >= uint64(1<<32) { - nl.NewRtAttrChild(options, nl.TCA_TBF_RATE64, nl.Uint64Attr(tbf.Rate)) + if qdisc.Rate >= uint64(1<<32) { + nl.NewRtAttrChild(options, nl.TCA_TBF_RATE64, nl.Uint64Attr(qdisc.Rate)) } - if tbf.Peakrate >= uint64(1<<32) { - nl.NewRtAttrChild(options, nl.TCA_TBF_PRATE64, nl.Uint64Attr(tbf.Peakrate)) + if qdisc.Peakrate >= uint64(1<<32) { + nl.NewRtAttrChild(options, nl.TCA_TBF_PRATE64, nl.Uint64Attr(qdisc.Peakrate)) } - if tbf.Peakrate > 0 { - nl.NewRtAttrChild(options, nl.TCA_TBF_PBURST, nl.Uint32Attr(tbf.Minburst)) + if qdisc.Peakrate > 0 { + nl.NewRtAttrChild(options, nl.TCA_TBF_PBURST, nl.Uint32Attr(qdisc.Minburst)) } - } else if htb, ok := qdisc.(*Htb); ok { + case *Htb: opt := nl.TcHtbGlob{} - opt.Version = htb.Version - opt.Rate2Quantum = htb.Rate2Quantum - opt.Defcls = htb.Defcls + opt.Version = qdisc.Version + opt.Rate2Quantum = qdisc.Rate2Quantum + opt.Defcls = qdisc.Defcls // TODO: Handle Debug properly. For now default to 0 - opt.Debug = htb.Debug - opt.DirectPkts = htb.DirectPkts + opt.Debug = qdisc.Debug + opt.DirectPkts = qdisc.DirectPkts nl.NewRtAttrChild(options, nl.TCA_HTB_INIT, opt.Serialize()) // nl.NewRtAttrChild(options, nl.TCA_HTB_DIRECT_QLEN, opt.Serialize()) - } else if netem, ok := qdisc.(*Netem); ok { + case *Netem: opt := nl.TcNetemQopt{} - opt.Latency = netem.Latency - opt.Limit = netem.Limit - opt.Loss = netem.Loss - opt.Gap = netem.Gap - opt.Duplicate = netem.Duplicate - opt.Jitter = netem.Jitter + opt.Latency = qdisc.Latency + opt.Limit = qdisc.Limit + opt.Loss = qdisc.Loss + opt.Gap = qdisc.Gap + opt.Duplicate = qdisc.Duplicate + opt.Jitter = qdisc.Jitter options = nl.NewRtAttr(nl.TCA_OPTIONS, opt.Serialize()) // Correlation corr := nl.TcNetemCorr{} - corr.DelayCorr = netem.DelayCorr - corr.LossCorr = netem.LossCorr - corr.DupCorr = netem.DuplicateCorr + corr.DelayCorr = qdisc.DelayCorr + corr.LossCorr = qdisc.LossCorr + corr.DupCorr = qdisc.DuplicateCorr if corr.DelayCorr > 0 || corr.LossCorr > 0 || corr.DupCorr > 0 { nl.NewRtAttrChild(options, nl.TCA_NETEM_CORR, corr.Serialize()) } // Corruption corruption := nl.TcNetemCorrupt{} - corruption.Probability = netem.CorruptProb - corruption.Correlation = netem.CorruptCorr + corruption.Probability = qdisc.CorruptProb + corruption.Correlation = qdisc.CorruptCorr if corruption.Probability > 0 { nl.NewRtAttrChild(options, nl.TCA_NETEM_CORRUPT, corruption.Serialize()) } // Reorder reorder := nl.TcNetemReorder{} - reorder.Probability = netem.ReorderProb - reorder.Correlation = netem.ReorderCorr + reorder.Probability = qdisc.ReorderProb + reorder.Correlation = qdisc.ReorderCorr if reorder.Probability > 0 { nl.NewRtAttrChild(options, nl.TCA_NETEM_REORDER, reorder.Serialize()) } - } else if _, ok := qdisc.(*Ingress); ok { + case *Ingress: // ingress filters must use the proper handle if qdisc.Attrs().Parent != HANDLE_INGRESS { return fmt.Errorf("Ingress filters must set Parent to HANDLE_INGRESS") diff --git a/components/engine/vendor/github.com/vishvananda/netlink/route.go b/components/engine/vendor/github.com/vishvananda/netlink/route.go index 03ac4b2391..68c6a2230d 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/route.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/route.go @@ -16,6 +16,7 @@ type Destination interface { Decode([]byte) error Encode() ([]byte, error) String() string + Equal(Destination) bool } type Encap interface { @@ -23,6 +24,7 @@ type Encap interface { Decode([]byte) error Encode() ([]byte, error) String() string + Equal(Encap) bool } // Route represents a netlink route. @@ -72,6 +74,25 @@ func (r Route) String() string { return fmt.Sprintf("{%s}", strings.Join(elems, " ")) } +func (r Route) Equal(x Route) bool { + return r.LinkIndex == x.LinkIndex && + r.ILinkIndex == x.ILinkIndex && + r.Scope == x.Scope && + ipNetEqual(r.Dst, x.Dst) && + r.Src.Equal(x.Src) && + r.Gw.Equal(x.Gw) && + nexthopInfoSlice(r.MultiPath).Equal(x.MultiPath) && + r.Protocol == x.Protocol && + r.Priority == x.Priority && + r.Table == x.Table && + r.Type == x.Type && + r.Tos == x.Tos && + r.Flags == x.Flags && + (r.MPLSDst == x.MPLSDst || (r.MPLSDst != nil && x.MPLSDst != nil && *r.MPLSDst == *x.MPLSDst)) && + (r.NewDst == x.NewDst || (r.NewDst != nil && r.NewDst.Equal(x.NewDst))) && + (r.Encap == x.Encap || (r.Encap != nil && r.Encap.Equal(x.Encap))) +} + func (r *Route) SetFlag(flag NextHopFlag) { r.Flags |= int(flag) } @@ -110,7 +131,46 @@ func (n *NexthopInfo) String() string { elems = append(elems, fmt.Sprintf("Encap: %s", n.Encap)) } elems = append(elems, fmt.Sprintf("Weight: %d", n.Hops+1)) - elems = append(elems, fmt.Sprintf("Gw: %d", n.Gw)) + elems = append(elems, fmt.Sprintf("Gw: %s", n.Gw)) elems = append(elems, fmt.Sprintf("Flags: %s", n.ListFlags())) return fmt.Sprintf("{%s}", strings.Join(elems, " ")) } + +func (n NexthopInfo) Equal(x NexthopInfo) bool { + return n.LinkIndex == x.LinkIndex && + n.Hops == x.Hops && + n.Gw.Equal(x.Gw) && + n.Flags == x.Flags && + (n.NewDst == x.NewDst || (n.NewDst != nil && n.NewDst.Equal(x.NewDst))) && + (n.Encap == x.Encap || (n.Encap != nil && n.Encap.Equal(x.Encap))) +} + +type nexthopInfoSlice []*NexthopInfo + +func (n nexthopInfoSlice) Equal(x []*NexthopInfo) bool { + if len(n) != len(x) { + return false + } + for i := range n { + if n[i] == nil || x[i] == nil { + return false + } + if !n[i].Equal(*x[i]) { + return false + } + } + return true +} + +// ipNetEqual returns true iff both IPNet are equal +func ipNetEqual(ipn1 *net.IPNet, ipn2 *net.IPNet) bool { + if ipn1 == ipn2 { + return true + } + if ipn1 == nil || ipn2 == nil { + return false + } + m1, _ := ipn1.Mask.Size() + m2, _ := ipn2.Mask.Size() + return m1 == m2 && ipn1.IP.Equal(ipn2.IP) +} diff --git a/components/engine/vendor/github.com/vishvananda/netlink/route_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/route_linux.go index cd739e7146..9234c6986d 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/route_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/route_linux.go @@ -86,6 +86,34 @@ func (d *MPLSDestination) String() string { return strings.Join(s, "/") } +func (d *MPLSDestination) Equal(x Destination) bool { + o, ok := x.(*MPLSDestination) + if !ok { + return false + } + if d == nil && o == nil { + return true + } + if d == nil || o == nil { + return false + } + if d.Labels == nil && o.Labels == nil { + return true + } + if d.Labels == nil || o.Labels == nil { + return false + } + if len(d.Labels) != len(o.Labels) { + return false + } + for i := range d.Labels { + if d.Labels[i] != o.Labels[i] { + return false + } + } + return true +} + type MPLSEncap struct { Labels []int } @@ -129,6 +157,34 @@ func (e *MPLSEncap) String() string { return strings.Join(s, "/") } +func (e *MPLSEncap) Equal(x Encap) bool { + o, ok := x.(*MPLSEncap) + if !ok { + return false + } + if e == nil && o == nil { + return true + } + if e == nil || o == nil { + return false + } + if e.Labels == nil && o.Labels == nil { + return true + } + if e.Labels == nil || o.Labels == nil { + return false + } + if len(e.Labels) != len(o.Labels) { + return false + } + for i := range e.Labels { + if e.Labels[i] != o.Labels[i] { + return false + } + } + return true +} + // RouteAdd will add a route to the system. // Equivalent to: `ip route add $route` func RouteAdd(route *Route) error { @@ -421,19 +477,8 @@ func (h *Handle) RouteListFiltered(family int, filter *Route, filterMask uint64) continue case filterMask&RT_FILTER_DST != 0: if filter.MPLSDst == nil || route.MPLSDst == nil || (*filter.MPLSDst) != (*route.MPLSDst) { - if filter.Dst == nil { - if route.Dst != nil { - continue - } - } else { - if route.Dst == nil { - continue - } - aMaskLen, aMaskBits := route.Dst.Mask.Size() - bMaskLen, bMaskBits := filter.Dst.Mask.Size() - if !(route.Dst.IP.Equal(filter.Dst.IP) && aMaskLen == bMaskLen && aMaskBits == bMaskBits) { - continue - } + if !ipNetEqual(route.Dst, filter.Dst) { + continue } } } @@ -633,16 +678,34 @@ func (h *Handle) RouteGet(destination net.IP) ([]Route, error) { // RouteSubscribe takes a chan down which notifications will be sent // when routes are added or deleted. Close the 'done' chan to stop subscription. func RouteSubscribe(ch chan<- RouteUpdate, done <-chan struct{}) error { - return routeSubscribeAt(netns.None(), netns.None(), ch, done) + return routeSubscribeAt(netns.None(), netns.None(), ch, done, nil) } // RouteSubscribeAt works like RouteSubscribe plus it allows the caller // to choose the network namespace in which to subscribe (ns). func RouteSubscribeAt(ns netns.NsHandle, ch chan<- RouteUpdate, done <-chan struct{}) error { - return routeSubscribeAt(ns, netns.None(), ch, done) + return routeSubscribeAt(ns, netns.None(), ch, done, nil) } -func routeSubscribeAt(newNs, curNs netns.NsHandle, ch chan<- RouteUpdate, done <-chan struct{}) error { +// RouteSubscribeOptions contains a set of options to use with +// RouteSubscribeWithOptions. +type RouteSubscribeOptions struct { + Namespace *netns.NsHandle + ErrorCallback func(error) +} + +// RouteSubscribeWithOptions work like RouteSubscribe but enable to +// provide additional options to modify the behavior. Currently, the +// namespace can be provided as well as an error callback. +func RouteSubscribeWithOptions(ch chan<- RouteUpdate, done <-chan struct{}, options RouteSubscribeOptions) error { + if options.Namespace == nil { + none := netns.None() + options.Namespace = &none + } + return routeSubscribeAt(*options.Namespace, netns.None(), ch, done, options.ErrorCallback) +} + +func routeSubscribeAt(newNs, curNs netns.NsHandle, ch chan<- RouteUpdate, done <-chan struct{}, cberr func(error)) error { s, err := nl.SubscribeAt(newNs, curNs, syscall.NETLINK_ROUTE, syscall.RTNLGRP_IPV4_ROUTE, syscall.RTNLGRP_IPV6_ROUTE) if err != nil { return err @@ -658,11 +721,17 @@ func routeSubscribeAt(newNs, curNs netns.NsHandle, ch chan<- RouteUpdate, done < for { msgs, err := s.Receive() if err != nil { + if cberr != nil { + cberr(err) + } return } for _, m := range msgs { route, err := deserializeRoute(m.Data) if err != nil { + if cberr != nil { + cberr(err) + } return } ch <- RouteUpdate{Type: m.Header.Type, Route: route} diff --git a/components/engine/vendor/github.com/vishvananda/netlink/rule.go b/components/engine/vendor/github.com/vishvananda/netlink/rule.go index f0243defd7..e4d9168d6c 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/rule.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/rule.go @@ -8,6 +8,7 @@ import ( // Rule represents a netlink rule. type Rule struct { Priority int + Family int Table int Mark int Mask int diff --git a/components/engine/vendor/github.com/vishvananda/netlink/rule_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/rule_linux.go index f9cdc855f1..cbd91a56bb 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/rule_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/rule_linux.go @@ -37,6 +37,9 @@ func (h *Handle) RuleDel(rule *Rule) error { func ruleHandle(rule *Rule, req *nl.NetlinkRequest) error { msg := nl.NewRtMsg() msg.Family = syscall.AF_INET + if rule.Family != 0 { + msg.Family = uint8(rule.Family) + } var dstFamily uint8 var rtAttrs []*nl.RtAttr