From 52f9a6e75eb366f2bae145ae479cc7114fad803c Mon Sep 17 00:00:00 2001 From: Asad Saeeduddin Date: Mon, 9 Oct 2017 20:22:44 -0400 Subject: [PATCH 01/16] Change multi-type parameters to single type Signed-off-by: Asad Saeeduddin Upstream-commit: c6d32bebe5b6a0919fcb530692092dd5516e5b54 Component: engine --- components/engine/api/swagger.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index c3b9c29244..078517eef6 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -836,9 +836,7 @@ definitions: type: "string" Cmd: description: "Command to run specified as a string or an array of strings." - type: - - "array" - - "string" + type: "array" items: type: "string" Healthcheck: @@ -866,9 +864,7 @@ definitions: The entry point for the container as a string or an array of strings. If the array consists of exactly one empty string (`[""]`) then the entry point is reset to system default (i.e., the entry point used by docker when there is no `ENTRYPOINT` instruction in the `Dockerfile`). - type: - - "array" - - "string" + type: "array" items: type: "string" NetworkDisabled: From 52ebdfbe5db4b1444e245e41d0cb2c30c46703da Mon Sep 17 00:00:00 2001 From: y00316549 Date: Fri, 24 Nov 2017 13:41:08 +0800 Subject: [PATCH 02/16] Close pipe in chrootarchive.invokeUnpack when cmd.Start()/json.NewEncoder failed. Signed-off-by: y00316549 Upstream-commit: f5f8f008608e579e9164647910ef6f976e91e297 Component: engine --- components/engine/pkg/chrootarchive/archive_unix.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/pkg/chrootarchive/archive_unix.go b/components/engine/pkg/chrootarchive/archive_unix.go index f2325abd74..75c010e98f 100644 --- a/components/engine/pkg/chrootarchive/archive_unix.go +++ b/components/engine/pkg/chrootarchive/archive_unix.go @@ -66,10 +66,12 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T cmd.Stderr = output if err := cmd.Start(); err != nil { + w.Close() return fmt.Errorf("Untar error on re-exec cmd: %v", err) } //write the options to the pipe for the untar exec to read if err := json.NewEncoder(w).Encode(options); err != nil { + w.Close() return fmt.Errorf("Untar json encode to pipe failed: %v", err) } w.Close() From 403fcf5047d6d6afc91f0731455c0ea9f06a1ef9 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 4 Dec 2017 14:45:26 -0800 Subject: [PATCH 03/16] Perform fsmagic detection on driver's home-dir if it exists The fsmagic check was always performed on "data-root" (`/var/lib/docker`), not on the storage-driver's home directory (e.g. `/var/lib/docker/`). This caused detection to be done on the wrong filesystem in situations where `/var/lib/docker/` was a mount, and a different filesystem than `/var/lib/docker` itself. This patch checks if the storage-driver's home directory exists, and only falls back to `/var/lib/docker` if it doesn't exist. Signed-off-by: Sebastiaan van Stijn Upstream-commit: f9c8fa305e1501d8056f8744cb193a720aab0e13 Component: engine --- components/engine/daemon/graphdriver/aufs/aufs.go | 11 ++++++++++- components/engine/daemon/graphdriver/btrfs/btrfs.go | 11 ++++++++++- components/engine/daemon/graphdriver/driver_linux.go | 4 +--- .../engine/daemon/graphdriver/overlay/overlay.go | 12 +++++++++++- .../engine/daemon/graphdriver/overlay2/overlay.go | 11 ++++++++++- 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/components/engine/daemon/graphdriver/aufs/aufs.go b/components/engine/daemon/graphdriver/aufs/aufs.go index 11e763d025..248b8bf88d 100644 --- a/components/engine/daemon/graphdriver/aufs/aufs.go +++ b/components/engine/daemon/graphdriver/aufs/aufs.go @@ -89,7 +89,16 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, graphdriver.ErrNotSupported } - fsMagic, err := graphdriver.GetFSMagic(root) + // Perform feature detection on /var/lib/docker/aufs if it's an existing directory. + // This covers situations where /var/lib/docker/aufs is a mount, and on a different + // filesystem than /var/lib/docker. + // If the path does not exist, fall back to using /var/lib/docker for feature detection. + testdir := root + if _, err := os.Stat(testdir); os.IsNotExist(err) { + testdir = filepath.Dir(testdir) + } + + fsMagic, err := graphdriver.GetFSMagic(testdir) if err != nil { return nil, err } diff --git a/components/engine/daemon/graphdriver/btrfs/btrfs.go b/components/engine/daemon/graphdriver/btrfs/btrfs.go index 0dabf711dd..57313c94c6 100644 --- a/components/engine/daemon/graphdriver/btrfs/btrfs.go +++ b/components/engine/daemon/graphdriver/btrfs/btrfs.go @@ -51,7 +51,16 @@ type btrfsOptions struct { // An error is returned if BTRFS is not supported. func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { - fsMagic, err := graphdriver.GetFSMagic(home) + // Perform feature detection on /var/lib/docker/btrfs if it's an existing directory. + // This covers situations where /var/lib/docker/btrfs is a mount, and on a different + // filesystem than /var/lib/docker. + // If the path does not exist, fall back to using /var/lib/docker for feature detection. + testdir := home + if _, err := os.Stat(testdir); os.IsNotExist(err) { + testdir = filepath.Dir(testdir) + } + + fsMagic, err := graphdriver.GetFSMagic(testdir) if err != nil { return nil, err } diff --git a/components/engine/daemon/graphdriver/driver_linux.go b/components/engine/daemon/graphdriver/driver_linux.go index aa3cfc9f79..f59862db85 100644 --- a/components/engine/daemon/graphdriver/driver_linux.go +++ b/components/engine/daemon/graphdriver/driver_linux.go @@ -3,8 +3,6 @@ package graphdriver import ( - "path/filepath" - "github.com/docker/docker/pkg/mount" "golang.org/x/sys/unix" ) @@ -82,7 +80,7 @@ var ( // GetFSMagic returns the filesystem id given the path. func GetFSMagic(rootpath string) (FsMagic, error) { var buf unix.Statfs_t - if err := unix.Statfs(filepath.Dir(rootpath), &buf); err != nil { + if err := unix.Statfs(rootpath, &buf); err != nil { return 0, err } return FsMagic(buf.Type), nil diff --git a/components/engine/daemon/graphdriver/overlay/overlay.go b/components/engine/daemon/graphdriver/overlay/overlay.go index 83a1677fe2..5c21068c0f 100644 --- a/components/engine/daemon/graphdriver/overlay/overlay.go +++ b/components/engine/daemon/graphdriver/overlay/overlay.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "strconv" "github.com/docker/docker/daemon/graphdriver" @@ -119,7 +120,16 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, graphdriver.ErrNotSupported } - fsMagic, err := graphdriver.GetFSMagic(home) + // Perform feature detection on /var/lib/docker/overlay if it's an existing directory. + // This covers situations where /var/lib/docker/overlay is a mount, and on a different + // filesystem than /var/lib/docker. + // If the path does not exist, fall back to using /var/lib/docker for feature detection. + testdir := home + if _, err := os.Stat(testdir); os.IsNotExist(err) { + testdir = filepath.Dir(testdir) + } + + fsMagic, err := graphdriver.GetFSMagic(testdir) if err != nil { return nil, err } diff --git a/components/engine/daemon/graphdriver/overlay2/overlay.go b/components/engine/daemon/graphdriver/overlay2/overlay.go index e660d8058a..1313a89c69 100644 --- a/components/engine/daemon/graphdriver/overlay2/overlay.go +++ b/components/engine/daemon/graphdriver/overlay2/overlay.go @@ -136,7 +136,16 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - fsMagic, err := graphdriver.GetFSMagic(home) + // Perform feature detection on /var/lib/docker/overlay2 if it's an existing directory. + // This covers situations where /var/lib/docker/overlay2 is a mount, and on a different + // filesystem than /var/lib/docker. + // If the path does not exist, fall back to using /var/lib/docker for feature detection. + testdir := home + if _, err := os.Stat(testdir); os.IsNotExist(err) { + testdir = filepath.Dir(testdir) + } + + fsMagic, err := graphdriver.GetFSMagic(testdir) if err != nil { return nil, err } From e4dde678751286b751cc7739e87205e5170efa99 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 16 Nov 2017 01:48:43 +0100 Subject: [PATCH 04/16] Remove support for overlay/overlay2 without d_type Support for running overlay/overlay2 on a backing filesystem without d_type support (most likely: xfs, as ext4 supports this by default), was deprecated for some time. Running without d_type support is problematic, and can lead to difficult to debug issues ("invalid argument" errors, or unable to remove files from the container's filesystem). This patch turns the warning that was previously printed into an "unsupported" error, so that the overlay/overlay2 drivers are not automatically selected when detecting supported storage drivers. Signed-off-by: Sebastiaan van Stijn Upstream-commit: 0abb8dec3f730f3ad2cc9a161c97968a6bfd0631 Component: engine --- .../engine/daemon/graphdriver/driver.go | 18 ++-------- .../engine/daemon/graphdriver/errors.go | 36 +++++++++++++++++++ .../graphdriver/graphtest/graphtest_unix.go | 2 +- .../daemon/graphdriver/overlay/overlay.go | 19 +++++----- .../daemon/graphdriver/overlay2/overlay.go | 21 +++++------ .../graphdriver/overlayutils/overlayutils.go | 8 +++-- 6 files changed, 62 insertions(+), 42 deletions(-) create mode 100644 components/engine/daemon/graphdriver/errors.go diff --git a/components/engine/daemon/graphdriver/driver.go b/components/engine/daemon/graphdriver/driver.go index d08c6dc5b7..721f6609e9 100644 --- a/components/engine/daemon/graphdriver/driver.go +++ b/components/engine/daemon/graphdriver/driver.go @@ -1,7 +1,6 @@ package graphdriver import ( - "errors" "fmt" "io" "os" @@ -28,13 +27,6 @@ const ( var ( // All registered drivers drivers map[string]InitFunc - - // ErrNotSupported returned when driver is not supported. - ErrNotSupported = errors.New("driver not supported") - // ErrPrerequisites returned when driver does not meet prerequisites. - ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)") - // ErrIncompatibleFS returned when file system is not supported. - ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver") ) //CreateOpts contains optional arguments for Create() and CreateReadWrite() @@ -248,7 +240,7 @@ func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, err for _, name := range list { driver, err := getBuiltinDriver(name, config.Root, config.DriverOptions, config.UIDMaps, config.GIDMaps) if err != nil { - if isDriverNotSupported(err) { + if IsDriverNotSupported(err) { continue } return nil, err @@ -260,7 +252,7 @@ func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, err for name, initFunc := range drivers { driver, err := initFunc(filepath.Join(config.Root, name), config.DriverOptions, config.UIDMaps, config.GIDMaps) if err != nil { - if isDriverNotSupported(err) { + if IsDriverNotSupported(err) { continue } return nil, err @@ -270,12 +262,6 @@ func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, err return nil, fmt.Errorf("No supported storage backend found") } -// isDriverNotSupported returns true if the error initializing -// the graph driver is a non-supported error. -func isDriverNotSupported(err error) bool { - return err == ErrNotSupported || err == ErrPrerequisites || err == ErrIncompatibleFS -} - // scanPriorDrivers returns an un-ordered scan of directories of prior storage drivers func scanPriorDrivers(root string) map[string]bool { driversMap := make(map[string]bool) diff --git a/components/engine/daemon/graphdriver/errors.go b/components/engine/daemon/graphdriver/errors.go new file mode 100644 index 0000000000..dd52ee4ada --- /dev/null +++ b/components/engine/daemon/graphdriver/errors.go @@ -0,0 +1,36 @@ +package graphdriver + +const ( + // ErrNotSupported returned when driver is not supported. + ErrNotSupported NotSupportedError = "driver not supported" + // ErrPrerequisites returned when driver does not meet prerequisites. + ErrPrerequisites NotSupportedError = "prerequisites for driver not satisfied (wrong filesystem?)" + // ErrIncompatibleFS returned when file system is not supported. + ErrIncompatibleFS NotSupportedError = "backing file system is unsupported for this graph driver" +) + +// ErrUnSupported signals that the graph-driver is not supported on the current configuration +type ErrUnSupported interface { + NotSupported() +} + +// NotSupportedError signals that the graph-driver is not supported on the current configuration +type NotSupportedError string + +func (e NotSupportedError) Error() string { + return string(e) +} + +// NotSupported signals that a graph-driver is not supported. +func (e NotSupportedError) NotSupported() {} + +// IsDriverNotSupported returns true if the error initializing +// the graph driver is a non-supported error. +func IsDriverNotSupported(err error) bool { + switch err.(type) { + case ErrUnSupported: + return true + default: + return false + } +} diff --git a/components/engine/daemon/graphdriver/graphtest/graphtest_unix.go b/components/engine/daemon/graphdriver/graphtest/graphtest_unix.go index c25d4826fc..da9443e098 100644 --- a/components/engine/daemon/graphdriver/graphtest/graphtest_unix.go +++ b/components/engine/daemon/graphdriver/graphtest/graphtest_unix.go @@ -42,7 +42,7 @@ func newDriver(t testing.TB, name string, options []string) *Driver { d, err := graphdriver.GetDriver(name, nil, graphdriver.Options{DriverOptions: options, Root: root}) if err != nil { t.Logf("graphdriver: %v\n", err) - if err == graphdriver.ErrNotSupported || err == graphdriver.ErrPrerequisites || err == graphdriver.ErrIncompatibleFS { + if graphdriver.IsDriverNotSupported(err) { t.Skipf("Driver %s not supported", name) } t.Fatal(err) diff --git a/components/engine/daemon/graphdriver/overlay/overlay.go b/components/engine/daemon/graphdriver/overlay/overlay.go index 5c21068c0f..b148431863 100644 --- a/components/engine/daemon/graphdriver/overlay/overlay.go +++ b/components/engine/daemon/graphdriver/overlay/overlay.go @@ -138,11 +138,19 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap } switch fsMagic { - case graphdriver.FsMagicAufs, graphdriver.FsMagicBtrfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicZfs, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs: + case graphdriver.FsMagicAufs, graphdriver.FsMagicBtrfs, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs, graphdriver.FsMagicOverlay, graphdriver.FsMagicZfs: logrus.Errorf("'overlay' is not supported over %s", backingFs) return nil, graphdriver.ErrIncompatibleFS } + supportsDType, err := fsutils.SupportsDType(testdir) + if err != nil { + return nil, err + } + if !supportsDType { + return nil, overlayutils.ErrDTypeNotSupported("overlay", backingFs) + } + rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) if err != nil { return nil, err @@ -156,15 +164,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - supportsDType, err := fsutils.SupportsDType(home) - if err != nil { - return nil, err - } - if !supportsDType { - // not a fatal error until v17.12 (#27443) - logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs)) - } - d := &Driver{ home: home, uidMaps: uidMaps, diff --git a/components/engine/daemon/graphdriver/overlay2/overlay.go b/components/engine/daemon/graphdriver/overlay2/overlay.go index 1313a89c69..ea707a0af6 100644 --- a/components/engine/daemon/graphdriver/overlay2/overlay.go +++ b/components/engine/daemon/graphdriver/overlay2/overlay.go @@ -153,9 +153,8 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap backingFs = fsName } - // check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs switch fsMagic { - case graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs: + case graphdriver.FsMagicAufs, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs, graphdriver.FsMagicOverlay, graphdriver.FsMagicZfs: logrus.Errorf("'overlay2' is not supported over %s", backingFs) return nil, graphdriver.ErrIncompatibleFS case graphdriver.FsMagicBtrfs: @@ -174,12 +173,19 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap if opts.overrideKernelCheck { logrus.Warn("Using pre-4.0.0 kernel for overlay2, mount failures may require kernel update") } else { - if err := supportsMultipleLowerDir(filepath.Dir(home)); err != nil { + if err := supportsMultipleLowerDir(testdir); err != nil { logrus.Debugf("Multiple lower dirs not supported: %v", err) return nil, graphdriver.ErrNotSupported } } } + supportsDType, err := fsutils.SupportsDType(testdir) + if err != nil { + return nil, err + } + if !supportsDType { + return nil, overlayutils.ErrDTypeNotSupported("overlay2", backingFs) + } rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) if err != nil { @@ -194,15 +200,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - supportsDType, err := fsutils.SupportsDType(home) - if err != nil { - return nil, err - } - if !supportsDType { - // not a fatal error until v17.12 (#27443) - logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay2", backingFs)) - } - d := &Driver{ home: home, uidMaps: uidMaps, diff --git a/components/engine/daemon/graphdriver/overlayutils/overlayutils.go b/components/engine/daemon/graphdriver/overlayutils/overlayutils.go index 7491c3457c..9f71c60d51 100644 --- a/components/engine/daemon/graphdriver/overlayutils/overlayutils.go +++ b/components/engine/daemon/graphdriver/overlayutils/overlayutils.go @@ -3,8 +3,9 @@ package overlayutils import ( - "errors" "fmt" + + "github.com/docker/docker/daemon/graphdriver" ) // ErrDTypeNotSupported denotes that the backing filesystem doesn't support d_type. @@ -13,6 +14,7 @@ func ErrDTypeNotSupported(driver, backingFs string) error { if backingFs == "xfs" { msg += " Reformat the filesystem with ftype=1 to enable d_type support." } - msg += " Running without d_type support will no longer be supported in Docker 17.12." - return errors.New(msg) + msg += " Backing filesystems without d_type support are not supported." + + return graphdriver.NotSupportedError(msg) } From 0c8a47d0194c93b2de4c1be39d0e50e0716114a0 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 4 Dec 2017 17:02:52 -0800 Subject: [PATCH 05/16] Allow existing setups to continue using d_type Even though it's highly discouraged, there are existing installs that are running overlay/overlay2 on filesystems without d_type support. This patch allows the daemon to start in such cases, instead of refusing to start without an option to override. For fresh installs, backing filesystems without d_type support will still cause the overlay/overlay2 drivers to be marked as "unsupported", and skipped during the automatic selection. This feature is only to keep backward compatibility, but will be removed at some point. Signed-off-by: Sebastiaan van Stijn Upstream-commit: 0a4e793a3da9ba6d20bccfb83f7c48e20a76d895 Component: engine --- components/engine/daemon/graphdriver/driver.go | 12 ++++++++++++ .../engine/daemon/graphdriver/overlay/overlay.go | 6 +++++- .../engine/daemon/graphdriver/overlay2/overlay.go | 6 +++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/components/engine/daemon/graphdriver/driver.go b/components/engine/daemon/graphdriver/driver.go index 721f6609e9..ceb49b9818 100644 --- a/components/engine/daemon/graphdriver/driver.go +++ b/components/engine/daemon/graphdriver/driver.go @@ -277,6 +277,18 @@ func scanPriorDrivers(root string) map[string]bool { return driversMap } +// IsInitialized checks if the driver's home-directory exists and is non-empty. +func IsInitialized(driverHome string) bool { + _, err := os.Stat(driverHome) + if os.IsNotExist(err) { + return false + } + if err != nil { + logrus.Warnf("graphdriver.IsInitialized: stat failed: %v", err) + } + return !isEmptyDir(driverHome) +} + // isEmptyDir checks if a directory is empty. It is used to check if prior // storage-driver directories exist. If an error occurs, it also assumes the // directory is not empty (which preserves the behavior _before_ this check diff --git a/components/engine/daemon/graphdriver/overlay/overlay.go b/components/engine/daemon/graphdriver/overlay/overlay.go index b148431863..fce775ea6b 100644 --- a/components/engine/daemon/graphdriver/overlay/overlay.go +++ b/components/engine/daemon/graphdriver/overlay/overlay.go @@ -148,7 +148,11 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } if !supportsDType { - return nil, overlayutils.ErrDTypeNotSupported("overlay", backingFs) + if !graphdriver.IsInitialized(home) { + return nil, overlayutils.ErrDTypeNotSupported("overlay", backingFs) + } + // allow running without d_type only for existing setups (#27443) + logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs)) } rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) diff --git a/components/engine/daemon/graphdriver/overlay2/overlay.go b/components/engine/daemon/graphdriver/overlay2/overlay.go index ea707a0af6..f1731ea935 100644 --- a/components/engine/daemon/graphdriver/overlay2/overlay.go +++ b/components/engine/daemon/graphdriver/overlay2/overlay.go @@ -184,7 +184,11 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } if !supportsDType { - return nil, overlayutils.ErrDTypeNotSupported("overlay2", backingFs) + if !graphdriver.IsInitialized(home) { + return nil, overlayutils.ErrDTypeNotSupported("overlay2", backingFs) + } + // allow running without d_type only for existing setups (#27443) + logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay2", backingFs)) } rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) From abf2603ea496aa0130b5dc288cba0407ff22631d Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Tue, 5 Dec 2017 12:31:02 -0300 Subject: [PATCH 06/16] Fix link in Contributing doc Signed-off-by: Lucas Molas Upstream-commit: 1f0572fd1a28d2b5b4566156fa449bb709362b38 Component: engine --- components/engine/docs/contributing/set-up-dev-env.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/components/engine/docs/contributing/set-up-dev-env.md b/components/engine/docs/contributing/set-up-dev-env.md index acd6888cd4..430e000bfe 100644 --- a/components/engine/docs/contributing/set-up-dev-env.md +++ b/components/engine/docs/contributing/set-up-dev-env.md @@ -10,8 +10,7 @@ Moby's development environment is itself, ultimately a Docker container. You use the `moby/moby` repository and its `Dockerfile` to create a Docker image, run a Docker container, and develop code in the container. -If you followed the procedures that -set up Git for contributing, you should have a fork of the `moby/moby` +If you followed the procedures that [set up Git for contributing](./set-up-git.md), you should have a fork of the `moby/moby` repository. You also created a branch called `dry-run-test`. In this section, you continue working with your fork on this branch. @@ -106,8 +105,7 @@ can take over 15 minutes to complete. ``` If you are following along with this guide, you created a `dry-run-test` - branch when you - set up Git for contributing. + branch when you [set up Git for contributing](./set-up-git.md). 3. Ensure you are on your `dry-run-test` branch. From e96b772de5ed355fa434ad9b8f9373a4fc5efcd0 Mon Sep 17 00:00:00 2001 From: Anshul Pundir Date: Mon, 4 Dec 2017 16:38:37 -0800 Subject: [PATCH 07/16] Swarmkit vendoring. Signed-off-by: Anshul Pundir Upstream-commit: 9fdbb32fcee24410c64c7897c0d9667202aa7e4f Component: engine --- components/engine/vendor.conf | 2 +- .../docker/swarmkit/agent/exec/controller.go | 4 +- .../github.com/docker/swarmkit/api/ca.pb.go | 11 +- .../docker/swarmkit/api/control.pb.go | 5 +- .../docker/swarmkit/api/dispatcher.pb.go | 5 +- .../docker/swarmkit/api/health.pb.go | 5 +- .../docker/swarmkit/api/logbroker.pb.go | 9 +- .../github.com/docker/swarmkit/api/raft.pb.go | 572 +++++++++++++--- .../github.com/docker/swarmkit/api/raft.proto | 18 + .../docker/swarmkit/api/resource.pb.go | 5 +- .../docker/swarmkit/api/types.pb.go | 645 +++++++++--------- .../docker/swarmkit/api/types.proto | 11 +- .../docker/swarmkit/api/watch.pb.go | 5 +- .../github.com/docker/swarmkit/ca/auth.go | 32 +- .../github.com/docker/swarmkit/ca/server.go | 18 +- .../manager/controlapi/ca_rotation.go | 25 +- .../swarmkit/manager/controlapi/cluster.go | 28 +- .../swarmkit/manager/controlapi/common.go | 20 +- .../swarmkit/manager/controlapi/config.go | 28 +- .../swarmkit/manager/controlapi/network.go | 50 +- .../swarmkit/manager/controlapi/node.go | 34 +- .../swarmkit/manager/controlapi/secret.go | 30 +- .../swarmkit/manager/controlapi/service.go | 108 +-- .../swarmkit/manager/controlapi/task.go | 10 +- .../swarmkit/manager/dispatcher/dispatcher.go | 35 +- .../swarmkit/manager/dispatcher/nodes.go | 13 +- .../docker/swarmkit/manager/health/health.go | 4 +- .../swarmkit/manager/logbroker/broker.go | 17 +- .../docker/swarmkit/manager/manager.go | 15 +- .../manager/orchestrator/global/global.go | 4 +- .../orchestrator/replicated/services.go | 44 +- .../swarmkit/manager/orchestrator/service.go | 25 +- .../orchestrator/taskreaper/task_reaper.go | 93 ++- .../swarmkit/manager/resourceapi/allocator.go | 12 +- .../swarmkit/manager/scheduler/filter.go | 13 +- .../swarmkit/manager/state/raft/raft.go | 130 +++- .../swarmkit/manager/state/raft/storage.go | 18 +- .../manager/state/raft/transport/peer.go | 118 +++- .../manager/state/raft/transport/transport.go | 1 - .../swarmkit/manager/state/store/memory.go | 41 +- .../docker/swarmkit/manager/watchapi/watch.go | 7 +- 41 files changed, 1541 insertions(+), 729 deletions(-) diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 120dd9d378..8d1bed0745 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -114,7 +114,7 @@ github.com/dmcgowan/go-tar go1.10 github.com/stevvooe/ttrpc 76e68349ad9ab4d03d764c713826d31216715e4f # cluster -github.com/docker/swarmkit de950a7ed842c7b7e47e9451cde9bf8f96031894 +github.com/docker/swarmkit 4429c763170d9ca96929249353c3270c19e7d39e github.com/gogo/protobuf v0.4 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b6506826e diff --git a/components/engine/vendor/github.com/docker/swarmkit/agent/exec/controller.go b/components/engine/vendor/github.com/docker/swarmkit/agent/exec/controller.go index 9b4fc7bca1..c9e9343fd7 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/agent/exec/controller.go +++ b/components/engine/vendor/github.com/docker/swarmkit/agent/exec/controller.go @@ -288,7 +288,9 @@ func Do(ctx context.Context, task *api.Task, ctlr Controller) (*api.TaskStatus, status.PortStatus = portStatus }() - if task.DesiredState == api.TaskStateShutdown { + // this branch bounds the largest state achievable in the agent as SHUTDOWN, which + // is exactly the correct behavior for the agent. + if task.DesiredState >= api.TaskStateShutdown { if status.State >= api.TaskStateCompleted { return noop() } diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/ca.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/ca.pb.go index caaa06cc83..7d289126a1 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/ca.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/ca.pb.go @@ -131,6 +131,8 @@ LeaveResponse ProcessRaftMessageRequest ProcessRaftMessageResponse + StreamRaftMessageRequest + StreamRaftMessageResponse ResolveAddressRequest ResolveAddressResponse InternalRaftRequest @@ -235,6 +237,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -984,12 +987,12 @@ func NewRaftProxyCAServer(local CAServer, connSelector raftselector.ConnProvider redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) @@ -1126,12 +1129,12 @@ func NewRaftProxyNodeCAServer(local NodeCAServer, connSelector raftselector.Conn redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/control.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/control.pb.go index 13ef4822cb..fb28c5d330 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/control.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/control.pb.go @@ -19,6 +19,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -5859,12 +5860,12 @@ func NewRaftProxyControlServer(local ControlServer, connSelector raftselector.Co redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/dispatcher.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/dispatcher.pb.go index 8e9b038cad..120df8811f 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/dispatcher.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/dispatcher.pb.go @@ -24,6 +24,7 @@ import github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -1602,12 +1603,12 @@ func NewRaftProxyDispatcherServer(local DispatcherServer, connSelector raftselec redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/health.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/health.pb.go index 757db6acbf..ed7df73054 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/health.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/health.pb.go @@ -17,6 +17,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -286,12 +287,12 @@ func NewRaftProxyHealthServer(local HealthServer, connSelector raftselector.Conn redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/logbroker.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/logbroker.pb.go index 58515aa11e..1108088fba 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/logbroker.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/logbroker.pb.go @@ -20,6 +20,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -1273,12 +1274,12 @@ func NewRaftProxyLogsServer(local LogsServer, connSelector raftselector.ConnProv redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) @@ -1396,12 +1397,12 @@ func NewRaftProxyLogBrokerServer(local LogBrokerServer, connSelector raftselecto redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/raft.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/raft.pb.go index 4710ee6b01..ddee3e18a6 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/raft.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/raft.pb.go @@ -21,6 +21,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -133,6 +134,23 @@ func (m *ProcessRaftMessageResponse) Reset() { *m = ProcessRa func (*ProcessRaftMessageResponse) ProtoMessage() {} func (*ProcessRaftMessageResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{6} } +// Raft message streaming request. +type StreamRaftMessageRequest struct { + Message *raftpb.Message `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` +} + +func (m *StreamRaftMessageRequest) Reset() { *m = StreamRaftMessageRequest{} } +func (*StreamRaftMessageRequest) ProtoMessage() {} +func (*StreamRaftMessageRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{7} } + +// Raft message streaming response. +type StreamRaftMessageResponse struct { +} + +func (m *StreamRaftMessageResponse) Reset() { *m = StreamRaftMessageResponse{} } +func (*StreamRaftMessageResponse) ProtoMessage() {} +func (*StreamRaftMessageResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{8} } + type ResolveAddressRequest struct { // raft_id is the ID to resolve to an address. RaftID uint64 `protobuf:"varint,1,opt,name=raft_id,json=raftId,proto3" json:"raft_id,omitempty"` @@ -140,7 +158,7 @@ type ResolveAddressRequest struct { func (m *ResolveAddressRequest) Reset() { *m = ResolveAddressRequest{} } func (*ResolveAddressRequest) ProtoMessage() {} -func (*ResolveAddressRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{7} } +func (*ResolveAddressRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{9} } type ResolveAddressResponse struct { // Addr specifies the address of the member @@ -149,7 +167,7 @@ type ResolveAddressResponse struct { func (m *ResolveAddressResponse) Reset() { *m = ResolveAddressResponse{} } func (*ResolveAddressResponse) ProtoMessage() {} -func (*ResolveAddressResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{8} } +func (*ResolveAddressResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{10} } // Contains one of many protobuf encoded objects to replicate // over the raft backend with a request ID to track when the @@ -161,7 +179,7 @@ type InternalRaftRequest struct { func (m *InternalRaftRequest) Reset() { *m = InternalRaftRequest{} } func (*InternalRaftRequest) ProtoMessage() {} -func (*InternalRaftRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{9} } +func (*InternalRaftRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{11} } // StoreAction defines a target and operation to apply on the storage system. type StoreAction struct { @@ -181,7 +199,7 @@ type StoreAction struct { func (m *StoreAction) Reset() { *m = StoreAction{} } func (*StoreAction) ProtoMessage() {} -func (*StoreAction) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{10} } +func (*StoreAction) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{12} } type isStoreAction_Target interface { isStoreAction_Target() @@ -512,6 +530,8 @@ func init() { proto.RegisterType((*LeaveResponse)(nil), "docker.swarmkit.v1.LeaveResponse") proto.RegisterType((*ProcessRaftMessageRequest)(nil), "docker.swarmkit.v1.ProcessRaftMessageRequest") proto.RegisterType((*ProcessRaftMessageResponse)(nil), "docker.swarmkit.v1.ProcessRaftMessageResponse") + proto.RegisterType((*StreamRaftMessageRequest)(nil), "docker.swarmkit.v1.StreamRaftMessageRequest") + proto.RegisterType((*StreamRaftMessageResponse)(nil), "docker.swarmkit.v1.StreamRaftMessageResponse") proto.RegisterType((*ResolveAddressRequest)(nil), "docker.swarmkit.v1.ResolveAddressRequest") proto.RegisterType((*ResolveAddressResponse)(nil), "docker.swarmkit.v1.ResolveAddressResponse") proto.RegisterType((*InternalRaftRequest)(nil), "docker.swarmkit.v1.InternalRaftRequest") @@ -539,6 +559,14 @@ func (p *authenticatedWrapperRaftServer) ProcessRaftMessage(ctx context.Context, return p.local.ProcessRaftMessage(ctx, r) } +func (p *authenticatedWrapperRaftServer) StreamRaftMessage(stream Raft_StreamRaftMessageServer) error { + + if err := p.authorize(stream.Context(), []string{"swarm-manager"}); err != nil { + return err + } + return p.local.StreamRaftMessage(stream) +} + func (p *authenticatedWrapperRaftServer) ResolveAddress(ctx context.Context, r *ResolveAddressRequest) (*ResolveAddressResponse, error) { if err := p.authorize(ctx, []string{"swarm-manager"}); err != nil { @@ -673,6 +701,16 @@ func (m *ProcessRaftMessageResponse) Copy() *ProcessRaftMessageResponse { } func (m *ProcessRaftMessageResponse) CopyFrom(src interface{}) {} +func (m *StreamRaftMessageResponse) Copy() *StreamRaftMessageResponse { + if m == nil { + return nil + } + o := &StreamRaftMessageResponse{} + o.CopyFrom(m) + return o +} + +func (m *StreamRaftMessageResponse) CopyFrom(src interface{}) {} func (m *ResolveAddressRequest) Copy() *ResolveAddressRequest { if m == nil { return nil @@ -813,6 +851,12 @@ type RaftClient interface { // ProcessRaftMessage sends a raft message to be processed on a raft member, it is // called from the RaftMember willing to send a message to its destination ('To' field) ProcessRaftMessage(ctx context.Context, in *ProcessRaftMessageRequest, opts ...grpc.CallOption) (*ProcessRaftMessageResponse, error) + // StreamRaftMessage accepts a stream of raft messages of type StreamRaftMessageRequest + // to be processed on a raft member, returning a StreamRaftMessageResponse + // when processing of the streamed messages is complete. A single stream corresponds + // to a single raft message, which may be disassembled and streamed as individual messages. + // It is called from the Raft leader, which uses it to stream messages to a raft member. + StreamRaftMessage(ctx context.Context, opts ...grpc.CallOption) (Raft_StreamRaftMessageClient, error) // ResolveAddress returns the address where the node with the given ID can be reached. ResolveAddress(ctx context.Context, in *ResolveAddressRequest, opts ...grpc.CallOption) (*ResolveAddressResponse, error) } @@ -834,6 +878,40 @@ func (c *raftClient) ProcessRaftMessage(ctx context.Context, in *ProcessRaftMess return out, nil } +func (c *raftClient) StreamRaftMessage(ctx context.Context, opts ...grpc.CallOption) (Raft_StreamRaftMessageClient, error) { + stream, err := grpc.NewClientStream(ctx, &_Raft_serviceDesc.Streams[0], c.cc, "/docker.swarmkit.v1.Raft/StreamRaftMessage", opts...) + if err != nil { + return nil, err + } + x := &raftStreamRaftMessageClient{stream} + return x, nil +} + +type Raft_StreamRaftMessageClient interface { + Send(*StreamRaftMessageRequest) error + CloseAndRecv() (*StreamRaftMessageResponse, error) + grpc.ClientStream +} + +type raftStreamRaftMessageClient struct { + grpc.ClientStream +} + +func (x *raftStreamRaftMessageClient) Send(m *StreamRaftMessageRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *raftStreamRaftMessageClient) CloseAndRecv() (*StreamRaftMessageResponse, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(StreamRaftMessageResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + func (c *raftClient) ResolveAddress(ctx context.Context, in *ResolveAddressRequest, opts ...grpc.CallOption) (*ResolveAddressResponse, error) { out := new(ResolveAddressResponse) err := grpc.Invoke(ctx, "/docker.swarmkit.v1.Raft/ResolveAddress", in, out, c.cc, opts...) @@ -849,6 +927,12 @@ type RaftServer interface { // ProcessRaftMessage sends a raft message to be processed on a raft member, it is // called from the RaftMember willing to send a message to its destination ('To' field) ProcessRaftMessage(context.Context, *ProcessRaftMessageRequest) (*ProcessRaftMessageResponse, error) + // StreamRaftMessage accepts a stream of raft messages of type StreamRaftMessageRequest + // to be processed on a raft member, returning a StreamRaftMessageResponse + // when processing of the streamed messages is complete. A single stream corresponds + // to a single raft message, which may be disassembled and streamed as individual messages. + // It is called from the Raft leader, which uses it to stream messages to a raft member. + StreamRaftMessage(Raft_StreamRaftMessageServer) error // ResolveAddress returns the address where the node with the given ID can be reached. ResolveAddress(context.Context, *ResolveAddressRequest) (*ResolveAddressResponse, error) } @@ -875,6 +959,32 @@ func _Raft_ProcessRaftMessage_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _Raft_StreamRaftMessage_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(RaftServer).StreamRaftMessage(&raftStreamRaftMessageServer{stream}) +} + +type Raft_StreamRaftMessageServer interface { + SendAndClose(*StreamRaftMessageResponse) error + Recv() (*StreamRaftMessageRequest, error) + grpc.ServerStream +} + +type raftStreamRaftMessageServer struct { + grpc.ServerStream +} + +func (x *raftStreamRaftMessageServer) SendAndClose(m *StreamRaftMessageResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *raftStreamRaftMessageServer) Recv() (*StreamRaftMessageRequest, error) { + m := new(StreamRaftMessageRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + func _Raft_ResolveAddress_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ResolveAddressRequest) if err := dec(in); err != nil { @@ -906,7 +1016,13 @@ var _Raft_serviceDesc = grpc.ServiceDesc{ Handler: _Raft_ResolveAddress_Handler, }, }, - Streams: []grpc.StreamDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamRaftMessage", + Handler: _Raft_StreamRaftMessage_Handler, + ClientStreams: true, + }, + }, Metadata: "github.com/docker/swarmkit/api/raft.proto", } @@ -1212,6 +1328,52 @@ func (m *ProcessRaftMessageResponse) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *StreamRaftMessageRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StreamRaftMessageRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Message != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintRaft(dAtA, i, uint64(m.Message.Size())) + n4, err := m.Message.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n4 + } + return i, nil +} + +func (m *StreamRaftMessageResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StreamRaftMessageResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + func (m *ResolveAddressRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1315,11 +1477,11 @@ func (m *StoreAction) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintRaft(dAtA, i, uint64(m.Action)) } if m.Target != nil { - nn4, err := m.Target.MarshalTo(dAtA[i:]) + nn5, err := m.Target.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += nn4 + i += nn5 } return i, nil } @@ -1330,11 +1492,11 @@ func (m *StoreAction_Node) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Node.Size())) - n5, err := m.Node.MarshalTo(dAtA[i:]) + n6, err := m.Node.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n5 + i += n6 } return i, nil } @@ -1344,11 +1506,11 @@ func (m *StoreAction_Service) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintRaft(dAtA, i, uint64(m.Service.Size())) - n6, err := m.Service.MarshalTo(dAtA[i:]) + n7, err := m.Service.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n6 + i += n7 } return i, nil } @@ -1358,11 +1520,11 @@ func (m *StoreAction_Task) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x22 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Task.Size())) - n7, err := m.Task.MarshalTo(dAtA[i:]) + n8, err := m.Task.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n7 + i += n8 } return i, nil } @@ -1372,11 +1534,11 @@ func (m *StoreAction_Network) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintRaft(dAtA, i, uint64(m.Network.Size())) - n8, err := m.Network.MarshalTo(dAtA[i:]) + n9, err := m.Network.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n8 + i += n9 } return i, nil } @@ -1386,11 +1548,11 @@ func (m *StoreAction_Cluster) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x32 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Cluster.Size())) - n9, err := m.Cluster.MarshalTo(dAtA[i:]) + n10, err := m.Cluster.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n9 + i += n10 } return i, nil } @@ -1400,11 +1562,11 @@ func (m *StoreAction_Secret) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x3a i++ i = encodeVarintRaft(dAtA, i, uint64(m.Secret.Size())) - n10, err := m.Secret.MarshalTo(dAtA[i:]) + n11, err := m.Secret.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n10 + i += n11 } return i, nil } @@ -1414,11 +1576,11 @@ func (m *StoreAction_Resource) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x42 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Resource.Size())) - n11, err := m.Resource.MarshalTo(dAtA[i:]) + n12, err := m.Resource.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n11 + i += n12 } return i, nil } @@ -1428,11 +1590,11 @@ func (m *StoreAction_Extension) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x4a i++ i = encodeVarintRaft(dAtA, i, uint64(m.Extension.Size())) - n12, err := m.Extension.MarshalTo(dAtA[i:]) + n13, err := m.Extension.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n12 + i += n13 } return i, nil } @@ -1442,11 +1604,11 @@ func (m *StoreAction_Config) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x52 i++ i = encodeVarintRaft(dAtA, i, uint64(m.Config.Size())) - n13, err := m.Config.MarshalTo(dAtA[i:]) + n14, err := m.Config.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n14 } return i, nil } @@ -1488,12 +1650,12 @@ func NewRaftProxyRaftServer(local RaftServer, connSelector raftselector.ConnProv redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) @@ -1585,6 +1747,63 @@ func (p *raftProxyRaftServer) ProcessRaftMessage(ctx context.Context, r *Process return resp, err } +type Raft_StreamRaftMessageServerWrapper struct { + Raft_StreamRaftMessageServer + ctx context.Context +} + +func (s Raft_StreamRaftMessageServerWrapper) Context() context.Context { + return s.ctx +} + +func (p *raftProxyRaftServer) StreamRaftMessage(stream Raft_StreamRaftMessageServer) error { + ctx := stream.Context() + conn, err := p.connSelector.LeaderConn(ctx) + if err != nil { + if err == raftselector.ErrIsLeader { + ctx, err = p.runCtxMods(ctx, p.localCtxMods) + if err != nil { + return err + } + streamWrapper := Raft_StreamRaftMessageServerWrapper{ + Raft_StreamRaftMessageServer: stream, + ctx: ctx, + } + return p.local.StreamRaftMessage(streamWrapper) + } + return err + } + ctx, err = p.runCtxMods(ctx, p.remoteCtxMods) + if err != nil { + return err + } + clientStream, err := NewRaftClient(conn).StreamRaftMessage(ctx) + + if err != nil { + return err + } + + for { + msg, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + return err + } + if err := clientStream.Send(msg); err != nil { + return err + } + } + + reply, err := clientStream.CloseAndRecv() + if err != nil { + return err + } + + return stream.SendAndClose(reply) +} + func (p *raftProxyRaftServer) ResolveAddress(ctx context.Context, r *ResolveAddressRequest) (*ResolveAddressResponse, error) { conn, err := p.connSelector.LeaderConn(ctx) @@ -1630,12 +1849,12 @@ func NewRaftProxyRaftMembershipServer(local RaftMembershipServer, connSelector r redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) @@ -1843,6 +2062,22 @@ func (m *ProcessRaftMessageResponse) Size() (n int) { return n } +func (m *StreamRaftMessageRequest) Size() (n int) { + var l int + _ = l + if m.Message != nil { + l = m.Message.Size() + n += 1 + l + sovRaft(uint64(l)) + } + return n +} + +func (m *StreamRaftMessageResponse) Size() (n int) { + var l int + _ = l + return n +} + func (m *ResolveAddressRequest) Size() (n int) { var l int _ = l @@ -2057,6 +2292,25 @@ func (this *ProcessRaftMessageResponse) String() string { }, "") return s } +func (this *StreamRaftMessageRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&StreamRaftMessageRequest{`, + `Message:` + strings.Replace(fmt.Sprintf("%v", this.Message), "Message", "raftpb.Message", 1) + `,`, + `}`, + }, "") + return s +} +func (this *StreamRaftMessageResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&StreamRaftMessageResponse{`, + `}`, + }, "") + return s +} func (this *ResolveAddressRequest) String() string { if this == nil { return "nil" @@ -2861,6 +3115,139 @@ func (m *ProcessRaftMessageResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *StreamRaftMessageRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StreamRaftMessageRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StreamRaftMessageRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRaft + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Message == nil { + m.Message = &raftpb.Message{} + } + if err := m.Message.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRaft(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRaft + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *StreamRaftMessageResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StreamRaftMessageResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StreamRaftMessageResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipRaft(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRaft + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *ResolveAddressRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -3574,66 +3961,69 @@ var ( func init() { proto.RegisterFile("github.com/docker/swarmkit/api/raft.proto", fileDescriptorRaft) } var fileDescriptorRaft = []byte{ - // 974 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x96, 0x4f, 0x6f, 0x1b, 0x45, - 0x18, 0xc6, 0x77, 0xd7, 0x5b, 0x27, 0x79, 0xd3, 0x26, 0xd1, 0x94, 0x84, 0xed, 0x52, 0x1c, 0x77, - 0x8b, 0x84, 0x13, 0x9a, 0xb5, 0x30, 0x48, 0x45, 0x85, 0x1e, 0x62, 0xc7, 0x92, 0x4d, 0x5b, 0xa7, - 0xda, 0x24, 0xd0, 0x5b, 0x58, 0xef, 0x4e, 0xdc, 0xc5, 0xf6, 0x8e, 0x99, 0x19, 0x3b, 0x70, 0x41, - 0x3d, 0xa2, 0x5c, 0x39, 0x80, 0x90, 0x7a, 0x82, 0x73, 0x3f, 0x00, 0x1f, 0x00, 0x45, 0x9c, 0xb8, - 0xc1, 0x29, 0xa2, 0xfe, 0x00, 0xf0, 0x15, 0xd0, 0xcc, 0xee, 0x3a, 0xc6, 0x59, 0x3b, 0xb9, 0x24, - 0xa3, 0x9d, 0xdf, 0xf3, 0x3e, 0xef, 0x3b, 0x7f, 0xde, 0x31, 0x6c, 0xb4, 0x02, 0xfe, 0xbc, 0xdf, - 0xb4, 0x3d, 0xd2, 0x2d, 0xfa, 0xc4, 0x6b, 0x63, 0x5a, 0x64, 0xc7, 0x2e, 0xed, 0xb6, 0x03, 0x5e, - 0x74, 0x7b, 0x41, 0x91, 0xba, 0x47, 0xdc, 0xee, 0x51, 0xc2, 0x09, 0x42, 0xd1, 0xbc, 0x9d, 0xcc, - 0xdb, 0x83, 0xf7, 0xcd, 0x7b, 0x97, 0xc8, 0x49, 0xf3, 0x4b, 0xec, 0x71, 0x16, 0x45, 0x30, 0x37, - 0x2f, 0xa1, 0xf9, 0x37, 0x3d, 0x9c, 0xb0, 0x5b, 0x63, 0xac, 0x47, 0x28, 0x26, 0xac, 0x88, 0xb9, - 0xe7, 0xcb, 0x84, 0xe4, 0x9f, 0x5e, 0x73, 0x2c, 0x39, 0xf3, 0x8d, 0x16, 0x69, 0x11, 0x39, 0x2c, - 0x8a, 0x51, 0xfc, 0xf5, 0xfe, 0x0c, 0x43, 0x49, 0x34, 0xfb, 0x47, 0xc5, 0x5e, 0xa7, 0xdf, 0x0a, - 0xc2, 0xf8, 0x5f, 0x24, 0xb4, 0x5e, 0xa9, 0x00, 0x8e, 0x7b, 0xc4, 0x9f, 0xe0, 0x6e, 0x13, 0x53, - 0x74, 0x17, 0xe6, 0x84, 0xd7, 0x61, 0xe0, 0x1b, 0x6a, 0x5e, 0x2d, 0xe8, 0x65, 0x18, 0x9e, 0xad, - 0x67, 0x05, 0x50, 0xdf, 0x71, 0xb2, 0x62, 0xaa, 0xee, 0x0b, 0x28, 0x24, 0x3e, 0x16, 0x90, 0x96, - 0x57, 0x0b, 0x0b, 0x11, 0xd4, 0x20, 0x3e, 0x16, 0x90, 0x98, 0xaa, 0xfb, 0x08, 0x81, 0xee, 0xfa, - 0x3e, 0x35, 0x32, 0x82, 0x70, 0xe4, 0x18, 0x95, 0x21, 0xcb, 0xb8, 0xcb, 0xfb, 0xcc, 0xd0, 0xf3, - 0x6a, 0x61, 0xb1, 0xf4, 0x8e, 0x7d, 0x71, 0xa5, 0xed, 0xf3, 0x6c, 0xf6, 0x24, 0x5b, 0xd6, 0x4f, - 0xcf, 0xd6, 0x15, 0x27, 0x56, 0x5a, 0x77, 0x60, 0xf1, 0x53, 0x12, 0x84, 0x0e, 0xfe, 0xaa, 0x8f, - 0x19, 0x1f, 0xd9, 0xa8, 0xe7, 0x36, 0xd6, 0x4f, 0x2a, 0x5c, 0x8f, 0x18, 0xd6, 0x23, 0x21, 0xc3, - 0x57, 0xab, 0xea, 0x23, 0x98, 0xeb, 0x4a, 0x5b, 0x66, 0x68, 0xf9, 0x4c, 0x61, 0xb1, 0x94, 0x9b, - 0x9d, 0x9d, 0x93, 0xe0, 0xe8, 0x3d, 0x58, 0xa6, 0xb8, 0x4b, 0x06, 0xd8, 0x3f, 0x4c, 0x22, 0x64, - 0xf2, 0x99, 0x82, 0x5e, 0xd6, 0x56, 0x14, 0x67, 0x29, 0x9e, 0x8a, 0x44, 0xcc, 0x2a, 0xc3, 0xf5, - 0xc7, 0xd8, 0x1d, 0xe0, 0xa4, 0x80, 0x12, 0xe8, 0x62, 0xc5, 0x64, 0x62, 0x97, 0x7b, 0x4a, 0xd6, - 0x5a, 0x86, 0x1b, 0x71, 0x8c, 0xa8, 0x40, 0xeb, 0x31, 0xdc, 0x7a, 0x4a, 0x89, 0x87, 0x19, 0x8b, - 0x58, 0xc6, 0xdc, 0xd6, 0xc8, 0x61, 0x43, 0x14, 0x26, 0xbf, 0xc4, 0x26, 0xcb, 0x76, 0x74, 0xac, - 0xec, 0x04, 0x4c, 0xe6, 0x1f, 0xe8, 0x2f, 0x7e, 0xb0, 0x14, 0xeb, 0x36, 0x98, 0x69, 0xd1, 0x62, - 0xaf, 0x4f, 0x60, 0xd5, 0xc1, 0x8c, 0x74, 0x06, 0x78, 0xdb, 0xf7, 0xa9, 0x80, 0x62, 0x9f, 0xab, - 0xac, 0xb2, 0x75, 0x0f, 0xd6, 0x26, 0xd5, 0xf1, 0x26, 0xa5, 0xed, 0x64, 0x07, 0x6e, 0xd6, 0x43, - 0x8e, 0x69, 0xe8, 0x76, 0x44, 0x9c, 0xc4, 0x69, 0x0d, 0xb4, 0x91, 0x49, 0x76, 0x78, 0xb6, 0xae, - 0xd5, 0x77, 0x1c, 0x2d, 0xf0, 0xd1, 0x43, 0xc8, 0xba, 0x1e, 0x0f, 0x48, 0x18, 0xef, 0xe0, 0x7a, - 0xda, 0x6a, 0xee, 0x71, 0x42, 0xf1, 0xb6, 0xc4, 0x92, 0xa3, 0x15, 0x89, 0xac, 0xdf, 0x74, 0x58, - 0x1c, 0x9b, 0x45, 0x1f, 0x8f, 0xc2, 0x09, 0xab, 0xa5, 0xd2, 0xdd, 0x4b, 0xc2, 0x3d, 0x0a, 0x42, - 0x3f, 0x09, 0x86, 0xec, 0x78, 0x5f, 0x35, 0xb9, 0xe4, 0x46, 0x9a, 0x54, 0xdc, 0x98, 0x9a, 0x12, - 0xed, 0x29, 0xba, 0x0f, 0x73, 0x0c, 0xd3, 0x41, 0xe0, 0x61, 0x79, 0x65, 0x16, 0x4b, 0x6f, 0xa5, - 0xba, 0x45, 0x48, 0x4d, 0x71, 0x12, 0x5a, 0x18, 0x71, 0x97, 0xb5, 0xe3, 0x2b, 0x95, 0x6a, 0xb4, - 0xef, 0xb2, 0xb6, 0x30, 0x12, 0x9c, 0x30, 0x0a, 0x31, 0x3f, 0x26, 0xb4, 0x6d, 0x5c, 0x9b, 0x6e, - 0xd4, 0x88, 0x10, 0x61, 0x14, 0xd3, 0x42, 0xe8, 0x75, 0xfa, 0x8c, 0x63, 0x6a, 0x64, 0xa7, 0x0b, - 0x2b, 0x11, 0x22, 0x84, 0x31, 0x8d, 0x3e, 0x84, 0x2c, 0xc3, 0x1e, 0xc5, 0xdc, 0x98, 0x93, 0x3a, - 0x33, 0xbd, 0x32, 0x41, 0xd4, 0xc4, 0x45, 0x97, 0x23, 0xf4, 0x00, 0xe6, 0x29, 0x66, 0xa4, 0x4f, - 0x3d, 0x6c, 0xcc, 0x4b, 0xdd, 0xed, 0xd4, 0xcb, 0x11, 0x33, 0x35, 0xc5, 0x19, 0xf1, 0xe8, 0x21, - 0x2c, 0xe0, 0xaf, 0x39, 0x0e, 0x99, 0xd8, 0xbc, 0x05, 0x29, 0x7e, 0x3b, 0x4d, 0x5c, 0x4d, 0xa0, - 0x9a, 0xe2, 0x9c, 0x2b, 0x44, 0xc2, 0x1e, 0x09, 0x8f, 0x82, 0x96, 0x01, 0xd3, 0x13, 0xae, 0x48, - 0x42, 0x24, 0x1c, 0xb1, 0xe5, 0x79, 0xc8, 0x72, 0x97, 0xb6, 0x30, 0xdf, 0xfc, 0x57, 0x85, 0xe5, - 0x89, 0x73, 0x81, 0xde, 0x85, 0xb9, 0x83, 0xc6, 0xa3, 0xc6, 0xee, 0xe7, 0x8d, 0x15, 0xc5, 0x34, - 0x4f, 0x5e, 0xe6, 0xd7, 0x26, 0x88, 0x83, 0xb0, 0x1d, 0x92, 0xe3, 0x10, 0x95, 0xe0, 0xe6, 0xde, - 0xfe, 0xae, 0x53, 0x3d, 0xdc, 0xae, 0xec, 0xd7, 0x77, 0x1b, 0x87, 0x15, 0xa7, 0xba, 0xbd, 0x5f, - 0x5d, 0x51, 0xcd, 0x5b, 0x27, 0x2f, 0xf3, 0xab, 0x13, 0xa2, 0x0a, 0xc5, 0x2e, 0xc7, 0x17, 0x34, - 0x07, 0x4f, 0x77, 0x84, 0x46, 0x4b, 0xd5, 0x1c, 0xf4, 0xfc, 0x34, 0x8d, 0x53, 0x7d, 0xb2, 0xfb, - 0x59, 0x75, 0x25, 0x93, 0xaa, 0x71, 0x64, 0x13, 0x33, 0xdf, 0xfc, 0xee, 0xe7, 0x9c, 0xf2, 0xeb, - 0x2f, 0xb9, 0xc9, 0xea, 0x4a, 0xdf, 0x6b, 0xa0, 0x8b, 0x1b, 0x8a, 0x4e, 0x54, 0x40, 0x17, 0x9b, - 0x07, 0xda, 0x4a, 0x5b, 0xc1, 0xa9, 0x2d, 0xcb, 0xb4, 0xaf, 0x8a, 0xc7, 0x3d, 0x69, 0xf5, 0xf7, - 0x57, 0xff, 0xfc, 0xa8, 0x2d, 0xc3, 0x0d, 0xc9, 0x6f, 0x75, 0xdd, 0xd0, 0x6d, 0x61, 0x8a, 0xbe, - 0x85, 0xa5, 0xff, 0x37, 0x1b, 0xb4, 0x31, 0xed, 0x08, 0x5d, 0x68, 0x67, 0xe6, 0xe6, 0x55, 0xd0, - 0x99, 0xfe, 0xa5, 0x3f, 0x55, 0x58, 0x3a, 0x6f, 0xde, 0xec, 0x79, 0xd0, 0x43, 0x5f, 0x80, 0x2e, - 0x9e, 0x26, 0x94, 0xda, 0x9a, 0xc6, 0x1e, 0x36, 0x33, 0x3f, 0x1d, 0x98, 0x5d, 0xb4, 0x07, 0xd7, - 0xe4, 0xe3, 0x80, 0x52, 0x23, 0x8c, 0xbf, 0x3d, 0xe6, 0x9d, 0x19, 0xc4, 0x4c, 0x93, 0xb2, 0x71, - 0xfa, 0x3a, 0xa7, 0xfc, 0xf5, 0x3a, 0xa7, 0xbc, 0x18, 0xe6, 0xd4, 0xd3, 0x61, 0x4e, 0xfd, 0x63, - 0x98, 0x53, 0xff, 0x1e, 0xe6, 0xd4, 0x67, 0x99, 0x67, 0x7a, 0x33, 0x2b, 0x7f, 0x5b, 0x7c, 0xf0, - 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x64, 0x01, 0xa3, 0x4f, 0x74, 0x09, 0x00, 0x00, + // 1015 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x96, 0xc1, 0x6e, 0x1b, 0x45, + 0x18, 0xc7, 0x77, 0xed, 0xad, 0xd3, 0x7c, 0x69, 0x93, 0x30, 0x25, 0x61, 0xb3, 0x2d, 0x8e, 0xbb, + 0x45, 0xc2, 0x09, 0xc9, 0x5a, 0x18, 0xa4, 0xa2, 0x42, 0x0f, 0x71, 0x62, 0x29, 0x26, 0xad, 0x53, + 0x6d, 0x12, 0xe8, 0x2d, 0xac, 0x77, 0x27, 0xee, 0x62, 0x7b, 0xc7, 0xcc, 0x8c, 0x1d, 0xb8, 0xa0, + 0x1e, 0x21, 0x2f, 0x00, 0x42, 0xaa, 0x38, 0xc0, 0xb9, 0x0f, 0xc0, 0x03, 0xa0, 0x88, 0x13, 0x37, + 0x38, 0x45, 0xd4, 0x0f, 0x00, 0xaf, 0x80, 0x66, 0x76, 0xd7, 0x31, 0xf6, 0xda, 0xf1, 0x81, 0x4b, + 0x32, 0xda, 0xf9, 0xfd, 0xbf, 0xff, 0x37, 0x33, 0xdf, 0x7c, 0x63, 0x58, 0xab, 0xfb, 0xfc, 0x59, + 0xa7, 0x66, 0xb9, 0xa4, 0x55, 0xf0, 0x88, 0xdb, 0xc0, 0xb4, 0xc0, 0x4e, 0x1d, 0xda, 0x6a, 0xf8, + 0xbc, 0xe0, 0xb4, 0xfd, 0x02, 0x75, 0x4e, 0xb8, 0xd5, 0xa6, 0x84, 0x13, 0x84, 0xc2, 0x79, 0x2b, + 0x9e, 0xb7, 0xba, 0xef, 0x1a, 0x1b, 0x57, 0xc8, 0x49, 0xed, 0x73, 0xec, 0x72, 0x16, 0x46, 0x30, + 0xd6, 0xaf, 0xa0, 0xf9, 0x57, 0x6d, 0x1c, 0xb3, 0x9b, 0x03, 0xac, 0x4b, 0x28, 0x26, 0xac, 0x80, + 0xb9, 0xeb, 0xc9, 0x84, 0xe4, 0x9f, 0x76, 0x6d, 0x20, 0x39, 0xe3, 0xf5, 0x3a, 0xa9, 0x13, 0x39, + 0x2c, 0x88, 0x51, 0xf4, 0xf5, 0xfe, 0x04, 0x43, 0x49, 0xd4, 0x3a, 0x27, 0x85, 0x76, 0xb3, 0x53, + 0xf7, 0x83, 0xe8, 0x5f, 0x28, 0x34, 0x5f, 0xaa, 0x00, 0xb6, 0x73, 0xc2, 0x1f, 0xe3, 0x56, 0x0d, + 0x53, 0x74, 0x0f, 0x66, 0x84, 0xd7, 0xb1, 0xef, 0xe9, 0x6a, 0x4e, 0xcd, 0x6b, 0x25, 0xe8, 0x5d, + 0xac, 0x66, 0x04, 0x50, 0xd9, 0xb1, 0x33, 0x62, 0xaa, 0xe2, 0x09, 0x28, 0x20, 0x1e, 0x16, 0x50, + 0x2a, 0xa7, 0xe6, 0x67, 0x43, 0xa8, 0x4a, 0x3c, 0x2c, 0x20, 0x31, 0x55, 0xf1, 0x10, 0x02, 0xcd, + 0xf1, 0x3c, 0xaa, 0xa7, 0x05, 0x61, 0xcb, 0x31, 0x2a, 0x41, 0x86, 0x71, 0x87, 0x77, 0x98, 0xae, + 0xe5, 0xd4, 0xfc, 0x5c, 0xf1, 0x2d, 0x6b, 0x74, 0xa7, 0xad, 0xcb, 0x6c, 0x0e, 0x24, 0x5b, 0xd2, + 0xce, 0x2f, 0x56, 0x15, 0x3b, 0x52, 0x9a, 0x77, 0x61, 0xee, 0x63, 0xe2, 0x07, 0x36, 0xfe, 0xa2, + 0x83, 0x19, 0xef, 0xdb, 0xa8, 0x97, 0x36, 0xe6, 0x0f, 0x2a, 0xdc, 0x08, 0x19, 0xd6, 0x26, 0x01, + 0xc3, 0xd3, 0xad, 0xea, 0x03, 0x98, 0x69, 0x49, 0x5b, 0xa6, 0xa7, 0x72, 0xe9, 0xfc, 0x5c, 0x31, + 0x3b, 0x39, 0x3b, 0x3b, 0xc6, 0xd1, 0x3b, 0xb0, 0x40, 0x71, 0x8b, 0x74, 0xb1, 0x77, 0x1c, 0x47, + 0x48, 0xe7, 0xd2, 0x79, 0xad, 0x94, 0x5a, 0x54, 0xec, 0xf9, 0x68, 0x2a, 0x14, 0x31, 0xb3, 0x04, + 0x37, 0x1e, 0x61, 0xa7, 0x8b, 0xe3, 0x05, 0x14, 0x41, 0x13, 0x3b, 0x26, 0x13, 0xbb, 0xda, 0x53, + 0xb2, 0xe6, 0x02, 0xdc, 0x8c, 0x62, 0x84, 0x0b, 0x34, 0x1f, 0xc1, 0xca, 0x13, 0x4a, 0x5c, 0xcc, + 0x58, 0xc8, 0x32, 0xe6, 0xd4, 0xfb, 0x0e, 0x6b, 0x62, 0x61, 0xf2, 0x4b, 0x64, 0xb2, 0x60, 0x85, + 0x65, 0x65, 0xc5, 0x60, 0x3c, 0xff, 0x40, 0x7b, 0xfe, 0x9d, 0xa9, 0x98, 0x77, 0xc0, 0x48, 0x8a, + 0x16, 0x79, 0xed, 0x81, 0x7e, 0xc0, 0x29, 0x76, 0x5a, 0xff, 0x87, 0xd5, 0x6d, 0x58, 0x49, 0x08, + 0x16, 0x39, 0x7d, 0x04, 0x4b, 0x36, 0x66, 0xa4, 0xd9, 0xc5, 0x5b, 0x9e, 0x47, 0x45, 0x3a, 0x91, + 0xcd, 0x34, 0xe7, 0x69, 0x6e, 0xc0, 0xf2, 0xb0, 0x3a, 0x2a, 0x87, 0xa4, 0x9a, 0x69, 0xc2, 0xad, + 0x4a, 0xc0, 0x31, 0x0d, 0x9c, 0xa6, 0x88, 0x13, 0x3b, 0x2d, 0x43, 0xaa, 0x6f, 0x92, 0xe9, 0x5d, + 0xac, 0xa6, 0x2a, 0x3b, 0x76, 0xca, 0xf7, 0xd0, 0x43, 0xc8, 0x38, 0x2e, 0xf7, 0x49, 0x10, 0xd5, + 0xca, 0x6a, 0xd2, 0xb9, 0x1d, 0x70, 0x42, 0xf1, 0x96, 0xc4, 0xe2, 0x22, 0x0e, 0x45, 0xe6, 0xaf, + 0x1a, 0xcc, 0x0d, 0xcc, 0xa2, 0x0f, 0xfb, 0xe1, 0x84, 0xd5, 0x7c, 0xf1, 0xde, 0x15, 0xe1, 0xf6, + 0xfc, 0xc0, 0x8b, 0x83, 0x21, 0x2b, 0xaa, 0xa0, 0x94, 0xdc, 0x71, 0x3d, 0x49, 0x2a, 0xee, 0xe6, + 0xae, 0x12, 0x56, 0x0f, 0xba, 0x0f, 0x33, 0x0c, 0xd3, 0xae, 0xef, 0x62, 0x79, 0x39, 0xe7, 0x8a, + 0xb7, 0x13, 0xdd, 0x42, 0x64, 0x57, 0xb1, 0x63, 0x5a, 0x18, 0x71, 0x87, 0x35, 0xa2, 0xcb, 0x9b, + 0x68, 0x74, 0xe8, 0xb0, 0x86, 0x30, 0x12, 0x9c, 0x30, 0x0a, 0x30, 0x3f, 0x25, 0xb4, 0xa1, 0x5f, + 0x1b, 0x6f, 0x54, 0x0d, 0x11, 0x61, 0x14, 0xd1, 0x42, 0xe8, 0x36, 0x3b, 0x8c, 0x63, 0xaa, 0x67, + 0xc6, 0x0b, 0xb7, 0x43, 0x44, 0x08, 0x23, 0x1a, 0xbd, 0x0f, 0x19, 0x86, 0x5d, 0x8a, 0xb9, 0x3e, + 0x23, 0x75, 0x46, 0xf2, 0xca, 0x04, 0xb1, 0x2b, 0x5a, 0x8a, 0x1c, 0xa1, 0x07, 0x70, 0x9d, 0x62, + 0x46, 0x3a, 0xd4, 0xc5, 0xfa, 0x75, 0xa9, 0xbb, 0x93, 0x78, 0x0d, 0x23, 0x66, 0x57, 0xb1, 0xfb, + 0x3c, 0x7a, 0x08, 0xb3, 0xf8, 0x4b, 0x8e, 0x03, 0x26, 0x0e, 0x6f, 0x56, 0x8a, 0xdf, 0x4c, 0x12, + 0x97, 0x63, 0x68, 0x57, 0xb1, 0x2f, 0x15, 0x22, 0x61, 0x97, 0x04, 0x27, 0x7e, 0x5d, 0x87, 0xf1, + 0x09, 0x6f, 0x4b, 0x42, 0x24, 0x1c, 0xb2, 0xa5, 0xeb, 0x90, 0xe1, 0x0e, 0xad, 0x63, 0xbe, 0xfe, + 0x8f, 0x0a, 0x0b, 0x43, 0x75, 0x81, 0xde, 0x86, 0x99, 0xa3, 0xea, 0x5e, 0x75, 0xff, 0xd3, 0xea, + 0xa2, 0x62, 0x18, 0x67, 0x2f, 0x72, 0xcb, 0x43, 0xc4, 0x51, 0xd0, 0x08, 0xc8, 0x69, 0x80, 0x8a, + 0x70, 0xeb, 0xe0, 0x70, 0xdf, 0x2e, 0x1f, 0x6f, 0x6d, 0x1f, 0x56, 0xf6, 0xab, 0xc7, 0xdb, 0x76, + 0x79, 0xeb, 0xb0, 0xbc, 0xa8, 0x1a, 0x2b, 0x67, 0x2f, 0x72, 0x4b, 0x43, 0xa2, 0x6d, 0x8a, 0x1d, + 0x8e, 0x47, 0x34, 0x47, 0x4f, 0x76, 0x84, 0x26, 0x95, 0xa8, 0x39, 0x6a, 0x7b, 0x49, 0x1a, 0xbb, + 0xfc, 0x78, 0xff, 0x93, 0xf2, 0x62, 0x3a, 0x51, 0x63, 0xcb, 0x76, 0x69, 0xbc, 0xf1, 0xcd, 0x4f, + 0x59, 0xe5, 0x97, 0x9f, 0xb3, 0xc3, 0xab, 0x2b, 0xfe, 0x98, 0x06, 0x4d, 0xdc, 0x50, 0x74, 0xa6, + 0x02, 0x1a, 0x6d, 0x53, 0x68, 0x33, 0x69, 0x07, 0xc7, 0x36, 0x47, 0xc3, 0x9a, 0x16, 0x8f, 0x7a, + 0xd2, 0xd2, 0x6f, 0x2f, 0xff, 0xfe, 0x3e, 0xb5, 0x00, 0x37, 0x25, 0xbf, 0xd9, 0x72, 0x02, 0xa7, + 0x8e, 0x29, 0xfa, 0x56, 0x85, 0xd7, 0x46, 0x1a, 0x19, 0xda, 0x48, 0xbe, 0xc6, 0xc9, 0xcd, 0xd3, + 0xd8, 0x9c, 0x92, 0x9e, 0x98, 0x49, 0x5e, 0x45, 0x5f, 0xc3, 0xfc, 0x7f, 0x1b, 0x1f, 0x5a, 0x1b, + 0x57, 0xce, 0x23, 0xad, 0xd5, 0x58, 0x9f, 0x06, 0x9d, 0x98, 0x41, 0xf1, 0x0f, 0x15, 0xe6, 0x2f, + 0x9f, 0x2c, 0xf6, 0xcc, 0x6f, 0xa3, 0xcf, 0x40, 0x13, 0x0f, 0x32, 0x4a, 0x6c, 0x93, 0x03, 0xcf, + 0xb9, 0x91, 0x1b, 0x0f, 0x4c, 0x3e, 0x00, 0x17, 0xae, 0xc9, 0x27, 0x11, 0x25, 0x46, 0x18, 0x7c, + 0x71, 0x8d, 0xbb, 0x13, 0x88, 0x89, 0x26, 0x25, 0xfd, 0xfc, 0x55, 0x56, 0xf9, 0xf3, 0x55, 0x56, + 0x79, 0xde, 0xcb, 0xaa, 0xe7, 0xbd, 0xac, 0xfa, 0x7b, 0x2f, 0xab, 0xfe, 0xd5, 0xcb, 0xaa, 0x4f, + 0xd3, 0x4f, 0xb5, 0x5a, 0x46, 0xfe, 0xa2, 0x7a, 0xef, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x3e, + 0x7a, 0x8b, 0xe7, 0x6a, 0x0a, 0x00, 0x00, } diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/raft.proto b/components/engine/vendor/github.com/docker/swarmkit/api/raft.proto index b39831584d..b351c15bde 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/raft.proto +++ b/components/engine/vendor/github.com/docker/swarmkit/api/raft.proto @@ -16,6 +16,15 @@ service Raft { option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" }; }; + // StreamRaftMessage accepts a stream of raft messages of type StreamRaftMessageRequest + // to be processed on a raft member, returning a StreamRaftMessageResponse + // when processing of the streamed messages is complete. A single stream corresponds + // to a single raft message, which may be disassembled and streamed as individual messages. + // It is called from the Raft leader, which uses it to stream messages to a raft member. + rpc StreamRaftMessage(stream StreamRaftMessageRequest) returns (StreamRaftMessageResponse) { + option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" }; + }; + // ResolveAddress returns the address where the node with the given ID can be reached. rpc ResolveAddress(ResolveAddressRequest) returns (ResolveAddressResponse) { option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" }; @@ -82,6 +91,15 @@ message ProcessRaftMessageRequest { message ProcessRaftMessageResponse {} +// Raft message streaming request. +message StreamRaftMessageRequest { + option (docker.protobuf.plugin.deepcopy) = false; + raftpb.Message message = 1; +} + +// Raft message streaming response. +message StreamRaftMessageResponse {} + message ResolveAddressRequest { // raft_id is the ID to resolve to an address. uint64 raft_id = 1; diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/resource.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/resource.pb.go index ead5d27ef4..d4e27f3362 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/resource.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/resource.pb.go @@ -19,6 +19,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -403,12 +404,12 @@ func NewRaftProxyResourceAllocatorServer(local ResourceAllocatorServer, connSele redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/types.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/types.pb.go index 33e2281b67..67243dc88e 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/types.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/types.pb.go @@ -72,7 +72,16 @@ const ( TaskStateShutdown TaskState = 640 TaskStateFailed TaskState = 704 TaskStateRejected TaskState = 768 - TaskStateOrphaned TaskState = 832 + // TaskStateRemove is used to correctly handle service deletions and scale + // downs. This allows us to keep track of tasks that have been marked for + // deletion, but can't yet be removed because the agent is in the process of + // shutting them down. Once the agent has shut down tasks with desired state + // REMOVE, the task reaper is responsible for removing them. + TaskStateRemove TaskState = 800 + // TaskStateOrphaned is used to free up resources associated with service + // tasks on unresponsive nodes without having to delete those tasks. This + // state is directly assigned to the task by the orchestrator. + TaskStateOrphaned TaskState = 832 ) var TaskState_name = map[int32]string{ @@ -88,6 +97,7 @@ var TaskState_name = map[int32]string{ 640: "SHUTDOWN", 704: "FAILED", 768: "REJECTED", + 800: "REMOVE", 832: "ORPHANED", } var TaskState_value = map[string]int32{ @@ -103,6 +113,7 @@ var TaskState_value = map[string]int32{ "SHUTDOWN": 640, "FAILED": 704, "REJECTED": 768, + "REMOVE": 800, "ORPHANED": 832, } @@ -17029,319 +17040,321 @@ var ( func init() { proto.RegisterFile("github.com/docker/swarmkit/api/types.proto", fileDescriptorTypes) } var fileDescriptorTypes = []byte{ - // 5020 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x5a, 0x4d, 0x6c, 0x24, 0x49, - 0x56, 0x76, 0xfd, 0xba, 0xea, 0x55, 0xd9, 0x4e, 0x47, 0x7b, 0x7b, 0xdc, 0xb5, 0xdd, 0x76, 0x4d, - 0xce, 0xf4, 0xce, 0x6c, 0x6f, 0x53, 0xfd, 0xb7, 0xbb, 0xea, 0x99, 0x61, 0x77, 0xa6, 0xfe, 0x6c, - 0xd7, 0xb6, 0x5d, 0x55, 0x8a, 0x2a, 0x77, 0xef, 0x22, 0x41, 0x2a, 0x9d, 0x19, 0x2e, 0xe7, 0x38, - 0x2b, 0xa3, 0xc8, 0xcc, 0xb2, 0xbb, 0x58, 0x10, 0x2d, 0x0e, 0x80, 0x7c, 0x82, 0xdb, 0x22, 0x64, - 0x2e, 0x70, 0x42, 0x48, 0x1c, 0x40, 0x42, 0x70, 0x1a, 0x24, 0x0e, 0x7b, 0x83, 0x05, 0x09, 0xad, - 0x40, 0x32, 0xac, 0x0f, 0xdc, 0x56, 0x70, 0x59, 0x71, 0x01, 0x09, 0xc5, 0x4f, 0x66, 0xa5, 0xab, - 0xd3, 0x76, 0x0f, 0xb3, 0x17, 0xbb, 0xe2, 0xbd, 0xef, 0xbd, 0x78, 0xf1, 0x22, 0xe2, 0xc5, 0x7b, - 0x11, 0x09, 0xf7, 0x06, 0x96, 0x7f, 0x30, 0xde, 0xab, 0x18, 0x74, 0xf8, 0xc0, 0xa4, 0xc6, 0x21, - 0x71, 0x1f, 0x78, 0xc7, 0xba, 0x3b, 0x3c, 0xb4, 0xfc, 0x07, 0xfa, 0xc8, 0x7a, 0xe0, 0x4f, 0x46, - 0xc4, 0xab, 0x8c, 0x5c, 0xea, 0x53, 0x84, 0x04, 0xa0, 0x12, 0x00, 0x2a, 0x47, 0x8f, 0x4a, 0xeb, - 0x03, 0x4a, 0x07, 0x36, 0x79, 0xc0, 0x11, 0x7b, 0xe3, 0xfd, 0x07, 0xbe, 0x35, 0x24, 0x9e, 0xaf, - 0x0f, 0x47, 0x42, 0xa8, 0xb4, 0x36, 0x0b, 0x30, 0xc7, 0xae, 0xee, 0x5b, 0xd4, 0x91, 0xfc, 0x95, - 0x01, 0x1d, 0x50, 0xfe, 0xf3, 0x01, 0xfb, 0x25, 0xa8, 0xea, 0x3a, 0xcc, 0x3f, 0x27, 0xae, 0x67, - 0x51, 0x07, 0xad, 0x40, 0xc6, 0x72, 0x4c, 0xf2, 0x72, 0x35, 0x51, 0x4e, 0xbc, 0x9f, 0xc6, 0xa2, - 0xa1, 0x3e, 0x04, 0x68, 0xb1, 0x1f, 0x4d, 0xc7, 0x77, 0x27, 0x48, 0x81, 0xd4, 0x21, 0x99, 0x70, - 0x44, 0x1e, 0xb3, 0x9f, 0x8c, 0x72, 0xa4, 0xdb, 0xab, 0x49, 0x41, 0x39, 0xd2, 0x6d, 0xf5, 0x27, - 0x09, 0x28, 0x54, 0x1d, 0x87, 0xfa, 0xbc, 0x77, 0x0f, 0x21, 0x48, 0x3b, 0xfa, 0x90, 0x48, 0x21, - 0xfe, 0x1b, 0xd5, 0x21, 0x6b, 0xeb, 0x7b, 0xc4, 0xf6, 0x56, 0x93, 0xe5, 0xd4, 0xfb, 0x85, 0xc7, - 0x5f, 0xab, 0xbc, 0x3e, 0xe4, 0x4a, 0x44, 0x49, 0x65, 0x9b, 0xa3, 0xb9, 0x11, 0x58, 0x8a, 0xa2, - 0x6f, 0xc3, 0xbc, 0xe5, 0x98, 0x96, 0x41, 0xbc, 0xd5, 0x34, 0xd7, 0xb2, 0x16, 0xa7, 0x65, 0x6a, - 0x7d, 0x2d, 0xfd, 0xc3, 0xb3, 0xf5, 0x39, 0x1c, 0x08, 0x95, 0x3e, 0x80, 0x42, 0x44, 0x6d, 0xcc, - 0xd8, 0x56, 0x20, 0x73, 0xa4, 0xdb, 0x63, 0x22, 0x47, 0x27, 0x1a, 0x1f, 0x26, 0x9f, 0x26, 0xd4, - 0x4f, 0x60, 0xa5, 0xad, 0x0f, 0x89, 0xb9, 0x49, 0x1c, 0xe2, 0x5a, 0x06, 0x26, 0x1e, 0x1d, 0xbb, - 0x06, 0x61, 0x63, 0x3d, 0xb4, 0x1c, 0x33, 0x18, 0x2b, 0xfb, 0x1d, 0xaf, 0x45, 0xad, 0xc3, 0x5b, - 0x0d, 0xcb, 0x33, 0x5c, 0xe2, 0x93, 0xcf, 0xad, 0x24, 0x15, 0x28, 0x39, 0x4b, 0xc0, 0xd2, 0xac, - 0xf4, 0x2f, 0xc1, 0x0d, 0xe6, 0x62, 0x53, 0x73, 0x25, 0x45, 0xf3, 0x46, 0xc4, 0xe0, 0xca, 0x0a, - 0x8f, 0xdf, 0x8f, 0xf3, 0x50, 0xdc, 0x48, 0xb6, 0xe6, 0xf0, 0x32, 0x57, 0x13, 0x10, 0x7a, 0x23, - 0x62, 0x20, 0x03, 0x6e, 0x9a, 0xd2, 0xe8, 0x19, 0xf5, 0x49, 0xae, 0x3e, 0x76, 0x1a, 0x2f, 0x19, - 0xe6, 0xd6, 0x1c, 0x5e, 0x09, 0x94, 0x45, 0x3b, 0xa9, 0x01, 0xe4, 0x02, 0xdd, 0xea, 0x0f, 0x12, - 0x90, 0x0f, 0x98, 0x1e, 0xfa, 0x2a, 0xe4, 0x1d, 0xdd, 0xa1, 0x9a, 0x31, 0x1a, 0x7b, 0x7c, 0x40, - 0xa9, 0x5a, 0xf1, 0xfc, 0x6c, 0x3d, 0xd7, 0xd6, 0x1d, 0x5a, 0xef, 0xee, 0x7a, 0x38, 0xc7, 0xd8, - 0xf5, 0xd1, 0xd8, 0x43, 0x6f, 0x43, 0x71, 0x48, 0x86, 0xd4, 0x9d, 0x68, 0x7b, 0x13, 0x9f, 0x78, - 0xd2, 0x6d, 0x05, 0x41, 0xab, 0x31, 0x12, 0xfa, 0x16, 0xcc, 0x0f, 0x84, 0x49, 0xab, 0x29, 0xbe, - 0x7c, 0xde, 0x89, 0xb3, 0x7e, 0xc6, 0x6a, 0x1c, 0xc8, 0xa8, 0xbf, 0x97, 0x80, 0x95, 0x90, 0x4a, - 0x7e, 0x75, 0x6c, 0xb9, 0x64, 0x48, 0x1c, 0xdf, 0x43, 0xdf, 0x80, 0xac, 0x6d, 0x0d, 0x2d, 0xdf, - 0x93, 0x3e, 0xbf, 0x13, 0xa7, 0x36, 0x1c, 0x14, 0x96, 0x60, 0x54, 0x85, 0xa2, 0x4b, 0x3c, 0xe2, - 0x1e, 0x89, 0x15, 0x2f, 0x3d, 0x7a, 0x8d, 0xf0, 0x05, 0x11, 0x75, 0x03, 0x72, 0x5d, 0x5b, 0xf7, - 0xf7, 0xa9, 0x3b, 0x44, 0x2a, 0x14, 0x75, 0xd7, 0x38, 0xb0, 0x7c, 0x62, 0xf8, 0x63, 0x37, 0xd8, - 0x7d, 0x17, 0x68, 0xe8, 0x26, 0x24, 0xa9, 0xe8, 0x28, 0x5f, 0xcb, 0x9e, 0x9f, 0xad, 0x27, 0x3b, - 0x3d, 0x9c, 0xa4, 0x9e, 0xfa, 0x11, 0x2c, 0x77, 0xed, 0xf1, 0xc0, 0x72, 0x1a, 0xc4, 0x33, 0x5c, - 0x6b, 0xc4, 0xb4, 0xb3, 0x55, 0xc9, 0x62, 0x54, 0xb0, 0x2a, 0xd9, 0xef, 0x70, 0x6b, 0x27, 0xa7, - 0x5b, 0x5b, 0xfd, 0x9d, 0x24, 0x2c, 0x37, 0x9d, 0x81, 0xe5, 0x90, 0xa8, 0xf4, 0x5d, 0x58, 0x24, - 0x9c, 0xa8, 0x1d, 0x89, 0x70, 0x23, 0xf5, 0x2c, 0x08, 0x6a, 0x10, 0x83, 0x5a, 0x33, 0x71, 0xe1, - 0x51, 0xdc, 0xf0, 0x5f, 0xd3, 0x1e, 0x1b, 0x1d, 0x9a, 0x30, 0x3f, 0xe2, 0x83, 0xf0, 0xe4, 0xf4, - 0xde, 0x8d, 0xd3, 0xf5, 0xda, 0x38, 0x83, 0x20, 0x21, 0x65, 0xbf, 0x48, 0x90, 0xf8, 0xb3, 0x24, - 0x2c, 0xb5, 0xa9, 0x79, 0xc1, 0x0f, 0x25, 0xc8, 0x1d, 0x50, 0xcf, 0x8f, 0x04, 0xc4, 0xb0, 0x8d, - 0x9e, 0x42, 0x6e, 0x24, 0xa7, 0x4f, 0xce, 0xfe, 0xed, 0x78, 0x93, 0x05, 0x06, 0x87, 0x68, 0xf4, - 0x11, 0xe4, 0x83, 0x2d, 0xc3, 0x46, 0xfb, 0x06, 0x0b, 0x67, 0x8a, 0x47, 0xdf, 0x82, 0xac, 0x98, - 0x84, 0xd5, 0x34, 0x97, 0xbc, 0xfb, 0x46, 0x3e, 0xc7, 0x52, 0x08, 0x6d, 0x42, 0xce, 0xb7, 0x3d, - 0xcd, 0x72, 0xf6, 0xe9, 0x6a, 0x86, 0x2b, 0x58, 0x8f, 0x0d, 0x32, 0xd4, 0x24, 0xfd, 0xed, 0x5e, - 0xcb, 0xd9, 0xa7, 0xb5, 0xc2, 0xf9, 0xd9, 0xfa, 0xbc, 0x6c, 0xe0, 0x79, 0xdf, 0xf6, 0xd8, 0x0f, - 0xf5, 0xf7, 0x13, 0x50, 0x88, 0xa0, 0xd0, 0x1d, 0x00, 0xdf, 0x1d, 0x7b, 0xbe, 0xe6, 0x52, 0xea, - 0x73, 0x67, 0x15, 0x71, 0x9e, 0x53, 0x30, 0xa5, 0x3e, 0xaa, 0xc0, 0x0d, 0x83, 0xb8, 0xbe, 0x66, - 0x79, 0xde, 0x98, 0xb8, 0x9a, 0x37, 0xde, 0xfb, 0x94, 0x18, 0x3e, 0x77, 0x5c, 0x11, 0x2f, 0x33, - 0x56, 0x8b, 0x73, 0x7a, 0x82, 0x81, 0x9e, 0xc0, 0xcd, 0x28, 0x7e, 0x34, 0xde, 0xb3, 0x2d, 0x43, - 0x63, 0x93, 0x99, 0xe2, 0x22, 0x37, 0xa6, 0x22, 0x5d, 0xce, 0x7b, 0x46, 0x26, 0xea, 0x8f, 0x13, - 0xa0, 0x60, 0x7d, 0xdf, 0xdf, 0x21, 0xc3, 0x3d, 0xe2, 0xf6, 0x7c, 0xdd, 0x1f, 0x7b, 0xe8, 0x26, - 0x64, 0x6d, 0xa2, 0x9b, 0xc4, 0xe5, 0x46, 0xe5, 0xb0, 0x6c, 0xa1, 0x5d, 0xb6, 0x83, 0x75, 0xe3, - 0x40, 0xdf, 0xb3, 0x6c, 0xcb, 0x9f, 0x70, 0x53, 0x16, 0xe3, 0x97, 0xf0, 0xac, 0xce, 0x0a, 0x8e, - 0x08, 0xe2, 0x0b, 0x6a, 0xd0, 0x2a, 0xcc, 0x0f, 0x89, 0xe7, 0xe9, 0x03, 0xc2, 0x2d, 0xcd, 0xe3, - 0xa0, 0xa9, 0x7e, 0x04, 0xc5, 0xa8, 0x1c, 0x2a, 0xc0, 0xfc, 0x6e, 0xfb, 0x59, 0xbb, 0xf3, 0xa2, - 0xad, 0xcc, 0xa1, 0x25, 0x28, 0xec, 0xb6, 0x71, 0xb3, 0x5a, 0xdf, 0xaa, 0xd6, 0xb6, 0x9b, 0x4a, - 0x02, 0x2d, 0x40, 0x7e, 0xda, 0x4c, 0xaa, 0x7f, 0x91, 0x00, 0x60, 0xee, 0x96, 0x83, 0xfa, 0x10, - 0x32, 0x9e, 0xaf, 0xfb, 0x62, 0x55, 0x2e, 0x3e, 0x7e, 0xf7, 0xb2, 0x39, 0x94, 0xf6, 0xb2, 0x7f, - 0x04, 0x0b, 0x91, 0xa8, 0x85, 0xc9, 0x0b, 0x16, 0xb2, 0x00, 0xa1, 0x9b, 0xa6, 0x2b, 0x0d, 0xe7, - 0xbf, 0xd5, 0x8f, 0x20, 0xc3, 0xa5, 0x2f, 0x9a, 0x9b, 0x83, 0x74, 0x83, 0xfd, 0x4a, 0xa0, 0x3c, - 0x64, 0x70, 0xb3, 0xda, 0xf8, 0x9e, 0x92, 0x44, 0x0a, 0x14, 0x1b, 0xad, 0x5e, 0xbd, 0xd3, 0x6e, - 0x37, 0xeb, 0xfd, 0x66, 0x43, 0x49, 0xa9, 0x77, 0x21, 0xd3, 0x1a, 0x32, 0xcd, 0xb7, 0xd9, 0x92, - 0xdf, 0x27, 0x2e, 0x71, 0x8c, 0x60, 0x27, 0x4d, 0x09, 0xea, 0x4f, 0x0b, 0x90, 0xd9, 0xa1, 0x63, - 0xc7, 0x47, 0x8f, 0x23, 0x61, 0x6b, 0x31, 0x3e, 0x43, 0xe0, 0xc0, 0x4a, 0x7f, 0x32, 0x22, 0x32, - 0xac, 0xdd, 0x84, 0xac, 0xd8, 0x1c, 0x72, 0x38, 0xb2, 0xc5, 0xe8, 0xbe, 0xee, 0x0e, 0x88, 0x2f, - 0xc7, 0x23, 0x5b, 0xe8, 0x7d, 0x76, 0x62, 0xe9, 0x26, 0x75, 0xec, 0x09, 0xdf, 0x43, 0x39, 0x71, - 0x2c, 0x61, 0xa2, 0x9b, 0x1d, 0xc7, 0x9e, 0xe0, 0x90, 0x8b, 0xb6, 0xa0, 0xb8, 0x67, 0x39, 0xa6, - 0x46, 0x47, 0x22, 0xc8, 0x67, 0x2e, 0xdf, 0x71, 0xc2, 0xaa, 0x9a, 0xe5, 0x98, 0x1d, 0x01, 0xc6, - 0x85, 0xbd, 0x69, 0x03, 0xb5, 0x61, 0xf1, 0x88, 0xda, 0xe3, 0x21, 0x09, 0x75, 0x65, 0xb9, 0xae, - 0xf7, 0x2e, 0xd7, 0xf5, 0x9c, 0xe3, 0x03, 0x6d, 0x0b, 0x47, 0xd1, 0x26, 0x7a, 0x06, 0x0b, 0xfe, - 0x70, 0xb4, 0xef, 0x85, 0xea, 0xe6, 0xb9, 0xba, 0xaf, 0x5c, 0xe1, 0x30, 0x06, 0x0f, 0xb4, 0x15, - 0xfd, 0x48, 0x0b, 0x6d, 0x42, 0xc1, 0xa0, 0x8e, 0x67, 0x79, 0x3e, 0x71, 0x8c, 0xc9, 0x6a, 0x8e, - 0xfb, 0xfe, 0x8a, 0x51, 0xd6, 0xa7, 0x60, 0x1c, 0x95, 0x2c, 0xfd, 0x56, 0x0a, 0x0a, 0x11, 0x17, - 0xa0, 0x1e, 0x14, 0x46, 0x2e, 0x1d, 0xe9, 0x03, 0x7e, 0xe2, 0xc9, 0x49, 0x7d, 0xf4, 0x46, 0xee, - 0xab, 0x74, 0xa7, 0x82, 0x38, 0xaa, 0x45, 0x3d, 0x4d, 0x42, 0x21, 0xc2, 0x44, 0xf7, 0x20, 0x87, - 0xbb, 0xb8, 0xf5, 0xbc, 0xda, 0x6f, 0x2a, 0x73, 0xa5, 0xdb, 0x27, 0xa7, 0xe5, 0x55, 0xae, 0x2d, - 0xaa, 0xa0, 0xeb, 0x5a, 0x47, 0x6c, 0x0d, 0xbf, 0x0f, 0xf3, 0x01, 0x34, 0x51, 0xfa, 0xf2, 0xc9, - 0x69, 0xf9, 0xad, 0x59, 0x68, 0x04, 0x89, 0x7b, 0x5b, 0x55, 0xdc, 0x6c, 0x28, 0xc9, 0x78, 0x24, - 0xee, 0x1d, 0xe8, 0x2e, 0x31, 0xd1, 0x57, 0x20, 0x2b, 0x81, 0xa9, 0x52, 0xe9, 0xe4, 0xb4, 0x7c, - 0x73, 0x16, 0x38, 0xc5, 0xe1, 0xde, 0x76, 0xf5, 0x79, 0x53, 0x49, 0xc7, 0xe3, 0x70, 0xcf, 0xd6, - 0x8f, 0x08, 0x7a, 0x17, 0x32, 0x02, 0x96, 0x29, 0xdd, 0x3a, 0x39, 0x2d, 0x7f, 0xe9, 0x35, 0x75, - 0x0c, 0x55, 0x5a, 0xfd, 0xdd, 0x3f, 0x5e, 0x9b, 0xfb, 0x9b, 0x3f, 0x59, 0x53, 0x66, 0xd9, 0xa5, - 0xff, 0x49, 0xc0, 0xc2, 0x85, 0xb5, 0x83, 0x54, 0xc8, 0x3a, 0xd4, 0xa0, 0x23, 0x71, 0x10, 0xe6, - 0x6a, 0x70, 0x7e, 0xb6, 0x9e, 0x6d, 0xd3, 0x3a, 0x1d, 0x4d, 0xb0, 0xe4, 0xa0, 0x67, 0x33, 0x47, - 0xf9, 0x93, 0x37, 0x5c, 0x98, 0xb1, 0x87, 0xf9, 0xc7, 0xb0, 0x60, 0xba, 0xd6, 0x11, 0x71, 0x35, - 0x83, 0x3a, 0xfb, 0xd6, 0x40, 0x1e, 0x72, 0xa5, 0xd8, 0x7c, 0x93, 0x03, 0x71, 0x51, 0x08, 0xd4, - 0x39, 0xfe, 0x0b, 0x1c, 0xe3, 0xa5, 0xe7, 0x50, 0x8c, 0x2e, 0x75, 0x76, 0x2e, 0x79, 0xd6, 0xaf, - 0x11, 0x99, 0x58, 0xf2, 0x34, 0x14, 0xe7, 0x19, 0x45, 0xa4, 0x95, 0xef, 0x41, 0x7a, 0x48, 0x4d, - 0xa1, 0x67, 0xa1, 0x76, 0x83, 0x65, 0x13, 0xff, 0x72, 0xb6, 0x5e, 0xa0, 0x5e, 0x65, 0xc3, 0xb2, - 0xc9, 0x0e, 0x35, 0x09, 0xe6, 0x00, 0xf5, 0x08, 0xd2, 0x2c, 0xe6, 0xa0, 0x2f, 0x43, 0xba, 0xd6, - 0x6a, 0x37, 0x94, 0xb9, 0xd2, 0xf2, 0xc9, 0x69, 0x79, 0x81, 0xbb, 0x84, 0x31, 0xd8, 0xda, 0x45, - 0xeb, 0x90, 0x7d, 0xde, 0xd9, 0xde, 0xdd, 0x61, 0xcb, 0xeb, 0xc6, 0xc9, 0x69, 0x79, 0x29, 0x64, - 0x0b, 0xa7, 0xa1, 0x3b, 0x90, 0xe9, 0xef, 0x74, 0x37, 0x7a, 0x4a, 0xb2, 0x84, 0x4e, 0x4e, 0xcb, - 0x8b, 0x21, 0x9f, 0xdb, 0x5c, 0x5a, 0x96, 0xb3, 0x9a, 0x0f, 0xe9, 0xea, 0x8f, 0x12, 0x50, 0x88, - 0x6c, 0x38, 0xb6, 0x30, 0x1b, 0xcd, 0x8d, 0xea, 0xee, 0x76, 0x5f, 0x99, 0x8b, 0x2c, 0xcc, 0x08, - 0xa4, 0x41, 0xf6, 0xf5, 0xb1, 0xcd, 0xe2, 0x1c, 0xd4, 0x3b, 0xed, 0x5e, 0xab, 0xd7, 0x6f, 0xb6, - 0xfb, 0x4a, 0xa2, 0xb4, 0x7a, 0x72, 0x5a, 0x5e, 0x99, 0x05, 0x6f, 0x8c, 0x6d, 0x9b, 0x2d, 0xcd, - 0x7a, 0xb5, 0xbe, 0xc5, 0xd7, 0xfa, 0x74, 0x69, 0x46, 0x50, 0x75, 0xdd, 0x38, 0x20, 0x26, 0xba, - 0x0f, 0xf9, 0x46, 0x73, 0xbb, 0xb9, 0x59, 0xe5, 0xd1, 0xbd, 0x74, 0xe7, 0xe4, 0xb4, 0x7c, 0xeb, - 0xf5, 0xde, 0x6d, 0x32, 0xd0, 0x7d, 0x62, 0xce, 0x2c, 0xd1, 0x08, 0x44, 0xfd, 0x59, 0x12, 0x16, - 0x30, 0x2b, 0x87, 0x5d, 0xbf, 0x4b, 0x6d, 0xcb, 0x98, 0xa0, 0x2e, 0xe4, 0x0d, 0xea, 0x98, 0x56, - 0x24, 0x4e, 0x3c, 0xbe, 0x24, 0x25, 0x9a, 0x4a, 0x05, 0xad, 0x7a, 0x20, 0x89, 0xa7, 0x4a, 0xd0, - 0x03, 0xc8, 0x98, 0xc4, 0xd6, 0x27, 0x32, 0x37, 0xbb, 0x55, 0x11, 0x05, 0x77, 0x25, 0x28, 0xb8, - 0x2b, 0x0d, 0x59, 0x70, 0x63, 0x81, 0xe3, 0x35, 0x88, 0xfe, 0x52, 0xd3, 0x7d, 0x9f, 0x0c, 0x47, - 0xbe, 0x48, 0xcc, 0xd2, 0xb8, 0x30, 0xd4, 0x5f, 0x56, 0x25, 0x09, 0x3d, 0x82, 0xec, 0xb1, 0xe5, - 0x98, 0xf4, 0x58, 0xe6, 0x5e, 0x57, 0x28, 0x95, 0x40, 0xf5, 0x84, 0xa5, 0x24, 0x33, 0x66, 0xb2, - 0x35, 0xd4, 0xee, 0xb4, 0x9b, 0xc1, 0x1a, 0x92, 0xfc, 0x8e, 0xd3, 0xa6, 0x0e, 0xdb, 0xff, 0xd0, - 0x69, 0x6b, 0x1b, 0xd5, 0xd6, 0xf6, 0x2e, 0x66, 0xeb, 0x68, 0xe5, 0xe4, 0xb4, 0xac, 0x84, 0x90, - 0x0d, 0xdd, 0xb2, 0x59, 0x31, 0x70, 0x0b, 0x52, 0xd5, 0xf6, 0xf7, 0x94, 0x64, 0x49, 0x39, 0x39, - 0x2d, 0x17, 0x43, 0x76, 0xd5, 0x99, 0x4c, 0xfd, 0x3e, 0xdb, 0xaf, 0xfa, 0xf7, 0x29, 0x28, 0xee, - 0x8e, 0x4c, 0xdd, 0x27, 0x62, 0x9f, 0xa1, 0x32, 0x14, 0x46, 0xba, 0xab, 0xdb, 0x36, 0xb1, 0x2d, - 0x6f, 0x28, 0xaf, 0x12, 0xa2, 0x24, 0xf4, 0xc1, 0x9b, 0xba, 0xb1, 0x96, 0x63, 0x7b, 0xe7, 0x07, - 0xff, 0xb6, 0x9e, 0x08, 0x1c, 0xba, 0x0b, 0x8b, 0xfb, 0xc2, 0x5a, 0x4d, 0x37, 0xf8, 0xc4, 0xa6, - 0xf8, 0xc4, 0x56, 0xe2, 0x26, 0x36, 0x6a, 0x56, 0x45, 0x0e, 0xb2, 0xca, 0xa5, 0xf0, 0xc2, 0x7e, - 0xb4, 0x89, 0x9e, 0xc0, 0xfc, 0x90, 0x3a, 0x96, 0x4f, 0xdd, 0xeb, 0x67, 0x21, 0x40, 0xa2, 0x7b, - 0xb0, 0xcc, 0x26, 0x37, 0xb0, 0x87, 0xb3, 0xf9, 0x71, 0x9e, 0xc4, 0x4b, 0x43, 0xfd, 0xa5, 0xec, - 0x10, 0x33, 0x32, 0xaa, 0x41, 0x86, 0xba, 0x2c, 0x5f, 0xcc, 0x72, 0x73, 0xef, 0x5f, 0x6b, 0xae, - 0x68, 0x74, 0x98, 0x0c, 0x16, 0xa2, 0xea, 0x37, 0x61, 0xe1, 0xc2, 0x20, 0x58, 0x9a, 0xd4, 0xad, - 0xee, 0xf6, 0x9a, 0xca, 0x1c, 0x2a, 0x42, 0xae, 0xde, 0x69, 0xf7, 0x5b, 0xed, 0x5d, 0x96, 0xe7, - 0x15, 0x21, 0x87, 0x3b, 0xdb, 0xdb, 0xb5, 0x6a, 0xfd, 0x99, 0x92, 0x54, 0x2b, 0x50, 0x88, 0x68, - 0x43, 0x8b, 0x00, 0xbd, 0x7e, 0xa7, 0xab, 0x6d, 0xb4, 0x70, 0xaf, 0x2f, 0xb2, 0xc4, 0x5e, 0xbf, - 0x8a, 0xfb, 0x92, 0x90, 0x50, 0xff, 0x33, 0x19, 0xcc, 0xa8, 0x4c, 0x0c, 0x6b, 0x17, 0x13, 0xc3, - 0x2b, 0x8c, 0x97, 0xa9, 0xe1, 0xb4, 0x11, 0x26, 0x88, 0x1f, 0x00, 0xf0, 0x85, 0x43, 0x4c, 0x4d, - 0xf7, 0xe5, 0xc4, 0x97, 0x5e, 0x73, 0x72, 0x3f, 0xb8, 0xd1, 0xc2, 0x79, 0x89, 0xae, 0xfa, 0xe8, - 0x5b, 0x50, 0x34, 0xe8, 0x70, 0x64, 0x13, 0x29, 0x9c, 0xba, 0x56, 0xb8, 0x10, 0xe2, 0xab, 0x7e, - 0x34, 0x35, 0x4d, 0x5f, 0x4c, 0x9e, 0x7f, 0x3b, 0x11, 0x78, 0x26, 0x26, 0x1b, 0x2d, 0x42, 0x6e, - 0xb7, 0xdb, 0xa8, 0xf6, 0x5b, 0xed, 0x4d, 0x25, 0x81, 0x00, 0xb2, 0xdc, 0xd5, 0x0d, 0x25, 0xc9, - 0xb2, 0xe8, 0x7a, 0x67, 0xa7, 0xbb, 0xdd, 0xe4, 0x11, 0x0b, 0xad, 0x80, 0x12, 0x38, 0x5b, 0xe3, - 0x8e, 0x6c, 0x36, 0x94, 0x34, 0xba, 0x01, 0x4b, 0x21, 0x55, 0x4a, 0x66, 0xd0, 0x4d, 0x40, 0x21, - 0x71, 0xaa, 0x22, 0xab, 0xfe, 0x06, 0x2c, 0xd5, 0xa9, 0xe3, 0xeb, 0x96, 0x13, 0x56, 0x18, 0x8f, - 0xd9, 0xa0, 0x25, 0x49, 0xb3, 0xe4, 0x4d, 0x50, 0x6d, 0xe9, 0xfc, 0x6c, 0xbd, 0x10, 0x42, 0x5b, - 0x0d, 0x9e, 0x2a, 0xc9, 0x86, 0xc9, 0xf6, 0xef, 0xc8, 0x32, 0xb9, 0x73, 0x33, 0xb5, 0xf9, 0xf3, - 0xb3, 0xf5, 0x54, 0xb7, 0xd5, 0xc0, 0x8c, 0x86, 0xbe, 0x0c, 0x79, 0xf2, 0xd2, 0xf2, 0x35, 0x83, - 0x9d, 0x4b, 0xcc, 0x81, 0x19, 0x9c, 0x63, 0x84, 0x3a, 0x3b, 0x86, 0x6a, 0x00, 0x5d, 0xea, 0xfa, - 0xb2, 0xe7, 0xaf, 0x43, 0x66, 0x44, 0x5d, 0x7e, 0x77, 0x71, 0xe9, 0x8d, 0x1a, 0x83, 0x8b, 0x85, - 0x8a, 0x05, 0x58, 0xfd, 0x83, 0x14, 0x40, 0x5f, 0xf7, 0x0e, 0xa5, 0x92, 0xa7, 0x90, 0x0f, 0x6f, - 0x27, 0xe5, 0x25, 0xc8, 0x95, 0xb3, 0x1d, 0x82, 0xd1, 0x93, 0x60, 0xb1, 0x89, 0xda, 0x29, 0xb6, - 0x88, 0x0d, 0x3a, 0x8a, 0x2b, 0x3f, 0x2e, 0x16, 0x48, 0xec, 0x98, 0x27, 0xae, 0x2b, 0x67, 0x9e, - 0xfd, 0x44, 0x75, 0x7e, 0x2c, 0x08, 0xa7, 0xc9, 0xec, 0x3b, 0xf6, 0xda, 0x67, 0x66, 0x46, 0xb6, - 0xe6, 0xf0, 0x54, 0x0e, 0x7d, 0x0c, 0x05, 0x36, 0x6e, 0xcd, 0xe3, 0x3c, 0x99, 0x78, 0x5f, 0xea, - 0x2a, 0xa1, 0x01, 0xc3, 0x68, 0xea, 0xe5, 0x3b, 0x00, 0xfa, 0x68, 0x64, 0x5b, 0xc4, 0xd4, 0xf6, - 0x26, 0x3c, 0xd3, 0xce, 0xe3, 0xbc, 0xa4, 0xd4, 0x26, 0x6c, 0xbb, 0x04, 0x6c, 0xdd, 0xe7, 0xd9, - 0xf3, 0x35, 0x0e, 0x94, 0xe8, 0xaa, 0x5f, 0x53, 0x60, 0xd1, 0x1d, 0x3b, 0xcc, 0xa1, 0xd2, 0x3a, - 0xf5, 0xcf, 0x93, 0xf0, 0x56, 0x9b, 0xf8, 0xc7, 0xd4, 0x3d, 0xac, 0xfa, 0xbe, 0x6e, 0x1c, 0x0c, - 0x89, 0x23, 0xa7, 0x2f, 0x52, 0xd0, 0x24, 0x2e, 0x14, 0x34, 0xab, 0x30, 0xaf, 0xdb, 0x96, 0xee, - 0x11, 0x91, 0xbc, 0xe5, 0x71, 0xd0, 0x64, 0x65, 0x17, 0x2b, 0xe2, 0x88, 0xe7, 0x11, 0x71, 0xaf, - 0xc2, 0x0c, 0x0f, 0x08, 0xe8, 0xfb, 0x70, 0x53, 0xa6, 0x69, 0x7a, 0xd8, 0x15, 0x2b, 0x28, 0x82, - 0x0b, 0xda, 0x66, 0x6c, 0x55, 0x19, 0x6f, 0x9c, 0xcc, 0xe3, 0xa6, 0xe4, 0xce, 0xc8, 0x97, 0x59, - 0xe1, 0x8a, 0x19, 0xc3, 0x2a, 0x6d, 0xc2, 0xad, 0x4b, 0x45, 0x3e, 0xd7, 0xbd, 0xcd, 0x3f, 0x25, - 0x01, 0x5a, 0xdd, 0xea, 0x8e, 0x74, 0x52, 0x03, 0xb2, 0xfb, 0xfa, 0xd0, 0xb2, 0x27, 0x57, 0x45, - 0xc0, 0x29, 0xbe, 0x52, 0x15, 0xee, 0xd8, 0xe0, 0x32, 0x58, 0xca, 0xf2, 0x9a, 0x72, 0xbc, 0xe7, - 0x10, 0x3f, 0xac, 0x29, 0x79, 0x8b, 0x99, 0xe1, 0xea, 0x4e, 0xb8, 0x74, 0x45, 0x83, 0x4d, 0x00, - 0x4b, 0x79, 0x8e, 0xf5, 0x49, 0x10, 0xb6, 0x64, 0x13, 0x6d, 0xf1, 0xdb, 0x51, 0xe2, 0x1e, 0x11, - 0x73, 0x35, 0xc3, 0x9d, 0x7a, 0x9d, 0x3d, 0x58, 0xc2, 0x85, 0xef, 0x42, 0xe9, 0xd2, 0x47, 0x3c, - 0x65, 0x9a, 0xb2, 0x3e, 0x97, 0x8f, 0x1e, 0xc2, 0xc2, 0x85, 0x71, 0xbe, 0x56, 0xcc, 0xb7, 0xba, - 0xcf, 0xbf, 0xae, 0xa4, 0xe5, 0xaf, 0x6f, 0x2a, 0x59, 0xf5, 0x4f, 0x53, 0x22, 0xd0, 0x48, 0xaf, - 0xc6, 0xbf, 0x0a, 0xe4, 0xf8, 0xea, 0x36, 0xa8, 0x2d, 0x03, 0xc0, 0x7b, 0x57, 0xc7, 0x1f, 0x56, - 0xd3, 0x71, 0x38, 0x0e, 0x05, 0xd1, 0x3a, 0x14, 0xc4, 0x2a, 0xd6, 0xd8, 0x86, 0xe3, 0x6e, 0x5d, - 0xc0, 0x20, 0x48, 0x4c, 0x12, 0xdd, 0x85, 0x45, 0x7e, 0xf9, 0xe3, 0x1d, 0x10, 0x53, 0x60, 0xd2, - 0x1c, 0xb3, 0x10, 0x52, 0x39, 0x6c, 0x07, 0x8a, 0x92, 0xa0, 0xf1, 0x7c, 0x3e, 0xc3, 0x0d, 0xba, - 0x77, 0x9d, 0x41, 0x42, 0x84, 0xa7, 0xf9, 0x85, 0xd1, 0xb4, 0xa1, 0x36, 0x20, 0x17, 0x18, 0x8b, - 0x56, 0x21, 0xd5, 0xaf, 0x77, 0x95, 0xb9, 0xd2, 0xd2, 0xc9, 0x69, 0xb9, 0x10, 0x90, 0xfb, 0xf5, - 0x2e, 0xe3, 0xec, 0x36, 0xba, 0x4a, 0xe2, 0x22, 0x67, 0xb7, 0xd1, 0x2d, 0xa5, 0x59, 0x0e, 0xa6, - 0xee, 0x43, 0x21, 0xd2, 0x03, 0x7a, 0x07, 0xe6, 0x5b, 0xed, 0x4d, 0xdc, 0xec, 0xf5, 0x94, 0xb9, - 0xd2, 0xcd, 0x93, 0xd3, 0x32, 0x8a, 0x70, 0x5b, 0xce, 0x80, 0xcd, 0x0f, 0xba, 0x03, 0xe9, 0xad, - 0x0e, 0x3b, 0xdb, 0x45, 0x01, 0x11, 0x41, 0x6c, 0x51, 0xcf, 0x2f, 0xdd, 0x90, 0xc9, 0x5d, 0x54, - 0xb1, 0xfa, 0x87, 0x09, 0xc8, 0x8a, 0xcd, 0x14, 0x3b, 0x51, 0x55, 0x98, 0x0f, 0xae, 0x09, 0x44, - 0x71, 0xf7, 0xde, 0xe5, 0x85, 0x58, 0x45, 0xd6, 0x4d, 0x62, 0xf9, 0x05, 0x72, 0xa5, 0x0f, 0xa1, - 0x18, 0x65, 0x7c, 0xae, 0xc5, 0xf7, 0x7d, 0x28, 0xb0, 0xf5, 0x1d, 0x14, 0x64, 0x8f, 0x21, 0x2b, - 0x02, 0x42, 0x78, 0xd6, 0x5c, 0x5e, 0x15, 0x4a, 0x24, 0x7a, 0x0a, 0xf3, 0xa2, 0x92, 0x0c, 0x6e, - 0x87, 0xd7, 0xae, 0xde, 0x45, 0x38, 0x80, 0xab, 0x1f, 0x43, 0xba, 0x4b, 0x88, 0xcb, 0x7c, 0xef, - 0x50, 0x93, 0x4c, 0x8f, 0x67, 0x59, 0x04, 0x9b, 0xa4, 0xd5, 0x60, 0x45, 0xb0, 0x49, 0x5a, 0x66, - 0x78, 0xff, 0x95, 0x8c, 0xdc, 0x7f, 0xf5, 0xa1, 0xf8, 0x82, 0x58, 0x83, 0x03, 0x9f, 0x98, 0x5c, - 0xd1, 0x7d, 0x48, 0x8f, 0x48, 0x68, 0xfc, 0x6a, 0xec, 0x02, 0x23, 0xc4, 0xc5, 0x1c, 0xc5, 0xe2, - 0xc8, 0x31, 0x97, 0x96, 0x4f, 0x1a, 0xb2, 0xa5, 0xfe, 0x63, 0x12, 0x16, 0x5b, 0x9e, 0x37, 0xd6, - 0x1d, 0x23, 0xc8, 0xdc, 0xbe, 0x7d, 0x31, 0x73, 0x8b, 0x7d, 0xfb, 0xb9, 0x28, 0x72, 0xf1, 0x5a, - 0x4f, 0x9e, 0x9e, 0xc9, 0xf0, 0xf4, 0x54, 0x7f, 0x9a, 0x08, 0xee, 0xee, 0xee, 0x46, 0xb6, 0xbb, - 0xa8, 0x03, 0xa3, 0x9a, 0xc8, 0xae, 0x73, 0xe8, 0xd0, 0x63, 0x07, 0xbd, 0x0d, 0x19, 0xdc, 0x6c, - 0x37, 0x5f, 0x28, 0x09, 0xb1, 0x3c, 0x2f, 0x80, 0x30, 0x71, 0xc8, 0x31, 0xd3, 0xd4, 0x6d, 0xb6, - 0x1b, 0x2c, 0xd3, 0x4a, 0xc6, 0x68, 0xea, 0x12, 0xc7, 0xb4, 0x9c, 0x01, 0x7a, 0x07, 0xb2, 0xad, - 0x5e, 0x6f, 0x97, 0x97, 0x89, 0x6f, 0x9d, 0x9c, 0x96, 0x6f, 0x5c, 0x40, 0xf1, 0x7b, 0x5b, 0x93, - 0x81, 0x58, 0x99, 0xc3, 0x72, 0xb0, 0x18, 0x10, 0xcb, 0x9f, 0x05, 0x08, 0x77, 0xfa, 0xd5, 0x7e, - 0x53, 0xc9, 0xc4, 0x80, 0x30, 0x65, 0x7f, 0xe5, 0x76, 0xfb, 0xd7, 0x24, 0x28, 0x55, 0xc3, 0x20, - 0x23, 0x9f, 0xf1, 0x65, 0x65, 0xd9, 0x87, 0xdc, 0x88, 0xfd, 0xb2, 0x48, 0x90, 0x25, 0x3d, 0x8d, - 0x7d, 0xbd, 0x9c, 0x91, 0xab, 0x60, 0x6a, 0x93, 0xaa, 0x39, 0xb4, 0x3c, 0xcf, 0xa2, 0x8e, 0xa0, - 0xe1, 0x50, 0x53, 0xe9, 0xbf, 0x12, 0x70, 0x23, 0x06, 0x81, 0x1e, 0x42, 0xda, 0xa5, 0x76, 0x30, - 0x87, 0xb7, 0x2f, 0xbb, 0x96, 0x65, 0xa2, 0x98, 0x23, 0xd1, 0x1a, 0x80, 0x3e, 0xf6, 0xa9, 0xce, - 0xfb, 0xe7, 0xb3, 0x97, 0xc3, 0x11, 0x0a, 0x7a, 0x01, 0x59, 0x8f, 0x18, 0x2e, 0x09, 0x72, 0xe9, - 0x8f, 0xff, 0xbf, 0xd6, 0x57, 0x7a, 0x5c, 0x0d, 0x96, 0xea, 0x4a, 0x15, 0xc8, 0x0a, 0x0a, 0x5b, - 0xf6, 0xa6, 0xee, 0xeb, 0xf2, 0xd2, 0x9e, 0xff, 0x66, 0xab, 0x49, 0xb7, 0x07, 0xc1, 0x6a, 0xd2, - 0xed, 0x81, 0xfa, 0x77, 0x49, 0x80, 0xe6, 0x4b, 0x9f, 0xb8, 0x8e, 0x6e, 0xd7, 0xab, 0xa8, 0x19, - 0x89, 0xfe, 0x62, 0xb4, 0x5f, 0x8d, 0x7d, 0x89, 0x08, 0x25, 0x2a, 0xf5, 0x6a, 0x4c, 0xfc, 0xbf, - 0x05, 0xa9, 0xb1, 0x2b, 0x1f, 0xa4, 0x45, 0x1e, 0xbc, 0x8b, 0xb7, 0x31, 0xa3, 0xa1, 0xe6, 0x34, - 0x6c, 0xa5, 0x2e, 0x7f, 0x76, 0x8e, 0x74, 0x10, 0x1b, 0xba, 0xd8, 0xce, 0x37, 0x74, 0xcd, 0x20, - 0xf2, 0xe4, 0x28, 0x8a, 0x9d, 0x5f, 0xaf, 0xd6, 0x89, 0xeb, 0xe3, 0xac, 0xa1, 0xb3, 0xff, 0x5f, - 0x28, 0xbe, 0xdd, 0x07, 0x98, 0x0e, 0x0d, 0xad, 0x41, 0xa6, 0xbe, 0xd1, 0xeb, 0x6d, 0x2b, 0x73, - 0x22, 0x80, 0x4f, 0x59, 0x9c, 0xac, 0xfe, 0x75, 0x12, 0x72, 0xf5, 0xaa, 0x3c, 0x56, 0xeb, 0xa0, - 0xf0, 0xa8, 0xc4, 0x9f, 0x3a, 0xc8, 0xcb, 0x91, 0xe5, 0x4e, 0x64, 0x60, 0xb9, 0xa2, 0xa8, 0x5d, - 0x64, 0x22, 0xcc, 0xea, 0x26, 0x17, 0x40, 0x18, 0x8a, 0x44, 0x3a, 0x41, 0x33, 0xf4, 0x20, 0xc6, - 0xaf, 0x5d, 0xed, 0x2c, 0x51, 0x9e, 0x4c, 0xdb, 0x1e, 0x2e, 0x04, 0x4a, 0xea, 0xba, 0x87, 0x3e, - 0x80, 0x25, 0xcf, 0x1a, 0x38, 0x96, 0x33, 0xd0, 0x02, 0xe7, 0xf1, 0x77, 0x97, 0xda, 0xf2, 0xf9, - 0xd9, 0xfa, 0x42, 0x4f, 0xb0, 0xa4, 0x0f, 0x17, 0x24, 0xb2, 0xce, 0x5d, 0x89, 0xbe, 0x09, 0x8b, - 0x11, 0x51, 0xe6, 0x45, 0xe1, 0x76, 0xe5, 0xfc, 0x6c, 0xbd, 0x18, 0x4a, 0x3e, 0x23, 0x13, 0x5c, - 0x0c, 0x05, 0x9f, 0x11, 0x7e, 0xff, 0xb2, 0x4f, 0x5d, 0x83, 0x68, 0x2e, 0xdf, 0xd3, 0xfc, 0x04, - 0x4f, 0xe3, 0x02, 0xa7, 0x89, 0x6d, 0xae, 0x3e, 0x87, 0x1b, 0x1d, 0xd7, 0x38, 0x20, 0x9e, 0x2f, - 0x5c, 0x21, 0xbd, 0xf8, 0x31, 0xdc, 0xf6, 0x75, 0xef, 0x50, 0x3b, 0xb0, 0x3c, 0x9f, 0xba, 0x13, - 0xcd, 0x25, 0x3e, 0x71, 0x18, 0x5f, 0xe3, 0x8f, 0xb5, 0xf2, 0xd2, 0xef, 0x16, 0xc3, 0x6c, 0x09, - 0x08, 0x0e, 0x10, 0xdb, 0x0c, 0xa0, 0xb6, 0xa0, 0xc8, 0xca, 0x14, 0x79, 0x71, 0xc6, 0x46, 0x0f, - 0x36, 0x1d, 0x68, 0x6f, 0x7c, 0x4c, 0xe5, 0x6d, 0x3a, 0x10, 0x3f, 0xd5, 0xef, 0x82, 0xd2, 0xb0, - 0xbc, 0x91, 0xee, 0x1b, 0x07, 0xc1, 0x6d, 0x26, 0x6a, 0x80, 0x72, 0x40, 0x74, 0xd7, 0xdf, 0x23, - 0xba, 0xaf, 0x8d, 0x88, 0x6b, 0x51, 0xf3, 0xfa, 0x59, 0x5e, 0x0a, 0x45, 0xba, 0x5c, 0x42, 0xfd, - 0xef, 0x04, 0x00, 0xd6, 0xf7, 0x83, 0x8c, 0xec, 0x6b, 0xb0, 0xec, 0x39, 0xfa, 0xc8, 0x3b, 0xa0, - 0xbe, 0x66, 0x39, 0x3e, 0x71, 0x8f, 0x74, 0x5b, 0x5e, 0xe0, 0x28, 0x01, 0xa3, 0x25, 0xe9, 0xe8, - 0x3e, 0xa0, 0x43, 0x42, 0x46, 0x1a, 0xb5, 0x4d, 0x2d, 0x60, 0x8a, 0xa7, 0xe4, 0x34, 0x56, 0x18, - 0xa7, 0x63, 0x9b, 0xbd, 0x80, 0x8e, 0x6a, 0xb0, 0xc6, 0x86, 0x4f, 0x1c, 0xdf, 0xb5, 0x88, 0xa7, - 0xed, 0x53, 0x57, 0xf3, 0x6c, 0x7a, 0xac, 0xed, 0x53, 0xdb, 0xa6, 0xc7, 0xc4, 0x0d, 0xee, 0xc6, - 0x4a, 0x36, 0x1d, 0x34, 0x05, 0x68, 0x83, 0xba, 0x3d, 0x9b, 0x1e, 0x6f, 0x04, 0x08, 0x96, 0xb6, - 0x4d, 0xc7, 0xec, 0x5b, 0xc6, 0x61, 0x90, 0xb6, 0x85, 0xd4, 0xbe, 0x65, 0x1c, 0xa2, 0x77, 0x60, - 0x81, 0xd8, 0x84, 0x5f, 0x91, 0x08, 0x54, 0x86, 0xa3, 0x8a, 0x01, 0x91, 0x81, 0xd4, 0x4f, 0x40, - 0x69, 0x3a, 0x86, 0x3b, 0x19, 0x45, 0xe6, 0xfc, 0x3e, 0x20, 0x16, 0x24, 0x35, 0x9b, 0x1a, 0x87, - 0xda, 0x50, 0x77, 0xf4, 0x01, 0xb3, 0x4b, 0xbc, 0xf0, 0x29, 0x8c, 0xb3, 0x4d, 0x8d, 0xc3, 0x1d, - 0x49, 0x57, 0x3f, 0x00, 0xe8, 0x8d, 0x5c, 0xa2, 0x9b, 0x1d, 0x96, 0x4d, 0x30, 0xd7, 0xf1, 0x96, - 0x66, 0xca, 0x17, 0x52, 0xea, 0xca, 0xad, 0xae, 0x08, 0x46, 0x23, 0xa4, 0xab, 0xbf, 0x0c, 0x37, - 0xba, 0xb6, 0x6e, 0xf0, 0xaf, 0x05, 0xba, 0xe1, 0x93, 0x15, 0x7a, 0x0a, 0x59, 0x01, 0x95, 0x33, - 0x19, 0xbb, 0xdd, 0xa6, 0x7d, 0x6e, 0xcd, 0x61, 0x89, 0xaf, 0x15, 0x01, 0xa6, 0x7a, 0xd4, 0xbf, - 0x4c, 0x40, 0x3e, 0xd4, 0x8f, 0xca, 0xe2, 0x25, 0xc6, 0x77, 0x75, 0xcb, 0x91, 0x55, 0x7d, 0x1e, - 0x47, 0x49, 0xa8, 0x05, 0x85, 0x51, 0x28, 0x7d, 0x65, 0x3e, 0x17, 0x63, 0x35, 0x8e, 0xca, 0xa2, - 0x0f, 0x21, 0x1f, 0x3c, 0x49, 0x07, 0x11, 0xf6, 0xea, 0x17, 0xec, 0x29, 0x5c, 0xfd, 0x36, 0xc0, - 0x77, 0xa8, 0xe5, 0xf4, 0xe9, 0x21, 0x71, 0xf8, 0x13, 0x2b, 0xab, 0x09, 0x49, 0xe0, 0x45, 0xd9, - 0xe2, 0xa5, 0xbe, 0x98, 0x82, 0xf0, 0xa5, 0x51, 0x34, 0xd5, 0xbf, 0x4d, 0x42, 0x16, 0x53, 0xea, - 0xd7, 0xab, 0xa8, 0x0c, 0x59, 0x19, 0x27, 0xf8, 0xf9, 0x53, 0xcb, 0x9f, 0x9f, 0xad, 0x67, 0x44, - 0x80, 0xc8, 0x18, 0x3c, 0x32, 0x44, 0x22, 0x78, 0xf2, 0xb2, 0x08, 0x8e, 0x1e, 0x42, 0x51, 0x82, - 0xb4, 0x03, 0xdd, 0x3b, 0x10, 0x05, 0x5a, 0x6d, 0xf1, 0xfc, 0x6c, 0x1d, 0x04, 0x72, 0x4b, 0xf7, - 0x0e, 0x30, 0x08, 0x34, 0xfb, 0x8d, 0x9a, 0x50, 0xf8, 0x94, 0x5a, 0x8e, 0xe6, 0xf3, 0x41, 0xc8, - 0xcb, 0xc4, 0xd8, 0x79, 0x9c, 0x0e, 0x55, 0x7e, 0x6f, 0x00, 0x9f, 0x4e, 0x07, 0xdf, 0x84, 0x05, - 0x97, 0x52, 0x5f, 0x84, 0x2d, 0x8b, 0x3a, 0xf2, 0x9e, 0xa2, 0x1c, 0x7b, 0x7d, 0x4d, 0xa9, 0x8f, - 0x25, 0x0e, 0x17, 0xdd, 0x48, 0x0b, 0x3d, 0x84, 0x15, 0x5b, 0xf7, 0x7c, 0x8d, 0xc7, 0x3b, 0x73, - 0xaa, 0x2d, 0xcb, 0xb7, 0x1a, 0x62, 0xbc, 0x0d, 0xce, 0x0a, 0x24, 0xd4, 0x7f, 0x4e, 0x40, 0x81, - 0x0d, 0xc6, 0xda, 0xb7, 0x0c, 0x96, 0xe4, 0x7d, 0xfe, 0xdc, 0xe3, 0x16, 0xa4, 0x0c, 0xcf, 0x95, - 0x4e, 0xe5, 0x87, 0x6f, 0xbd, 0x87, 0x31, 0xa3, 0xa1, 0x4f, 0x20, 0x2b, 0xef, 0x4b, 0x44, 0xda, - 0xa1, 0x5e, 0x9f, 0x8e, 0x4a, 0xdf, 0x48, 0x39, 0xbe, 0x96, 0xa7, 0xd6, 0x89, 0x43, 0x00, 0x47, - 0x49, 0xe8, 0x26, 0x24, 0x0d, 0xe1, 0x2e, 0xf9, 0x41, 0x4b, 0xbd, 0x8d, 0x93, 0x86, 0xa3, 0xfe, - 0x28, 0x01, 0x0b, 0xd3, 0x0d, 0xcf, 0x56, 0xc0, 0x6d, 0xc8, 0x7b, 0xe3, 0x3d, 0x6f, 0xe2, 0xf9, - 0x64, 0x18, 0x3c, 0x1f, 0x87, 0x04, 0xd4, 0x82, 0xbc, 0x6e, 0x0f, 0xa8, 0x6b, 0xf9, 0x07, 0x43, - 0x59, 0x89, 0xc6, 0xa7, 0x0a, 0x51, 0x9d, 0x95, 0x6a, 0x20, 0x82, 0xa7, 0xd2, 0xc1, 0xb9, 0x2f, - 0xbe, 0x31, 0xe0, 0xe7, 0xfe, 0xdb, 0x50, 0xb4, 0xf5, 0x21, 0xbf, 0x40, 0xf2, 0xad, 0xa1, 0x18, - 0x47, 0x1a, 0x17, 0x24, 0xad, 0x6f, 0x0d, 0x89, 0xaa, 0x42, 0x3e, 0x54, 0x86, 0x96, 0xa0, 0x50, - 0x6d, 0xf6, 0xb4, 0x47, 0x8f, 0x9f, 0x6a, 0x9b, 0xf5, 0x1d, 0x65, 0x4e, 0xe6, 0xa6, 0x7f, 0x95, - 0x80, 0x05, 0x19, 0x8e, 0x64, 0xbe, 0xff, 0x0e, 0xcc, 0xbb, 0xfa, 0xbe, 0x1f, 0x54, 0x24, 0x69, - 0xb1, 0xaa, 0x59, 0x84, 0x67, 0x15, 0x09, 0x63, 0xc5, 0x57, 0x24, 0x91, 0x0f, 0x1a, 0x52, 0x57, - 0x7e, 0xd0, 0x90, 0xfe, 0xb9, 0x7c, 0xd0, 0xa0, 0xfe, 0x26, 0xc0, 0x86, 0x65, 0x93, 0xbe, 0xb8, - 0x6b, 0x8a, 0xab, 0x2f, 0x59, 0x0e, 0x27, 0xef, 0x32, 0x83, 0x1c, 0xae, 0xd5, 0xc0, 0x8c, 0xc6, - 0x58, 0x03, 0xcb, 0x94, 0x9b, 0x91, 0xb3, 0x36, 0x19, 0x6b, 0x60, 0x99, 0xe1, 0xcb, 0x5b, 0xfa, - 0xba, 0x97, 0xb7, 0xd3, 0x04, 0x2c, 0xc9, 0xdc, 0x35, 0x0c, 0xbf, 0x5f, 0x85, 0xbc, 0x48, 0x63, - 0xa7, 0x05, 0x1d, 0x7f, 0xc4, 0x17, 0xb8, 0x56, 0x03, 0xe7, 0x04, 0xbb, 0x65, 0xa2, 0x75, 0x28, - 0x48, 0x68, 0xe4, 0xe3, 0x27, 0x10, 0xa4, 0x36, 0x33, 0xff, 0xeb, 0x90, 0xde, 0xb7, 0x6c, 0x22, - 0x17, 0x7a, 0x6c, 0x00, 0x98, 0x3a, 0x60, 0x6b, 0x0e, 0x73, 0x74, 0x2d, 0x17, 0x5c, 0xc6, 0x71, - 0xfb, 0x64, 0xd9, 0x19, 0xb5, 0x4f, 0x54, 0xa0, 0x33, 0xf6, 0x09, 0x1c, 0xb3, 0x4f, 0xb0, 0x85, - 0x7d, 0x12, 0x1a, 0xb5, 0x4f, 0x90, 0x7e, 0x2e, 0xf6, 0x6d, 0xc3, 0xcd, 0x9a, 0xad, 0x1b, 0x87, - 0xb6, 0xe5, 0xf9, 0xc4, 0x8c, 0x46, 0x8c, 0xc7, 0x90, 0xbd, 0x90, 0x74, 0x5e, 0x75, 0x6b, 0x29, - 0x91, 0xea, 0x7f, 0x24, 0xa0, 0xb8, 0x45, 0x74, 0xdb, 0x3f, 0x98, 0x5e, 0x0d, 0xf9, 0xc4, 0xf3, - 0xe5, 0x61, 0xc5, 0x7f, 0xa3, 0x6f, 0x40, 0x2e, 0xcc, 0x49, 0xae, 0x7d, 0x7f, 0x0b, 0xa1, 0xe8, - 0x09, 0xcc, 0xb3, 0x3d, 0x46, 0xc7, 0x41, 0xb1, 0x73, 0xd5, 0xd3, 0x8e, 0x44, 0xb2, 0x43, 0xc6, - 0x25, 0x3c, 0x09, 0xe1, 0x4b, 0x29, 0x83, 0x83, 0x26, 0xfa, 0x45, 0x28, 0xf2, 0x97, 0x89, 0x20, - 0xe7, 0xca, 0x5c, 0xa7, 0xb3, 0x20, 0x1e, 0x17, 0x45, 0xbe, 0xf5, 0xbf, 0x09, 0x58, 0xd9, 0xd1, - 0x27, 0x7b, 0x44, 0x86, 0x0d, 0x62, 0x62, 0x62, 0x50, 0xd7, 0x44, 0xdd, 0x68, 0xb8, 0xb9, 0xe2, - 0xad, 0x32, 0x4e, 0x38, 0x3e, 0xea, 0x04, 0x05, 0x58, 0x32, 0x52, 0x80, 0xad, 0x40, 0xc6, 0xa1, - 0x8e, 0x41, 0x64, 0x2c, 0x12, 0x0d, 0xd5, 0x8a, 0x86, 0x9a, 0x52, 0xf8, 0x8c, 0xc8, 0x1f, 0x01, - 0xdb, 0xd4, 0x0f, 0x7b, 0x43, 0x9f, 0x40, 0xa9, 0xd7, 0xac, 0xe3, 0x66, 0xbf, 0xd6, 0xf9, 0xae, - 0xd6, 0xab, 0x6e, 0xf7, 0xaa, 0x8f, 0x1f, 0x6a, 0xdd, 0xce, 0xf6, 0xf7, 0x1e, 0x3d, 0x79, 0xf8, - 0x0d, 0x25, 0x51, 0x2a, 0x9f, 0x9c, 0x96, 0x6f, 0xb7, 0xab, 0xf5, 0x6d, 0xb1, 0x63, 0xf6, 0xe8, - 0xcb, 0x9e, 0x6e, 0x7b, 0xfa, 0xe3, 0x87, 0x5d, 0x6a, 0x4f, 0x18, 0x86, 0x2d, 0xeb, 0x62, 0xf4, - 0xbc, 0x8a, 0x1e, 0xc3, 0x89, 0x4b, 0x8f, 0xe1, 0xe9, 0x69, 0x9e, 0xbc, 0xe4, 0x34, 0xdf, 0x80, - 0x15, 0xc3, 0xa5, 0x9e, 0xa7, 0xb1, 0xec, 0x9f, 0x98, 0x33, 0xf5, 0xc5, 0x97, 0xce, 0xcf, 0xd6, - 0x97, 0xeb, 0x8c, 0xdf, 0xe3, 0x6c, 0xa9, 0x7e, 0xd9, 0x88, 0x90, 0x78, 0x4f, 0xea, 0x1f, 0xa5, - 0x58, 0x22, 0x65, 0x1d, 0x59, 0x36, 0x19, 0x10, 0x0f, 0x3d, 0x87, 0x25, 0xc3, 0x25, 0x26, 0x4b, - 0xeb, 0x75, 0x3b, 0xfa, 0x11, 0xed, 0x2f, 0xc4, 0xe6, 0x34, 0xa1, 0x60, 0xa5, 0x1e, 0x4a, 0xf5, - 0x46, 0xc4, 0xc0, 0x8b, 0xc6, 0x85, 0x36, 0xfa, 0x14, 0x96, 0x3c, 0x62, 0x5b, 0xce, 0xf8, 0xa5, - 0x66, 0x50, 0xc7, 0x27, 0x2f, 0x83, 0x17, 0xb1, 0xeb, 0xf4, 0xf6, 0x9a, 0xdb, 0x4c, 0xaa, 0x2e, - 0x84, 0x6a, 0xe8, 0xfc, 0x6c, 0x7d, 0xf1, 0x22, 0x0d, 0x2f, 0x4a, 0xcd, 0xb2, 0x5d, 0x6a, 0xc3, - 0xe2, 0x45, 0x6b, 0xd0, 0x8a, 0xdc, 0xfb, 0x3c, 0x84, 0x04, 0x7b, 0x1b, 0xdd, 0x86, 0x9c, 0x4b, - 0x06, 0x96, 0xe7, 0xbb, 0xc2, 0xcd, 0x8c, 0x13, 0x52, 0xd8, 0xce, 0x17, 0x5f, 0x40, 0x95, 0x7e, - 0x1d, 0x66, 0x7a, 0x64, 0x9b, 0xc5, 0xb4, 0x3c, 0x7d, 0x4f, 0xaa, 0xcc, 0xe1, 0xa0, 0xc9, 0xd6, - 0xe0, 0xd8, 0x0b, 0x13, 0x35, 0xfe, 0x9b, 0xd1, 0x78, 0x46, 0x21, 0xbf, 0x07, 0xe3, 0x39, 0x43, - 0xf0, 0x61, 0x69, 0x3a, 0xf2, 0x61, 0xe9, 0x0a, 0x64, 0x6c, 0x72, 0x44, 0x6c, 0x71, 0x96, 0x63, - 0xd1, 0xb8, 0xf7, 0x10, 0x8a, 0xc1, 0x17, 0x8c, 0xfc, 0xcb, 0x89, 0x1c, 0xa4, 0xfb, 0xd5, 0xde, - 0x33, 0x65, 0x0e, 0x01, 0x64, 0xc5, 0xe2, 0x14, 0xaf, 0x75, 0xf5, 0x4e, 0x7b, 0xa3, 0xb5, 0xa9, - 0x24, 0xef, 0xfd, 0x2c, 0x05, 0xf9, 0xf0, 0xbd, 0x88, 0x9d, 0x1d, 0xed, 0xe6, 0x8b, 0x60, 0x75, - 0x87, 0xf4, 0x36, 0x39, 0x46, 0x6f, 0x4f, 0x6f, 0xa1, 0x3e, 0x11, 0x0f, 0xe4, 0x21, 0x3b, 0xb8, - 0x81, 0x7a, 0x17, 0x72, 0xd5, 0x5e, 0xaf, 0xb5, 0xd9, 0x6e, 0x36, 0x94, 0xcf, 0x12, 0xa5, 0x2f, - 0x9d, 0x9c, 0x96, 0x97, 0x43, 0x50, 0xd5, 0x13, 0x8b, 0x8f, 0xa3, 0xea, 0xf5, 0x66, 0xb7, 0xdf, - 0x6c, 0x28, 0xaf, 0x92, 0xb3, 0x28, 0x7e, 0xab, 0xc2, 0x3f, 0xdd, 0xc9, 0x77, 0x71, 0xb3, 0x5b, - 0xc5, 0xac, 0xc3, 0xcf, 0x92, 0xe2, 0x72, 0x6c, 0xda, 0xa3, 0x4b, 0x46, 0xba, 0xcb, 0xfa, 0x5c, - 0x0b, 0xbe, 0x85, 0x7b, 0x95, 0x12, 0x9f, 0x77, 0x4c, 0x1f, 0xbf, 0x88, 0x6e, 0x4e, 0x58, 0x6f, - 0xfc, 0xd5, 0x91, 0xab, 0x49, 0xcd, 0xf4, 0xd6, 0x63, 0xb1, 0x87, 0x69, 0x51, 0x61, 0x1e, 0xef, - 0xb6, 0xdb, 0x0c, 0xf4, 0x2a, 0x3d, 0x33, 0x3a, 0x3c, 0x76, 0x58, 0xc5, 0x8c, 0xee, 0x42, 0x2e, - 0x78, 0x94, 0x54, 0x3e, 0x4b, 0xcf, 0x18, 0x54, 0x0f, 0x5e, 0x54, 0x79, 0x87, 0x5b, 0xbb, 0x7d, - 0xfe, 0xa9, 0xde, 0xab, 0xcc, 0x6c, 0x87, 0x07, 0x63, 0xdf, 0xa4, 0xc7, 0x0e, 0xdb, 0xb3, 0xf2, - 0x1e, 0xee, 0xb3, 0x8c, 0xb8, 0xb4, 0x08, 0x31, 0xf2, 0x12, 0xee, 0x5d, 0xc8, 0xe1, 0xe6, 0x77, - 0xc4, 0x57, 0x7d, 0xaf, 0xb2, 0x33, 0x7a, 0x30, 0xf9, 0x94, 0x18, 0xb2, 0xb7, 0x0e, 0xee, 0x6e, - 0x55, 0xb9, 0xcb, 0x67, 0x51, 0x1d, 0x77, 0x74, 0xa0, 0x3b, 0xc4, 0x9c, 0x7e, 0xe3, 0x12, 0xb2, - 0xee, 0xfd, 0x0a, 0xe4, 0x82, 0xcc, 0x14, 0xad, 0x41, 0xf6, 0x45, 0x07, 0x3f, 0x6b, 0x62, 0x65, - 0x4e, 0xf8, 0x30, 0xe0, 0xbc, 0x10, 0x35, 0x45, 0x19, 0xe6, 0x77, 0xaa, 0xed, 0xea, 0x66, 0x13, - 0x07, 0x57, 0xe4, 0x01, 0x40, 0xa6, 0x57, 0x25, 0x45, 0x76, 0x10, 0xea, 0xac, 0xad, 0xfe, 0xf0, - 0x27, 0x6b, 0x73, 0x3f, 0xfe, 0xc9, 0xda, 0xdc, 0xab, 0xf3, 0xb5, 0xc4, 0x0f, 0xcf, 0xd7, 0x12, - 0xff, 0x70, 0xbe, 0x96, 0xf8, 0xf7, 0xf3, 0xb5, 0xc4, 0x5e, 0x96, 0x1f, 0x02, 0x4f, 0xfe, 0x2f, - 0x00, 0x00, 0xff, 0xff, 0x4b, 0xdb, 0xdc, 0xec, 0xf0, 0x31, 0x00, 0x00, + // 5043 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x7a, 0x4d, 0x6c, 0x24, 0x49, + 0x56, 0xbf, 0xeb, 0xd3, 0x55, 0xaf, 0xca, 0x76, 0x3a, 0xda, 0xdb, 0xe3, 0xae, 0xed, 0xb6, 0x6b, + 0x72, 0xa6, 0x77, 0x66, 0x7b, 0xfb, 0x5f, 0xfd, 0xb5, 0xbb, 0xea, 0x99, 0xf9, 0xef, 0xce, 0xd4, + 0x97, 0xed, 0xda, 0xb6, 0xab, 0x4a, 0x51, 0xe5, 0xee, 0x5d, 0x24, 0x48, 0xa5, 0x33, 0xc3, 0xe5, + 0x1c, 0x67, 0x65, 0x14, 0x99, 0x59, 0x76, 0x17, 0x0b, 0xa2, 0xc5, 0x01, 0x90, 0x4f, 0x70, 0x62, + 0x11, 0x32, 0x42, 0x82, 0x13, 0x42, 0xe2, 0x00, 0x12, 0x82, 0xd3, 0x20, 0x71, 0xd8, 0x1b, 0x2c, + 0x48, 0x68, 0x05, 0x92, 0x61, 0x7d, 0xe0, 0xb6, 0x82, 0x0b, 0xe2, 0x02, 0x12, 0x8a, 0x8f, 0xcc, + 0x4a, 0x57, 0xa7, 0xed, 0x1e, 0x66, 0x2f, 0x76, 0xc5, 0x7b, 0xbf, 0xf7, 0xe2, 0xc5, 0x8b, 0x88, + 0x17, 0xef, 0x45, 0x24, 0xdc, 0x1b, 0x58, 0xfe, 0xc1, 0x78, 0xaf, 0x62, 0xd0, 0xe1, 0x03, 0x93, + 0x1a, 0x87, 0xc4, 0x7d, 0xe0, 0x1d, 0xeb, 0xee, 0xf0, 0xd0, 0xf2, 0x1f, 0xe8, 0x23, 0xeb, 0x81, + 0x3f, 0x19, 0x11, 0xaf, 0x32, 0x72, 0xa9, 0x4f, 0x11, 0x12, 0x80, 0x4a, 0x00, 0xa8, 0x1c, 0x3d, + 0x2a, 0xad, 0x0f, 0x28, 0x1d, 0xd8, 0xe4, 0x01, 0x47, 0xec, 0x8d, 0xf7, 0x1f, 0xf8, 0xd6, 0x90, + 0x78, 0xbe, 0x3e, 0x1c, 0x09, 0xa1, 0xd2, 0xda, 0x2c, 0xc0, 0x1c, 0xbb, 0xba, 0x6f, 0x51, 0x47, + 0xf2, 0x57, 0x06, 0x74, 0x40, 0xf9, 0xcf, 0x07, 0xec, 0x97, 0xa0, 0xaa, 0xeb, 0x30, 0xff, 0x9c, + 0xb8, 0x9e, 0x45, 0x1d, 0xb4, 0x02, 0x19, 0xcb, 0x31, 0xc9, 0xcb, 0xd5, 0x44, 0x39, 0xf1, 0x7e, + 0x1a, 0x8b, 0x86, 0xfa, 0x10, 0xa0, 0xc5, 0x7e, 0x34, 0x1d, 0xdf, 0x9d, 0x20, 0x05, 0x52, 0x87, + 0x64, 0xc2, 0x11, 0x79, 0xcc, 0x7e, 0x32, 0xca, 0x91, 0x6e, 0xaf, 0x26, 0x05, 0xe5, 0x48, 0xb7, + 0xd5, 0x9f, 0x24, 0xa0, 0x50, 0x75, 0x1c, 0xea, 0xf3, 0xde, 0x3d, 0x84, 0x20, 0xed, 0xe8, 0x43, + 0x22, 0x85, 0xf8, 0x6f, 0x54, 0x87, 0xac, 0xad, 0xef, 0x11, 0xdb, 0x5b, 0x4d, 0x96, 0x53, 0xef, + 0x17, 0x1e, 0x7f, 0xad, 0xf2, 0xfa, 0x90, 0x2b, 0x11, 0x25, 0x95, 0x6d, 0x8e, 0xe6, 0x46, 0x60, + 0x29, 0x8a, 0xbe, 0x0d, 0xf3, 0x96, 0x63, 0x5a, 0x06, 0xf1, 0x56, 0xd3, 0x5c, 0xcb, 0x5a, 0x9c, + 0x96, 0xa9, 0xf5, 0xb5, 0xf4, 0x0f, 0xcf, 0xd6, 0xe7, 0x70, 0x20, 0x54, 0xfa, 0x00, 0x0a, 0x11, + 0xb5, 0x31, 0x63, 0x5b, 0x81, 0xcc, 0x91, 0x6e, 0x8f, 0x89, 0x1c, 0x9d, 0x68, 0x7c, 0x98, 0x7c, + 0x9a, 0x50, 0x3f, 0x81, 0x95, 0xb6, 0x3e, 0x24, 0xe6, 0x26, 0x71, 0x88, 0x6b, 0x19, 0x98, 0x78, + 0x74, 0xec, 0x1a, 0x84, 0x8d, 0xf5, 0xd0, 0x72, 0xcc, 0x60, 0xac, 0xec, 0x77, 0xbc, 0x16, 0xb5, + 0x0e, 0x6f, 0x35, 0x2c, 0xcf, 0x70, 0x89, 0x4f, 0x3e, 0xb7, 0x92, 0x54, 0xa0, 0xe4, 0x2c, 0x01, + 0x4b, 0xb3, 0xd2, 0x3f, 0x07, 0x37, 0x98, 0x8b, 0x4d, 0xcd, 0x95, 0x14, 0xcd, 0x1b, 0x11, 0x83, + 0x2b, 0x2b, 0x3c, 0x7e, 0x3f, 0xce, 0x43, 0x71, 0x23, 0xd9, 0x9a, 0xc3, 0xcb, 0x5c, 0x4d, 0x40, + 0xe8, 0x8d, 0x88, 0x81, 0x0c, 0xb8, 0x69, 0x4a, 0xa3, 0x67, 0xd4, 0x27, 0xb9, 0xfa, 0xd8, 0x69, + 0xbc, 0x64, 0x98, 0x5b, 0x73, 0x78, 0x25, 0x50, 0x16, 0xed, 0xa4, 0x06, 0x90, 0x0b, 0x74, 0xab, + 0x3f, 0x48, 0x40, 0x3e, 0x60, 0x7a, 0xe8, 0xab, 0x90, 0x77, 0x74, 0x87, 0x6a, 0xc6, 0x68, 0xec, + 0xf1, 0x01, 0xa5, 0x6a, 0xc5, 0xf3, 0xb3, 0xf5, 0x5c, 0x5b, 0x77, 0x68, 0xbd, 0xbb, 0xeb, 0xe1, + 0x1c, 0x63, 0xd7, 0x47, 0x63, 0x0f, 0xbd, 0x0d, 0xc5, 0x21, 0x19, 0x52, 0x77, 0xa2, 0xed, 0x4d, + 0x7c, 0xe2, 0x49, 0xb7, 0x15, 0x04, 0xad, 0xc6, 0x48, 0xe8, 0x5b, 0x30, 0x3f, 0x10, 0x26, 0xad, + 0xa6, 0xf8, 0xf2, 0x79, 0x27, 0xce, 0xfa, 0x19, 0xab, 0x71, 0x20, 0xa3, 0xfe, 0x56, 0x02, 0x56, + 0x42, 0x2a, 0xf9, 0xc5, 0xb1, 0xe5, 0x92, 0x21, 0x71, 0x7c, 0x0f, 0x7d, 0x03, 0xb2, 0xb6, 0x35, + 0xb4, 0x7c, 0x4f, 0xfa, 0xfc, 0x4e, 0x9c, 0xda, 0x70, 0x50, 0x58, 0x82, 0x51, 0x15, 0x8a, 0x2e, + 0xf1, 0x88, 0x7b, 0x24, 0x56, 0xbc, 0xf4, 0xe8, 0x35, 0xc2, 0x17, 0x44, 0xd4, 0x0d, 0xc8, 0x75, + 0x6d, 0xdd, 0xdf, 0xa7, 0xee, 0x10, 0xa9, 0x50, 0xd4, 0x5d, 0xe3, 0xc0, 0xf2, 0x89, 0xe1, 0x8f, + 0xdd, 0x60, 0xf7, 0x5d, 0xa0, 0xa1, 0x9b, 0x90, 0xa4, 0xa2, 0xa3, 0x7c, 0x2d, 0x7b, 0x7e, 0xb6, + 0x9e, 0xec, 0xf4, 0x70, 0x92, 0x7a, 0xea, 0x47, 0xb0, 0xdc, 0xb5, 0xc7, 0x03, 0xcb, 0x69, 0x10, + 0xcf, 0x70, 0xad, 0x11, 0xd3, 0xce, 0x56, 0x25, 0x8b, 0x51, 0xc1, 0xaa, 0x64, 0xbf, 0xc3, 0xad, + 0x9d, 0x9c, 0x6e, 0x6d, 0xf5, 0x37, 0x92, 0xb0, 0xdc, 0x74, 0x06, 0x96, 0x43, 0xa2, 0xd2, 0x77, + 0x61, 0x91, 0x70, 0xa2, 0x76, 0x24, 0xc2, 0x8d, 0xd4, 0xb3, 0x20, 0xa8, 0x41, 0x0c, 0x6a, 0xcd, + 0xc4, 0x85, 0x47, 0x71, 0xc3, 0x7f, 0x4d, 0x7b, 0x6c, 0x74, 0x68, 0xc2, 0xfc, 0x88, 0x0f, 0xc2, + 0x93, 0xd3, 0x7b, 0x37, 0x4e, 0xd7, 0x6b, 0xe3, 0x0c, 0x82, 0x84, 0x94, 0xfd, 0x22, 0x41, 0xe2, + 0x4f, 0x92, 0xb0, 0xd4, 0xa6, 0xe6, 0x05, 0x3f, 0x94, 0x20, 0x77, 0x40, 0x3d, 0x3f, 0x12, 0x10, + 0xc3, 0x36, 0x7a, 0x0a, 0xb9, 0x91, 0x9c, 0x3e, 0x39, 0xfb, 0xb7, 0xe3, 0x4d, 0x16, 0x18, 0x1c, + 0xa2, 0xd1, 0x47, 0x90, 0x0f, 0xb6, 0x0c, 0x1b, 0xed, 0x1b, 0x2c, 0x9c, 0x29, 0x1e, 0x7d, 0x0b, + 0xb2, 0x62, 0x12, 0x56, 0xd3, 0x5c, 0xf2, 0xee, 0x1b, 0xf9, 0x1c, 0x4b, 0x21, 0xb4, 0x09, 0x39, + 0xdf, 0xf6, 0x34, 0xcb, 0xd9, 0xa7, 0xab, 0x19, 0xae, 0x60, 0x3d, 0x36, 0xc8, 0x50, 0x93, 0xf4, + 0xb7, 0x7b, 0x2d, 0x67, 0x9f, 0xd6, 0x0a, 0xe7, 0x67, 0xeb, 0xf3, 0xb2, 0x81, 0xe7, 0x7d, 0xdb, + 0x63, 0x3f, 0xd4, 0xdf, 0x4e, 0x40, 0x21, 0x82, 0x42, 0x77, 0x00, 0x7c, 0x77, 0xec, 0xf9, 0x9a, + 0x4b, 0xa9, 0xcf, 0x9d, 0x55, 0xc4, 0x79, 0x4e, 0xc1, 0x94, 0xfa, 0xa8, 0x02, 0x37, 0x0c, 0xe2, + 0xfa, 0x9a, 0xe5, 0x79, 0x63, 0xe2, 0x6a, 0xde, 0x78, 0xef, 0x53, 0x62, 0xf8, 0xdc, 0x71, 0x45, + 0xbc, 0xcc, 0x58, 0x2d, 0xce, 0xe9, 0x09, 0x06, 0x7a, 0x02, 0x37, 0xa3, 0xf8, 0xd1, 0x78, 0xcf, + 0xb6, 0x0c, 0x8d, 0x4d, 0x66, 0x8a, 0x8b, 0xdc, 0x98, 0x8a, 0x74, 0x39, 0xef, 0x19, 0x99, 0xa8, + 0x3f, 0x4e, 0x80, 0x82, 0xf5, 0x7d, 0x7f, 0x87, 0x0c, 0xf7, 0x88, 0xdb, 0xf3, 0x75, 0x7f, 0xec, + 0xa1, 0x9b, 0x90, 0xb5, 0x89, 0x6e, 0x12, 0x97, 0x1b, 0x95, 0xc3, 0xb2, 0x85, 0x76, 0xd9, 0x0e, + 0xd6, 0x8d, 0x03, 0x7d, 0xcf, 0xb2, 0x2d, 0x7f, 0xc2, 0x4d, 0x59, 0x8c, 0x5f, 0xc2, 0xb3, 0x3a, + 0x2b, 0x38, 0x22, 0x88, 0x2f, 0xa8, 0x41, 0xab, 0x30, 0x3f, 0x24, 0x9e, 0xa7, 0x0f, 0x08, 0xb7, + 0x34, 0x8f, 0x83, 0xa6, 0xfa, 0x11, 0x14, 0xa3, 0x72, 0xa8, 0x00, 0xf3, 0xbb, 0xed, 0x67, 0xed, + 0xce, 0x8b, 0xb6, 0x32, 0x87, 0x96, 0xa0, 0xb0, 0xdb, 0xc6, 0xcd, 0x6a, 0x7d, 0xab, 0x5a, 0xdb, + 0x6e, 0x2a, 0x09, 0xb4, 0x00, 0xf9, 0x69, 0x33, 0xa9, 0xfe, 0x59, 0x02, 0x80, 0xb9, 0x5b, 0x0e, + 0xea, 0x43, 0xc8, 0x78, 0xbe, 0xee, 0x8b, 0x55, 0xb9, 0xf8, 0xf8, 0xdd, 0xcb, 0xe6, 0x50, 0xda, + 0xcb, 0xfe, 0x11, 0x2c, 0x44, 0xa2, 0x16, 0x26, 0x2f, 0x58, 0xc8, 0x02, 0x84, 0x6e, 0x9a, 0xae, + 0x34, 0x9c, 0xff, 0x56, 0x3f, 0x82, 0x0c, 0x97, 0xbe, 0x68, 0x6e, 0x0e, 0xd2, 0x0d, 0xf6, 0x2b, + 0x81, 0xf2, 0x90, 0xc1, 0xcd, 0x6a, 0xe3, 0x7b, 0x4a, 0x12, 0x29, 0x50, 0x6c, 0xb4, 0x7a, 0xf5, + 0x4e, 0xbb, 0xdd, 0xac, 0xf7, 0x9b, 0x0d, 0x25, 0xa5, 0xde, 0x85, 0x4c, 0x6b, 0xc8, 0x34, 0xdf, + 0x66, 0x4b, 0x7e, 0x9f, 0xb8, 0xc4, 0x31, 0x82, 0x9d, 0x34, 0x25, 0xa8, 0x3f, 0x2d, 0x40, 0x66, + 0x87, 0x8e, 0x1d, 0x1f, 0x3d, 0x8e, 0x84, 0xad, 0xc5, 0xf8, 0x0c, 0x81, 0x03, 0x2b, 0xfd, 0xc9, + 0x88, 0xc8, 0xb0, 0x76, 0x13, 0xb2, 0x62, 0x73, 0xc8, 0xe1, 0xc8, 0x16, 0xa3, 0xfb, 0xba, 0x3b, + 0x20, 0xbe, 0x1c, 0x8f, 0x6c, 0xa1, 0xf7, 0xd9, 0x89, 0xa5, 0x9b, 0xd4, 0xb1, 0x27, 0x7c, 0x0f, + 0xe5, 0xc4, 0xb1, 0x84, 0x89, 0x6e, 0x76, 0x1c, 0x7b, 0x82, 0x43, 0x2e, 0xda, 0x82, 0xe2, 0x9e, + 0xe5, 0x98, 0x1a, 0x1d, 0x89, 0x20, 0x9f, 0xb9, 0x7c, 0xc7, 0x09, 0xab, 0x6a, 0x96, 0x63, 0x76, + 0x04, 0x18, 0x17, 0xf6, 0xa6, 0x0d, 0xd4, 0x86, 0xc5, 0x23, 0x6a, 0x8f, 0x87, 0x24, 0xd4, 0x95, + 0xe5, 0xba, 0xde, 0xbb, 0x5c, 0xd7, 0x73, 0x8e, 0x0f, 0xb4, 0x2d, 0x1c, 0x45, 0x9b, 0xe8, 0x19, + 0x2c, 0xf8, 0xc3, 0xd1, 0xbe, 0x17, 0xaa, 0x9b, 0xe7, 0xea, 0xbe, 0x72, 0x85, 0xc3, 0x18, 0x3c, + 0xd0, 0x56, 0xf4, 0x23, 0x2d, 0xb4, 0x09, 0x05, 0x83, 0x3a, 0x9e, 0xe5, 0xf9, 0xc4, 0x31, 0x26, + 0xab, 0x39, 0xee, 0xfb, 0x2b, 0x46, 0x59, 0x9f, 0x82, 0x71, 0x54, 0xb2, 0xf4, 0x6b, 0x29, 0x28, + 0x44, 0x5c, 0x80, 0x7a, 0x50, 0x18, 0xb9, 0x74, 0xa4, 0x0f, 0xf8, 0x89, 0x27, 0x27, 0xf5, 0xd1, + 0x1b, 0xb9, 0xaf, 0xd2, 0x9d, 0x0a, 0xe2, 0xa8, 0x16, 0xf5, 0x34, 0x09, 0x85, 0x08, 0x13, 0xdd, + 0x83, 0x1c, 0xee, 0xe2, 0xd6, 0xf3, 0x6a, 0xbf, 0xa9, 0xcc, 0x95, 0x6e, 0x9f, 0x9c, 0x96, 0x57, + 0xb9, 0xb6, 0xa8, 0x82, 0xae, 0x6b, 0x1d, 0xb1, 0x35, 0xfc, 0x3e, 0xcc, 0x07, 0xd0, 0x44, 0xe9, + 0xcb, 0x27, 0xa7, 0xe5, 0xb7, 0x66, 0xa1, 0x11, 0x24, 0xee, 0x6d, 0x55, 0x71, 0xb3, 0xa1, 0x24, + 0xe3, 0x91, 0xb8, 0x77, 0xa0, 0xbb, 0xc4, 0x44, 0x5f, 0x81, 0xac, 0x04, 0xa6, 0x4a, 0xa5, 0x93, + 0xd3, 0xf2, 0xcd, 0x59, 0xe0, 0x14, 0x87, 0x7b, 0xdb, 0xd5, 0xe7, 0x4d, 0x25, 0x1d, 0x8f, 0xc3, + 0x3d, 0x5b, 0x3f, 0x22, 0xe8, 0x5d, 0xc8, 0x08, 0x58, 0xa6, 0x74, 0xeb, 0xe4, 0xb4, 0xfc, 0xa5, + 0xd7, 0xd4, 0x31, 0x54, 0x69, 0xf5, 0x37, 0xff, 0x70, 0x6d, 0xee, 0xaf, 0xfe, 0x68, 0x4d, 0x99, + 0x65, 0x97, 0xfe, 0x3b, 0x01, 0x0b, 0x17, 0xd6, 0x0e, 0x52, 0x21, 0xeb, 0x50, 0x83, 0x8e, 0xc4, + 0x41, 0x98, 0xab, 0xc1, 0xf9, 0xd9, 0x7a, 0xb6, 0x4d, 0xeb, 0x74, 0x34, 0xc1, 0x92, 0x83, 0x9e, + 0xcd, 0x1c, 0xe5, 0x4f, 0xde, 0x70, 0x61, 0xc6, 0x1e, 0xe6, 0x1f, 0xc3, 0x82, 0xe9, 0x5a, 0x47, + 0xc4, 0xd5, 0x0c, 0xea, 0xec, 0x5b, 0x03, 0x79, 0xc8, 0x95, 0x62, 0xf3, 0x4d, 0x0e, 0xc4, 0x45, + 0x21, 0x50, 0xe7, 0xf8, 0x2f, 0x70, 0x8c, 0x97, 0x9e, 0x43, 0x31, 0xba, 0xd4, 0xd9, 0xb9, 0xe4, + 0x59, 0xbf, 0x44, 0x64, 0x62, 0xc9, 0xd3, 0x50, 0x9c, 0x67, 0x14, 0x91, 0x56, 0xbe, 0x07, 0xe9, + 0x21, 0x35, 0x85, 0x9e, 0x85, 0xda, 0x0d, 0x96, 0x4d, 0xfc, 0xd3, 0xd9, 0x7a, 0x81, 0x7a, 0x95, + 0x0d, 0xcb, 0x26, 0x3b, 0xd4, 0x24, 0x98, 0x03, 0xd4, 0x23, 0x48, 0xb3, 0x98, 0x83, 0xbe, 0x0c, + 0xe9, 0x5a, 0xab, 0xdd, 0x50, 0xe6, 0x4a, 0xcb, 0x27, 0xa7, 0xe5, 0x05, 0xee, 0x12, 0xc6, 0x60, + 0x6b, 0x17, 0xad, 0x43, 0xf6, 0x79, 0x67, 0x7b, 0x77, 0x87, 0x2d, 0xaf, 0x1b, 0x27, 0xa7, 0xe5, + 0xa5, 0x90, 0x2d, 0x9c, 0x86, 0xee, 0x40, 0xa6, 0xbf, 0xd3, 0xdd, 0xe8, 0x29, 0xc9, 0x12, 0x3a, + 0x39, 0x2d, 0x2f, 0x86, 0x7c, 0x6e, 0x73, 0x69, 0x59, 0xce, 0x6a, 0x3e, 0xa4, 0xab, 0x3f, 0x4a, + 0x40, 0x21, 0xb2, 0xe1, 0xd8, 0xc2, 0x6c, 0x34, 0x37, 0xaa, 0xbb, 0xdb, 0x7d, 0x65, 0x2e, 0xb2, + 0x30, 0x23, 0x90, 0x06, 0xd9, 0xd7, 0xc7, 0x36, 0x8b, 0x73, 0x50, 0xef, 0xb4, 0x7b, 0xad, 0x5e, + 0xbf, 0xd9, 0xee, 0x2b, 0x89, 0xd2, 0xea, 0xc9, 0x69, 0x79, 0x65, 0x16, 0xbc, 0x31, 0xb6, 0x6d, + 0xb6, 0x34, 0xeb, 0xd5, 0xfa, 0x16, 0x5f, 0xeb, 0xd3, 0xa5, 0x19, 0x41, 0xd5, 0x75, 0xe3, 0x80, + 0x98, 0xe8, 0x3e, 0xe4, 0x1b, 0xcd, 0xed, 0xe6, 0x66, 0x95, 0x47, 0xf7, 0xd2, 0x9d, 0x93, 0xd3, + 0xf2, 0xad, 0xd7, 0x7b, 0xb7, 0xc9, 0x40, 0xf7, 0x89, 0x39, 0xb3, 0x44, 0x23, 0x10, 0xf5, 0x3f, + 0x93, 0xb0, 0x80, 0x59, 0x39, 0xec, 0xfa, 0x5d, 0x6a, 0x5b, 0xc6, 0x04, 0x75, 0x21, 0x6f, 0x50, + 0xc7, 0xb4, 0x22, 0x71, 0xe2, 0xf1, 0x25, 0x29, 0xd1, 0x54, 0x2a, 0x68, 0xd5, 0x03, 0x49, 0x3c, + 0x55, 0x82, 0x1e, 0x40, 0xc6, 0x24, 0xb6, 0x3e, 0x91, 0xb9, 0xd9, 0xad, 0x8a, 0x28, 0xb8, 0x2b, + 0x41, 0xc1, 0x5d, 0x69, 0xc8, 0x82, 0x1b, 0x0b, 0x1c, 0xaf, 0x41, 0xf4, 0x97, 0x9a, 0xee, 0xfb, + 0x64, 0x38, 0xf2, 0x45, 0x62, 0x96, 0xc6, 0x85, 0xa1, 0xfe, 0xb2, 0x2a, 0x49, 0xe8, 0x11, 0x64, + 0x8f, 0x2d, 0xc7, 0xa4, 0xc7, 0x32, 0xf7, 0xba, 0x42, 0xa9, 0x04, 0xaa, 0x27, 0x2c, 0x25, 0x99, + 0x31, 0x93, 0xad, 0xa1, 0x76, 0xa7, 0xdd, 0x0c, 0xd6, 0x90, 0xe4, 0x77, 0x9c, 0x36, 0x75, 0xd8, + 0xfe, 0x87, 0x4e, 0x5b, 0xdb, 0xa8, 0xb6, 0xb6, 0x77, 0x31, 0x5b, 0x47, 0x2b, 0x27, 0xa7, 0x65, + 0x25, 0x84, 0x6c, 0xe8, 0x96, 0xcd, 0x8a, 0x81, 0x5b, 0x90, 0xaa, 0xb6, 0xbf, 0xa7, 0x24, 0x4b, + 0xca, 0xc9, 0x69, 0xb9, 0x18, 0xb2, 0xab, 0xce, 0x64, 0xea, 0xf7, 0xd9, 0x7e, 0xd5, 0xbf, 0x4d, + 0x41, 0x71, 0x77, 0x64, 0xea, 0x3e, 0x11, 0xfb, 0x0c, 0x95, 0xa1, 0x30, 0xd2, 0x5d, 0xdd, 0xb6, + 0x89, 0x6d, 0x79, 0x43, 0x79, 0x95, 0x10, 0x25, 0xa1, 0x0f, 0xde, 0xd4, 0x8d, 0xb5, 0x1c, 0xdb, + 0x3b, 0x3f, 0xf8, 0x97, 0xf5, 0x44, 0xe0, 0xd0, 0x5d, 0x58, 0xdc, 0x17, 0xd6, 0x6a, 0xba, 0xc1, + 0x27, 0x36, 0xc5, 0x27, 0xb6, 0x12, 0x37, 0xb1, 0x51, 0xb3, 0x2a, 0x72, 0x90, 0x55, 0x2e, 0x85, + 0x17, 0xf6, 0xa3, 0x4d, 0xf4, 0x04, 0xe6, 0x87, 0xd4, 0xb1, 0x7c, 0xea, 0x5e, 0x3f, 0x0b, 0x01, + 0x12, 0xdd, 0x83, 0x65, 0x36, 0xb9, 0x81, 0x3d, 0x9c, 0xcd, 0x8f, 0xf3, 0x24, 0x5e, 0x1a, 0xea, + 0x2f, 0x65, 0x87, 0x98, 0x91, 0x51, 0x0d, 0x32, 0xd4, 0x65, 0xf9, 0x62, 0x96, 0x9b, 0x7b, 0xff, + 0x5a, 0x73, 0x45, 0xa3, 0xc3, 0x64, 0xb0, 0x10, 0x55, 0xbf, 0x09, 0x0b, 0x17, 0x06, 0xc1, 0xd2, + 0xa4, 0x6e, 0x75, 0xb7, 0xd7, 0x54, 0xe6, 0x50, 0x11, 0x72, 0xf5, 0x4e, 0xbb, 0xdf, 0x6a, 0xef, + 0xb2, 0x3c, 0xaf, 0x08, 0x39, 0xdc, 0xd9, 0xde, 0xae, 0x55, 0xeb, 0xcf, 0x94, 0xa4, 0x5a, 0x81, + 0x42, 0x44, 0x1b, 0x5a, 0x04, 0xe8, 0xf5, 0x3b, 0x5d, 0x6d, 0xa3, 0x85, 0x7b, 0x7d, 0x91, 0x25, + 0xf6, 0xfa, 0x55, 0xdc, 0x97, 0x84, 0x84, 0xfa, 0xef, 0xc9, 0x60, 0x46, 0x65, 0x62, 0x58, 0xbb, + 0x98, 0x18, 0x5e, 0x61, 0xbc, 0x4c, 0x0d, 0xa7, 0x8d, 0x30, 0x41, 0xfc, 0x00, 0x80, 0x2f, 0x1c, + 0x62, 0x6a, 0xba, 0x2f, 0x27, 0xbe, 0xf4, 0x9a, 0x93, 0xfb, 0xc1, 0x8d, 0x16, 0xce, 0x4b, 0x74, + 0xd5, 0x47, 0xdf, 0x82, 0xa2, 0x41, 0x87, 0x23, 0x9b, 0x48, 0xe1, 0xd4, 0xb5, 0xc2, 0x85, 0x10, + 0x5f, 0xf5, 0xa3, 0xa9, 0x69, 0xfa, 0x62, 0xf2, 0xfc, 0xeb, 0x89, 0xc0, 0x33, 0x31, 0xd9, 0x68, + 0x11, 0x72, 0xbb, 0xdd, 0x46, 0xb5, 0xdf, 0x6a, 0x6f, 0x2a, 0x09, 0x04, 0x90, 0xe5, 0xae, 0x6e, + 0x28, 0x49, 0x96, 0x45, 0xd7, 0x3b, 0x3b, 0xdd, 0xed, 0x26, 0x8f, 0x58, 0x68, 0x05, 0x94, 0xc0, + 0xd9, 0x1a, 0x77, 0x64, 0xb3, 0xa1, 0xa4, 0xd1, 0x0d, 0x58, 0x0a, 0xa9, 0x52, 0x32, 0x83, 0x6e, + 0x02, 0x0a, 0x89, 0x53, 0x15, 0x59, 0xf5, 0x57, 0x60, 0xa9, 0x4e, 0x1d, 0x5f, 0xb7, 0x9c, 0xb0, + 0xc2, 0x78, 0xcc, 0x06, 0x2d, 0x49, 0x9a, 0x25, 0x6f, 0x82, 0x6a, 0x4b, 0xe7, 0x67, 0xeb, 0x85, + 0x10, 0xda, 0x6a, 0xf0, 0x54, 0x49, 0x36, 0x4c, 0xb6, 0x7f, 0x47, 0x96, 0xc9, 0x9d, 0x9b, 0xa9, + 0xcd, 0x9f, 0x9f, 0xad, 0xa7, 0xba, 0xad, 0x06, 0x66, 0x34, 0xf4, 0x65, 0xc8, 0x93, 0x97, 0x96, + 0xaf, 0x19, 0xec, 0x5c, 0x62, 0x0e, 0xcc, 0xe0, 0x1c, 0x23, 0xd4, 0xd9, 0x31, 0x54, 0x03, 0xe8, + 0x52, 0xd7, 0x97, 0x3d, 0x7f, 0x1d, 0x32, 0x23, 0xea, 0xf2, 0xbb, 0x8b, 0x4b, 0x6f, 0xd4, 0x18, + 0x5c, 0x2c, 0x54, 0x2c, 0xc0, 0xea, 0xef, 0xa6, 0x00, 0xfa, 0xba, 0x77, 0x28, 0x95, 0x3c, 0x85, + 0x7c, 0x78, 0x3b, 0x29, 0x2f, 0x41, 0xae, 0x9c, 0xed, 0x10, 0x8c, 0x9e, 0x04, 0x8b, 0x4d, 0xd4, + 0x4e, 0xb1, 0x45, 0x6c, 0xd0, 0x51, 0x5c, 0xf9, 0x71, 0xb1, 0x40, 0x62, 0xc7, 0x3c, 0x71, 0x5d, + 0x39, 0xf3, 0xec, 0x27, 0xaa, 0xf3, 0x63, 0x41, 0x38, 0x4d, 0x66, 0xdf, 0xb1, 0xd7, 0x3e, 0x33, + 0x33, 0xb2, 0x35, 0x87, 0xa7, 0x72, 0xe8, 0x63, 0x28, 0xb0, 0x71, 0x6b, 0x1e, 0xe7, 0xc9, 0xc4, + 0xfb, 0x52, 0x57, 0x09, 0x0d, 0x18, 0x46, 0x53, 0x2f, 0xdf, 0x01, 0xd0, 0x47, 0x23, 0xdb, 0x22, + 0xa6, 0xb6, 0x37, 0xe1, 0x99, 0x76, 0x1e, 0xe7, 0x25, 0xa5, 0x36, 0x61, 0xdb, 0x25, 0x60, 0xeb, + 0x3e, 0xcf, 0x9e, 0xaf, 0x71, 0xa0, 0x44, 0x57, 0xfd, 0x9a, 0x02, 0x8b, 0xee, 0xd8, 0x61, 0x0e, + 0x95, 0xd6, 0xa9, 0x7f, 0x9a, 0x84, 0xb7, 0xda, 0xc4, 0x3f, 0xa6, 0xee, 0x61, 0xd5, 0xf7, 0x75, + 0xe3, 0x60, 0x48, 0x1c, 0x39, 0x7d, 0x91, 0x82, 0x26, 0x71, 0xa1, 0xa0, 0x59, 0x85, 0x79, 0xdd, + 0xb6, 0x74, 0x8f, 0x88, 0xe4, 0x2d, 0x8f, 0x83, 0x26, 0x2b, 0xbb, 0x58, 0x11, 0x47, 0x3c, 0x8f, + 0x88, 0x7b, 0x15, 0x66, 0x78, 0x40, 0x40, 0xdf, 0x87, 0x9b, 0x32, 0x4d, 0xd3, 0xc3, 0xae, 0x58, + 0x41, 0x11, 0x5c, 0xd0, 0x36, 0x63, 0xab, 0xca, 0x78, 0xe3, 0x64, 0x1e, 0x37, 0x25, 0x77, 0x46, + 0xbe, 0xcc, 0x0a, 0x57, 0xcc, 0x18, 0x56, 0x69, 0x13, 0x6e, 0x5d, 0x2a, 0xf2, 0xb9, 0xee, 0x6d, + 0xfe, 0x21, 0x09, 0xd0, 0xea, 0x56, 0x77, 0xa4, 0x93, 0x1a, 0x90, 0xdd, 0xd7, 0x87, 0x96, 0x3d, + 0xb9, 0x2a, 0x02, 0x4e, 0xf1, 0x95, 0xaa, 0x70, 0xc7, 0x06, 0x97, 0xc1, 0x52, 0x96, 0xd7, 0x94, + 0xe3, 0x3d, 0x87, 0xf8, 0x61, 0x4d, 0xc9, 0x5b, 0xcc, 0x0c, 0x57, 0x77, 0xc2, 0xa5, 0x2b, 0x1a, + 0x6c, 0x02, 0x58, 0xca, 0x73, 0xac, 0x4f, 0x82, 0xb0, 0x25, 0x9b, 0x68, 0x8b, 0xdf, 0x8e, 0x12, + 0xf7, 0x88, 0x98, 0xab, 0x19, 0xee, 0xd4, 0xeb, 0xec, 0xc1, 0x12, 0x2e, 0x7c, 0x17, 0x4a, 0x97, + 0x3e, 0xe2, 0x29, 0xd3, 0x94, 0xf5, 0xb9, 0x7c, 0xf4, 0x10, 0x16, 0x2e, 0x8c, 0xf3, 0xb5, 0x62, + 0xbe, 0xd5, 0x7d, 0xfe, 0x75, 0x25, 0x2d, 0x7f, 0x7d, 0x53, 0xc9, 0xaa, 0x7f, 0x9c, 0x12, 0x81, + 0x46, 0x7a, 0x35, 0xfe, 0x55, 0x20, 0xc7, 0x57, 0xb7, 0x41, 0x6d, 0x19, 0x00, 0xde, 0xbb, 0x3a, + 0xfe, 0xb0, 0x9a, 0x8e, 0xc3, 0x71, 0x28, 0x88, 0xd6, 0xa1, 0x20, 0x56, 0xb1, 0xc6, 0x36, 0x1c, + 0x77, 0xeb, 0x02, 0x06, 0x41, 0x62, 0x92, 0xe8, 0x2e, 0x2c, 0xf2, 0xcb, 0x1f, 0xef, 0x80, 0x98, + 0x02, 0x93, 0xe6, 0x98, 0x85, 0x90, 0xca, 0x61, 0x3b, 0x50, 0x94, 0x04, 0x8d, 0xe7, 0xf3, 0x19, + 0x6e, 0xd0, 0xbd, 0xeb, 0x0c, 0x12, 0x22, 0x3c, 0xcd, 0x2f, 0x8c, 0xa6, 0x0d, 0xb5, 0x01, 0xb9, + 0xc0, 0x58, 0xb4, 0x0a, 0xa9, 0x7e, 0xbd, 0xab, 0xcc, 0x95, 0x96, 0x4e, 0x4e, 0xcb, 0x85, 0x80, + 0xdc, 0xaf, 0x77, 0x19, 0x67, 0xb7, 0xd1, 0x55, 0x12, 0x17, 0x39, 0xbb, 0x8d, 0x6e, 0x29, 0xcd, + 0x72, 0x30, 0x75, 0x1f, 0x0a, 0x91, 0x1e, 0xd0, 0x3b, 0x30, 0xdf, 0x6a, 0x6f, 0xe2, 0x66, 0xaf, + 0xa7, 0xcc, 0x95, 0x6e, 0x9e, 0x9c, 0x96, 0x51, 0x84, 0xdb, 0x72, 0x06, 0x6c, 0x7e, 0xd0, 0x1d, + 0x48, 0x6f, 0x75, 0xd8, 0xd9, 0x2e, 0x0a, 0x88, 0x08, 0x62, 0x8b, 0x7a, 0x7e, 0xe9, 0x86, 0x4c, + 0xee, 0xa2, 0x8a, 0xd5, 0xdf, 0x4b, 0x40, 0x56, 0x6c, 0xa6, 0xd8, 0x89, 0xaa, 0xc2, 0x7c, 0x70, + 0x4d, 0x20, 0x8a, 0xbb, 0xf7, 0x2e, 0x2f, 0xc4, 0x2a, 0xb2, 0x6e, 0x12, 0xcb, 0x2f, 0x90, 0x2b, + 0x7d, 0x08, 0xc5, 0x28, 0xe3, 0x73, 0x2d, 0xbe, 0xef, 0x43, 0x81, 0xad, 0xef, 0xa0, 0x20, 0x7b, + 0x0c, 0x59, 0x11, 0x10, 0xc2, 0xb3, 0xe6, 0xf2, 0xaa, 0x50, 0x22, 0xd1, 0x53, 0x98, 0x17, 0x95, + 0x64, 0x70, 0x3b, 0xbc, 0x76, 0xf5, 0x2e, 0xc2, 0x01, 0x5c, 0xfd, 0x18, 0xd2, 0x5d, 0x42, 0x5c, + 0xe6, 0x7b, 0x87, 0x9a, 0x64, 0x7a, 0x3c, 0xcb, 0x22, 0xd8, 0x24, 0xad, 0x06, 0x2b, 0x82, 0x4d, + 0xd2, 0x32, 0xc3, 0xfb, 0xaf, 0x64, 0xe4, 0xfe, 0xab, 0x0f, 0xc5, 0x17, 0xc4, 0x1a, 0x1c, 0xf8, + 0xc4, 0xe4, 0x8a, 0xee, 0x43, 0x7a, 0x44, 0x42, 0xe3, 0x57, 0x63, 0x17, 0x18, 0x21, 0x2e, 0xe6, + 0x28, 0x16, 0x47, 0x8e, 0xb9, 0xb4, 0x7c, 0xd2, 0x90, 0x2d, 0xf5, 0xef, 0x93, 0xb0, 0xd8, 0xf2, + 0xbc, 0xb1, 0xee, 0x18, 0x41, 0xe6, 0xf6, 0xed, 0x8b, 0x99, 0x5b, 0xec, 0xdb, 0xcf, 0x45, 0x91, + 0x8b, 0xd7, 0x7a, 0xf2, 0xf4, 0x4c, 0x86, 0xa7, 0xa7, 0xfa, 0xd3, 0x44, 0x70, 0x77, 0x77, 0x37, + 0xb2, 0xdd, 0x45, 0x1d, 0x18, 0xd5, 0x44, 0x76, 0x9d, 0x43, 0x87, 0x1e, 0x3b, 0xe8, 0x6d, 0xc8, + 0xe0, 0x66, 0xbb, 0xf9, 0x42, 0x49, 0x88, 0xe5, 0x79, 0x01, 0x84, 0x89, 0x43, 0x8e, 0x99, 0xa6, + 0x6e, 0xb3, 0xdd, 0x60, 0x99, 0x56, 0x32, 0x46, 0x53, 0x97, 0x38, 0xa6, 0xe5, 0x0c, 0xd0, 0x3b, + 0x90, 0x6d, 0xf5, 0x7a, 0xbb, 0xbc, 0x4c, 0x7c, 0xeb, 0xe4, 0xb4, 0x7c, 0xe3, 0x02, 0x8a, 0xdf, + 0xdb, 0x9a, 0x0c, 0xc4, 0xca, 0x1c, 0x96, 0x83, 0xc5, 0x80, 0x58, 0xfe, 0x2c, 0x40, 0xb8, 0xd3, + 0xaf, 0xf6, 0x9b, 0x4a, 0x26, 0x06, 0x84, 0x29, 0xfb, 0x2b, 0xb7, 0xdb, 0x3f, 0x27, 0x41, 0xa9, + 0x1a, 0x06, 0x19, 0xf9, 0x8c, 0x2f, 0x2b, 0xcb, 0x3e, 0xe4, 0x46, 0xec, 0x97, 0x45, 0x82, 0x2c, + 0xe9, 0x69, 0xec, 0xeb, 0xe5, 0x8c, 0x5c, 0x05, 0x53, 0x9b, 0x54, 0xcd, 0xa1, 0xe5, 0x79, 0x16, + 0x75, 0x04, 0x0d, 0x87, 0x9a, 0x4a, 0xff, 0x91, 0x80, 0x1b, 0x31, 0x08, 0xf4, 0x10, 0xd2, 0x2e, + 0xb5, 0x83, 0x39, 0xbc, 0x7d, 0xd9, 0xb5, 0x2c, 0x13, 0xc5, 0x1c, 0x89, 0xd6, 0x00, 0xf4, 0xb1, + 0x4f, 0x75, 0xde, 0x3f, 0x9f, 0xbd, 0x1c, 0x8e, 0x50, 0xd0, 0x0b, 0xc8, 0x7a, 0xc4, 0x70, 0x49, + 0x90, 0x4b, 0x7f, 0xfc, 0x7f, 0xb5, 0xbe, 0xd2, 0xe3, 0x6a, 0xb0, 0x54, 0x57, 0xaa, 0x40, 0x56, + 0x50, 0xd8, 0xb2, 0x37, 0x75, 0x5f, 0x97, 0x97, 0xf6, 0xfc, 0x37, 0x5b, 0x4d, 0xba, 0x3d, 0x08, + 0x56, 0x93, 0x6e, 0x0f, 0xd4, 0xbf, 0x49, 0x02, 0x34, 0x5f, 0xfa, 0xc4, 0x75, 0x74, 0xbb, 0x5e, + 0x45, 0xcd, 0x48, 0xf4, 0x17, 0xa3, 0xfd, 0x6a, 0xec, 0x4b, 0x44, 0x28, 0x51, 0xa9, 0x57, 0x63, + 0xe2, 0xff, 0x2d, 0x48, 0x8d, 0x5d, 0xf9, 0x20, 0x2d, 0xf2, 0xe0, 0x5d, 0xbc, 0x8d, 0x19, 0x0d, + 0x35, 0xa7, 0x61, 0x2b, 0x75, 0xf9, 0xb3, 0x73, 0xa4, 0x83, 0xd8, 0xd0, 0xc5, 0x76, 0xbe, 0xa1, + 0x6b, 0x06, 0x91, 0x27, 0x47, 0x51, 0xec, 0xfc, 0x7a, 0xb5, 0x4e, 0x5c, 0x1f, 0x67, 0x0d, 0x9d, + 0xfd, 0xff, 0x42, 0xf1, 0xed, 0x3e, 0xc0, 0x74, 0x68, 0x68, 0x0d, 0x32, 0xf5, 0x8d, 0x5e, 0x6f, + 0x5b, 0x99, 0x13, 0x01, 0x7c, 0xca, 0xe2, 0x64, 0xf5, 0x2f, 0x93, 0x90, 0xab, 0x57, 0xe5, 0xb1, + 0x5a, 0x07, 0x85, 0x47, 0x25, 0xfe, 0xd4, 0x41, 0x5e, 0x8e, 0x2c, 0x77, 0x22, 0x03, 0xcb, 0x15, + 0x45, 0xed, 0x22, 0x13, 0x61, 0x56, 0x37, 0xb9, 0x00, 0xc2, 0x50, 0x24, 0xd2, 0x09, 0x9a, 0xa1, + 0x07, 0x31, 0x7e, 0xed, 0x6a, 0x67, 0x89, 0xf2, 0x64, 0xda, 0xf6, 0x70, 0x21, 0x50, 0x52, 0xd7, + 0x3d, 0xf4, 0x01, 0x2c, 0x79, 0xd6, 0xc0, 0xb1, 0x9c, 0x81, 0x16, 0x38, 0x8f, 0xbf, 0xbb, 0xd4, + 0x96, 0xcf, 0xcf, 0xd6, 0x17, 0x7a, 0x82, 0x25, 0x7d, 0xb8, 0x20, 0x91, 0x75, 0xee, 0x4a, 0xf4, + 0x4d, 0x58, 0x8c, 0x88, 0x32, 0x2f, 0x0a, 0xb7, 0x2b, 0xe7, 0x67, 0xeb, 0xc5, 0x50, 0xf2, 0x19, + 0x99, 0xe0, 0x62, 0x28, 0xf8, 0x8c, 0xf0, 0xfb, 0x97, 0x7d, 0xea, 0x1a, 0x44, 0x73, 0xf9, 0x9e, + 0xe6, 0x27, 0x78, 0x1a, 0x17, 0x38, 0x4d, 0x6c, 0x73, 0xf5, 0x39, 0xdc, 0xe8, 0xb8, 0xc6, 0x01, + 0xf1, 0x7c, 0xe1, 0x0a, 0xe9, 0xc5, 0x8f, 0xe1, 0xb6, 0xaf, 0x7b, 0x87, 0xda, 0x81, 0xe5, 0xf9, + 0xd4, 0x9d, 0x68, 0x2e, 0xf1, 0x89, 0xc3, 0xf8, 0x1a, 0x7f, 0xac, 0x95, 0x97, 0x7e, 0xb7, 0x18, + 0x66, 0x4b, 0x40, 0x70, 0x80, 0xd8, 0x66, 0x00, 0xb5, 0x05, 0x45, 0x56, 0xa6, 0xc8, 0x8b, 0x33, + 0x36, 0x7a, 0xb0, 0xe9, 0x40, 0x7b, 0xe3, 0x63, 0x2a, 0x6f, 0xd3, 0x81, 0xf8, 0xa9, 0x7e, 0x17, + 0x94, 0x86, 0xe5, 0x8d, 0x74, 0xdf, 0x38, 0x08, 0x6e, 0x33, 0x51, 0x03, 0x94, 0x03, 0xa2, 0xbb, + 0xfe, 0x1e, 0xd1, 0x7d, 0x6d, 0x44, 0x5c, 0x8b, 0x9a, 0xd7, 0xcf, 0xf2, 0x52, 0x28, 0xd2, 0xe5, + 0x12, 0xea, 0x7f, 0x25, 0x00, 0xb0, 0xbe, 0x1f, 0x64, 0x64, 0x5f, 0x83, 0x65, 0xcf, 0xd1, 0x47, + 0xde, 0x01, 0xf5, 0x35, 0xcb, 0xf1, 0x89, 0x7b, 0xa4, 0xdb, 0xf2, 0x02, 0x47, 0x09, 0x18, 0x2d, + 0x49, 0x47, 0xf7, 0x01, 0x1d, 0x12, 0x32, 0xd2, 0xa8, 0x6d, 0x6a, 0x01, 0x53, 0x3c, 0x25, 0xa7, + 0xb1, 0xc2, 0x38, 0x1d, 0xdb, 0xec, 0x05, 0x74, 0x54, 0x83, 0x35, 0x36, 0x7c, 0xe2, 0xf8, 0xae, + 0x45, 0x3c, 0x6d, 0x9f, 0xba, 0x9a, 0x67, 0xd3, 0x63, 0x6d, 0x9f, 0xda, 0x36, 0x3d, 0x26, 0x6e, + 0x70, 0x37, 0x56, 0xb2, 0xe9, 0xa0, 0x29, 0x40, 0x1b, 0xd4, 0xed, 0xd9, 0xf4, 0x78, 0x23, 0x40, + 0xb0, 0xb4, 0x6d, 0x3a, 0x66, 0xdf, 0x32, 0x0e, 0x83, 0xb4, 0x2d, 0xa4, 0xf6, 0x2d, 0xe3, 0x10, + 0xbd, 0x03, 0x0b, 0xc4, 0x26, 0xfc, 0x8a, 0x44, 0xa0, 0x32, 0x1c, 0x55, 0x0c, 0x88, 0x0c, 0xa4, + 0x7e, 0x02, 0x4a, 0xd3, 0x31, 0xdc, 0xc9, 0x28, 0x32, 0xe7, 0xf7, 0x01, 0xb1, 0x20, 0xa9, 0xd9, + 0xd4, 0x38, 0xd4, 0x86, 0xba, 0xa3, 0x0f, 0x98, 0x5d, 0xe2, 0x85, 0x4f, 0x61, 0x9c, 0x6d, 0x6a, + 0x1c, 0xee, 0x48, 0xba, 0xfa, 0x01, 0x40, 0x6f, 0xe4, 0x12, 0xdd, 0xec, 0xb0, 0x6c, 0x82, 0xb9, + 0x8e, 0xb7, 0x34, 0x53, 0xbe, 0x90, 0x52, 0x57, 0x6e, 0x75, 0x45, 0x30, 0x1a, 0x21, 0x5d, 0xfd, + 0x79, 0xb8, 0xd1, 0xb5, 0x75, 0x83, 0x7f, 0x2d, 0xd0, 0x0d, 0x9f, 0xac, 0xd0, 0x53, 0xc8, 0x0a, + 0xa8, 0x9c, 0xc9, 0xd8, 0xed, 0x36, 0xed, 0x73, 0x6b, 0x0e, 0x4b, 0x7c, 0xad, 0x08, 0x30, 0xd5, + 0xa3, 0xfe, 0x79, 0x02, 0xf2, 0xa1, 0x7e, 0x54, 0x16, 0x2f, 0x31, 0xbe, 0xab, 0x5b, 0x8e, 0xac, + 0xea, 0xf3, 0x38, 0x4a, 0x42, 0x2d, 0x28, 0x8c, 0x42, 0xe9, 0x2b, 0xf3, 0xb9, 0x18, 0xab, 0x71, + 0x54, 0x16, 0x7d, 0x08, 0xf9, 0xe0, 0x49, 0x3a, 0x88, 0xb0, 0x57, 0xbf, 0x60, 0x4f, 0xe1, 0xea, + 0xb7, 0x01, 0xbe, 0x43, 0x2d, 0xa7, 0x4f, 0x0f, 0x89, 0xc3, 0x9f, 0x58, 0x59, 0x4d, 0x48, 0x02, + 0x2f, 0xca, 0x16, 0x2f, 0xf5, 0xc5, 0x14, 0x84, 0x2f, 0x8d, 0xa2, 0xa9, 0xfe, 0x75, 0x12, 0xb2, + 0x98, 0x52, 0xbf, 0x5e, 0x45, 0x65, 0xc8, 0xca, 0x38, 0xc1, 0xcf, 0x9f, 0x5a, 0xfe, 0xfc, 0x6c, + 0x3d, 0x23, 0x02, 0x44, 0xc6, 0xe0, 0x91, 0x21, 0x12, 0xc1, 0x93, 0x97, 0x45, 0x70, 0xf4, 0x10, + 0x8a, 0x12, 0xa4, 0x1d, 0xe8, 0xde, 0x81, 0x28, 0xd0, 0x6a, 0x8b, 0xe7, 0x67, 0xeb, 0x20, 0x90, + 0x5b, 0xba, 0x77, 0x80, 0x41, 0xa0, 0xd9, 0x6f, 0xd4, 0x84, 0xc2, 0xa7, 0xd4, 0x72, 0x34, 0x9f, + 0x0f, 0x42, 0x5e, 0x26, 0xc6, 0xce, 0xe3, 0x74, 0xa8, 0xf2, 0x7b, 0x03, 0xf8, 0x74, 0x3a, 0xf8, + 0x26, 0x2c, 0xb8, 0x94, 0xfa, 0x22, 0x6c, 0x59, 0xd4, 0x91, 0xf7, 0x14, 0xe5, 0xd8, 0xeb, 0x6b, + 0x4a, 0x7d, 0x2c, 0x71, 0xb8, 0xe8, 0x46, 0x5a, 0xe8, 0x21, 0xac, 0xd8, 0xba, 0xe7, 0x6b, 0x3c, + 0xde, 0x99, 0x53, 0x6d, 0x59, 0xbe, 0xd5, 0x10, 0xe3, 0x6d, 0x70, 0x56, 0x20, 0xa1, 0xfe, 0x63, + 0x02, 0x0a, 0x6c, 0x30, 0xd6, 0xbe, 0x65, 0xb0, 0x24, 0xef, 0xf3, 0xe7, 0x1e, 0xb7, 0x20, 0x65, + 0x78, 0xae, 0x74, 0x2a, 0x3f, 0x7c, 0xeb, 0x3d, 0x8c, 0x19, 0x0d, 0x7d, 0x02, 0x59, 0x79, 0x5f, + 0x22, 0xd2, 0x0e, 0xf5, 0xfa, 0x74, 0x54, 0xfa, 0x46, 0xca, 0xf1, 0xb5, 0x3c, 0xb5, 0x4e, 0x1c, + 0x02, 0x38, 0x4a, 0x42, 0x37, 0x21, 0x69, 0x08, 0x77, 0xc9, 0x0f, 0x5a, 0xea, 0x6d, 0x9c, 0x34, + 0x1c, 0xf5, 0x47, 0x09, 0x58, 0x98, 0x6e, 0x78, 0xb6, 0x02, 0x6e, 0x43, 0xde, 0x1b, 0xef, 0x79, + 0x13, 0xcf, 0x27, 0xc3, 0xe0, 0xf9, 0x38, 0x24, 0xa0, 0x16, 0xe4, 0x75, 0x7b, 0x40, 0x5d, 0xcb, + 0x3f, 0x18, 0xca, 0x4a, 0x34, 0x3e, 0x55, 0x88, 0xea, 0xac, 0x54, 0x03, 0x11, 0x3c, 0x95, 0x0e, + 0xce, 0x7d, 0xf1, 0x8d, 0x01, 0x3f, 0xf7, 0xdf, 0x86, 0xa2, 0xad, 0x0f, 0xf9, 0x05, 0x92, 0x6f, + 0x0d, 0xc5, 0x38, 0xd2, 0xb8, 0x20, 0x69, 0x7d, 0x6b, 0x48, 0x54, 0x15, 0xf2, 0xa1, 0x32, 0xb4, + 0x04, 0x85, 0x6a, 0xb3, 0xa7, 0x3d, 0x7a, 0xfc, 0x54, 0xdb, 0xac, 0xef, 0x28, 0x73, 0x32, 0x37, + 0xfd, 0x8b, 0x04, 0x2c, 0xc8, 0x70, 0x24, 0xf3, 0xfd, 0x77, 0x60, 0xde, 0xd5, 0xf7, 0xfd, 0xa0, + 0x22, 0x49, 0x8b, 0x55, 0xcd, 0x22, 0x3c, 0xab, 0x48, 0x18, 0x2b, 0xbe, 0x22, 0x89, 0x7c, 0xd0, + 0x90, 0xba, 0xf2, 0x83, 0x86, 0xf4, 0xcf, 0xe4, 0x83, 0x06, 0xf5, 0x57, 0x01, 0x36, 0x2c, 0x9b, + 0xf4, 0xc5, 0x5d, 0x53, 0x5c, 0x7d, 0xc9, 0x72, 0x38, 0x79, 0x97, 0x19, 0xe4, 0x70, 0xad, 0x06, + 0x66, 0x34, 0xc6, 0x1a, 0x58, 0xa6, 0xdc, 0x8c, 0x9c, 0xb5, 0xc9, 0x58, 0x03, 0xcb, 0x0c, 0x5f, + 0xde, 0xd2, 0xd7, 0xbd, 0xbc, 0x9d, 0x26, 0x60, 0x49, 0xe6, 0xae, 0x61, 0xf8, 0xfd, 0x2a, 0xe4, + 0x45, 0x1a, 0x3b, 0x2d, 0xe8, 0xf8, 0x23, 0xbe, 0xc0, 0xb5, 0x1a, 0x38, 0x27, 0xd8, 0x2d, 0x13, + 0xad, 0x43, 0x41, 0x42, 0x23, 0x1f, 0x3f, 0x81, 0x20, 0xb5, 0x99, 0xf9, 0x5f, 0x87, 0xf4, 0xbe, + 0x65, 0x13, 0xb9, 0xd0, 0x63, 0x03, 0xc0, 0xd4, 0x01, 0x5b, 0x73, 0x98, 0xa3, 0x6b, 0xb9, 0xe0, + 0x32, 0x8e, 0xdb, 0x27, 0xcb, 0xce, 0xa8, 0x7d, 0xa2, 0x02, 0x9d, 0xb1, 0x4f, 0xe0, 0x98, 0x7d, + 0x82, 0x2d, 0xec, 0x93, 0xd0, 0xa8, 0x7d, 0x82, 0xf4, 0x33, 0xb1, 0x6f, 0x1b, 0x6e, 0xd6, 0x6c, + 0xdd, 0x38, 0xb4, 0x2d, 0xcf, 0x27, 0x66, 0x34, 0x62, 0x3c, 0x86, 0xec, 0x85, 0xa4, 0xf3, 0xaa, + 0x5b, 0x4b, 0x89, 0x54, 0xff, 0x2d, 0x01, 0xc5, 0x2d, 0xa2, 0xdb, 0xfe, 0xc1, 0xf4, 0x6a, 0xc8, + 0x27, 0x9e, 0x2f, 0x0f, 0x2b, 0xfe, 0x1b, 0x7d, 0x03, 0x72, 0x61, 0x4e, 0x72, 0xed, 0xfb, 0x5b, + 0x08, 0x45, 0x4f, 0x60, 0x9e, 0xed, 0x31, 0x3a, 0x0e, 0x8a, 0x9d, 0xab, 0x9e, 0x76, 0x24, 0x92, + 0x1d, 0x32, 0x2e, 0xe1, 0x49, 0x08, 0x5f, 0x4a, 0x19, 0x1c, 0x34, 0xd1, 0xff, 0x87, 0x22, 0x7f, + 0x99, 0x08, 0x72, 0xae, 0xcc, 0x75, 0x3a, 0x0b, 0xe2, 0x71, 0x51, 0xe4, 0x5b, 0xff, 0x93, 0x80, + 0x95, 0x1d, 0x7d, 0xb2, 0x47, 0x64, 0xd8, 0x20, 0x26, 0x26, 0x06, 0x75, 0x4d, 0xd4, 0x8d, 0x86, + 0x9b, 0x2b, 0xde, 0x2a, 0xe3, 0x84, 0xe3, 0xa3, 0x4e, 0x50, 0x80, 0x25, 0x23, 0x05, 0xd8, 0x0a, + 0x64, 0x1c, 0xea, 0x18, 0x44, 0xc6, 0x22, 0xd1, 0x50, 0xad, 0x68, 0xa8, 0x29, 0x85, 0xcf, 0x88, + 0xfc, 0x11, 0xb0, 0x4d, 0xfd, 0xb0, 0x37, 0xf4, 0x09, 0x94, 0x7a, 0xcd, 0x3a, 0x6e, 0xf6, 0x6b, + 0x9d, 0xef, 0x6a, 0xbd, 0xea, 0x76, 0xaf, 0xfa, 0xf8, 0xa1, 0xd6, 0xed, 0x6c, 0x7f, 0xef, 0xd1, + 0x93, 0x87, 0xdf, 0x50, 0x12, 0xa5, 0xf2, 0xc9, 0x69, 0xf9, 0x76, 0xbb, 0x5a, 0xdf, 0x16, 0x3b, + 0x66, 0x8f, 0xbe, 0xec, 0xe9, 0xb6, 0xa7, 0x3f, 0x7e, 0xd8, 0xa5, 0xf6, 0x84, 0x61, 0xd8, 0xb2, + 0x2e, 0x46, 0xcf, 0xab, 0xe8, 0x31, 0x9c, 0xb8, 0xf4, 0x18, 0x9e, 0x9e, 0xe6, 0xc9, 0x4b, 0x4e, + 0xf3, 0x0d, 0x58, 0x31, 0x5c, 0xea, 0x79, 0x1a, 0xcb, 0xfe, 0x89, 0x39, 0x53, 0x5f, 0x7c, 0xe9, + 0xfc, 0x6c, 0x7d, 0xb9, 0xce, 0xf8, 0x3d, 0xce, 0x96, 0xea, 0x97, 0x8d, 0x08, 0x89, 0xf7, 0xa4, + 0xfe, 0x7e, 0x8a, 0x25, 0x52, 0xd6, 0x91, 0x65, 0x93, 0x01, 0xf1, 0xd0, 0x73, 0x58, 0x32, 0x5c, + 0x62, 0xb2, 0xb4, 0x5e, 0xb7, 0xa3, 0x1f, 0xd1, 0xfe, 0xbf, 0xd8, 0x9c, 0x26, 0x14, 0xac, 0xd4, + 0x43, 0xa9, 0xde, 0x88, 0x18, 0x78, 0xd1, 0xb8, 0xd0, 0x46, 0x9f, 0xc2, 0x92, 0x47, 0x6c, 0xcb, + 0x19, 0xbf, 0xd4, 0x0c, 0xea, 0xf8, 0xe4, 0x65, 0xf0, 0x22, 0x76, 0x9d, 0xde, 0x5e, 0x73, 0x9b, + 0x49, 0xd5, 0x85, 0x50, 0x0d, 0x9d, 0x9f, 0xad, 0x2f, 0x5e, 0xa4, 0xe1, 0x45, 0xa9, 0x59, 0xb6, + 0x4b, 0x6d, 0x58, 0xbc, 0x68, 0x0d, 0x5a, 0x91, 0x7b, 0x9f, 0x87, 0x90, 0x60, 0x6f, 0xa3, 0xdb, + 0x90, 0x73, 0xc9, 0xc0, 0xf2, 0x7c, 0x57, 0xb8, 0x99, 0x71, 0x42, 0x0a, 0xdb, 0xf9, 0xe2, 0x0b, + 0xa8, 0xd2, 0x2f, 0xc3, 0x4c, 0x8f, 0x6c, 0xb3, 0x98, 0x96, 0xa7, 0xef, 0x49, 0x95, 0x39, 0x1c, + 0x34, 0xd9, 0x1a, 0x1c, 0x7b, 0x61, 0xa2, 0xc6, 0x7f, 0x33, 0x1a, 0xcf, 0x28, 0xe4, 0xf7, 0x60, + 0x3c, 0x67, 0x08, 0x3e, 0x2c, 0x4d, 0x47, 0x3e, 0x2c, 0x5d, 0x81, 0x8c, 0x4d, 0x8e, 0x88, 0x2d, + 0xce, 0x72, 0x2c, 0x1a, 0xf7, 0x1e, 0x42, 0x31, 0xf8, 0x82, 0x91, 0x7f, 0x39, 0x91, 0x83, 0x74, + 0xbf, 0xda, 0x7b, 0xa6, 0xcc, 0x21, 0x80, 0xac, 0x58, 0x9c, 0xe2, 0xb5, 0xae, 0xde, 0x69, 0x6f, + 0xb4, 0x36, 0x95, 0xe4, 0xbd, 0xdf, 0x49, 0x43, 0x3e, 0x7c, 0x2f, 0x62, 0x67, 0x47, 0xbb, 0xf9, + 0x22, 0x58, 0xdd, 0x21, 0xbd, 0x4d, 0x8e, 0xd1, 0xdb, 0xd3, 0x5b, 0xa8, 0x4f, 0xc4, 0x03, 0x79, + 0xc8, 0x0e, 0x6e, 0xa0, 0xde, 0x85, 0x5c, 0xb5, 0xd7, 0x6b, 0x6d, 0xb6, 0x9b, 0x0d, 0xe5, 0xb3, + 0x44, 0xe9, 0x4b, 0x27, 0xa7, 0xe5, 0xe5, 0x10, 0x54, 0xf5, 0xc4, 0xe2, 0xe3, 0xa8, 0x7a, 0xbd, + 0xd9, 0xed, 0x37, 0x1b, 0xca, 0xab, 0xe4, 0x2c, 0x8a, 0xdf, 0xaa, 0xf0, 0x4f, 0x77, 0xf2, 0x5d, + 0xdc, 0xec, 0x56, 0x31, 0xeb, 0xf0, 0xb3, 0xa4, 0xb8, 0x1c, 0x9b, 0xf6, 0xe8, 0x92, 0x91, 0xee, + 0xb2, 0x3e, 0xd7, 0x82, 0x6f, 0xe1, 0x5e, 0xa5, 0xc4, 0xe7, 0x1d, 0xd3, 0xc7, 0x2f, 0xa2, 0x9b, + 0x13, 0xd6, 0x1b, 0x7f, 0x75, 0xe4, 0x6a, 0x52, 0x33, 0xbd, 0xf5, 0x58, 0xec, 0x61, 0x5a, 0x54, + 0x98, 0xc7, 0xbb, 0xed, 0x36, 0x03, 0xbd, 0x4a, 0xcf, 0x8c, 0x0e, 0x8f, 0x1d, 0x56, 0x31, 0xa3, + 0xbb, 0x90, 0x0b, 0x1e, 0x25, 0x95, 0xcf, 0xd2, 0x33, 0x06, 0xd5, 0x83, 0x17, 0x55, 0xde, 0xe1, + 0xd6, 0x6e, 0x9f, 0x7f, 0xaa, 0xf7, 0x2a, 0x33, 0xdb, 0xe1, 0xc1, 0xd8, 0x37, 0xe9, 0xb1, 0xc3, + 0xf6, 0xac, 0xbc, 0x87, 0xfb, 0x2c, 0x23, 0x2e, 0x2d, 0x42, 0x8c, 0xbc, 0x84, 0x7b, 0x17, 0x72, + 0xb8, 0xf9, 0x1d, 0xf1, 0x55, 0xdf, 0xab, 0xec, 0x8c, 0x1e, 0x4c, 0x3e, 0x25, 0x06, 0xeb, 0xad, + 0x0c, 0x59, 0xdc, 0xdc, 0xe9, 0x3c, 0x6f, 0x2a, 0x7f, 0x90, 0x9d, 0xd1, 0x83, 0xc9, 0x90, 0xf2, + 0x6f, 0x9b, 0x72, 0x1d, 0xdc, 0xdd, 0xaa, 0xf2, 0x49, 0x99, 0xd5, 0xd3, 0x71, 0x47, 0x07, 0xba, + 0x43, 0xcc, 0xe9, 0x57, 0x30, 0x21, 0xeb, 0xde, 0x2f, 0x40, 0x2e, 0xc8, 0x5d, 0xd1, 0x1a, 0x64, + 0x5f, 0x74, 0xf0, 0xb3, 0x26, 0x56, 0xe6, 0x84, 0x97, 0x03, 0xce, 0x0b, 0x51, 0x75, 0x94, 0x61, + 0x7e, 0xa7, 0xda, 0xae, 0x6e, 0x36, 0x71, 0x70, 0x89, 0x1e, 0x00, 0x64, 0x02, 0x56, 0x52, 0x64, + 0x07, 0xa1, 0xce, 0xda, 0xea, 0x0f, 0x7f, 0xb2, 0x36, 0xf7, 0xe3, 0x9f, 0xac, 0xcd, 0xbd, 0x3a, + 0x5f, 0x4b, 0xfc, 0xf0, 0x7c, 0x2d, 0xf1, 0x77, 0xe7, 0x6b, 0x89, 0x7f, 0x3d, 0x5f, 0x4b, 0xec, + 0x65, 0xf9, 0x31, 0xf1, 0xe4, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x08, 0x6c, 0xc9, 0x88, 0x12, + 0x32, 0x00, 0x00, } diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/types.proto b/components/engine/vendor/github.com/docker/swarmkit/api/types.proto index 635d12b200..3b60c544c3 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/types.proto +++ b/components/engine/vendor/github.com/docker/swarmkit/api/types.proto @@ -458,7 +458,16 @@ enum TaskState { SHUTDOWN = 640 [(gogoproto.enumvalue_customname)="TaskStateShutdown"]; // orchestrator requested shutdown FAILED = 704 [(gogoproto.enumvalue_customname)="TaskStateFailed"]; // task execution failed with error REJECTED = 768 [(gogoproto.enumvalue_customname)="TaskStateRejected"]; // task could not be executed here. - ORPHANED = 832 [(gogoproto.enumvalue_customname)="TaskStateOrphaned"]; // The node on which this task is scheduled is Down for too long + // TaskStateRemove is used to correctly handle service deletions and scale + // downs. This allows us to keep track of tasks that have been marked for + // deletion, but can't yet be removed because the agent is in the process of + // shutting them down. Once the agent has shut down tasks with desired state + // REMOVE, the task reaper is responsible for removing them. + REMOVE = 800 [(gogoproto.enumvalue_customname)="TaskStateRemove"]; + // TaskStateOrphaned is used to free up resources associated with service + // tasks on unresponsive nodes without having to delete those tasks. This + // state is directly assigned to the task by the orchestrator. + ORPHANED = 832 [(gogoproto.enumvalue_customname)="TaskStateOrphaned"]; // NOTE(stevvooe): The state of a task is actually a lamport clock, in that // given two observations, the greater of the two can be considered diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/watch.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/watch.pb.go index 58fa8917e5..78f0014c7a 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/watch.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/watch.pb.go @@ -19,6 +19,7 @@ import ( import raftselector "github.com/docker/swarmkit/manager/raftselector" import codes "google.golang.org/grpc/codes" +import status "google.golang.org/grpc/status" import metadata "google.golang.org/grpc/metadata" import transport "google.golang.org/grpc/transport" import rafttime "time" @@ -2043,12 +2044,12 @@ func NewRaftProxyWatchServer(local WatchServer, connSelector raftselector.ConnPr redirectChecker := func(ctx context.Context) (context.Context, error) { s, ok := transport.StreamFromContext(ctx) if !ok { - return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context") } addr := s.ServerTransport().RemoteAddr().String() md, ok := metadata.FromContext(ctx) if ok && len(md["redirect"]) != 0 { - return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) } if !ok { md = metadata.New(map[string]string{}) diff --git a/components/engine/vendor/github.com/docker/swarmkit/ca/auth.go b/components/engine/vendor/github.com/docker/swarmkit/ca/auth.go index 488d34dd33..df4547fb13 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/ca/auth.go +++ b/components/engine/vendor/github.com/docker/swarmkit/ca/auth.go @@ -10,10 +10,10 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" ) type localRequestKeyType struct{} @@ -52,13 +52,13 @@ func LogTLSState(ctx context.Context, tlsState *tls.ConnectionState) { // getCertificateSubject extracts the subject from a verified client certificate func getCertificateSubject(tlsState *tls.ConnectionState) (pkix.Name, error) { if tlsState == nil { - return pkix.Name{}, grpc.Errorf(codes.PermissionDenied, "request is not using TLS") + return pkix.Name{}, status.Errorf(codes.PermissionDenied, "request is not using TLS") } if len(tlsState.PeerCertificates) == 0 { - return pkix.Name{}, grpc.Errorf(codes.PermissionDenied, "no client certificates in request") + return pkix.Name{}, status.Errorf(codes.PermissionDenied, "no client certificates in request") } if len(tlsState.VerifiedChains) == 0 { - return pkix.Name{}, grpc.Errorf(codes.PermissionDenied, "no verified chains for remote certificate") + return pkix.Name{}, status.Errorf(codes.PermissionDenied, "no verified chains for remote certificate") } return tlsState.VerifiedChains[0][0].Subject, nil @@ -67,11 +67,11 @@ func getCertificateSubject(tlsState *tls.ConnectionState) (pkix.Name, error) { func tlsConnStateFromContext(ctx context.Context) (*tls.ConnectionState, error) { peer, ok := peer.FromContext(ctx) if !ok { - return nil, grpc.Errorf(codes.PermissionDenied, "Permission denied: no peer info") + return nil, status.Errorf(codes.PermissionDenied, "Permission denied: no peer info") } tlsInfo, ok := peer.AuthInfo.(credentials.TLSInfo) if !ok { - return nil, grpc.Errorf(codes.PermissionDenied, "Permission denied: peer didn't not present valid peer certificate") + return nil, status.Errorf(codes.PermissionDenied, "Permission denied: peer didn't not present valid peer certificate") } return &tlsInfo.State, nil } @@ -98,21 +98,21 @@ func AuthorizeOrgAndRole(ctx context.Context, org string, blacklistedCerts map[s return authorizeOrg(certSubj, org, blacklistedCerts) } - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of OUs: %v", ou) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of OUs: %v", ou) } // authorizeOrg takes in a certificate subject and an organization, and returns // the Node ID of the node. func authorizeOrg(certSubj pkix.Name, org string, blacklistedCerts map[string]*api.BlacklistedCertificate) (string, error) { if _, ok := blacklistedCerts[certSubj.CommonName]; ok { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: node %s was removed from swarm", certSubj.CommonName) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: node %s was removed from swarm", certSubj.CommonName) } if len(certSubj.Organization) > 0 && certSubj.Organization[0] == org { return certSubj.CommonName, nil } - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of organization: %s", org) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of organization: %s", org) } // AuthorizeForwardedRoleAndOrg checks for proper roles and organization of caller. The RPC may have @@ -123,7 +123,7 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde if isForwardedRequest(ctx) { _, err := AuthorizeOrgAndRole(ctx, org, blacklistedCerts, forwarderRoles...) if err != nil { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarder role: %v", err) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarder role: %v", err) } // This was a forwarded request. Authorize the forwarder, and @@ -132,15 +132,15 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde _, forwardedID, forwardedOrg, forwardedOUs := forwardedTLSInfoFromContext(ctx) if len(forwardedOUs) == 0 || forwardedID == "" || forwardedOrg == "" { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request") + return "", status.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request") } if !intersectArrays(forwardedOUs, authorizedRoles) { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarded role, expecting: %v", authorizedRoles) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarded role, expecting: %v", authorizedRoles) } if forwardedOrg != org { - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: organization mismatch, expecting: %s", org) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: organization mismatch, expecting: %s", org) } return forwardedID, nil @@ -152,7 +152,7 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde return nodeID, nil } - return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized peer role: %v", err) + return "", status.Errorf(codes.PermissionDenied, "Permission denied: unauthorized peer role: %v", err) } // intersectArrays returns true when there is at least one element in common @@ -219,7 +219,7 @@ func RemoteNode(ctx context.Context) (RemoteNodeInfo, error) { peer, ok := peer.FromContext(ctx) if !ok { - return RemoteNodeInfo{}, grpc.Errorf(codes.PermissionDenied, "Permission denied: no peer info") + return RemoteNodeInfo{}, status.Errorf(codes.PermissionDenied, "Permission denied: no peer info") } directInfo := RemoteNodeInfo{ @@ -232,7 +232,7 @@ func RemoteNode(ctx context.Context) (RemoteNodeInfo, error) { if isForwardedRequest(ctx) { remoteAddr, cn, org, ous := forwardedTLSInfoFromContext(ctx) if len(ous) == 0 || cn == "" || org == "" { - return RemoteNodeInfo{}, grpc.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request") + return RemoteNodeInfo{}, status.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request") } return RemoteNodeInfo{ Roles: ous, diff --git a/components/engine/vendor/github.com/docker/swarmkit/ca/server.go b/components/engine/vendor/github.com/docker/swarmkit/ca/server.go index 16dbedc8da..a456df7900 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/ca/server.go +++ b/components/engine/vendor/github.com/docker/swarmkit/ca/server.go @@ -16,8 +16,8 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -149,7 +149,7 @@ func (s *Server) GetUnlockKey(ctx context.Context, request *api.GetUnlockKeyRequ // NodeCertificateStatus returns the current issuance status of an issuance request identified by the nodeID func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCertificateStatusRequest) (*api.NodeCertificateStatusResponse, error) { if request.NodeID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, codes.InvalidArgument.String()) + return nil, status.Errorf(codes.InvalidArgument, codes.InvalidArgument.String()) } serverCtx, err := s.isRunningLocked() @@ -180,7 +180,7 @@ func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCer // This node ID doesn't exist if node == nil { - return nil, grpc.Errorf(codes.NotFound, codes.NotFound.String()) + return nil, status.Errorf(codes.NotFound, codes.NotFound.String()) } log.G(ctx).WithFields(logrus.Fields{ @@ -236,7 +236,7 @@ func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCer func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNodeCertificateRequest) (*api.IssueNodeCertificateResponse, error) { // First, let's see if the remote node is presenting a non-empty CSR if len(request.CSR) == 0 { - return nil, grpc.Errorf(codes.InvalidArgument, codes.InvalidArgument.String()) + return nil, status.Errorf(codes.InvalidArgument, codes.InvalidArgument.String()) } if err := s.isReadyLocked(); err != nil { @@ -295,7 +295,7 @@ func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNod s.mu.Unlock() if role < 0 { - return nil, grpc.Errorf(codes.InvalidArgument, "A valid join token is necessary to join this cluster") + return nil, status.Errorf(codes.InvalidArgument, "A valid join token is necessary to join this cluster") } // Max number of collisions of ID or CN to tolerate before giving up @@ -369,7 +369,7 @@ func (s *Server) issueRenewCertificate(ctx context.Context, nodeID string, csr [ "method": "issueRenewCertificate", }).Warnf("node does not exist") // If this node doesn't exist, we shouldn't be renewing a certificate for it - return grpc.Errorf(codes.NotFound, "node %s not found when attempting to renew certificate", nodeID) + return status.Errorf(codes.NotFound, "node %s not found when attempting to renew certificate", nodeID) } // Create a new Certificate entry for this node with the new CSR and a RENEW state @@ -594,7 +594,7 @@ func (s *Server) isRunningLocked() (context.Context, error) { s.mu.Lock() if !s.isRunning() { s.mu.Unlock() - return nil, grpc.Errorf(codes.Aborted, "CA signer is stopped") + return nil, status.Errorf(codes.Aborted, "CA signer is stopped") } ctx := s.ctx s.mu.Unlock() @@ -605,10 +605,10 @@ func (s *Server) isReadyLocked() error { s.mu.Lock() defer s.mu.Unlock() if !s.isRunning() { - return grpc.Errorf(codes.Aborted, "CA signer is stopped") + return status.Errorf(codes.Aborted, "CA signer is stopped") } if s.joinTokens == nil { - return grpc.Errorf(codes.Aborted, "CA signer is still starting") + return status.Errorf(codes.Aborted, "CA signer is still starting") } return nil } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/ca_rotation.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/ca_rotation.go index 5e8fa43ce3..d39c7d2b69 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/ca_rotation.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/ca_rotation.go @@ -10,13 +10,12 @@ import ( "net/url" "time" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "github.com/cloudflare/cfssl/helpers" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/log" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var minRootExpiration = 1 * helpers.OneYear @@ -60,7 +59,7 @@ func newRootRotationObject(ctx context.Context, securityConfig *ca.SecurityConfi crossSignedCert, err = oldRootCA.CrossSignCACertificate(rootCert) } case !newRootHasSigner: // the original CA and the new CA both require external CAs - return nil, grpc.Errorf(codes.InvalidArgument, "rotating from one external CA to a different external CA is not supported") + return nil, status.Errorf(codes.InvalidArgument, "rotating from one external CA to a different external CA is not supported") default: // We need the same credentials but to connect to the original URLs (in case we are in the middle of a root rotation already) var urls []string @@ -70,7 +69,7 @@ func newRootRotationObject(ctx context.Context, securityConfig *ca.SecurityConfi } } if len(urls) == 0 { - return nil, grpc.Errorf(codes.InvalidArgument, + return nil, status.Errorf(codes.InvalidArgument, "must provide an external CA for the current external root CA to generate a cross-signed certificate") } rootPool := x509.NewCertPool() @@ -83,7 +82,7 @@ func newRootRotationObject(ctx context.Context, securityConfig *ca.SecurityConfi if err != nil { log.G(ctx).WithError(err).Error("unable to generate a cross-signed certificate for root rotation") - return nil, grpc.Errorf(codes.Internal, "unable to generate a cross-signed certificate for root rotation") + return nil, status.Errorf(codes.Internal, "unable to generate a cross-signed certificate for root rotation") } copied := apiRootCA.Copy() @@ -146,7 +145,7 @@ func validateHasAtLeastOneExternalCA(ctx context.Context, externalCAs map[string } } } - return nil, grpc.Errorf(codes.InvalidArgument, "there must be at least one valid, reachable external CA corresponding to the %s CA certificate", desc) + return nil, status.Errorf(codes.InvalidArgument, "there must be at least one valid, reachable external CA corresponding to the %s CA certificate", desc) } // validates that the list of external CAs have valid certs associated with them, and produce a mapping of subject/pubkey:external @@ -193,7 +192,7 @@ func validateCAConfig(ctx context.Context, securityConfig *ca.SecurityConfig, cl newConfig.SigningCACert = ca.NormalizePEMs(newConfig.SigningCACert) // ensure this is normalized before we use it if len(newConfig.SigningCAKey) > 0 && len(newConfig.SigningCACert) == 0 { - return nil, grpc.Errorf(codes.InvalidArgument, "if a signing CA key is provided, the signing CA cert must also be provided") + return nil, status.Errorf(codes.InvalidArgument, "if a signing CA key is provided, the signing CA cert must also be provided") } normalizedRootCA := ca.NormalizePEMs(cluster.RootCA.CACert) @@ -216,7 +215,7 @@ func validateCAConfig(ctx context.Context, securityConfig *ca.SecurityConfig, cl if cluster.RootCA.LastForcedRotation != newConfig.ForceRotate { newRootCA, err := ca.CreateRootCA(ca.DefaultRootCN) if err != nil { - return nil, grpc.Errorf(codes.Internal, err.Error()) + return nil, status.Errorf(codes.Internal, err.Error()) } return newRootRotationObject(ctx, securityConfig, &cluster.RootCA, newRootCA, oldCertExtCAs, newConfig.ForceRotate) } @@ -240,21 +239,21 @@ func validateCAConfig(ctx context.Context, securityConfig *ca.SecurityConfig, cl } newRootCA, err := ca.NewRootCA(newConfig.SigningCACert, signingCert, newConfig.SigningCAKey, ca.DefaultNodeCertExpiration, nil) if err != nil { - return nil, grpc.Errorf(codes.InvalidArgument, err.Error()) + return nil, status.Errorf(codes.InvalidArgument, err.Error()) } if len(newRootCA.Pool.Subjects()) != 1 { - return nil, grpc.Errorf(codes.InvalidArgument, "the desired CA certificate cannot contain multiple certificates") + return nil, status.Errorf(codes.InvalidArgument, "the desired CA certificate cannot contain multiple certificates") } parsedCert, err := helpers.ParseCertificatePEM(newConfig.SigningCACert) if err != nil { - return nil, grpc.Errorf(codes.InvalidArgument, "could not parse the desired CA certificate") + return nil, status.Errorf(codes.InvalidArgument, "could not parse the desired CA certificate") } // The new certificate's expiry must be at least one year away if parsedCert.NotAfter.Before(time.Now().Add(minRootExpiration)) { - return nil, grpc.Errorf(codes.InvalidArgument, "CA certificate expires too soon") + return nil, status.Errorf(codes.InvalidArgument, "CA certificate expires too soon") } if !hasSigningKey(newConfig) { diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go index 329313a950..0876113d70 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/cluster.go @@ -11,8 +11,8 @@ import ( "github.com/docker/swarmkit/manager/state/store" gogotypes "github.com/gogo/protobuf/types" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -23,17 +23,17 @@ const ( func validateClusterSpec(spec *api.ClusterSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } // Validate that expiry time being provided is valid, and over our minimum if spec.CAConfig.NodeCertExpiry != nil { expiry, err := gogotypes.DurationFromProto(spec.CAConfig.NodeCertExpiry) if err != nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if expiry < ca.MinNodeCertExpiration { - return grpc.Errorf(codes.InvalidArgument, "minimum certificate expiry time is: %s", ca.MinNodeCertExpiration) + return status.Errorf(codes.InvalidArgument, "minimum certificate expiry time is: %s", ca.MinNodeCertExpiration) } } @@ -42,7 +42,7 @@ func validateClusterSpec(spec *api.ClusterSpec) error { if len(spec.AcceptancePolicy.Policies) > 0 { for _, policy := range spec.AcceptancePolicy.Policies { if policy.Secret != nil && strings.ToLower(policy.Secret.Alg) != "bcrypt" { - return grpc.Errorf(codes.InvalidArgument, "hashing algorithm is not supported: %s", policy.Secret.Alg) + return status.Errorf(codes.InvalidArgument, "hashing algorithm is not supported: %s", policy.Secret.Alg) } } } @@ -51,13 +51,17 @@ func validateClusterSpec(spec *api.ClusterSpec) error { if spec.Dispatcher.HeartbeatPeriod != nil { heartbeatPeriod, err := gogotypes.DurationFromProto(spec.Dispatcher.HeartbeatPeriod) if err != nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if heartbeatPeriod < 0 { - return grpc.Errorf(codes.InvalidArgument, "heartbeat time period cannot be a negative duration") + return status.Errorf(codes.InvalidArgument, "heartbeat time period cannot be a negative duration") } } + if spec.Annotations.Name != store.DefaultClusterName { + return status.Errorf(codes.InvalidArgument, "modification of cluster name is not allowed") + } + return nil } @@ -66,7 +70,7 @@ func validateClusterSpec(spec *api.ClusterSpec) error { // - Returns `NotFound` if the Cluster is not found. func (s *Server) GetCluster(ctx context.Context, request *api.GetClusterRequest) (*api.GetClusterResponse, error) { if request.ClusterID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var cluster *api.Cluster @@ -74,7 +78,7 @@ func (s *Server) GetCluster(ctx context.Context, request *api.GetClusterRequest) cluster = store.GetCluster(tx, request.ClusterID) }) if cluster == nil { - return nil, grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID) + return nil, status.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID) } redactedClusters := redactClusters([]*api.Cluster{cluster}) @@ -92,7 +96,7 @@ func (s *Server) GetCluster(ctx context.Context, request *api.GetClusterRequest) // - Returns an error if the update fails. func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRequest) (*api.UpdateClusterResponse, error) { if request.ClusterID == "" || request.ClusterVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateClusterSpec(request.Spec); err != nil { return nil, err @@ -102,7 +106,7 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe err := s.store.Update(func(tx store.Tx) error { cluster = store.GetCluster(tx, request.ClusterID) if cluster == nil { - return grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID) + return status.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID) } // This ensures that we have the current rootCA with which to generate tokens (expiration doesn't matter // for generating the tokens) @@ -110,7 +114,7 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe if err != nil { log.G(ctx).WithField( "method", "(*controlapi.Server).UpdateCluster").WithError(err).Error("invalid cluster root CA") - return grpc.Errorf(codes.Internal, "error loading cluster rootCA for update") + return status.Errorf(codes.Internal, "error loading cluster rootCA for update") } cluster.Meta.Version = *request.ClusterVersion diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/common.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/common.go index c01662379e..9e52179464 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/common.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/common.go @@ -10,8 +10,8 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/allocator" "github.com/docker/swarmkit/manager/state/store" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var isValidDNSName = regexp.MustCompile(`^[a-zA-Z0-9](?:[-_]*[A-Za-z0-9]+)*$`) @@ -70,25 +70,25 @@ func filterMatchLabels(match map[string]string, candidates map[string]string) bo func validateAnnotations(m api.Annotations) error { if m.Name == "" { - return grpc.Errorf(codes.InvalidArgument, "meta: name must be provided") + return status.Errorf(codes.InvalidArgument, "meta: name must be provided") } if !isValidDNSName.MatchString(m.Name) { // if the name doesn't match the regex - return grpc.Errorf(codes.InvalidArgument, "name must be valid as a DNS name component") + return status.Errorf(codes.InvalidArgument, "name must be valid as a DNS name component") } if len(m.Name) > 63 { // DNS labels are limited to 63 characters - return grpc.Errorf(codes.InvalidArgument, "name must be 63 characters or fewer") + return status.Errorf(codes.InvalidArgument, "name must be 63 characters or fewer") } return nil } func validateConfigOrSecretAnnotations(m api.Annotations) error { if m.Name == "" { - return grpc.Errorf(codes.InvalidArgument, "name must be provided") + return status.Errorf(codes.InvalidArgument, "name must be provided") } else if len(m.Name) > 64 || !isValidConfigOrSecretName.MatchString(m.Name) { // if the name doesn't match the regex - return grpc.Errorf(codes.InvalidArgument, + return status.Errorf(codes.InvalidArgument, "invalid name, only 64 [a-zA-Z0-9-_.] characters allowed, and the start and end character must be [a-zA-Z0-9]") } return nil @@ -102,7 +102,7 @@ func validateDriver(driver *api.Driver, pg plugingetter.PluginGetter, pluginType } if driver.Name == "" { - return grpc.Errorf(codes.InvalidArgument, "driver name: if driver is specified name is required") + return status.Errorf(codes.InvalidArgument, "driver name: if driver is specified name is required") } // First check against the known drivers @@ -119,16 +119,16 @@ func validateDriver(driver *api.Driver, pg plugingetter.PluginGetter, pluginType } if pg == nil { - return grpc.Errorf(codes.InvalidArgument, "plugin %s not supported", driver.Name) + return status.Errorf(codes.InvalidArgument, "plugin %s not supported", driver.Name) } p, err := pg.Get(driver.Name, pluginType, plugingetter.Lookup) if err != nil { - return grpc.Errorf(codes.InvalidArgument, "error during lookup of plugin %s", driver.Name) + return status.Errorf(codes.InvalidArgument, "error during lookup of plugin %s", driver.Name) } if p.IsV1() { - return grpc.Errorf(codes.InvalidArgument, "legacy plugin %s of type %s is not supported in swarm mode", driver.Name, pluginType) + return status.Errorf(codes.InvalidArgument, "legacy plugin %s of type %s is not supported in swarm mode", driver.Name, pluginType) } return nil diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/config.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/config.go index d0fe8a56c1..ae08885b00 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/config.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/config.go @@ -10,8 +10,8 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/sirupsen/logrus" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // MaxConfigSize is the maximum byte length of the `Config.Spec.Data` field. @@ -32,7 +32,7 @@ func configFromConfigSpec(spec *api.ConfigSpec) *api.Config { // - Returns an error if getting fails. func (s *Server) GetConfig(ctx context.Context, request *api.GetConfigRequest) (*api.GetConfigResponse, error) { if request.ConfigID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, "config ID must be provided") + return nil, status.Errorf(codes.InvalidArgument, "config ID must be provided") } var config *api.Config @@ -41,7 +41,7 @@ func (s *Server) GetConfig(ctx context.Context, request *api.GetConfigRequest) ( }) if config == nil { - return nil, grpc.Errorf(codes.NotFound, "config %s not found", request.ConfigID) + return nil, status.Errorf(codes.NotFound, "config %s not found", request.ConfigID) } return &api.GetConfigResponse{Config: config}, nil @@ -53,21 +53,21 @@ func (s *Server) GetConfig(ctx context.Context, request *api.GetConfigRequest) ( // - Returns an error if the update fails. func (s *Server) UpdateConfig(ctx context.Context, request *api.UpdateConfigRequest) (*api.UpdateConfigResponse, error) { if request.ConfigID == "" || request.ConfigVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var config *api.Config err := s.store.Update(func(tx store.Tx) error { config = store.GetConfig(tx, request.ConfigID) if config == nil { - return grpc.Errorf(codes.NotFound, "config %s not found", request.ConfigID) + return status.Errorf(codes.NotFound, "config %s not found", request.ConfigID) } // Check if the Name is different than the current name, or the config is non-nil and different // than the current config if config.Spec.Annotations.Name != request.Spec.Annotations.Name || (request.Spec.Data != nil && !bytes.Equal(request.Spec.Data, config.Spec.Data)) { - return grpc.Errorf(codes.InvalidArgument, "only updates to Labels are allowed") + return status.Errorf(codes.InvalidArgument, "only updates to Labels are allowed") } // We only allow updating Labels @@ -164,7 +164,7 @@ func (s *Server) CreateConfig(ctx context.Context, request *api.CreateConfigRequ switch err { case store.ErrNameConflict: - return nil, grpc.Errorf(codes.AlreadyExists, "config %s already exists", request.Spec.Annotations.Name) + return nil, status.Errorf(codes.AlreadyExists, "config %s already exists", request.Spec.Annotations.Name) case nil: log.G(ctx).WithFields(logrus.Fields{ "config.Name": request.Spec.Annotations.Name, @@ -184,20 +184,20 @@ func (s *Server) CreateConfig(ctx context.Context, request *api.CreateConfigRequ // - Returns an error if the deletion fails. func (s *Server) RemoveConfig(ctx context.Context, request *api.RemoveConfigRequest) (*api.RemoveConfigResponse, error) { if request.ConfigID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, "config ID must be provided") + return nil, status.Errorf(codes.InvalidArgument, "config ID must be provided") } err := s.store.Update(func(tx store.Tx) error { // Check if the config exists config := store.GetConfig(tx, request.ConfigID) if config == nil { - return grpc.Errorf(codes.NotFound, "could not find config %s", request.ConfigID) + return status.Errorf(codes.NotFound, "could not find config %s", request.ConfigID) } // Check if any services currently reference this config, return error if so services, err := store.FindServices(tx, store.ByReferencedConfigID(request.ConfigID)) if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using config %s: %v", request.ConfigID, err) + return status.Errorf(codes.Internal, "could not find services using config %s: %v", request.ConfigID, err) } if len(services) != 0 { @@ -213,14 +213,14 @@ func (s *Server) RemoveConfig(ctx context.Context, request *api.RemoveConfigRequ serviceStr = "service" } - return grpc.Errorf(codes.InvalidArgument, "config '%s' is in use by the following %s: %v", configName, serviceStr, serviceNameStr) + return status.Errorf(codes.InvalidArgument, "config '%s' is in use by the following %s: %v", configName, serviceStr, serviceNameStr) } return store.DeleteConfig(tx, request.ConfigID) }) switch err { case store.ErrNotExist: - return nil, grpc.Errorf(codes.NotFound, "config %s not found", request.ConfigID) + return nil, status.Errorf(codes.NotFound, "config %s not found", request.ConfigID) case nil: log.G(ctx).WithFields(logrus.Fields{ "config.ID": request.ConfigID, @@ -235,14 +235,14 @@ func (s *Server) RemoveConfig(ctx context.Context, request *api.RemoveConfigRequ func validateConfigSpec(spec *api.ConfigSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateConfigOrSecretAnnotations(spec.Annotations); err != nil { return err } if len(spec.Data) >= MaxConfigSize || len(spec.Data) < 1 { - return grpc.Errorf(codes.InvalidArgument, "config data must be larger than 0 and less than %d bytes", MaxConfigSize) + return status.Errorf(codes.InvalidArgument, "config data must be larger than 0 and less than %d bytes", MaxConfigSize) } return nil } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/network.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/network.go index b150de0e78..481b0cfe4b 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/network.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/network.go @@ -12,39 +12,39 @@ import ( "github.com/docker/swarmkit/manager/allocator/networkallocator" "github.com/docker/swarmkit/manager/state/store" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func validateIPAMConfiguration(ipamConf *api.IPAMConfig) error { if ipamConf == nil { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: cannot be empty") + return status.Errorf(codes.InvalidArgument, "ipam configuration: cannot be empty") } _, subnet, err := net.ParseCIDR(ipamConf.Subnet) if err != nil { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: invalid subnet %s", ipamConf.Subnet) + return status.Errorf(codes.InvalidArgument, "ipam configuration: invalid subnet %s", ipamConf.Subnet) } if ipamConf.Range != "" { ip, _, err := net.ParseCIDR(ipamConf.Range) if err != nil { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: invalid range %s", ipamConf.Range) + return status.Errorf(codes.InvalidArgument, "ipam configuration: invalid range %s", ipamConf.Range) } if !subnet.Contains(ip) { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain range %s", ipamConf.Subnet, ipamConf.Range) + return status.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain range %s", ipamConf.Subnet, ipamConf.Range) } } if ipamConf.Gateway != "" { ip := net.ParseIP(ipamConf.Gateway) if ip == nil { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: invalid gateway %s", ipamConf.Gateway) + return status.Errorf(codes.InvalidArgument, "ipam configuration: invalid gateway %s", ipamConf.Gateway) } if !subnet.Contains(ip) { - return grpc.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain gateway %s", ipamConf.Subnet, ipamConf.Gateway) + return status.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain gateway %s", ipamConf.Subnet, ipamConf.Gateway) } } @@ -73,15 +73,15 @@ func validateIPAM(ipam *api.IPAMOptions, pg plugingetter.PluginGetter) error { func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if spec.Ingress && spec.DriverConfig != nil && spec.DriverConfig.Name != "overlay" { - return grpc.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network") + return status.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network") } if spec.Attachable && spec.Ingress { - return grpc.Errorf(codes.InvalidArgument, "ingress network cannot be attachable") + return status.Errorf(codes.InvalidArgument, "ingress network cannot be attachable") } if err := validateAnnotations(spec.Annotations); err != nil { @@ -89,7 +89,7 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er } if _, ok := spec.Annotations.Labels[networkallocator.PredefinedLabel]; ok { - return grpc.Errorf(codes.PermissionDenied, "label %s is for internally created predefined networks and cannot be applied by users", + return status.Errorf(codes.PermissionDenied, "label %s is for internally created predefined networks and cannot be applied by users", networkallocator.PredefinedLabel) } if err := validateDriver(spec.DriverConfig, pg, driverapi.NetworkPluginEndpointType); err != nil { @@ -117,9 +117,9 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe err := s.store.Update(func(tx store.Tx) error { if request.Spec.Ingress { if n, err := allocator.GetIngressNetwork(s.store); err == nil { - return grpc.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID) + return status.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID) } else if err != allocator.ErrNoIngress { - return grpc.Errorf(codes.Internal, "failed ingress network presence check: %v", err) + return status.Errorf(codes.Internal, "failed ingress network presence check: %v", err) } } return store.CreateNetwork(tx, n) @@ -138,7 +138,7 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe // - Returns `NotFound` if the Network is not found. func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest) (*api.GetNetworkResponse, error) { if request.NetworkID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var n *api.Network @@ -146,7 +146,7 @@ func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest) n = store.GetNetwork(tx, request.NetworkID) }) if n == nil { - return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) + return nil, status.Errorf(codes.NotFound, "network %s not found", request.NetworkID) } return &api.GetNetworkResponse{ Network: n, @@ -159,7 +159,7 @@ func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest) // - Returns an error if the deletion fails. func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRequest) (*api.RemoveNetworkResponse, error) { if request.NetworkID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var ( @@ -171,7 +171,7 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe n = store.GetNetwork(tx, request.NetworkID) }) if n == nil { - return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) + return nil, status.Errorf(codes.NotFound, "network %s not found", request.NetworkID) } if allocator.IsIngressNetwork(n) { @@ -179,13 +179,13 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe } if v, ok := n.Spec.Annotations.Labels[networkallocator.PredefinedLabel]; ok && v == "true" { - return nil, grpc.Errorf(codes.FailedPrecondition, "network %s (%s) is a swarm predefined network and cannot be removed", + return nil, status.Errorf(codes.FailedPrecondition, "network %s (%s) is a swarm predefined network and cannot be removed", request.NetworkID, n.Spec.Annotations.Name) } if err := rm(n.ID); err != nil { if err == store.ErrNotExist { - return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) + return nil, status.Errorf(codes.NotFound, "network %s not found", request.NetworkID) } return nil, err } @@ -196,21 +196,21 @@ func (s *Server) removeNetwork(id string) error { return s.store.Update(func(tx store.Tx) error { services, err := store.FindServices(tx, store.ByReferencedNetworkID(id)) if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) + return status.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) } if len(services) != 0 { - return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID) + return status.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID) } tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(id)) if err != nil { - return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err) + return status.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err) } for _, t := range tasks { if t.DesiredState <= api.TaskStateRunning && t.Status.State <= api.TaskStateRunning { - return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID) + return status.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID) } } @@ -222,11 +222,11 @@ func (s *Server) removeIngressNetwork(id string) error { return s.store.Update(func(tx store.Tx) error { services, err := store.FindServices(tx, store.All) if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) + return status.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) } for _, srv := range services { if allocator.IsIngressNetworkNeeded(srv) { - return grpc.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID) + return status.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID) } } return store.DeleteNetwork(tx, id) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/node.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/node.go index bac6b8073d..e1fe3dec1d 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/node.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/node.go @@ -9,13 +9,13 @@ import ( "github.com/docker/swarmkit/manager/state/store" gogotypes "github.com/gogo/protobuf/types" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func validateNodeSpec(spec *api.NodeSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } return nil } @@ -25,7 +25,7 @@ func validateNodeSpec(spec *api.NodeSpec) error { // - Returns `NotFound` if the Node is not found. func (s *Server) GetNode(ctx context.Context, request *api.GetNodeRequest) (*api.GetNodeResponse, error) { if request.NodeID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var node *api.Node @@ -33,7 +33,7 @@ func (s *Server) GetNode(ctx context.Context, request *api.GetNodeRequest) (*api node = store.GetNode(tx, request.NodeID) }) if node == nil { - return nil, grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID) + return nil, status.Errorf(codes.NotFound, "node %s not found", request.NodeID) } if s.raft != nil { @@ -196,7 +196,7 @@ func (s *Server) ListNodes(ctx context.Context, request *api.ListNodesRequest) ( // - Returns an error if the update fails. func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) (*api.UpdateNodeResponse, error) { if request.NodeID == "" || request.NodeVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateNodeSpec(request.Spec); err != nil { return nil, err @@ -210,7 +210,7 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) err := s.store.Update(func(tx store.Tx) error { node = store.GetNode(tx, request.NodeID) if node == nil { - return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID) + return status.Errorf(codes.NotFound, "node %s not found", request.NodeID) } // Demotion sanity checks. @@ -218,20 +218,20 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) // Check for manager entries in Store. managers, err := store.FindNodes(tx, store.ByRole(api.NodeRoleManager)) if err != nil { - return grpc.Errorf(codes.Internal, "internal store error: %v", err) + return status.Errorf(codes.Internal, "internal store error: %v", err) } if len(managers) == 1 && managers[0].ID == node.ID { - return grpc.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm") + return status.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm") } // Check for node in memberlist if member = s.raft.GetMemberByNodeID(request.NodeID); member == nil { - return grpc.Errorf(codes.NotFound, "can't find manager in raft memberlist") + return status.Errorf(codes.NotFound, "can't find manager in raft memberlist") } // Quorum safeguard if !s.raft.CanRemoveMember(member.RaftID) { - return grpc.Errorf(codes.FailedPrecondition, "can't remove member from the raft: this would result in a loss of quorum") + return status.Errorf(codes.FailedPrecondition, "can't remove member from the raft: this would result in a loss of quorum") } } @@ -278,33 +278,33 @@ func removeNodeAttachments(tx store.Tx, nodeID string) error { // - Returns an error if the delete fails. func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest) (*api.RemoveNodeResponse, error) { if request.NodeID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } err := s.store.Update(func(tx store.Tx) error { node := store.GetNode(tx, request.NodeID) if node == nil { - return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID) + return status.Errorf(codes.NotFound, "node %s not found", request.NodeID) } if node.Spec.DesiredRole == api.NodeRoleManager { if s.raft == nil { - return grpc.Errorf(codes.FailedPrecondition, "node %s is a manager but cannot access node information from the raft memberlist", request.NodeID) + return status.Errorf(codes.FailedPrecondition, "node %s is a manager but cannot access node information from the raft memberlist", request.NodeID) } if member := s.raft.GetMemberByNodeID(request.NodeID); member != nil { - return grpc.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID) + return status.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID) } } if !request.Force && node.Status.State == api.NodeStatus_READY { - return grpc.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID) + return status.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID) } // lookup the cluster - clusters, err := store.FindClusters(tx, store.ByName("default")) + clusters, err := store.FindClusters(tx, store.ByName(store.DefaultClusterName)) if err != nil { return err } if len(clusters) != 1 { - return grpc.Errorf(codes.Internal, "could not fetch cluster object") + return status.Errorf(codes.Internal, "could not fetch cluster object") } cluster := clusters[0] diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go index bac4c102ef..fdcd2c412c 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/secret.go @@ -11,8 +11,8 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/sirupsen/logrus" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // assumes spec is not nil @@ -30,7 +30,7 @@ func secretFromSecretSpec(spec *api.SecretSpec) *api.Secret { // - Returns an error if getting fails. func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) (*api.GetSecretResponse, error) { if request.SecretID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, "secret ID must be provided") + return nil, status.Errorf(codes.InvalidArgument, "secret ID must be provided") } var secret *api.Secret @@ -39,7 +39,7 @@ func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) ( }) if secret == nil { - return nil, grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID) + return nil, status.Errorf(codes.NotFound, "secret %s not found", request.SecretID) } secret.Spec.Data = nil // clean the actual secret data so it's never returned @@ -52,20 +52,20 @@ func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) ( // - Returns an error if the update fails. func (s *Server) UpdateSecret(ctx context.Context, request *api.UpdateSecretRequest) (*api.UpdateSecretResponse, error) { if request.SecretID == "" || request.SecretVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var secret *api.Secret err := s.store.Update(func(tx store.Tx) error { secret = store.GetSecret(tx, request.SecretID) if secret == nil { - return grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID) + return status.Errorf(codes.NotFound, "secret %s not found", request.SecretID) } // Check if the Name is different than the current name, or the secret is non-nil and different // than the current secret if secret.Spec.Annotations.Name != request.Spec.Annotations.Name || (request.Spec.Data != nil && subtle.ConstantTimeCompare(request.Spec.Data, secret.Spec.Data) == 0) { - return grpc.Errorf(codes.InvalidArgument, "only updates to Labels are allowed") + return status.Errorf(codes.InvalidArgument, "only updates to Labels are allowed") } // We only allow updating Labels @@ -171,7 +171,7 @@ func (s *Server) CreateSecret(ctx context.Context, request *api.CreateSecretRequ switch err { case store.ErrNameConflict: - return nil, grpc.Errorf(codes.AlreadyExists, "secret %s already exists", request.Spec.Annotations.Name) + return nil, status.Errorf(codes.AlreadyExists, "secret %s already exists", request.Spec.Annotations.Name) case nil: secret.Spec.Data = nil // clean the actual secret data so it's never returned log.G(ctx).WithFields(logrus.Fields{ @@ -192,20 +192,20 @@ func (s *Server) CreateSecret(ctx context.Context, request *api.CreateSecretRequ // - Returns an error if the deletion fails. func (s *Server) RemoveSecret(ctx context.Context, request *api.RemoveSecretRequest) (*api.RemoveSecretResponse, error) { if request.SecretID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, "secret ID must be provided") + return nil, status.Errorf(codes.InvalidArgument, "secret ID must be provided") } err := s.store.Update(func(tx store.Tx) error { // Check if the secret exists secret := store.GetSecret(tx, request.SecretID) if secret == nil { - return grpc.Errorf(codes.NotFound, "could not find secret %s", request.SecretID) + return status.Errorf(codes.NotFound, "could not find secret %s", request.SecretID) } // Check if any services currently reference this secret, return error if so services, err := store.FindServices(tx, store.ByReferencedSecretID(request.SecretID)) if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using secret %s: %v", request.SecretID, err) + return status.Errorf(codes.Internal, "could not find services using secret %s: %v", request.SecretID, err) } if len(services) != 0 { @@ -221,14 +221,14 @@ func (s *Server) RemoveSecret(ctx context.Context, request *api.RemoveSecretRequ serviceStr = "service" } - return grpc.Errorf(codes.InvalidArgument, "secret '%s' is in use by the following %s: %v", secretName, serviceStr, serviceNameStr) + return status.Errorf(codes.InvalidArgument, "secret '%s' is in use by the following %s: %v", secretName, serviceStr, serviceNameStr) } return store.DeleteSecret(tx, request.SecretID) }) switch err { case store.ErrNotExist: - return nil, grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID) + return nil, status.Errorf(codes.NotFound, "secret %s not found", request.SecretID) case nil: log.G(ctx).WithFields(logrus.Fields{ "secret.ID": request.SecretID, @@ -243,7 +243,7 @@ func (s *Server) RemoveSecret(ctx context.Context, request *api.RemoveSecretRequ func validateSecretSpec(spec *api.SecretSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateConfigOrSecretAnnotations(spec.Annotations); err != nil { return err @@ -252,12 +252,12 @@ func validateSecretSpec(spec *api.SecretSpec) error { if spec.Driver != nil { // Ensure secret driver has a name if spec.Driver.Name == "" { - return grpc.Errorf(codes.InvalidArgument, "secret driver must have a name") + return status.Errorf(codes.InvalidArgument, "secret driver must have a name") } return nil } if err := validation.ValidateSecretPayload(spec.Data); err != nil { - return grpc.Errorf(codes.InvalidArgument, "%s", err.Error()) + return status.Errorf(codes.InvalidArgument, "%s", err.Error()) } return nil } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/service.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/service.go index 951c27b01e..e4c27df977 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/service.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/service.go @@ -19,8 +19,8 @@ import ( "github.com/docker/swarmkit/template" gogotypes "github.com/gogo/protobuf/types" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( @@ -37,11 +37,11 @@ func validateResources(r *api.Resources) error { } if r.NanoCPUs != 0 && r.NanoCPUs < 1e6 { - return grpc.Errorf(codes.InvalidArgument, "invalid cpu value %g: Must be at least %g", float64(r.NanoCPUs)/1e9, 1e6/1e9) + return status.Errorf(codes.InvalidArgument, "invalid cpu value %g: Must be at least %g", float64(r.NanoCPUs)/1e9, 1e6/1e9) } if r.MemoryBytes != 0 && r.MemoryBytes < 4*1024*1024 { - return grpc.Errorf(codes.InvalidArgument, "invalid memory value %d: Must be at least 4MiB", r.MemoryBytes) + return status.Errorf(codes.InvalidArgument, "invalid memory value %d: Must be at least 4MiB", r.MemoryBytes) } if err := genericresource.ValidateTask(r); err != nil { return nil @@ -70,7 +70,7 @@ func validateRestartPolicy(rp *api.RestartPolicy) error { return err } if delay < 0 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: restart-delay cannot be negative") + return status.Errorf(codes.InvalidArgument, "TaskSpec: restart-delay cannot be negative") } } @@ -80,7 +80,7 @@ func validateRestartPolicy(rp *api.RestartPolicy) error { return err } if win < 0 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: restart-window cannot be negative") + return status.Errorf(codes.InvalidArgument, "TaskSpec: restart-window cannot be negative") } } @@ -101,7 +101,7 @@ func validateUpdate(uc *api.UpdateConfig) error { } if uc.Delay < 0 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-delay cannot be negative") + return status.Errorf(codes.InvalidArgument, "TaskSpec: update-delay cannot be negative") } if uc.Monitor != nil { @@ -110,12 +110,12 @@ func validateUpdate(uc *api.UpdateConfig) error { return err } if monitor < 0 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-monitor cannot be negative") + return status.Errorf(codes.InvalidArgument, "TaskSpec: update-monitor cannot be negative") } } if uc.MaxFailureRatio < 0 || uc.MaxFailureRatio > 1 { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-maxfailureratio cannot be less than 0 or bigger than 1") + return status.Errorf(codes.InvalidArgument, "TaskSpec: update-maxfailureratio cannot be less than 0 or bigger than 1") } return nil @@ -147,7 +147,7 @@ func validateContainerSpec(taskSpec api.TaskSpec) error { LogDriver: taskSpec.LogDriver, }) if err != nil { - return grpc.Errorf(codes.InvalidArgument, err.Error()) + return status.Errorf(codes.InvalidArgument, err.Error()) } if err := validateImage(container.Image); err != nil { @@ -164,11 +164,11 @@ func validateContainerSpec(taskSpec api.TaskSpec) error { // validateImage validates image name in containerSpec func validateImage(image string) error { if image == "" { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: image reference must be provided") + return status.Errorf(codes.InvalidArgument, "ContainerSpec: image reference must be provided") } if _, err := reference.ParseNormalizedNamed(image); err != nil { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: %q is not a valid repository/tag", image) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: %q is not a valid repository/tag", image) } return nil } @@ -178,7 +178,7 @@ func validateMounts(mounts []api.Mount) error { mountMap := make(map[string]bool) for _, mount := range mounts { if _, exists := mountMap[mount.Target]; exists { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: duplicate mount point: %s", mount.Target) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: duplicate mount point: %s", mount.Target) } mountMap[mount.Target] = true } @@ -198,7 +198,7 @@ func validateHealthCheck(hc *api.HealthConfig) error { return err } if interval != 0 && interval < time.Duration(minimumDuration) { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: Interval in HealthConfig cannot be less than %s", minimumDuration) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: Interval in HealthConfig cannot be less than %s", minimumDuration) } } @@ -208,7 +208,7 @@ func validateHealthCheck(hc *api.HealthConfig) error { return err } if timeout != 0 && timeout < time.Duration(minimumDuration) { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: Timeout in HealthConfig cannot be less than %s", minimumDuration) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: Timeout in HealthConfig cannot be less than %s", minimumDuration) } } @@ -218,12 +218,12 @@ func validateHealthCheck(hc *api.HealthConfig) error { return err } if sp != 0 && sp < time.Duration(minimumDuration) { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: StartPeriod in HealthConfig cannot be less than %s", minimumDuration) + return status.Errorf(codes.InvalidArgument, "ContainerSpec: StartPeriod in HealthConfig cannot be less than %s", minimumDuration) } } if hc.Retries < 0 { - return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: Retries in HealthConfig cannot be negative") + return status.Errorf(codes.InvalidArgument, "ContainerSpec: Retries in HealthConfig cannot be negative") } return nil @@ -233,28 +233,28 @@ func validateGenericRuntimeSpec(taskSpec api.TaskSpec) error { generic := taskSpec.GetGeneric() if len(generic.Kind) < 3 { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime: Invalid name %q", generic.Kind) + return status.Errorf(codes.InvalidArgument, "Generic runtime: Invalid name %q", generic.Kind) } reservedNames := []string{"container", "attachment"} for _, n := range reservedNames { if strings.ToLower(generic.Kind) == n { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime: %q is a reserved name", generic.Kind) + return status.Errorf(codes.InvalidArgument, "Generic runtime: %q is a reserved name", generic.Kind) } } payload := generic.Payload if payload == nil { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime is missing payload") + return status.Errorf(codes.InvalidArgument, "Generic runtime is missing payload") } if payload.TypeUrl == "" { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime is missing payload type") + return status.Errorf(codes.InvalidArgument, "Generic runtime is missing payload type") } if len(payload.Value) == 0 { - return grpc.Errorf(codes.InvalidArgument, "Generic runtime has an empty payload") + return status.Errorf(codes.InvalidArgument, "Generic runtime has an empty payload") } return nil @@ -284,7 +284,7 @@ func validateTaskSpec(taskSpec api.TaskSpec) error { } if taskSpec.GetRuntime() == nil { - return grpc.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime") + return status.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime") } switch taskSpec.GetRuntime().(type) { @@ -297,7 +297,7 @@ func validateTaskSpec(taskSpec api.TaskSpec) error { return err } default: - return grpc.Errorf(codes.Unimplemented, "RuntimeSpec: unimplemented runtime in service spec") + return status.Errorf(codes.Unimplemented, "RuntimeSpec: unimplemented runtime in service spec") } return nil @@ -324,7 +324,7 @@ func validateEndpointSpec(epSpec *api.EndpointSpec) error { // for the backend network and hence we accept that configuration. if epSpec.Mode == api.ResolutionModeDNSRoundRobin && port.PublishMode == api.PublishModeIngress { - return grpc.Errorf(codes.InvalidArgument, "EndpointSpec: port published with ingress mode can't be used with dnsrr mode") + return status.Errorf(codes.InvalidArgument, "EndpointSpec: port published with ingress mode can't be used with dnsrr mode") } // If published port is not specified, it does not conflict @@ -335,7 +335,7 @@ func validateEndpointSpec(epSpec *api.EndpointSpec) error { portSpec := portSpec{publishedPort: port.PublishedPort, protocol: port.Protocol} if _, ok := portSet[portSpec]; ok { - return grpc.Errorf(codes.InvalidArgument, "EndpointSpec: duplicate published ports provided") + return status.Errorf(codes.InvalidArgument, "EndpointSpec: duplicate published ports provided") } portSet[portSpec] = struct{}{} @@ -358,23 +358,23 @@ func validateSecretRefsSpec(spec api.TaskSpec) error { for _, secretRef := range container.Secrets { // SecretID and SecretName are mandatory, we have invalid references without them if secretRef.SecretID == "" || secretRef.SecretName == "" { - return grpc.Errorf(codes.InvalidArgument, "malformed secret reference") + return status.Errorf(codes.InvalidArgument, "malformed secret reference") } // Every secret reference requires a Target if secretRef.GetTarget() == nil { - return grpc.Errorf(codes.InvalidArgument, "malformed secret reference, no target provided") + return status.Errorf(codes.InvalidArgument, "malformed secret reference, no target provided") } // If this is a file target, we will ensure filename uniqueness if secretRef.GetFile() != nil { fileName := secretRef.GetFile().Name if fileName == "" { - return grpc.Errorf(codes.InvalidArgument, "malformed file secret reference, invalid target file name provided") + return status.Errorf(codes.InvalidArgument, "malformed file secret reference, invalid target file name provided") } // If this target is already in use, we have conflicting targets if prevSecretName, ok := existingTargets[fileName]; ok { - return grpc.Errorf(codes.InvalidArgument, "secret references '%s' and '%s' have a conflicting target: '%s'", prevSecretName, secretRef.SecretName, fileName) + return status.Errorf(codes.InvalidArgument, "secret references '%s' and '%s' have a conflicting target: '%s'", prevSecretName, secretRef.SecretName, fileName) } existingTargets[fileName] = secretRef.SecretName @@ -398,12 +398,12 @@ func validateConfigRefsSpec(spec api.TaskSpec) error { for _, configRef := range container.Configs { // ConfigID and ConfigName are mandatory, we have invalid references without them if configRef.ConfigID == "" || configRef.ConfigName == "" { - return grpc.Errorf(codes.InvalidArgument, "malformed config reference") + return status.Errorf(codes.InvalidArgument, "malformed config reference") } // Every config reference requires a Target if configRef.GetTarget() == nil { - return grpc.Errorf(codes.InvalidArgument, "malformed config reference, no target provided") + return status.Errorf(codes.InvalidArgument, "malformed config reference, no target provided") } // If this is a file target, we will ensure filename uniqueness @@ -411,12 +411,12 @@ func validateConfigRefsSpec(spec api.TaskSpec) error { fileName := configRef.GetFile().Name // Validate the file name if fileName == "" { - return grpc.Errorf(codes.InvalidArgument, "malformed file config reference, invalid target file name provided") + return status.Errorf(codes.InvalidArgument, "malformed file config reference, invalid target file name provided") } // If this target is already in use, we have conflicting targets if prevConfigName, ok := existingTargets[fileName]; ok { - return grpc.Errorf(codes.InvalidArgument, "config references '%s' and '%s' have a conflicting target: '%s'", prevConfigName, configRef.ConfigName, fileName) + return status.Errorf(codes.InvalidArgument, "config references '%s' and '%s' have a conflicting target: '%s'", prevConfigName, configRef.ConfigName, fileName) } existingTargets[fileName] = configRef.ConfigName @@ -436,7 +436,7 @@ func (s *Server) validateNetworks(networks []*api.NetworkAttachmentConfig) error continue } if allocator.IsIngressNetwork(network) { - return grpc.Errorf(codes.InvalidArgument, + return status.Errorf(codes.InvalidArgument, "Service cannot be explicitly attached to the ingress network %q", network.Spec.Annotations.Name) } } @@ -448,11 +448,11 @@ func validateMode(s *api.ServiceSpec) error { switch m.(type) { case *api.ServiceSpec_Replicated: if int64(m.(*api.ServiceSpec_Replicated).Replicated.Replicas) < 0 { - return grpc.Errorf(codes.InvalidArgument, "Number of replicas must be non-negative") + return status.Errorf(codes.InvalidArgument, "Number of replicas must be non-negative") } case *api.ServiceSpec_Global: default: - return grpc.Errorf(codes.InvalidArgument, "Unrecognized service mode") + return status.Errorf(codes.InvalidArgument, "Unrecognized service mode") } return nil @@ -460,7 +460,7 @@ func validateMode(s *api.ServiceSpec) error { func validateServiceSpec(spec *api.ServiceSpec) error { if spec == nil { - return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateAnnotations(spec.Annotations); err != nil { return err @@ -536,7 +536,7 @@ func (s *Server) checkPortConflicts(spec *api.ServiceSpec, serviceID string) err switch pc.PublishMode { case api.PublishModeHost: if _, ok := ingressPorts[pcToStruct(pc)]; ok { - return grpc.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as a host-published port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID) + return status.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as a host-published port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID) } // Multiple services with same port in host publish mode can @@ -546,7 +546,7 @@ func (s *Server) checkPortConflicts(spec *api.ServiceSpec, serviceID string) err _, ingressConflict := ingressPorts[pcToStruct(pc)] _, hostModeConflict := hostModePorts[pcToStruct(pc)] if ingressConflict || hostModeConflict { - return grpc.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as an ingress port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID) + return status.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as an ingress port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID) } } @@ -598,7 +598,7 @@ func (s *Server) checkSecretExistence(tx store.Tx, spec *api.ServiceSpec) error secretStr = "secret" } - return grpc.Errorf(codes.InvalidArgument, "%s not found: %v", secretStr, strings.Join(failedSecrets, ", ")) + return status.Errorf(codes.InvalidArgument, "%s not found: %v", secretStr, strings.Join(failedSecrets, ", ")) } @@ -627,7 +627,7 @@ func (s *Server) checkConfigExistence(tx store.Tx, spec *api.ServiceSpec) error configStr = "config" } - return grpc.Errorf(codes.InvalidArgument, "%s not found: %v", configStr, strings.Join(failedConfigs, ", ")) + return status.Errorf(codes.InvalidArgument, "%s not found: %v", configStr, strings.Join(failedConfigs, ", ")) } @@ -662,7 +662,7 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe if allocator.IsIngressNetworkNeeded(service) { if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress { - return nil, grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") + return nil, status.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") } } @@ -694,7 +694,7 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe // - Returns `NotFound` if the Service is not found. func (s *Server) GetService(ctx context.Context, request *api.GetServiceRequest) (*api.GetServiceResponse, error) { if request.ServiceID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var service *api.Service @@ -702,7 +702,7 @@ func (s *Server) GetService(ctx context.Context, request *api.GetServiceRequest) service = store.GetService(tx, request.ServiceID) }) if service == nil { - return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID) + return nil, status.Errorf(codes.NotFound, "service %s not found", request.ServiceID) } if request.InsertDefaults { @@ -721,7 +721,7 @@ func (s *Server) GetService(ctx context.Context, request *api.GetServiceRequest) // - Returns an error if the update fails. func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRequest) (*api.UpdateServiceResponse, error) { if request.ServiceID == "" || request.ServiceVersion == nil { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } if err := validateServiceSpec(request.Spec); err != nil { return nil, err @@ -732,7 +732,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe service = store.GetService(tx, request.ServiceID) }) if service == nil { - return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID) + return nil, status.Errorf(codes.NotFound, "service %s not found", request.ServiceID) } if request.Spec.Endpoint != nil && !reflect.DeepEqual(request.Spec.Endpoint, service.Spec.Endpoint) { @@ -744,7 +744,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe err := s.store.Update(func(tx store.Tx) error { service = store.GetService(tx, request.ServiceID) if service == nil { - return grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID) + return status.Errorf(codes.NotFound, "service %s not found", request.ServiceID) } // It's not okay to update Service.Spec.Networks on its own. @@ -754,7 +754,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe if (len(request.Spec.Networks) != 0 || len(service.Spec.Networks) != 0) && !reflect.DeepEqual(request.Spec.Networks, service.Spec.Networks) && reflect.DeepEqual(request.Spec.Task.Networks, service.Spec.Task.Networks) { - return grpc.Errorf(codes.Unimplemented, errNetworkUpdateNotSupported.Error()) + return status.Errorf(codes.Unimplemented, errNetworkUpdateNotSupported.Error()) } // Check to see if all the secrets being added exist as objects @@ -773,18 +773,18 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe // with service mode change (comparing current config with previous config). // proper way to change service mode is to delete and re-add. if reflect.TypeOf(service.Spec.Mode) != reflect.TypeOf(request.Spec.Mode) { - return grpc.Errorf(codes.Unimplemented, errModeChangeNotAllowed.Error()) + return status.Errorf(codes.Unimplemented, errModeChangeNotAllowed.Error()) } if service.Spec.Annotations.Name != request.Spec.Annotations.Name { - return grpc.Errorf(codes.Unimplemented, errRenameNotSupported.Error()) + return status.Errorf(codes.Unimplemented, errRenameNotSupported.Error()) } service.Meta.Version = *request.ServiceVersion if request.Rollback == api.UpdateServiceRequest_PREVIOUS { if service.PreviousSpec == nil { - return grpc.Errorf(codes.FailedPrecondition, "service %s does not have a previous spec", request.ServiceID) + return status.Errorf(codes.FailedPrecondition, "service %s does not have a previous spec", request.ServiceID) } curSpec := service.Spec.Copy() @@ -815,7 +815,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe if allocator.IsIngressNetworkNeeded(service) { if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress { - return grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") + return status.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") } } @@ -836,7 +836,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe // - Returns an error if the deletion fails. func (s *Server) RemoveService(ctx context.Context, request *api.RemoveServiceRequest) (*api.RemoveServiceResponse, error) { if request.ServiceID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } err := s.store.Update(func(tx store.Tx) error { @@ -844,7 +844,7 @@ func (s *Server) RemoveService(ctx context.Context, request *api.RemoveServiceRe }) if err != nil { if err == store.ErrNotExist { - return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID) + return nil, status.Errorf(codes.NotFound, "service %s not found", request.ServiceID) } return nil, err } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/task.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/task.go index d2ae21522f..51b7bf82d8 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/task.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/task.go @@ -6,8 +6,8 @@ import ( "github.com/docker/swarmkit/manager/orchestrator" "github.com/docker/swarmkit/manager/state/store" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // GetTask returns a Task given a TaskID. @@ -15,7 +15,7 @@ import ( // - Returns `NotFound` if the Task is not found. func (s *Server) GetTask(ctx context.Context, request *api.GetTaskRequest) (*api.GetTaskResponse, error) { if request.TaskID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } var task *api.Task @@ -23,7 +23,7 @@ func (s *Server) GetTask(ctx context.Context, request *api.GetTaskRequest) (*api task = store.GetTask(tx, request.TaskID) }) if task == nil { - return nil, grpc.Errorf(codes.NotFound, "task %s not found", request.TaskID) + return nil, status.Errorf(codes.NotFound, "task %s not found", request.TaskID) } return &api.GetTaskResponse{ Task: task, @@ -36,7 +36,7 @@ func (s *Server) GetTask(ctx context.Context, request *api.GetTaskRequest) (*api // - Returns an error if the deletion fails. func (s *Server) RemoveTask(ctx context.Context, request *api.RemoveTaskRequest) (*api.RemoveTaskResponse, error) { if request.TaskID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } err := s.store.Update(func(tx store.Tx) error { @@ -44,7 +44,7 @@ func (s *Server) RemoveTask(ctx context.Context, request *api.RemoveTaskRequest) }) if err != nil { if err == store.ErrNotExist { - return nil, grpc.Errorf(codes.NotFound, "task %s not found", request.TaskID) + return nil, status.Errorf(codes.NotFound, "task %s not found", request.TaskID) } return nil, err } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go index 4de6a30c4d..13d68293ae 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go @@ -7,11 +7,8 @@ import ( "sync" "time" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/transport" - "github.com/docker/go-events" + "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/api/equality" "github.com/docker/swarmkit/ca" @@ -25,6 +22,9 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/grpc/transport" ) const ( @@ -66,8 +66,18 @@ var ( ErrSessionInvalid = errors.New("session invalid") // ErrNodeNotFound returned when the Node doesn't exist in raft. ErrNodeNotFound = errors.New("node not found") + + // Scheduling delay timer. + schedulingDelayTimer metrics.Timer ) +func init() { + ns := metrics.NewNamespace("swarm", "dispatcher", nil) + schedulingDelayTimer = ns.NewTimer("scheduling_delay", + "Scheduling delay is the time a task takes to go from NEW to RUNNING state.") + metrics.Register(ns) +} + // Config is configuration for Dispatcher. For default you should use // DefaultConfig. type Config struct { @@ -322,7 +332,7 @@ func (d *Dispatcher) isRunningLocked() (context.Context, error) { d.mu.Lock() if !d.isRunning() { d.mu.Unlock() - return nil, grpc.Errorf(codes.Aborted, "dispatcher is stopped") + return nil, status.Errorf(codes.Aborted, "dispatcher is stopped") } ctx := d.ctx d.mu.Unlock() @@ -556,7 +566,7 @@ func (d *Dispatcher) UpdateTaskStatus(ctx context.Context, r *api.UpdateTaskStat } if t.NodeID != nodeID { - err := grpc.Errorf(codes.PermissionDenied, "cannot update a task not assigned this node") + err := status.Errorf(codes.PermissionDenied, "cannot update a task not assigned this node") log.WithField("task.id", u.TaskID).Error(err) return nil, err } @@ -632,6 +642,17 @@ func (d *Dispatcher) processUpdates(ctx context.Context) { return nil } + // Update scheduling delay metric for running tasks. + // We use the status update time on the leader to calculate the scheduling delay. + // Because of this, the recorded scheduling delay will be an overestimate and include + // the network delay between the worker and the leader. + // This is not ideal, but its a known overestimation, rather than using the status update time + // from the worker node, which may cause unknown incorrect results due to possible clock skew. + if status.State == api.TaskStateRunning { + start := time.Unix(status.AppliedAt.GetSeconds(), int64(status.AppliedAt.GetNanos())) + schedulingDelayTimer.UpdateSince(start) + } + task.Status = *status task.Status.AppliedBy = d.securityConfig.ClientTLSCreds.NodeID() task.Status.AppliedAt = ptypes.MustTimestampProto(time.Now()) @@ -1189,7 +1210,7 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio log.WithError(err).Error("failed to remove node") } // still return an abort if the transport closure was ineffective. - return grpc.Errorf(codes.Aborted, "node must disconnect") + return status.Errorf(codes.Aborted, "node must disconnect") } for { diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/nodes.go b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/nodes.go index 8a0de558d6..cf35bb869a 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/nodes.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/dispatcher/nodes.go @@ -4,12 +4,11 @@ import ( "sync" "time" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/manager/dispatcher/heartbeat" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const rateLimitCount = 3 @@ -36,7 +35,7 @@ func (rn *registeredNode) checkSessionID(sessionID string) error { // changed. If it has, we will the stream and make the node // re-register. if sessionID == "" || rn.SessionID != sessionID { - return grpc.Errorf(codes.InvalidArgument, ErrSessionInvalid.Error()) + return status.Errorf(codes.InvalidArgument, ErrSessionInvalid.Error()) } return nil @@ -97,7 +96,7 @@ func (s *nodeStore) CheckRateLimit(id string) error { } existRn.Attempts++ if existRn.Attempts > rateLimitCount { - return grpc.Errorf(codes.Unavailable, "node %s exceeded rate limit count of registrations", id) + return status.Errorf(codes.Unavailable, "node %s exceeded rate limit count of registrations", id) } existRn.Registered = time.Now() } @@ -136,7 +135,7 @@ func (s *nodeStore) Get(id string) (*registeredNode, error) { rn, ok := s.nodes[id] s.mu.RUnlock() if !ok { - return nil, grpc.Errorf(codes.NotFound, ErrNodeNotRegistered.Error()) + return nil, status.Errorf(codes.NotFound, ErrNodeNotRegistered.Error()) } return rn, nil } @@ -146,7 +145,7 @@ func (s *nodeStore) GetWithSession(id, sid string) (*registeredNode, error) { rn, ok := s.nodes[id] s.mu.RUnlock() if !ok { - return nil, grpc.Errorf(codes.NotFound, ErrNodeNotRegistered.Error()) + return nil, status.Errorf(codes.NotFound, ErrNodeNotRegistered.Error()) } return rn, rn.checkSessionID(sid) } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/health/health.go b/components/engine/vendor/github.com/docker/swarmkit/manager/health/health.go index bf220bdc18..ef6658b09d 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/health/health.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/health/health.go @@ -12,8 +12,8 @@ import ( "github.com/docker/swarmkit/api" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // Server represents a Health Check server to check @@ -46,7 +46,7 @@ func (s *Server) Check(ctx context.Context, in *api.HealthCheckRequest) (*api.He Status: status, }, nil } - return nil, grpc.Errorf(codes.NotFound, "unknown service") + return nil, status.Errorf(codes.NotFound, "unknown service") } // SetServingStatus is called when need to reset the serving status of a service diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go b/components/engine/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go index 860b55c30e..de2b93633f 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/logbroker/broker.go @@ -6,9 +6,6 @@ import ( "io" "sync" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "github.com/docker/go-events" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" @@ -18,6 +15,8 @@ import ( "github.com/docker/swarmkit/watch" "github.com/sirupsen/logrus" "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( @@ -93,11 +92,11 @@ func (lb *LogBroker) Stop() error { func validateSelector(selector *api.LogSelector) error { if selector == nil { - return grpc.Errorf(codes.InvalidArgument, "log selector must be provided") + return status.Errorf(codes.InvalidArgument, "log selector must be provided") } if len(selector.ServiceIDs) == 0 && len(selector.TaskIDs) == 0 && len(selector.NodeIDs) == 0 { - return grpc.Errorf(codes.InvalidArgument, "log selector must not be empty") + return status.Errorf(codes.InvalidArgument, "log selector must not be empty") } return nil @@ -401,17 +400,17 @@ func (lb *LogBroker) PublishLogs(stream api.LogBroker_PublishLogsServer) (err er } if logMsg.SubscriptionID == "" { - return grpc.Errorf(codes.InvalidArgument, "missing subscription ID") + return status.Errorf(codes.InvalidArgument, "missing subscription ID") } if currentSubscription == nil { currentSubscription = lb.getSubscription(logMsg.SubscriptionID) if currentSubscription == nil { - return grpc.Errorf(codes.NotFound, "unknown subscription ID") + return status.Errorf(codes.NotFound, "unknown subscription ID") } } else { if logMsg.SubscriptionID != currentSubscription.message.ID { - return grpc.Errorf(codes.InvalidArgument, "different subscription IDs in the same session") + return status.Errorf(codes.InvalidArgument, "different subscription IDs in the same session") } } @@ -427,7 +426,7 @@ func (lb *LogBroker) PublishLogs(stream api.LogBroker_PublishLogsServer) (err er // Make sure logs are emitted using the right Node ID to avoid impersonation. for _, msg := range logMsg.Messages { if msg.Context.NodeID != remote.NodeID { - return grpc.Errorf(codes.PermissionDenied, "invalid NodeID: expected=%s;received=%s", remote.NodeID, msg.Context.NodeID) + return status.Errorf(codes.PermissionDenied, "invalid NodeID: expected=%s;received=%s", remote.NodeID, msg.Context.NodeID) } } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go b/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go index 39db22b09a..1ce727dcac 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go @@ -38,6 +38,7 @@ import ( "github.com/docker/swarmkit/manager/resourceapi" "github.com/docker/swarmkit/manager/scheduler" "github.com/docker/swarmkit/manager/state/raft" + "github.com/docker/swarmkit/manager/state/raft/transport" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/manager/watchapi" "github.com/docker/swarmkit/remotes" @@ -54,9 +55,6 @@ import ( const ( // defaultTaskHistoryRetentionLimit is the number of tasks to keep. defaultTaskHistoryRetentionLimit = 5 - - // Default value for grpc max message size. - grpcMaxMessageSize = 128 << 20 ) // RemoteAddrs provides a listening address and an optional advertise address @@ -234,7 +232,7 @@ func New(config *Config) (*Manager, error) { grpc.Creds(config.SecurityConfig.ServerTLSCreds), grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor), - grpc.MaxMsgSize(grpcMaxMessageSize), + grpc.MaxMsgSize(transport.GRPCMaxMsgSize), } m := &Manager{ @@ -404,7 +402,7 @@ func (m *Manager) Run(parent context.Context) error { ) m.raftNode.MemoryStore().View(func(readTx store.ReadTx) { - clusters, err = store.FindClusters(readTx, store.ByName("default")) + clusters, err = store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) }) @@ -954,13 +952,18 @@ func (m *Manager) becomeLeader(ctx context.Context) { // store. Don't check the error because // we expect this to fail unless this // is a brand new cluster. - store.CreateCluster(tx, defaultClusterObject( + err := store.CreateCluster(tx, defaultClusterObject( clusterID, initialCAConfig, raftCfg, api.EncryptionConfig{AutoLockManagers: m.config.AutoLockManagers}, unlockKeys, rootCA)) + + if err != nil && err != store.ErrExist { + log.G(ctx).WithError(err).Errorf("error creating cluster object") + } + // Add Node entry for ourself, if one // doesn't exist already. freshCluster := nil == store.CreateNode(tx, managerNode(nodeID, m.config.Availability)) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/global/global.go b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/global/global.go index a1d2873e80..f6217ff7b4 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/global/global.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/global/global.go @@ -73,7 +73,7 @@ func (g *Orchestrator) Run(ctx context.Context) error { var err error g.store.View(func(readTx store.ReadTx) { var clusters []*api.Cluster - clusters, err = store.FindClusters(readTx, store.ByName("default")) + clusters, err = store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) if len(clusters) != 1 { return // just pick up the cluster when it is created. @@ -147,7 +147,7 @@ func (g *Orchestrator) Run(ctx context.Context) error { if !orchestrator.IsGlobalService(v.Service) { continue } - orchestrator.DeleteServiceTasks(ctx, g.store, v.Service) + orchestrator.SetServiceTasksRemove(ctx, g.store, v.Service) // delete the service from service map delete(g.globalServices, v.Service.ID) g.restarts.ClearServiceHistory(v.Service.ID) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/replicated/services.go b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/replicated/services.go index f4d0511f80..04aea8795a 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/replicated/services.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/replicated/services.go @@ -17,7 +17,7 @@ import ( // responds to changes in individual tasks (or nodes which run them). func (r *Orchestrator) initCluster(readTx store.ReadTx) error { - clusters, err := store.FindClusters(readTx, store.ByName("default")) + clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) if err != nil { return err } @@ -50,7 +50,7 @@ func (r *Orchestrator) handleServiceEvent(ctx context.Context, event events.Even if !orchestrator.IsReplicatedService(v.Service) { return } - orchestrator.DeleteServiceTasks(ctx, r.store, v.Service) + orchestrator.SetServiceTasksRemove(ctx, r.store, v.Service) r.restarts.ClearServiceHistory(v.Service.ID) delete(r.reconcileServices, v.Service.ID) case api.EventCreateService: @@ -86,6 +86,12 @@ func (r *Orchestrator) resolveService(ctx context.Context, task *api.Task) *api. return service } +// reconcile decides what actions must be taken depending on the number of +// specificed slots and actual running slots. If the actual running slots are +// fewer than what is requested, it creates new tasks. If the actual running +// slots are more than requested, then it decides which slots must be removed +// and sets desired state of those tasks to REMOVE (the actual removal is handled +// by the task reaper, after the agent shuts the tasks down). func (r *Orchestrator) reconcile(ctx context.Context, service *api.Service) { runningSlots, deadSlots, err := r.updatableAndDeadSlots(ctx, service) if err != nil { @@ -157,7 +163,11 @@ func (r *Orchestrator) reconcile(ctx context.Context, service *api.Service) { r.updater.Update(ctx, r.cluster, service, sortedSlots[:specifiedSlots]) err = r.store.Batch(func(batch *store.Batch) error { r.deleteTasksMap(ctx, batch, deadSlots) - r.deleteTasks(ctx, batch, sortedSlots[specifiedSlots:]) + // for all slots that we are removing, we set the desired state of those tasks + // to REMOVE. Then, the agent is responsible for shutting them down, and the + // task reaper is responsible for actually removing them from the store after + // shutdown. + r.setTasksDesiredState(ctx, batch, sortedSlots[specifiedSlots:], api.TaskStateRemove) return nil }) if err != nil { @@ -198,10 +208,34 @@ func (r *Orchestrator) addTasks(ctx context.Context, batch *store.Batch, service } } -func (r *Orchestrator) deleteTasks(ctx context.Context, batch *store.Batch, slots []orchestrator.Slot) { +// setTasksDesiredState sets the desired state for all tasks for the given slots to the +// requested state +func (r *Orchestrator) setTasksDesiredState(ctx context.Context, batch *store.Batch, slots []orchestrator.Slot, newDesiredState api.TaskState) { for _, slot := range slots { for _, t := range slot { - r.deleteTask(ctx, batch, t) + err := batch.Update(func(tx store.Tx) error { + // time travel is not allowed. if the current desired state is + // above the one we're trying to go to we can't go backwards. + // we have nothing to do and we should skip to the next task + if t.DesiredState > newDesiredState { + // log a warning, though. we shouln't be trying to rewrite + // a state to an earlier state + log.G(ctx).Warnf( + "cannot update task %v in desired state %v to an earlier desired state %v", + t.ID, t.DesiredState, newDesiredState, + ) + return nil + } + // update desired state + t.DesiredState = newDesiredState + + return store.UpdateTask(tx, t) + }) + + // log an error if we get one + if err != nil { + log.G(ctx).WithError(err).Errorf("failed to update task to %v", newDesiredState.String()) + } } } } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/service.go b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/service.go index 4e52c83abf..7356c38cd5 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/service.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/service.go @@ -27,8 +27,10 @@ func IsGlobalService(service *api.Service) bool { return ok } -// DeleteServiceTasks deletes the tasks associated with a service. -func DeleteServiceTasks(ctx context.Context, s *store.MemoryStore, service *api.Service) { +// SetServiceTasksRemove sets the desired state of tasks associated with a service +// to REMOVE, so that they can be properly shut down by the agent and later removed +// by the task reaper. +func SetServiceTasksRemove(ctx context.Context, s *store.MemoryStore, service *api.Service) { var ( tasks []*api.Task err error @@ -44,8 +46,23 @@ func DeleteServiceTasks(ctx context.Context, s *store.MemoryStore, service *api. err = s.Batch(func(batch *store.Batch) error { for _, t := range tasks { err := batch.Update(func(tx store.Tx) error { - if err := store.DeleteTask(tx, t.ID); err != nil { - log.G(ctx).WithError(err).Errorf("failed to delete task") + // time travel is not allowed. if the current desired state is + // above the one we're trying to go to we can't go backwards. + // we have nothing to do and we should skip to the next task + if t.DesiredState > api.TaskStateRemove { + // log a warning, though. we shouln't be trying to rewrite + // a state to an earlier state + log.G(ctx).Warnf( + "cannot update task %v in desired state %v to an earlier desired state %v", + t.ID, t.DesiredState, api.TaskStateRemove, + ) + return nil + } + // update desired state to REMOVE + t.DesiredState = api.TaskStateRemove + + if err := store.UpdateTask(tx, t); err != nil { + log.G(ctx).WithError(err).Errorf("failed transaction: update task desired state to REMOVE") } return nil }) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go index 577319c8e8..bcef801f63 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/orchestrator/taskreaper/task_reaper.go @@ -23,12 +23,19 @@ const ( // exist for the same service/instance or service/nodeid combination. type TaskReaper struct { store *store.MemoryStore + // taskHistory is the number of tasks to keep taskHistory int64 - dirty map[orchestrator.SlotTuple]struct{} - orphaned []string - stopChan chan struct{} - doneChan chan struct{} + + // List of slot tubles to be inspected for task history cleanup. + dirty map[orchestrator.SlotTuple]struct{} + + // List of tasks collected for cleanup, which includes two kinds of tasks + // - serviceless orphaned tasks + // - tasks with desired state REMOVE that have already been shut down + cleanup []string + stopChan chan struct{} + doneChan chan struct{} } // New creates a new TaskReaper. @@ -41,7 +48,13 @@ func New(store *store.MemoryStore) *TaskReaper { } } -// Run is the TaskReaper's main loop. +// Run is the TaskReaper's watch loop which collects candidates for cleanup. +// Task history is mainly used in task restarts but is also available for administrative purposes. +// Note that the task history is stored per-slot-per-service for replicated services +// and per-node-per-service for global services. History does not apply to serviceless +// since they are not attached to a service. In addition, the TaskReaper watch loop is also +// responsible for cleaning up tasks associated with slots that were removed as part of +// service scale down or service removal. func (tr *TaskReaper) Run(ctx context.Context) { watcher, watchCancel := state.Watch(tr.store.WatchQueue(), api.EventCreateTask{}, api.EventUpdateTask{}, api.EventUpdateCluster{}) @@ -50,7 +63,8 @@ func (tr *TaskReaper) Run(ctx context.Context) { watchCancel() }() - var tasks []*api.Task + var orphanedTasks []*api.Task + var removeTasks []*api.Task tr.store.View(func(readTx store.ReadTx) { var err error @@ -59,29 +73,54 @@ func (tr *TaskReaper) Run(ctx context.Context) { tr.taskHistory = clusters[0].Spec.Orchestration.TaskHistoryRetentionLimit } - tasks, err = store.FindTasks(readTx, store.ByTaskState(api.TaskStateOrphaned)) + // On startup, scan the entire store and inspect orphaned tasks from previous life. + orphanedTasks, err = store.FindTasks(readTx, store.ByTaskState(api.TaskStateOrphaned)) if err != nil { log.G(ctx).WithError(err).Error("failed to find Orphaned tasks in task reaper init") } + removeTasks, err = store.FindTasks(readTx, store.ByDesiredState(api.TaskStateRemove)) + if err != nil { + log.G(ctx).WithError(err).Error("failed to find tasks with desired state REMOVE in task reaper init") + } }) - if len(tasks) > 0 { - for _, t := range tasks { - // Do not reap service tasks immediately + if len(orphanedTasks)+len(removeTasks) > 0 { + for _, t := range orphanedTasks { + // Do not reap service tasks immediately. + // Let them go through the regular history cleanup process + // of checking TaskHistoryRetentionLimit. if t.ServiceID != "" { continue } - tr.orphaned = append(tr.orphaned, t.ID) + // Serviceless tasks can be cleaned up right away since they are not attached to a service. + tr.cleanup = append(tr.cleanup, t.ID) } - - if len(tr.orphaned) > 0 { + // tasks with desired state REMOVE that have progressed beyond SHUTDOWN can be cleaned up + // right away + for _, t := range removeTasks { + if t.Status.State >= api.TaskStateShutdown { + tr.cleanup = append(tr.cleanup, t.ID) + } + } + // Clean up tasks in 'cleanup' right away + if len(tr.cleanup) > 0 { tr.tick() } } + // Clean up when we hit TaskHistoryRetentionLimit or when the timer expires, + // whichever happens first. timer := time.NewTimer(reaperBatchingInterval) + // Watch for: + // 1. EventCreateTask for cleaning slots, which is the best time to cleanup that node/slot. + // 2. EventUpdateTask for cleaning + // - serviceless orphaned tasks (when orchestrator updates the task status to ORPHANED) + // - tasks which have desired state REMOVE and have been shut down by the agent + // (these are tasks which are associated with slots removed as part of service + // remove or scale down) + // 3. EventUpdateCluster for TaskHistoryRetentionLimit update. for { select { case event := <-watcher: @@ -95,14 +134,21 @@ func (tr *TaskReaper) Run(ctx context.Context) { }] = struct{}{} case api.EventUpdateTask: t := v.Task + // add serviceless orphaned tasks if t.Status.State >= api.TaskStateOrphaned && t.ServiceID == "" { - tr.orphaned = append(tr.orphaned, t.ID) + tr.cleanup = append(tr.cleanup, t.ID) + } + // add tasks that have progressed beyond SHUTDOWN and have desired state REMOVE. These + // tasks are associated with slots that were removed as part of a service scale down + // or service removal. + if t.DesiredState == api.TaskStateRemove && t.Status.State >= api.TaskStateShutdown { + tr.cleanup = append(tr.cleanup, t.ID) } case api.EventUpdateCluster: tr.taskHistory = v.Cluster.Spec.Orchestration.TaskHistoryRetentionLimit } - if len(tr.dirty)+len(tr.orphaned) > maxDirty { + if len(tr.dirty)+len(tr.cleanup) > maxDirty { timer.Stop() tr.tick() } else { @@ -118,19 +164,22 @@ func (tr *TaskReaper) Run(ctx context.Context) { } } +// tick performs task history cleanup. func (tr *TaskReaper) tick() { - if len(tr.dirty) == 0 && len(tr.orphaned) == 0 { + if len(tr.dirty) == 0 && len(tr.cleanup) == 0 { return } defer func() { - tr.orphaned = nil + tr.cleanup = nil }() deleteTasks := make(map[string]struct{}) - for _, tID := range tr.orphaned { + for _, tID := range tr.cleanup { deleteTasks[tID] = struct{}{} } + + // Check history of dirty tasks for cleanup. tr.store.View(func(tx store.ReadTx) { for dirty := range tr.dirty { service := store.GetService(tx, dirty.ServiceID) @@ -141,8 +190,8 @@ func (tr *TaskReaper) tick() { taskHistory := tr.taskHistory // If MaxAttempts is set, keep at least one more than - // that number of tasks. This is necessary reconstruct - // restart history when the orchestrator starts up. + // that number of tasks (this overrides TaskHistoryRetentionLimit). + // This is necessary to reconstruct restart history when the orchestrator starts up. // TODO(aaronl): Consider hiding tasks beyond the normal // retention limit in the UI. // TODO(aaronl): There are some ways to cut down the @@ -156,6 +205,7 @@ func (tr *TaskReaper) tick() { taskHistory = int64(service.Spec.Task.Restart.MaxAttempts) + 1 } + // Negative value for TaskHistoryRetentionLimit is an indication to never clean up task history. if taskHistory < 0 { continue } @@ -164,6 +214,7 @@ func (tr *TaskReaper) tick() { switch service.Spec.GetMode().(type) { case *api.ServiceSpec_Replicated: + // Clean out the slot for which we received EventCreateTask. var err error historicTasks, err = store.FindTasks(tx, store.BySlot(dirty.ServiceID, dirty.Slot)) if err != nil { @@ -171,6 +222,7 @@ func (tr *TaskReaper) tick() { } case *api.ServiceSpec_Global: + // Clean out the node history in case of global services. tasksByNode, err := store.FindTasks(tx, store.ByNodeID(dirty.NodeID)) if err != nil { continue @@ -215,6 +267,7 @@ func (tr *TaskReaper) tick() { } }) + // Perform cleanup. if len(deleteTasks) > 0 { tr.store.Batch(func(batch *store.Batch) error { for taskID := range deleteTasks { diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/resourceapi/allocator.go b/components/engine/vendor/github.com/docker/swarmkit/manager/resourceapi/allocator.go index 87b01ebd3b..ec19fba850 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/resourceapi/allocator.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/resourceapi/allocator.go @@ -10,8 +10,8 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/protobuf/ptypes" "golang.org/x/net/context" - "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( @@ -50,11 +50,11 @@ func (ra *ResourceAllocator) AttachNetwork(ctx context.Context, request *api.Att } }) if network == nil { - return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.Config.Target) + return nil, status.Errorf(codes.NotFound, "network %s not found", request.Config.Target) } if !network.Spec.Attachable { - return nil, grpc.Errorf(codes.PermissionDenied, "network %s not manually attachable", request.Config.Target) + return nil, status.Errorf(codes.PermissionDenied, "network %s not manually attachable", request.Config.Target) } t := &api.Task{ @@ -98,7 +98,7 @@ func (ra *ResourceAllocator) AttachNetwork(ctx context.Context, request *api.Att // - Returns an error if the deletion fails. func (ra *ResourceAllocator) DetachNetwork(ctx context.Context, request *api.DetachNetworkRequest) (*api.DetachNetworkResponse, error) { if request.AttachmentID == "" { - return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) + return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } nodeInfo, err := ca.RemoteNode(ctx) @@ -109,10 +109,10 @@ func (ra *ResourceAllocator) DetachNetwork(ctx context.Context, request *api.Det if err := ra.store.Update(func(tx store.Tx) error { t := store.GetTask(tx, request.AttachmentID) if t == nil { - return grpc.Errorf(codes.NotFound, "attachment %s not found", request.AttachmentID) + return status.Errorf(codes.NotFound, "attachment %s not found", request.AttachmentID) } if t.NodeID != nodeInfo.NodeID { - return grpc.Errorf(codes.PermissionDenied, "attachment %s doesn't belong to this node", request.AttachmentID) + return status.Errorf(codes.PermissionDenied, "attachment %s doesn't belong to this node", request.AttachmentID) } return store.DeleteTask(tx, request.AttachmentID) diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go b/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go index 3b1c73fe2d..dab3c6695d 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/filter.go @@ -169,7 +169,10 @@ func (f *PluginFilter) Check(n *NodeInfo) bool { } } - if f.t.Spec.LogDriver != nil && f.t.Spec.LogDriver.Name != "none" { + // It's possible that the LogDriver object does not carry a name, just some + // configuration options. In that case, the plugin filter shouldn't fail to + // schedule the task + if f.t.Spec.LogDriver != nil && f.t.Spec.LogDriver.Name != "none" && f.t.Spec.LogDriver.Name != "" { // If there are no log driver types in the list at all, most likely this is // an older daemon that did not report this information. In this case don't filter if typeFound, exists := f.pluginExistsOnNode("Log", f.t.Spec.LogDriver.Name, nodePlugins); !exists && typeFound { @@ -294,6 +297,14 @@ func (f *PlatformFilter) platformEqual(imgPlatform, nodePlatform api.Platform) b nodePlatform.Architecture = "amd64" } + // normalize "aarch64" architectures to "arm64" + if imgPlatform.Architecture == "aarch64" { + imgPlatform.Architecture = "arm64" + } + if nodePlatform.Architecture == "aarch64" { + nodePlatform.Architecture = "arm64" + } + if (imgPlatform.Architecture == "" || imgPlatform.Architecture == nodePlatform.Architecture) && (imgPlatform.OS == "" || imgPlatform.OS == nodePlatform.OS) { return true } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go index 28c7cfa47e..56b7c7c966 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/raft.go @@ -2,6 +2,7 @@ package raft import ( "fmt" + "io" "math" "math/rand" "net" @@ -14,6 +15,7 @@ import ( "github.com/coreos/etcd/raft/raftpb" "github.com/docker/docker/pkg/signal" "github.com/docker/go-events" + "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/log" @@ -34,6 +36,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" ) var ( @@ -62,6 +65,9 @@ var ( // work around lint lostQuorumMessage = "The swarm does not have a leader. It's possible that too few managers are online. Make sure more than half of the managers are online." errLostQuorum = errors.New(lostQuorumMessage) + + // Timer to capture ProposeValue() latency. + proposeLatencyTimer metrics.Timer ) // LeadershipState indicates whether the node is a leader or follower. @@ -180,12 +186,9 @@ type NodeOptions struct { ClockSource clock.Clock // SendTimeout is the timeout on the sending messages to other raft // nodes. Leave this as 0 to get the default value. - SendTimeout time.Duration - // LargeSendTimeout is the timeout on the sending snapshots to other raft - // nodes. Leave this as 0 to get the default value. - LargeSendTimeout time.Duration - TLSCredentials credentials.TransportCredentials - KeyRotator EncryptionKeyRotator + SendTimeout time.Duration + TLSCredentials credentials.TransportCredentials + KeyRotator EncryptionKeyRotator // DisableStackDump prevents Run from dumping goroutine stacks when the // store becomes stuck. DisableStackDump bool @@ -193,6 +196,9 @@ type NodeOptions struct { func init() { rand.Seed(time.Now().UnixNano()) + ns := metrics.NewNamespace("swarm", "raft", nil) + proposeLatencyTimer = ns.NewTimer("transaction_latency", "Raft transaction latency.") + metrics.Register(ns) } // NewNode generates a new Raft node @@ -207,11 +213,6 @@ func NewNode(opts NodeOptions) *Node { if opts.SendTimeout == 0 { opts.SendTimeout = 2 * time.Second } - if opts.LargeSendTimeout == 0 { - // a "slow" 100Mbps connection can send over 240MB data in 20 seconds - // which is well over the gRPC message limit of 128MB allowed by SwarmKit - opts.LargeSendTimeout = 20 * time.Second - } raftStore := raft.NewMemoryStorage() @@ -357,7 +358,6 @@ func (n *Node) initTransport() { transportConfig := &transport.Config{ HeartbeatInterval: time.Duration(n.Config.ElectionTick) * n.opts.TickInterval, SendTimeout: n.opts.SendTimeout, - LargeSendTimeout: n.opts.LargeSendTimeout, Credentials: n.opts.TLSCredentials, Raft: n, } @@ -664,7 +664,7 @@ func (n *Node) Run(ctx context.Context) error { if n.snapshotInProgress == nil && (n.needsSnapshot(ctx) || raftConfig.SnapshotInterval > 0 && n.appliedIndex-n.snapshotMeta.Index >= raftConfig.SnapshotInterval) { - n.doSnapshot(ctx, raftConfig) + n.triggerSnapshot(ctx, raftConfig) } if wasLeader && atomic.LoadUint32(&n.signalledLeadership) != 1 { @@ -706,7 +706,7 @@ func (n *Node) Run(ctx context.Context) error { // there was a key rotation that took place before while the snapshot // was in progress - we have to take another snapshot and encrypt with the new key n.rotationQueued = false - n.doSnapshot(ctx, raftConfig) + n.triggerSnapshot(ctx, raftConfig) } case <-n.keyRotator.RotationNotify(): // There are 2 separate checks: rotationQueued, and n.needsSnapshot(). @@ -719,7 +719,7 @@ func (n *Node) Run(ctx context.Context) error { case n.snapshotInProgress != nil: n.rotationQueued = true case n.needsSnapshot(ctx): - n.doSnapshot(ctx, n.getCurrentRaftConfig()) + n.triggerSnapshot(ctx, n.getCurrentRaftConfig()) } case <-ctx.Done(): return nil @@ -929,11 +929,11 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons defer n.membershipLock.Unlock() if !n.IsMember() { - return nil, grpc.Errorf(codes.FailedPrecondition, "%s", ErrNoRaftMember.Error()) + return nil, status.Errorf(codes.FailedPrecondition, "%s", ErrNoRaftMember.Error()) } if !n.isLeader() { - return nil, grpc.Errorf(codes.FailedPrecondition, "%s", ErrLostLeadership.Error()) + return nil, status.Errorf(codes.FailedPrecondition, "%s", ErrLostLeadership.Error()) } remoteAddr := req.Addr @@ -944,7 +944,7 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons requestHost, requestPort, err := net.SplitHostPort(remoteAddr) if err != nil { - return nil, grpc.Errorf(codes.InvalidArgument, "invalid address %s in raft join request", remoteAddr) + return nil, status.Errorf(codes.InvalidArgument, "invalid address %s in raft join request", remoteAddr) } requestIP := net.ParseIP(requestHost) @@ -1118,7 +1118,7 @@ func (n *Node) UpdateNode(id uint64, addr string) { // membership to an active member of the raft func (n *Node) Leave(ctx context.Context, req *api.LeaveRequest) (*api.LeaveResponse, error) { if req.Node == nil { - return nil, grpc.Errorf(codes.InvalidArgument, "no node information provided") + return nil, status.Errorf(codes.InvalidArgument, "no node information provided") } nodeInfo, err := ca.RemoteNode(ctx) @@ -1302,6 +1302,82 @@ func (n *Node) reportNewAddress(ctx context.Context, id uint64) error { return n.transport.UpdatePeerAddr(id, newAddr) } +// StreamRaftMessage is the server endpoint for streaming Raft messages. +// It accepts a stream of raft messages to be processed on this raft member, +// returning a StreamRaftMessageResponse when processing of the streamed +// messages is complete. +// It is called from the Raft leader, which uses it to stream messages +// to this raft member. +// A single stream corresponds to a single raft message, +// which may be disassembled and streamed by the sender +// as individual messages. Therefore, each of the messages +// received by the stream will have the same raft message type and index. +// Currently, only messages of type raftpb.MsgSnap can be disassembled, sent +// and received on the stream. +func (n *Node) StreamRaftMessage(stream api.Raft_StreamRaftMessageServer) error { + // recvdMsg is the current messasge received from the stream. + // assembledMessage is where the data from recvdMsg is appended to. + var recvdMsg, assembledMessage *api.StreamRaftMessageRequest + var err error + + // First message index. + var raftMsgIndex uint64 + + for { + recvdMsg, err = stream.Recv() + if err == io.EOF { + break + } else if err != nil { + log.G(stream.Context()).WithError(err).Error("error while reading from stream") + return err + } + + // Initialized the message to be used for assembling + // the raft message. + if assembledMessage == nil { + // For all message types except raftpb.MsgSnap, + // we don't expect more than a single message + // on the stream so we'll get an EOF on the next Recv() + // and go on to process the received message. + assembledMessage = recvdMsg + raftMsgIndex = recvdMsg.Message.Index + continue + } + + // Verify raft message index. + if recvdMsg.Message.Index != raftMsgIndex { + errMsg := fmt.Sprintf("Raft message chunk with index %d is different from the previously received raft message index %d", + recvdMsg.Message.Index, raftMsgIndex) + log.G(stream.Context()).Errorf(errMsg) + return status.Errorf(codes.InvalidArgument, "%s", errMsg) + } + + // Verify that multiple message received on a stream + // can only be of type raftpb.MsgSnap. + if recvdMsg.Message.Type != raftpb.MsgSnap { + errMsg := fmt.Sprintf("Raft message chunk is not of type %d", + raftpb.MsgSnap) + log.G(stream.Context()).Errorf(errMsg) + return status.Errorf(codes.InvalidArgument, "%s", errMsg) + } + + // Append the received snapshot data. + assembledMessage.Message.Snapshot.Data = append(assembledMessage.Message.Snapshot.Data, recvdMsg.Message.Snapshot.Data...) + } + + // We should have the complete snapshot. Verify and process. + if err == io.EOF { + _, err = n.ProcessRaftMessage(stream.Context(), &api.ProcessRaftMessageRequest{Message: assembledMessage.Message}) + if err == nil { + // Translate the response of ProcessRaftMessage() from + // ProcessRaftMessageResponse to StreamRaftMessageResponse if needed. + return stream.SendAndClose(&api.StreamRaftMessageResponse{}) + } + } + + return err +} + // ProcessRaftMessage calls 'Step' which advances the // raft state machine with the provided message on the // receiving node @@ -1315,7 +1391,7 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa // a node in the remove set if n.cluster.IsIDRemoved(msg.Message.From) { n.processRaftMessageLogger(ctx, msg).Debug("received message from removed member") - return nil, grpc.Errorf(codes.NotFound, "%s", membership.ErrMemberRemoved.Error()) + return nil, status.Errorf(codes.NotFound, "%s", membership.ErrMemberRemoved.Error()) } ctx, cancel := n.WithContext(ctx) @@ -1393,7 +1469,7 @@ func (n *Node) ResolveAddress(ctx context.Context, msg *api.ResolveAddressReques member := n.cluster.GetMember(msg.RaftID) if member == nil { - return nil, grpc.Errorf(codes.NotFound, "member %x not found", msg.RaftID) + return nil, status.Errorf(codes.NotFound, "member %x not found", msg.RaftID) } return &api.ResolveAddressResponse{Addr: member.Addr}, nil } @@ -1497,9 +1573,11 @@ func (n *Node) registerNode(node *api.RaftMember) error { // ProposeValue calls Propose on the underlying raft library(etcd/raft) and waits // on the commit log action before returning a result func (n *Node) ProposeValue(ctx context.Context, storeAction []api.StoreAction, cb func()) error { + defer metrics.StartTimer(proposeLatencyTimer)() ctx, cancel := n.WithContext(ctx) defer cancel() _, err := n.processInternalRaftRequest(ctx, &api.InternalRaftRequest{Action: storeAction}, cb) + if err != nil { return err } @@ -1808,12 +1886,13 @@ func (n *Node) processEntry(ctx context.Context, entry raftpb.Entry) error { } if !n.wait.trigger(r.ID, r) { - log.G(ctx).Errorf("wait not found for raft request id %x", r.ID) - // There was no wait on this ID, meaning we don't have a // transaction in progress that would be committed to the - // memory store by the "trigger" call. Either a different node - // wrote this to raft, or we wrote it before losing the leader + // memory store by the "trigger" call. This could mean that: + // 1. Startup is in progress, and the raft WAL is being parsed, + // processed and applied to the store, or + // 2. Either a different node wrote this to raft, + // or we wrote it before losing the leader // position and cancelling the transaction. This entry still needs // to be committed since other nodes have already committed it. // Create a new transaction to commit this entry. @@ -1827,7 +1906,6 @@ func (n *Node) processEntry(ctx context.Context, entry raftpb.Entry) error { err := n.memoryStore.ApplyStoreActions(r.Action) if err != nil { log.G(ctx).WithError(err).Error("failed to apply actions from raft") - // TODO(anshul) return err here ? } } return nil diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/storage.go b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/storage.go index d12c43af2c..f538317c06 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/storage.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/storage.go @@ -5,6 +5,7 @@ import ( "github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft/raftpb" + "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/encryption" @@ -15,6 +16,18 @@ import ( "golang.org/x/net/context" ) +var ( + // Snapshot create latency timer. + snapshotLatencyTimer metrics.Timer +) + +func init() { + ns := metrics.NewNamespace("swarm", "raft", nil) + snapshotLatencyTimer = ns.NewTimer("snapshot_latency", + "Raft snapshot create latency.") + metrics.Register(ns) +} + func (n *Node) readFromDisk(ctx context.Context) (*raftpb.Snapshot, storage.WALData, error) { keys := n.keyRotator.GetKeys() @@ -169,7 +182,7 @@ func (n *Node) newRaftLogs(nodeID string) (raft.Peer, error) { return raft.Peer{ID: n.Config.ID, Context: metadata}, nil } -func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { +func (n *Node) triggerSnapshot(ctx context.Context, raftConfig api.RaftConfig) { snapshot := api.Snapshot{Version: api.Snapshot_V0} for _, member := range n.cluster.Members() { snapshot.Membership.Members = append(snapshot.Membership.Members, @@ -185,6 +198,9 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { n.asyncTasks.Add(1) n.snapshotInProgress = make(chan raftpb.SnapshotMetadata, 1) // buffered in case Shutdown is called during the snapshot go func(appliedIndex uint64, snapshotMeta raftpb.SnapshotMetadata) { + // Deferred latency capture. + defer metrics.StartTimer(snapshotLatencyTimer)() + defer func() { n.asyncTasks.Done() n.snapshotInProgress <- snapshotMeta diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go index 8c7ca75458..eb849c0803 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/peer.go @@ -18,6 +18,11 @@ import ( "github.com/pkg/errors" ) +const ( + // GRPCMaxMsgSize is the max allowed gRPC message size for raft messages. + GRPCMaxMsgSize = 4 << 20 +) + type peer struct { id uint64 @@ -132,17 +137,112 @@ func (p *peer) resolveAddr(ctx context.Context, id uint64) (string, error) { return resp.Addr, nil } -func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error { - timeout := p.tr.config.SendTimeout - // if a snapshot is being sent, set timeout to LargeSendTimeout because - // sending snapshots can take more time than other messages sent between peers. - // The same applies to AppendEntries as well, where messages can get large. - if m.Type == raftpb.MsgSnap || m.Type == raftpb.MsgApp { - timeout = p.tr.config.LargeSendTimeout +// Returns the raft message struct size (not including the payload size) for the given raftpb.Message. +// The payload is typically the snapshot or append entries. +func raftMessageStructSize(m *raftpb.Message) int { + return (&api.ProcessRaftMessageRequest{Message: m}).Size() - len(m.Snapshot.Data) +} + +// Returns the max allowable payload based on MaxRaftMsgSize and +// the struct size for the given raftpb.Message. +func raftMessagePayloadSize(m *raftpb.Message) int { + return GRPCMaxMsgSize - raftMessageStructSize(m) +} + +// Split a large raft message into smaller messages. +// Currently this means splitting the []Snapshot.Data into chunks whose size +// is dictacted by MaxRaftMsgSize. +func splitSnapshotData(ctx context.Context, m *raftpb.Message) []api.StreamRaftMessageRequest { + var messages []api.StreamRaftMessageRequest + if m.Type != raftpb.MsgSnap { + return messages } - ctx, cancel := context.WithTimeout(ctx, timeout) + + // get the size of the data to be split. + size := len(m.Snapshot.Data) + + // Get the max payload size. + payloadSize := raftMessagePayloadSize(m) + + // split the snapshot into smaller messages. + for snapDataIndex := 0; snapDataIndex < size; { + chunkSize := size - snapDataIndex + if chunkSize > payloadSize { + chunkSize = payloadSize + } + + raftMsg := *m + + // sub-slice for this snapshot chunk. + raftMsg.Snapshot.Data = m.Snapshot.Data[snapDataIndex : snapDataIndex+chunkSize] + + snapDataIndex += chunkSize + + // add message to the list of messages to be sent. + msg := api.StreamRaftMessageRequest{Message: &raftMsg} + messages = append(messages, msg) + } + + return messages +} + +// Function to check if this message needs to be split to be streamed +// (because it is larger than GRPCMaxMsgSize). +// Returns true if the message type is MsgSnap +// and size larger than MaxRaftMsgSize. +func needsSplitting(m *raftpb.Message) bool { + raftMsg := api.ProcessRaftMessageRequest{Message: m} + return m.Type == raftpb.MsgSnap && raftMsg.Size() > GRPCMaxMsgSize +} + +func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error { + ctx, cancel := context.WithTimeout(ctx, p.tr.config.SendTimeout) defer cancel() - _, err := api.NewRaftClient(p.conn()).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m}) + + var err error + var stream api.Raft_StreamRaftMessageClient + stream, err = api.NewRaftClient(p.conn()).StreamRaftMessage(ctx) + + if err == nil { + // Split the message if needed. + // Currently only supported for MsgSnap. + var msgs []api.StreamRaftMessageRequest + if needsSplitting(&m) { + msgs = splitSnapshotData(ctx, &m) + } else { + raftMsg := api.StreamRaftMessageRequest{Message: &m} + msgs = append(msgs, raftMsg) + } + + // Stream + for _, msg := range msgs { + err = stream.Send(&msg) + if err != nil { + log.G(ctx).WithError(err).Error("error streaming message to peer") + stream.CloseAndRecv() + break + } + } + + // Finished sending all the messages. + // Close and receive response. + if err == nil { + _, err = stream.CloseAndRecv() + + if err != nil { + log.G(ctx).WithError(err).Error("error receiving response") + } + } + } else { + log.G(ctx).WithError(err).Error("error sending message to peer") + } + + // Try doing a regular rpc if the receiver doesn't support streaming. + if grpc.Code(err) == codes.Unimplemented { + _, err = api.NewRaftClient(p.conn()).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m}) + } + + // Handle errors. if grpc.Code(err) == codes.NotFound && grpc.ErrorDesc(err) == membership.ErrMemberRemoved.Error() { p.tr.config.NodeRemoved() } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go index 6f096ef9b2..b741c4aa67 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/state/raft/transport/transport.go @@ -36,7 +36,6 @@ type Raft interface { type Config struct { HeartbeatInterval time.Duration SendTimeout time.Duration - LargeSendTimeout time.Duration Credentials credentials.TransportCredentials RaftID string diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/state/store/memory.go b/components/engine/vendor/github.com/docker/swarmkit/manager/state/store/memory.go index 01245a6966..e64565fae8 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/state/store/memory.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/state/store/memory.go @@ -11,6 +11,7 @@ import ( "time" "github.com/docker/go-events" + "github.com/docker/go-metrics" "github.com/docker/swarmkit/api" pb "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/state" @@ -76,8 +77,38 @@ var ( // WedgeTimeout is the maximum amount of time the store lock may be // held before declaring a suspected deadlock. WedgeTimeout = 30 * time.Second + + // update()/write tx latency timer. + updateLatencyTimer metrics.Timer + + // view()/read tx latency timer. + viewLatencyTimer metrics.Timer + + // lookup() latency timer. + lookupLatencyTimer metrics.Timer + + // Batch() latency timer. + batchLatencyTimer metrics.Timer + + // timer to capture the duration for which the memory store mutex is locked. + storeLockDurationTimer metrics.Timer ) +func init() { + ns := metrics.NewNamespace("swarm", "store", nil) + updateLatencyTimer = ns.NewTimer("write_tx_latency", + "Raft store write tx latency.") + viewLatencyTimer = ns.NewTimer("read_tx_latency", + "Raft store read tx latency.") + lookupLatencyTimer = ns.NewTimer("lookup_latency", + "Raft store read latency.") + batchLatencyTimer = ns.NewTimer("batch_latency", + "Raft store batch latency.") + storeLockDurationTimer = ns.NewTimer("memory_store_lock_duration", + "Duration for which the raft memory store lock was held.") + metrics.Register(ns) +} + func register(os ObjectStoreConfig) { objectStorers = append(objectStorers, os) schema.Tables[os.Table.Name] = os.Table @@ -94,8 +125,13 @@ func (m *timedMutex) Lock() { m.lockedAt.Store(time.Now()) } +// Unlocks the timedMutex and captures the duration +// for which it was locked in a metric. func (m *timedMutex) Unlock() { + unlockedTimestamp := m.lockedAt.Load() m.Mutex.Unlock() + lockedFor := time.Since(unlockedTimestamp.(time.Time)) + storeLockDurationTimer.Update(lockedFor) m.lockedAt.Store(time.Time{}) } @@ -184,6 +220,7 @@ type readTx struct { // View executes a read transaction. func (s *MemoryStore) View(cb func(ReadTx)) { + defer metrics.StartTimer(viewLatencyTimer)() memDBTx := s.memDB.Txn(false) readTx := readTx{ @@ -280,6 +317,7 @@ func applyStoreAction(tx Tx, sa api.StoreAction) error { } func (s *MemoryStore) update(proposer state.Proposer, cb func(Tx) error) error { + defer metrics.StartTimer(updateLatencyTimer)() s.updateLock.Lock() memDBTx := s.memDB.Txn(true) @@ -329,7 +367,6 @@ func (s *MemoryStore) update(proposer state.Proposer, cb func(Tx) error) error { } s.updateLock.Unlock() return err - } func (s *MemoryStore) updateLocal(cb func(Tx) error) error { @@ -458,6 +495,7 @@ func (batch *Batch) commit() error { // If Batch returns an error, no guarantees are made about how many updates // were committed successfully. func (s *MemoryStore) Batch(cb func(*Batch) error) error { + defer metrics.StartTimer(batchLatencyTimer)() s.updateLock.Lock() batch := Batch{ @@ -498,6 +536,7 @@ func (tx tx) changelistStoreActions() ([]api.StoreAction, error) { // lookup is an internal typed wrapper around memdb. func (tx readTx) lookup(table, index, id string) api.StoreObject { + defer metrics.StartTimer(lookupLatencyTimer)() j, err := tx.memDBTx.First(table, index, id) if err != nil { return nil diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/watchapi/watch.go b/components/engine/vendor/github.com/docker/swarmkit/manager/watchapi/watch.go index 53bed49f1c..223dcb55d6 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/watchapi/watch.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/watchapi/watch.go @@ -1,12 +1,11 @@ package watchapi import ( - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/state" "github.com/docker/swarmkit/manager/state/store" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // Watch starts a stream that returns any changes to objects that match @@ -26,7 +25,7 @@ func (s *Server) Watch(request *api.WatchRequest, stream api.Watch_WatchServer) watchArgs, err := api.ConvertWatchArgs(request.Entries) if err != nil { - return grpc.Errorf(codes.InvalidArgument, "%s", err.Error()) + return status.Errorf(codes.InvalidArgument, "%s", err.Error()) } watchArgs = append(watchArgs, state.EventCommit{}) From 70c95e6ecbd9058a0d51004224cd21aacfbdab12 Mon Sep 17 00:00:00 2001 From: Anshul Pundir Date: Tue, 5 Dec 2017 14:30:16 -0800 Subject: [PATCH 08/16] Change TestSwarmServiceTTY to only verify running containers since with swarmkit#b187b24 service deletions/scale-downs don't directly remove the container. Signed-off-by: Anshul Pundir Upstream-commit: c5c779adc8c207cfb77a9115450f3e4f1783b26e Component: engine --- components/engine/integration-cli/docker_cli_swarm_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/integration-cli/docker_cli_swarm_test.go b/components/engine/integration-cli/docker_cli_swarm_test.go index 9488fff45e..283f576f57 100644 --- a/components/engine/integration-cli/docker_cli_swarm_test.go +++ b/components/engine/integration-cli/docker_cli_swarm_test.go @@ -849,7 +849,7 @@ func (s *DockerSwarmSuite) TestSwarmServiceTTY(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) // We need to get the container id. - out, err = d.Cmd("ps", "-a", "-q", "--no-trunc") + out, err = d.Cmd("ps", "-q", "--no-trunc") c.Assert(err, checker.IsNil) id := strings.TrimSpace(out) @@ -872,7 +872,7 @@ func (s *DockerSwarmSuite) TestSwarmServiceTTY(c *check.C) { waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) // We need to get the container id. - out, err = d.Cmd("ps", "-a", "-q", "--no-trunc") + out, err = d.Cmd("ps", "-q", "--no-trunc") c.Assert(err, checker.IsNil) id = strings.TrimSpace(out) From 3ef7b79d9314e4f8986961de83e12a9b0ca3d349 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 5 Dec 2017 17:20:41 -0500 Subject: [PATCH 09/16] Fix codegen problems in swagger spec Code generation does't support multiple types Signed-off-by: Daniel Nephin Upstream-commit: 3648d883556e59392e3e5a05ef5de1b56b68c6fc Component: engine --- components/engine/api/swagger.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index 96386e0599..406b72a29b 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -254,6 +254,7 @@ definitions: properties: Propagation: description: "A propagation mode with the value `[r]private`, `[r]shared`, or `[r]slave`." + type: "string" enum: - "private" - "rprivate" @@ -823,9 +824,7 @@ definitions: type: "string" Cmd: description: "Command to run specified as a string or an array of strings." - type: - - "array" - - "string" + type: "array" items: type: "string" Healthcheck: @@ -853,9 +852,7 @@ definitions: The entry point for the container as a string or an array of strings. If the array consists of exactly one empty string (`[""]`) then the entry point is reset to system default (i.e., the entry point used by docker when there is no `ENTRYPOINT` instruction in the `Dockerfile`). - type: - - "array" - - "string" + type: "array" items: type: "string" NetworkDisabled: From a505f09c7f290166202e2d1475c4951118ddd342 Mon Sep 17 00:00:00 2001 From: Anshul Pundir Date: Wed, 6 Dec 2017 11:29:58 -0800 Subject: [PATCH 10/16] Validate/sanitize swarmkit spec name. Signed-off-by: Anshul Pundir Upstream-commit: 739c4cc6b2bab5c3e27656e0a95d858901ecf6b4 Component: engine --- components/engine/daemon/cluster/swarm.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/engine/daemon/cluster/swarm.go b/components/engine/daemon/cluster/swarm.go index 61223691bd..ec527ce59f 100644 --- a/components/engine/daemon/cluster/swarm.go +++ b/components/engine/daemon/cluster/swarm.go @@ -228,6 +228,13 @@ func (c *Cluster) Update(version uint64, spec types.Spec, flags types.UpdateFlag return err } + // Validate spec name. + if spec.Annotations.Name == "" { + spec.Annotations.Name = "default" + } else if spec.Annotations.Name != "default" { + return validationError{errors.New(`swarm spec must be named "default"`)} + } + // In update, client should provide the complete spec of the swarm, including // Name and Labels. If a field is specified with 0 or nil, then the default value // will be used to swarmkit. From 4d38282082b0d2a2a834e2cb20d905f1d47655c8 Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Fri, 1 Dec 2017 11:24:14 -0800 Subject: [PATCH 11/16] Add option to enable networkDB debug Add a new configuration option to allow the enabling of the networkDB debug. The option is only parsed using the reload event. This will protect the daemon on start or restart if the option is left behind in the config file Signed-off-by: Flavio Crisciani Upstream-commit: a97e45794ea8318a08daf763a5b63b04184a886b Component: engine --- components/engine/cmd/dockerd/config.go | 2 + components/engine/daemon/config/config.go | 41 +++++++------- components/engine/daemon/reload.go | 18 ++++++ components/engine/daemon/reload_test.go | 69 +++++++++++++++++++++++ 4 files changed, 110 insertions(+), 20 deletions(-) diff --git a/components/engine/cmd/dockerd/config.go b/components/engine/cmd/dockerd/config.go index b9d586a4ba..c4ae197335 100644 --- a/components/engine/cmd/dockerd/config.go +++ b/components/engine/cmd/dockerd/config.go @@ -59,6 +59,8 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) { flags.IntVar(&maxConcurrentDownloads, "max-concurrent-downloads", config.DefaultMaxConcurrentDownloads, "Set the max concurrent downloads for each pull") flags.IntVar(&maxConcurrentUploads, "max-concurrent-uploads", config.DefaultMaxConcurrentUploads, "Set the max concurrent uploads for each push") flags.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", defaultShutdownTimeout, "Set the default shutdown timeout") + flags.IntVar(&conf.NetworkDiagnosticPort, "network-diagnostic-port", 0, "TCP port number of the network diagnostic server") + flags.MarkHidden("network-diagnostic-port") flags.StringVar(&conf.SwarmDefaultAdvertiseAddr, "swarm-default-advertise-addr", "", "Set default address or interface for swarm advertised address") flags.BoolVar(&conf.Experimental, "experimental", false, "Enable experimental features") diff --git a/components/engine/daemon/config/config.go b/components/engine/daemon/config/config.go index 1e22a6fea7..199fae6cf1 100644 --- a/components/engine/daemon/config/config.go +++ b/components/engine/daemon/config/config.go @@ -85,26 +85,27 @@ type CommonTLSOptions struct { // It includes json tags to deserialize configuration from a file // using the same names that the flags in the command line use. type CommonConfig struct { - AuthzMiddleware *authorization.Middleware `json:"-"` - AuthorizationPlugins []string `json:"authorization-plugins,omitempty"` // AuthorizationPlugins holds list of authorization plugins - AutoRestart bool `json:"-"` - Context map[string][]string `json:"-"` - DisableBridge bool `json:"-"` - DNS []string `json:"dns,omitempty"` - DNSOptions []string `json:"dns-opts,omitempty"` - DNSSearch []string `json:"dns-search,omitempty"` - ExecOptions []string `json:"exec-opts,omitempty"` - GraphDriver string `json:"storage-driver,omitempty"` - GraphOptions []string `json:"storage-opts,omitempty"` - Labels []string `json:"labels,omitempty"` - Mtu int `json:"mtu,omitempty"` - Pidfile string `json:"pidfile,omitempty"` - RawLogs bool `json:"raw-logs,omitempty"` - RootDeprecated string `json:"graph,omitempty"` - Root string `json:"data-root,omitempty"` - ExecRoot string `json:"exec-root,omitempty"` - SocketGroup string `json:"group,omitempty"` - CorsHeaders string `json:"api-cors-header,omitempty"` + AuthzMiddleware *authorization.Middleware `json:"-"` + AuthorizationPlugins []string `json:"authorization-plugins,omitempty"` // AuthorizationPlugins holds list of authorization plugins + AutoRestart bool `json:"-"` + Context map[string][]string `json:"-"` + DisableBridge bool `json:"-"` + DNS []string `json:"dns,omitempty"` + DNSOptions []string `json:"dns-opts,omitempty"` + DNSSearch []string `json:"dns-search,omitempty"` + ExecOptions []string `json:"exec-opts,omitempty"` + GraphDriver string `json:"storage-driver,omitempty"` + GraphOptions []string `json:"storage-opts,omitempty"` + Labels []string `json:"labels,omitempty"` + Mtu int `json:"mtu,omitempty"` + NetworkDiagnosticPort int `json:"network-diagnostic-port,omitempty"` + Pidfile string `json:"pidfile,omitempty"` + RawLogs bool `json:"raw-logs,omitempty"` + RootDeprecated string `json:"graph,omitempty"` + Root string `json:"data-root,omitempty"` + ExecRoot string `json:"exec-root,omitempty"` + SocketGroup string `json:"group,omitempty"` + CorsHeaders string `json:"api-cors-header,omitempty"` // TrustKeyPath is used to generate the daemon ID and for signing schema 1 manifests // when pushing to a registry which does not support schema 2. This field is marked as diff --git a/components/engine/daemon/reload.go b/components/engine/daemon/reload.go index 0d16bc8d02..a20eb688f4 100644 --- a/components/engine/daemon/reload.go +++ b/components/engine/daemon/reload.go @@ -61,6 +61,9 @@ func (daemon *Daemon) Reload(conf *config.Config) (err error) { if err := daemon.reloadLiveRestore(conf, attributes); err != nil { return err } + if err := daemon.reloadNetworkDiagnosticPort(conf, attributes); err != nil { + return err + } return nil } @@ -308,3 +311,18 @@ func (daemon *Daemon) reloadLiveRestore(conf *config.Config, attributes map[stri attributes["live-restore"] = fmt.Sprintf("%t", daemon.configStore.LiveRestoreEnabled) return nil } + +// reloadNetworkDiagnosticPort updates the network controller starting the diagnose mode if the config is valid +func (daemon *Daemon) reloadNetworkDiagnosticPort(conf *config.Config, attributes map[string]string) error { + if conf == nil || daemon.netController == nil { + return nil + } + // Enable the network diagnose if the flag is set with a valid port withing the range + if conf.IsValueSet("network-diagnostic-port") && conf.NetworkDiagnosticPort > 0 && conf.NetworkDiagnosticPort < 65536 { + logrus.Warnf("Calling the diagnostic start with %d", conf.NetworkDiagnosticPort) + daemon.netController.StartDiagnose(conf.NetworkDiagnosticPort) + } else { + daemon.netController.StopDiagnose() + } + return nil +} diff --git a/components/engine/daemon/reload_test.go b/components/engine/daemon/reload_test.go index 96b1a2452d..03b249bada 100644 --- a/components/engine/daemon/reload_test.go +++ b/components/engine/daemon/reload_test.go @@ -10,6 +10,7 @@ import ( "github.com/docker/docker/pkg/discovery" _ "github.com/docker/docker/pkg/discovery/memory" "github.com/docker/docker/registry" + "github.com/docker/libnetwork" "github.com/stretchr/testify/assert" ) @@ -479,3 +480,71 @@ func TestDaemonDiscoveryReloadOnlyClusterAdvertise(t *testing.T) { t.Fatal(e) } } + +func TestDaemonReloadNetworkDiagnosticPort(t *testing.T) { + daemon := &Daemon{} + daemon.configStore = &config.Config{} + + valuesSet := make(map[string]interface{}) + valuesSet["network-diagnostic-port"] = 2000 + enableConfig := &config.Config{ + CommonConfig: config.CommonConfig{ + NetworkDiagnosticPort: 2000, + ValuesSet: valuesSet, + }, + } + disableConfig := &config.Config{ + CommonConfig: config.CommonConfig{}, + } + + netOptions, err := daemon.networkOptions(enableConfig, nil, nil) + if err != nil { + t.Fatal(err) + } + controller, err := libnetwork.New(netOptions...) + if err != nil { + t.Fatal(err) + } + daemon.netController = controller + + // Enable/Disable the server for some iterations + for i := 0; i < 10; i++ { + enableConfig.CommonConfig.NetworkDiagnosticPort++ + if err := daemon.Reload(enableConfig); err != nil { + t.Fatal(err) + } + // Check that the diagnose is enabled + if !daemon.netController.IsDiagnoseEnabled() { + t.Fatalf("diagnosed should be enable") + } + + // Reload + if err := daemon.Reload(disableConfig); err != nil { + t.Fatal(err) + } + // Check that the diagnose is disabled + if daemon.netController.IsDiagnoseEnabled() { + t.Fatalf("diagnosed should be disable") + } + } + + enableConfig.CommonConfig.NetworkDiagnosticPort++ + // 2 times the enable should not create problems + if err := daemon.Reload(enableConfig); err != nil { + t.Fatal(err) + } + // Check that the diagnose is enabled + if !daemon.netController.IsDiagnoseEnabled() { + t.Fatalf("diagnosed should be enable") + } + + // Check that another reload does not cause issues + if err := daemon.Reload(enableConfig); err != nil { + t.Fatal(err) + } + // Check that the diagnose is enable + if !daemon.netController.IsDiagnoseEnabled() { + t.Fatalf("diagnosed should be enable") + } + +} From 8afbe15ccfd08d7c0e26a30b5791306414af2f5e Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Fri, 1 Dec 2017 11:26:35 -0800 Subject: [PATCH 12/16] Vendoring of libnetwork Signed-off-by: Flavio Crisciani Upstream-commit: c1e1fb6fa5204bb7014ca9ebab9fe1804c1eb957 Component: engine --- components/engine/vendor.conf | 2 +- .../github.com/docker/libnetwork/agent.go | 8 +- .../docker/libnetwork/controller.go | 36 +++ .../docker/libnetwork/diagnose/diagnose.go | 133 ---------- .../docker/libnetwork/diagnose/server.go | 228 +++++++++++++++++ .../docker/libnetwork/diagnose/types.go | 122 +++++++++ .../libnetwork/drivers/bridge/bridge.go | 162 ++++++------ .../drivers/bridge/setup_firewalld.go | 2 +- .../libnetwork/drivers/overlay/ov_network.go | 11 + .../github.com/docker/libnetwork/ipvs/ipvs.go | 16 ++ .../docker/libnetwork/ipvs/netlink.go | 11 +- .../libnetwork/networkdb/event_delegate.go | 2 +- .../docker/libnetwork/networkdb/networkdb.go | 18 +- .../libnetwork/networkdb/networkdbdiagnose.go | 242 ++++++++++++++---- .../github.com/docker/libnetwork/vendor.conf | 2 +- 15 files changed, 714 insertions(+), 281 deletions(-) delete mode 100644 components/engine/vendor/github.com/docker/libnetwork/diagnose/diagnose.go create mode 100644 components/engine/vendor/github.com/docker/libnetwork/diagnose/server.go create mode 100644 components/engine/vendor/github.com/docker/libnetwork/diagnose/types.go diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 120dd9d378..f9cc8a7131 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -30,7 +30,7 @@ github.com/moby/buildkit aaff9d591ef128560018433fe61beb802e149de8 github.com/tonistiigi/fsutil dea3a0da73aee887fc02142d995be764106ac5e2 #get libnetwork packages -github.com/docker/libnetwork 64ae58878fc8f95e4a167499d654e13fa36abdc7 +github.com/docker/libnetwork 9bca9a4a220b158cc94402e0f8c2c7714eb6f503 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec diff --git a/components/engine/vendor/github.com/docker/libnetwork/agent.go b/components/engine/vendor/github.com/docker/libnetwork/agent.go index f12006583b..085b8c47ca 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/agent.go +++ b/components/engine/vendor/github.com/docker/libnetwork/agent.go @@ -293,11 +293,13 @@ func (c *controller) agentInit(listenAddr, bindAddrOrInterface, advertiseAddr, d c.Config().Daemon.NetworkControlPlaneMTU, netDBConf.PacketBufferSize) } nDB, err := networkdb.New(netDBConf) - if err != nil { return err } + // Register the diagnose handlers + c.DiagnoseServer.RegisterHandler(nDB, networkdb.NetDbPaths2Func) + var cancelList []func() ch, cancel := nDB.Watch(libnetworkEPTable, "", "") cancelList = append(cancelList, cancel) @@ -436,7 +438,7 @@ func (n *network) Services() map[string]ServiceInfo { for eid, value := range entries { var epRec EndpointRecord nid := n.ID() - if err := proto.Unmarshal(value.([]byte), &epRec); err != nil { + if err := proto.Unmarshal(value.Value, &epRec); err != nil { logrus.Errorf("Unmarshal of libnetworkEPTable failed for endpoint %s in network %s, %v", eid, nid, err) continue } @@ -461,7 +463,7 @@ func (n *network) Services() map[string]ServiceInfo { } entries := agent.networkDB.GetTableByNetwork(table.name, n.id) for key, value := range entries { - epID, info := d.DecodeTableEntry(table.name, key, value.([]byte)) + epID, info := d.DecodeTableEntry(table.name, key, value.Value) if ep, ok := eps[epID]; !ok { logrus.Errorf("Inconsistent driver and libnetwork state for endpoint %s", epID) } else { diff --git a/components/engine/vendor/github.com/docker/libnetwork/controller.go b/components/engine/vendor/github.com/docker/libnetwork/controller.go index 24494ae834..e9389487a3 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/controller.go +++ b/components/engine/vendor/github.com/docker/libnetwork/controller.go @@ -60,6 +60,7 @@ import ( "github.com/docker/libnetwork/cluster" "github.com/docker/libnetwork/config" "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/diagnose" "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/drvregistry" @@ -133,6 +134,13 @@ type NetworkController interface { // SetKeys configures the encryption key for gossip and overlay data path SetKeys(keys []*types.EncryptionKey) error + + // StartDiagnose start the network diagnose mode + StartDiagnose(port int) + // StopDiagnose start the network diagnose mode + StopDiagnose() + // IsDiagnoseEnabled returns true if the diagnose is enabled + IsDiagnoseEnabled() bool } // NetworkWalker is a client provided function which will be used to walk the Networks. @@ -167,6 +175,7 @@ type controller struct { agentStopDone chan struct{} keys []*types.EncryptionKey clusterConfigAvailable bool + DiagnoseServer *diagnose.Server sync.Mutex } @@ -185,7 +194,9 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { serviceBindings: make(map[serviceKey]*service), agentInitDone: make(chan struct{}), networkLocker: locker.New(), + DiagnoseServer: diagnose.New(), } + c.DiagnoseServer.Init() if err := c.initStores(); err != nil { return nil, err @@ -1291,3 +1302,28 @@ func (c *controller) Stop() { c.stopExternalKeyListener() osl.GC() } + +// StartDiagnose start the network diagnose mode +func (c *controller) StartDiagnose(port int) { + c.Lock() + if !c.DiagnoseServer.IsDebugEnable() { + c.DiagnoseServer.EnableDebug("127.0.0.1", port) + } + c.Unlock() +} + +// StopDiagnose start the network diagnose mode +func (c *controller) StopDiagnose() { + c.Lock() + if c.DiagnoseServer.IsDebugEnable() { + c.DiagnoseServer.DisableDebug() + } + c.Unlock() +} + +// IsDiagnoseEnabled returns true if the diagnose is enabled +func (c *controller) IsDiagnoseEnabled() bool { + c.Lock() + defer c.Unlock() + return c.DiagnoseServer.IsDebugEnable() +} diff --git a/components/engine/vendor/github.com/docker/libnetwork/diagnose/diagnose.go b/components/engine/vendor/github.com/docker/libnetwork/diagnose/diagnose.go deleted file mode 100644 index 9682fff78b..0000000000 --- a/components/engine/vendor/github.com/docker/libnetwork/diagnose/diagnose.go +++ /dev/null @@ -1,133 +0,0 @@ -package diagnose - -import ( - "fmt" - "net" - "net/http" - "sync" - - "github.com/sirupsen/logrus" -) - -// HTTPHandlerFunc TODO -type HTTPHandlerFunc func(interface{}, http.ResponseWriter, *http.Request) - -type httpHandlerCustom struct { - ctx interface{} - F func(interface{}, http.ResponseWriter, *http.Request) -} - -// ServeHTTP TODO -func (h httpHandlerCustom) ServeHTTP(w http.ResponseWriter, r *http.Request) { - h.F(h.ctx, w, r) -} - -var diagPaths2Func = map[string]HTTPHandlerFunc{ - "/": notImplemented, - "/help": help, - "/ready": ready, -} - -// Server when the debug is enabled exposes a -// This data structure is protected by the Agent mutex so does not require and additional mutex here -type Server struct { - sk net.Listener - port int - mux *http.ServeMux - registeredHanders []string - sync.Mutex -} - -// Init TODO -func (n *Server) Init() { - n.mux = http.NewServeMux() - - // Register local handlers - n.RegisterHandler(n, diagPaths2Func) -} - -// RegisterHandler TODO -func (n *Server) RegisterHandler(ctx interface{}, hdlrs map[string]HTTPHandlerFunc) { - n.Lock() - defer n.Unlock() - for path, fun := range hdlrs { - n.mux.Handle(path, httpHandlerCustom{ctx, fun}) - n.registeredHanders = append(n.registeredHanders, path) - } -} - -// EnableDebug opens a TCP socket to debug the passed network DB -func (n *Server) EnableDebug(ip string, port int) { - n.Lock() - defer n.Unlock() - - n.port = port - logrus.SetLevel(logrus.DebugLevel) - - if n.sk != nil { - logrus.Infof("The server is already up and running") - return - } - - logrus.Infof("Starting the server listening on %d for commands", port) - - // // Create the socket - // var err error - // n.sk, err = net.Listen("tcp", listeningAddr) - // if err != nil { - // log.Fatal(err) - // } - // - // go func() { - // http.Serve(n.sk, n.mux) - // }() - http.ListenAndServe(fmt.Sprintf(":%d", port), n.mux) -} - -// DisableDebug stop the dubug and closes the tcp socket -func (n *Server) DisableDebug() { - n.Lock() - defer n.Unlock() - n.sk.Close() - n.sk = nil -} - -// IsDebugEnable returns true when the debug is enabled -func (n *Server) IsDebugEnable() bool { - n.Lock() - defer n.Unlock() - return n.sk != nil -} - -func notImplemented(ctx interface{}, w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "URL path: %s no method implemented check /help\n", r.URL.Path) -} - -func help(ctx interface{}, w http.ResponseWriter, r *http.Request) { - n, ok := ctx.(*Server) - if ok { - for _, path := range n.registeredHanders { - fmt.Fprintf(w, "%s\n", path) - } - } -} - -func ready(ctx interface{}, w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "OK\n") -} - -// DebugHTTPForm TODO -func DebugHTTPForm(r *http.Request) { - r.ParseForm() - for k, v := range r.Form { - logrus.Debugf("Form[%q] = %q\n", k, v) - } -} - -// HTTPReplyError TODO -func HTTPReplyError(w http.ResponseWriter, message, usage string) { - fmt.Fprintf(w, "%s\n", message) - if usage != "" { - fmt.Fprintf(w, "Usage: %s\n", usage) - } -} diff --git a/components/engine/vendor/github.com/docker/libnetwork/diagnose/server.go b/components/engine/vendor/github.com/docker/libnetwork/diagnose/server.go new file mode 100644 index 0000000000..c841e51966 --- /dev/null +++ b/components/engine/vendor/github.com/docker/libnetwork/diagnose/server.go @@ -0,0 +1,228 @@ +package diagnose + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sync" + "sync/atomic" + + stackdump "github.com/docker/docker/pkg/signal" + "github.com/docker/libnetwork/common" + "github.com/sirupsen/logrus" +) + +// HTTPHandlerFunc TODO +type HTTPHandlerFunc func(interface{}, http.ResponseWriter, *http.Request) + +type httpHandlerCustom struct { + ctx interface{} + F func(interface{}, http.ResponseWriter, *http.Request) +} + +// ServeHTTP TODO +func (h httpHandlerCustom) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.F(h.ctx, w, r) +} + +var diagPaths2Func = map[string]HTTPHandlerFunc{ + "/": notImplemented, + "/help": help, + "/ready": ready, + "/stackdump": stackTrace, +} + +// Server when the debug is enabled exposes a +// This data structure is protected by the Agent mutex so does not require and additional mutex here +type Server struct { + enable int32 + srv *http.Server + port int + mux *http.ServeMux + registeredHanders map[string]bool + sync.Mutex +} + +// New creates a new diagnose server +func New() *Server { + return &Server{ + registeredHanders: make(map[string]bool), + } +} + +// Init initialize the mux for the http handling and register the base hooks +func (s *Server) Init() { + s.mux = http.NewServeMux() + + // Register local handlers + s.RegisterHandler(s, diagPaths2Func) +} + +// RegisterHandler allows to register new handlers to the mux and to a specific path +func (s *Server) RegisterHandler(ctx interface{}, hdlrs map[string]HTTPHandlerFunc) { + s.Lock() + defer s.Unlock() + for path, fun := range hdlrs { + if _, ok := s.registeredHanders[path]; ok { + continue + } + s.mux.Handle(path, httpHandlerCustom{ctx, fun}) + s.registeredHanders[path] = true + } +} + +// ServeHTTP this is the method called bu the ListenAndServe, and is needed to allow us to +// use our custom mux +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mux.ServeHTTP(w, r) +} + +// EnableDebug opens a TCP socket to debug the passed network DB +func (s *Server) EnableDebug(ip string, port int) { + s.Lock() + defer s.Unlock() + + s.port = port + + if s.enable == 1 { + logrus.Info("The server is already up and running") + return + } + + logrus.Infof("Starting the diagnose server listening on %d for commands", port) + srv := &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: s} + s.srv = srv + s.enable = 1 + go func(n *Server) { + // Ingore ErrServerClosed that is returned on the Shutdown call + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + logrus.Errorf("ListenAndServe error: %s", err) + atomic.SwapInt32(&n.enable, 0) + } + }(s) + +} + +// DisableDebug stop the dubug and closes the tcp socket +func (s *Server) DisableDebug() { + s.Lock() + defer s.Unlock() + + s.srv.Shutdown(context.Background()) + s.srv = nil + s.enable = 0 + logrus.Info("Disabling the diagnose server") +} + +// IsDebugEnable returns true when the debug is enabled +func (s *Server) IsDebugEnable() bool { + s.Lock() + defer s.Unlock() + return s.enable == 1 +} + +func notImplemented(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + _, json := ParseHTTPFormOptions(r) + rsp := WrongCommand("not implemented", fmt.Sprintf("URL path: %s no method implemented check /help\n", r.URL.Path)) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("command not implemented done") + + HTTPReply(w, rsp, json) +} + +func help(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + _, json := ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("help done") + + n, ok := ctx.(*Server) + var result string + if ok { + for path := range n.registeredHanders { + result += fmt.Sprintf("%s\n", path) + } + HTTPReply(w, CommandSucceed(&StringCmd{Info: result}), json) + } +} + +func ready(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + _, json := ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("ready done") + HTTPReply(w, CommandSucceed(&StringCmd{Info: "OK"}), json) +} + +func stackTrace(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + _, json := ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("stack trace") + + path, err := stackdump.DumpStacks("/tmp/") + if err != nil { + log.WithError(err).Error("failed to write goroutines dump") + HTTPReply(w, FailCommand(err), json) + } else { + log.Info("stack trace done") + HTTPReply(w, CommandSucceed(&StringCmd{Info: fmt.Sprintf("goroutine stacks written to %s", path)}), json) + } +} + +// DebugHTTPForm helper to print the form url parameters +func DebugHTTPForm(r *http.Request) { + for k, v := range r.Form { + logrus.Debugf("Form[%q] = %q\n", k, v) + } +} + +// JSONOutput contains details on JSON output printing +type JSONOutput struct { + enable bool + prettyPrint bool +} + +// ParseHTTPFormOptions easily parse the JSON printing options +func ParseHTTPFormOptions(r *http.Request) (bool, *JSONOutput) { + _, unsafe := r.Form["unsafe"] + v, json := r.Form["json"] + var pretty bool + if len(v) > 0 { + pretty = v[0] == "pretty" + } + return unsafe, &JSONOutput{enable: json, prettyPrint: pretty} +} + +// HTTPReply helper function that takes care of sending the message out +func HTTPReply(w http.ResponseWriter, r *HTTPResult, j *JSONOutput) (int, error) { + var response []byte + if j.enable { + w.Header().Set("Content-Type", "application/json") + var err error + if j.prettyPrint { + response, err = json.MarshalIndent(r, "", " ") + if err != nil { + response, _ = json.MarshalIndent(FailCommand(err), "", " ") + } + } else { + response, err = json.Marshal(r) + if err != nil { + response, _ = json.Marshal(FailCommand(err)) + } + } + } else { + response = []byte(r.String()) + } + return fmt.Fprint(w, string(response)) +} diff --git a/components/engine/vendor/github.com/docker/libnetwork/diagnose/types.go b/components/engine/vendor/github.com/docker/libnetwork/diagnose/types.go new file mode 100644 index 0000000000..982c54a2b2 --- /dev/null +++ b/components/engine/vendor/github.com/docker/libnetwork/diagnose/types.go @@ -0,0 +1,122 @@ +package diagnose + +import "fmt" + +// StringInterface interface that has to be implemented by messages +type StringInterface interface { + String() string +} + +// CommandSucceed creates a success message +func CommandSucceed(result StringInterface) *HTTPResult { + return &HTTPResult{ + Message: "OK", + Details: result, + } +} + +// FailCommand creates a failure message with error +func FailCommand(err error) *HTTPResult { + return &HTTPResult{ + Message: "FAIL", + Details: &ErrorCmd{Error: err.Error()}, + } +} + +// WrongCommand creates a wrong command response +func WrongCommand(message, usage string) *HTTPResult { + return &HTTPResult{ + Message: message, + Details: &UsageCmd{Usage: usage}, + } +} + +// HTTPResult Diagnose Server HTTP result operation +type HTTPResult struct { + Message string `json:"message"` + Details StringInterface `json:"details"` +} + +func (h *HTTPResult) String() string { + rsp := h.Message + if h.Details != nil { + rsp += "\n" + h.Details.String() + } + return rsp +} + +// UsageCmd command with usage field +type UsageCmd struct { + Usage string `json:"usage"` +} + +func (u *UsageCmd) String() string { + return "Usage: " + u.Usage +} + +// StringCmd command with info string +type StringCmd struct { + Info string `json:"info"` +} + +func (s *StringCmd) String() string { + return s.Info +} + +// ErrorCmd command with error +type ErrorCmd struct { + Error string `json:"error"` +} + +func (e *ErrorCmd) String() string { + return "Error: " + e.Error +} + +// TableObj network db table object +type TableObj struct { + Length int `json:"size"` + Elements []StringInterface `json:"entries"` +} + +func (t *TableObj) String() string { + output := fmt.Sprintf("total entries: %d\n", t.Length) + for _, e := range t.Elements { + output += e.String() + } + return output +} + +// PeerEntryObj entry in the networkdb peer table +type PeerEntryObj struct { + Index int `json:"-"` + Name string `json:"-=name"` + IP string `json:"ip"` +} + +func (p *PeerEntryObj) String() string { + return fmt.Sprintf("%d) %s -> %s\n", p.Index, p.Name, p.IP) +} + +// TableEntryObj network db table entry object +type TableEntryObj struct { + Index int `json:"-"` + Key string `json:"key"` + Value string `json:"value"` + Owner string `json:"owner"` +} + +func (t *TableEntryObj) String() string { + return fmt.Sprintf("%d) k:`%s` -> v:`%s` owner:`%s`\n", t.Index, t.Key, t.Value, t.Owner) +} + +// TableEndpointsResult fully typed message for proper unmarshaling on the client side +type TableEndpointsResult struct { + TableObj + Elements []TableEntryObj `json:"entries"` +} + +// TablePeersResult fully typed message for proper unmarshaling on the client side +type TablePeersResult struct { + TableObj + Elements []PeerEntryObj `json:"entries"` +} diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go index 1742c8df31..1fa8f0e921 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go @@ -42,6 +42,14 @@ const ( DefaultGatewayV6AuxKey = "DefaultGatewayIPv6" ) +type defaultBridgeNetworkConflict struct { + ID string +} + +func (d defaultBridgeNetworkConflict) Error() string { + return fmt.Sprintf("Stale default bridge network %s", d.ID) +} + type iptableCleanFunc func() error type iptablesCleanFuncs []iptableCleanFunc @@ -137,6 +145,7 @@ type driver struct { networks map[string]*bridgeNetwork store datastore.DataStore nlh *netlink.Handle + configNetwork sync.Mutex sync.Mutex } @@ -322,41 +331,6 @@ func (n *bridgeNetwork) isolateNetwork(others []*bridgeNetwork, enable bool) err return nil } -// Checks whether this network's configuration for the network with this id conflicts with any of the passed networks -func (c *networkConfiguration) conflictsWithNetworks(id string, others []*bridgeNetwork) error { - for _, nw := range others { - - nw.Lock() - nwID := nw.id - nwConfig := nw.config - nwBridge := nw.bridge - nw.Unlock() - - if nwID == id { - continue - } - // Verify the name (which may have been set by newInterface()) does not conflict with - // existing bridge interfaces. Ironically the system chosen name gets stored in the config... - // Basically we are checking if the two original configs were both empty. - if nwConfig.BridgeName == c.BridgeName { - return types.ForbiddenErrorf("conflicts with network %s (%s) by bridge name", nwID, nwConfig.BridgeName) - } - // If this network config specifies the AddressIPv4, we need - // to make sure it does not conflict with any previously allocated - // bridges. This could not be completely caught by the config conflict - // check, because networks which config does not specify the AddressIPv4 - // get their address and subnet selected by the driver (see electBridgeIPv4()) - if c.AddressIPv4 != nil && nwBridge.bridgeIPv4 != nil { - if nwBridge.bridgeIPv4.Contains(c.AddressIPv4.IP) || - c.AddressIPv4.Contains(nwBridge.bridgeIPv4.IP) { - return types.ForbiddenErrorf("conflicts with network %s (%s) by ip network", nwID, nwConfig.BridgeName) - } - } - } - - return nil -} - func (d *driver) configure(option map[string]interface{}) error { var ( config *configuration @@ -602,11 +576,27 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d return err } - err = config.processIPAM(id, ipV4Data, ipV6Data) - if err != nil { + if err = config.processIPAM(id, ipV4Data, ipV6Data); err != nil { return err } + // start the critical section, from this point onward we are dealing with the list of networks + // so to be consistent we cannot allow that the list changes + d.configNetwork.Lock() + defer d.configNetwork.Unlock() + + // check network conflicts + if err = d.checkConflict(config); err != nil { + nerr, ok := err.(defaultBridgeNetworkConflict) + if !ok { + return err + } + // Got a conflict with a stale default network, clean that up and continue + logrus.Warn(nerr) + d.deleteNetwork(nerr.ID) + } + + // there is no conflict, now create the network if err = d.createNetwork(config); err != nil { return err } @@ -614,33 +604,47 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d return d.storeUpdate(config) } +func (d *driver) checkConflict(config *networkConfiguration) error { + networkList := d.getNetworks() + for _, nw := range networkList { + nw.Lock() + nwConfig := nw.config + nw.Unlock() + if err := nwConfig.Conflicts(config); err != nil { + if config.DefaultBridge { + // We encountered and identified a stale default network + // We must delete it as libnetwork is the source of truth + // The default network being created must be the only one + // This can happen only from docker 1.12 on ward + logrus.Infof("Found stale default bridge network %s (%s)", nwConfig.ID, nwConfig.BridgeName) + return defaultBridgeNetworkConflict{nwConfig.ID} + } + + return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s", + config.ID, config.BridgeName, nwConfig.ID, nwConfig.BridgeName, err.Error()) + } + } + return nil +} + func (d *driver) createNetwork(config *networkConfiguration) error { var err error defer osl.InitOSContext()() networkList := d.getNetworks() - for i, nw := range networkList { - nw.Lock() - nwConfig := nw.config - nw.Unlock() - if err := nwConfig.Conflicts(config); err != nil { - if config.DefaultBridge { - // We encountered and identified a stale default network - // We must delete it as libnetwork is the source of thruth - // The default network being created must be the only one - // This can happen only from docker 1.12 on ward - logrus.Infof("Removing stale default bridge network %s (%s)", nwConfig.ID, nwConfig.BridgeName) - if err := d.DeleteNetwork(nwConfig.ID); err != nil { - logrus.Warnf("Failed to remove stale default network: %s (%s): %v. Will remove from store.", nwConfig.ID, nwConfig.BridgeName, err) - d.storeDelete(nwConfig) - } - networkList = append(networkList[:i], networkList[i+1:]...) - } else { - return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s", - config.ID, config.BridgeName, nwConfig.ID, nwConfig.BridgeName, err.Error()) - } - } + + // Initialize handle when needed + d.Lock() + if d.nlh == nil { + d.nlh = ns.NlHandle() + } + d.Unlock() + + // Create or retrieve the bridge L3 interface + bridgeIface, err := newInterface(d.nlh, config) + if err != nil { + return err } // Create and set network handler in driver @@ -649,6 +653,7 @@ func (d *driver) createNetwork(config *networkConfiguration) error { endpoints: make(map[string]*bridgeEndpoint), config: config, portMapper: portmapper.New(d.config.UserlandProxyPath), + bridge: bridgeIface, driver: d, } @@ -665,35 +670,15 @@ func (d *driver) createNetwork(config *networkConfiguration) error { } }() - // Initialize handle when needed - d.Lock() - if d.nlh == nil { - d.nlh = ns.NlHandle() - } - d.Unlock() - - // Create or retrieve the bridge L3 interface - bridgeIface, err := newInterface(d.nlh, config) - if err != nil { - return err - } - network.bridge = bridgeIface - - // Verify the network configuration does not conflict with previously installed - // networks. This step is needed now because driver might have now set the bridge - // name on this config struct. And because we need to check for possible address - // conflicts, so we need to check against operationa lnetworks. - if err = config.conflictsWithNetworks(config.ID, networkList); err != nil { - return err - } - + // Add inter-network communication rules. setupNetworkIsolationRules := func(config *networkConfiguration, i *bridgeInterface) error { if err := network.isolateNetwork(networkList, true); err != nil { - if err := network.isolateNetwork(networkList, false); err != nil { + if err = network.isolateNetwork(networkList, false); err != nil { logrus.Warnf("Failed on removing the inter-network iptables rules on cleanup: %v", err) } return err } + // register the cleanup function network.registerIptCleanFunc(func() error { nwList := d.getNetworks() return network.isolateNetwork(nwList, false) @@ -767,10 +752,17 @@ func (d *driver) createNetwork(config *networkConfiguration) error { } func (d *driver) DeleteNetwork(nid string) error { + + d.configNetwork.Lock() + defer d.configNetwork.Unlock() + + return d.deleteNetwork(nid) +} + +func (d *driver) deleteNetwork(nid string) error { var err error defer osl.InitOSContext()() - // Get network handler and remove it from driver d.Lock() n, ok := d.networks[nid] @@ -814,12 +806,6 @@ func (d *driver) DeleteNetwork(nid string) error { } }() - // Sanity check - if n == nil { - err = driverapi.ErrNoNetwork(nid) - return err - } - switch config.BridgeIfaceCreator { case ifaceCreatedByLibnetwork, ifaceCreatorUnknown: // We only delete the bridge if it was created by the bridge driver and diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_firewalld.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_firewalld.go index fc45a7e983..50cbdb1ddc 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_firewalld.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/bridge/setup_firewalld.go @@ -9,7 +9,7 @@ func (n *bridgeNetwork) setupFirewalld(config *networkConfiguration, i *bridgeIn d.Unlock() // Sanity check. - if driverConfig.EnableIPTables == false { + if !driverConfig.EnableIPTables { return IPTableCfgError(config.BridgeName) } diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go index 3fbfccf007..11314170b1 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go @@ -696,6 +696,12 @@ func (n *network) initSandbox(restore bool) error { var nlSock *nl.NetlinkSocket sbox.InvokeFunc(func() { nlSock, err = nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH) + if err != nil { + return + } + // set the receive timeout to not remain stuck on the RecvFrom if the fd gets closed + tv := syscall.NsecToTimeval(soTimeout.Nanoseconds()) + err = nlSock.SetReceiveTimeout(&tv) }) n.setNetlinkSocket(nlSock) @@ -721,6 +727,11 @@ func (n *network) watchMiss(nlSock *nl.NetlinkSocket) { // The netlink socket got closed, simply exit to not leak this goroutine return } + // When the receive timeout expires the receive will return EAGAIN + if err == syscall.EAGAIN { + // we continue here to avoid spam for timeouts + continue + } logrus.Errorf("Failed to receive from netlink: %v ", err) continue } diff --git a/components/engine/vendor/github.com/docker/libnetwork/ipvs/ipvs.go b/components/engine/vendor/github.com/docker/libnetwork/ipvs/ipvs.go index ebcdd808c3..effbb716eb 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ipvs/ipvs.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ipvs/ipvs.go @@ -5,12 +5,19 @@ package ipvs import ( "net" "syscall" + "time" "fmt" + "github.com/vishvananda/netlink/nl" "github.com/vishvananda/netns" ) +const ( + netlinkRecvSocketsTimeout = 3 * time.Second + netlinkSendSocketTimeout = 30 * time.Second +) + // Service defines an IPVS service in its entirety. type Service struct { // Virtual service address. @@ -82,6 +89,15 @@ func New(path string) (*Handle, error) { if err != nil { return nil, err } + // Add operation timeout to avoid deadlocks + tv := syscall.NsecToTimeval(netlinkSendSocketTimeout.Nanoseconds()) + if err := sock.SetSendTimeout(&tv); err != nil { + return nil, err + } + tv = syscall.NsecToTimeval(netlinkRecvSocketsTimeout.Nanoseconds()) + if err := sock.SetReceiveTimeout(&tv); err != nil { + return nil, err + } return &Handle{sock: sock}, nil } diff --git a/components/engine/vendor/github.com/docker/libnetwork/ipvs/netlink.go b/components/engine/vendor/github.com/docker/libnetwork/ipvs/netlink.go index 2089283d14..c062a1789d 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ipvs/netlink.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ipvs/netlink.go @@ -203,10 +203,6 @@ func newGenlRequest(familyID int, cmd uint8) *nl.NetlinkRequest { } func execute(s *nl.NetlinkSocket, req *nl.NetlinkRequest, resType uint16) ([][]byte, error) { - var ( - err error - ) - if err := s.Send(req); err != nil { return nil, err } @@ -222,6 +218,13 @@ done: for { msgs, err := s.Receive() if err != nil { + if s.GetFd() == -1 { + return nil, fmt.Errorf("Socket got closed on receive") + } + if err == syscall.EAGAIN { + // timeout fired + continue + } return nil, err } for _, m := range msgs { diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/event_delegate.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/event_delegate.go index 6075718a63..6e11805610 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/event_delegate.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/event_delegate.go @@ -72,7 +72,7 @@ func (e *eventDelegate) NotifyLeave(mn *memberlist.Node) { // If the node instead left because was going down, then it makes sense to just delete all its state e.nDB.Lock() defer e.nDB.Unlock() - e.nDB.deleteNetworkEntriesForNode(mn.Name) + e.nDB.deleteNodeFromNetworks(mn.Name) e.nDB.deleteNodeTableEntries(mn.Name) if n, ok := e.nDB.nodes[mn.Name]; ok { delete(e.nDB.nodes, mn.Name) diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go index 025e0ca843..9ec6beca35 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdb.go @@ -401,17 +401,23 @@ func (nDB *NetworkDB) UpdateEntry(tname, nid, key string, value []byte) error { return nil } +// TableElem elem +type TableElem struct { + Value []byte + owner string +} + // GetTableByNetwork walks the networkdb by the give table and network id and // returns a map of keys and values -func (nDB *NetworkDB) GetTableByNetwork(tname, nid string) map[string]interface{} { - entries := make(map[string]interface{}) +func (nDB *NetworkDB) GetTableByNetwork(tname, nid string) map[string]*TableElem { + entries := make(map[string]*TableElem) nDB.indexes[byTable].WalkPrefix(fmt.Sprintf("/%s/%s", tname, nid), func(k string, v interface{}) bool { entry := v.(*entry) if entry.deleting { return false } key := k[strings.LastIndex(k, "/")+1:] - entries[key] = entry.value + entries[key] = &TableElem{Value: entry.value, owner: entry.node} return false }) return entries @@ -445,7 +451,7 @@ func (nDB *NetworkDB) DeleteEntry(tname, nid, key string) error { return nil } -func (nDB *NetworkDB) deleteNetworkEntriesForNode(deletedNode string) { +func (nDB *NetworkDB) deleteNodeFromNetworks(deletedNode string) { for nid, nodes := range nDB.networkNodes { updatedNodes := make([]string, 0, len(nodes)) for _, node := range nodes { @@ -547,7 +553,9 @@ func (nDB *NetworkDB) deleteNodeTableEntries(node string) { nDB.deleteEntry(nid, tname, key) - nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, oldEntry.value)) + if !oldEntry.deleting { + nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, oldEntry.value)) + } return false }) } diff --git a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdbdiagnose.go b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdbdiagnose.go index 01429a5381..3c6032b4ea 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdbdiagnose.go +++ b/components/engine/vendor/github.com/docker/libnetwork/networkdb/networkdbdiagnose.go @@ -1,17 +1,19 @@ package networkdb import ( + "encoding/base64" "fmt" "net/http" "strings" - stackdump "github.com/docker/docker/pkg/signal" + "github.com/docker/libnetwork/common" "github.com/docker/libnetwork/diagnose" "github.com/sirupsen/logrus" ) const ( missingParameter = "missing parameter" + dbNotAvailable = "database not available" ) // NetDbPaths2Func TODO @@ -26,14 +28,21 @@ var NetDbPaths2Func = map[string]diagnose.HTTPHandlerFunc{ "/deleteentry": dbDeleteEntry, "/getentry": dbGetEntry, "/gettable": dbGetTable, - "/dump": dbStackTrace, } func dbJoin(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + _, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("join cluster") + if len(r.Form["members"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?members=ip1,ip2,...", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?members=ip1,ip2,...", r.URL.Path)) + log.Error("join cluster failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -41,51 +50,88 @@ func dbJoin(ctx interface{}, w http.ResponseWriter, r *http.Request) { if ok { err := nDB.Join(strings.Split(r.Form["members"][0], ",")) if err != nil { - fmt.Fprintf(w, "%s error in the DB join %s\n", r.URL.Path, err) + rsp := diagnose.FailCommand(fmt.Errorf("%s error in the DB join %s", r.URL.Path, err)) + log.WithError(err).Error("join cluster failed") + diagnose.HTTPReply(w, rsp, json) return } - fmt.Fprintf(w, "OK\n") + log.Info("join cluster done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbPeers(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + _, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("network peers") + if len(r.Form["nid"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?nid=test", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?nid=test", r.URL.Path)) + log.Error("network peers failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } nDB, ok := ctx.(*NetworkDB) if ok { peers := nDB.Peers(r.Form["nid"][0]) - fmt.Fprintf(w, "Network:%s Total peers: %d\n", r.Form["nid"], len(peers)) + rsp := &diagnose.TableObj{Length: len(peers)} for i, peerInfo := range peers { - fmt.Fprintf(w, "%d) %s -> %s\n", i, peerInfo.Name, peerInfo.IP) + rsp.Elements = append(rsp.Elements, &diagnose.PeerEntryObj{Index: i, Name: peerInfo.Name, IP: peerInfo.IP}) } + log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("network peers done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(rsp), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbClusterPeers(ctx interface{}, w http.ResponseWriter, r *http.Request) { + r.ParseForm() + diagnose.DebugHTTPForm(r) + _, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("cluster peers") + nDB, ok := ctx.(*NetworkDB) if ok { peers := nDB.ClusterPeers() - fmt.Fprintf(w, "Total peers: %d\n", len(peers)) + rsp := &diagnose.TableObj{Length: len(peers)} for i, peerInfo := range peers { - fmt.Fprintf(w, "%d) %s -> %s\n", i, peerInfo.Name, peerInfo.IP) + rsp.Elements = append(rsp.Elements, &diagnose.PeerEntryObj{Index: i, Name: peerInfo.Name, IP: peerInfo.IP}) } + log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("cluster peers done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(rsp), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbCreateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + unsafe, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("create entry") + if len(r.Form["tname"]) < 1 || len(r.Form["nid"]) < 1 || len(r.Form["key"]) < 1 || len(r.Form["value"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path)) + log.Error("create entry failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -93,25 +139,48 @@ func dbCreateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { nid := r.Form["nid"][0] key := r.Form["key"][0] value := r.Form["value"][0] + decodedValue := []byte(value) + if !unsafe { + var err error + decodedValue, err = base64.StdEncoding.DecodeString(value) + if err != nil { + log.WithError(err).Error("create entry failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) + return + } + } nDB, ok := ctx.(*NetworkDB) if ok { - if err := nDB.CreateEntry(tname, nid, key, []byte(value)); err != nil { - diagnose.HTTPReplyError(w, err.Error(), "") + if err := nDB.CreateEntry(tname, nid, key, decodedValue); err != nil { + rsp := diagnose.FailCommand(err) + diagnose.HTTPReply(w, rsp, json) + log.WithError(err).Error("create entry failed") return } - fmt.Fprintf(w, "OK\n") + log.Info("create entry done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbUpdateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + unsafe, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("update entry") + if len(r.Form["tname"]) < 1 || len(r.Form["nid"]) < 1 || len(r.Form["key"]) < 1 || len(r.Form["value"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path)) + log.Error("update entry failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -119,24 +188,46 @@ func dbUpdateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { nid := r.Form["nid"][0] key := r.Form["key"][0] value := r.Form["value"][0] + decodedValue := []byte(value) + if !unsafe { + var err error + decodedValue, err = base64.StdEncoding.DecodeString(value) + if err != nil { + log.WithError(err).Error("update entry failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) + return + } + } nDB, ok := ctx.(*NetworkDB) if ok { - if err := nDB.UpdateEntry(tname, nid, key, []byte(value)); err != nil { - diagnose.HTTPReplyError(w, err.Error(), "") + if err := nDB.UpdateEntry(tname, nid, key, decodedValue); err != nil { + log.WithError(err).Error("update entry failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) return } - fmt.Fprintf(w, "OK\n") + log.Info("update entry done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbDeleteEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + _, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("delete entry") + if len(r.Form["tname"]) < 1 || len(r.Form["nid"]) < 1 || len(r.Form["key"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path)) + log.Error("delete entry failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -148,20 +239,32 @@ func dbDeleteEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { if ok { err := nDB.DeleteEntry(tname, nid, key) if err != nil { - diagnose.HTTPReplyError(w, err.Error(), "") + log.WithError(err).Error("delete entry failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) return } - fmt.Fprintf(w, "OK\n") + log.Info("delete entry done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbGetEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + unsafe, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("get entry") + if len(r.Form["tname"]) < 1 || len(r.Form["nid"]) < 1 || len(r.Form["key"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path)) + log.Error("get entry failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -173,18 +276,39 @@ func dbGetEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) { if ok { value, err := nDB.GetEntry(tname, nid, key) if err != nil { - diagnose.HTTPReplyError(w, err.Error(), "") + log.WithError(err).Error("get entry failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) return } - fmt.Fprintf(w, "key:`%s` value:`%s`\n", key, string(value)) + + var encodedValue string + if unsafe { + encodedValue = string(value) + } else { + encodedValue = base64.StdEncoding.EncodeToString(value) + } + + rsp := &diagnose.TableEntryObj{Key: key, Value: encodedValue} + log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("update entry done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(rsp), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbJoinNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + _, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("join network") + if len(r.Form["nid"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path)) + log.Error("join network failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -193,18 +317,30 @@ func dbJoinNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) { nDB, ok := ctx.(*NetworkDB) if ok { if err := nDB.JoinNetwork(nid); err != nil { - diagnose.HTTPReplyError(w, err.Error(), "") + log.WithError(err).Error("join network failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) return } - fmt.Fprintf(w, "OK\n") + log.Info("join network done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbLeaveNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + _, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("leave network") + if len(r.Form["nid"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path)) + log.Error("leave network failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -213,19 +349,31 @@ func dbLeaveNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) { nDB, ok := ctx.(*NetworkDB) if ok { if err := nDB.LeaveNetwork(nid); err != nil { - diagnose.HTTPReplyError(w, err.Error(), "") + log.WithError(err).Error("leave network failed") + diagnose.HTTPReply(w, diagnose.FailCommand(err), json) return } - fmt.Fprintf(w, "OK\n") + log.Info("leave network done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json) + return } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } func dbGetTable(ctx interface{}, w http.ResponseWriter, r *http.Request) { r.ParseForm() diagnose.DebugHTTPForm(r) + unsafe, json := diagnose.ParseHTTPFormOptions(r) + + // audit logs + log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()}) + log.Info("get table") + if len(r.Form["tname"]) < 1 || len(r.Form["nid"]) < 1 { - diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id", r.URL.Path)) + rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id", r.URL.Path)) + log.Error("get table failed, wrong input") + diagnose.HTTPReply(w, rsp, json) return } @@ -235,20 +383,26 @@ func dbGetTable(ctx interface{}, w http.ResponseWriter, r *http.Request) { nDB, ok := ctx.(*NetworkDB) if ok { table := nDB.GetTableByNetwork(tname, nid) - fmt.Fprintf(w, "total elements: %d\n", len(table)) - i := 0 + rsp := &diagnose.TableObj{Length: len(table)} + var i = 0 for k, v := range table { - fmt.Fprintf(w, "%d) k:`%s` -> v:`%s`\n", i, k, string(v.([]byte))) - i++ + var encodedValue string + if unsafe { + encodedValue = string(v.Value) + } else { + encodedValue = base64.StdEncoding.EncodeToString(v.Value) + } + rsp.Elements = append(rsp.Elements, + &diagnose.TableEntryObj{ + Index: i, + Key: k, + Value: encodedValue, + Owner: v.owner, + }) } + log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("get table done") + diagnose.HTTPReply(w, diagnose.CommandSucceed(rsp), json) + return } -} - -func dbStackTrace(ctx interface{}, w http.ResponseWriter, r *http.Request) { - path, err := stackdump.DumpStacks("/tmp/") - if err != nil { - logrus.WithError(err).Error("failed to write goroutines dump") - } else { - fmt.Fprintf(w, "goroutine stacks written to %s", path) - } + diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json) } diff --git a/components/engine/vendor/github.com/docker/libnetwork/vendor.conf b/components/engine/vendor/github.com/docker/libnetwork/vendor.conf index f00d8ced44..bdcb5a1a54 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/vendor.conf +++ b/components/engine/vendor/github.com/docker/libnetwork/vendor.conf @@ -45,7 +45,7 @@ github.com/sirupsen/logrus v1.0.3 github.com/stretchr/testify dab07ac62d4905d3e48d17dc549c684ac3b7c15a github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16 github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065 -github.com/vishvananda/netlink bd6d5de5ccef2d66b0a26177928d0d8895d7f969 +github.com/vishvananda/netlink b2de5d10e38ecce8607e6b438b6d174f389a004e github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25 golang.org/x/crypto 558b6879de74bc843225cde5686419267ff707ca golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 From da56fc49673d3516c2e9ec2d39cc73768fe4b6bd Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Mon, 4 Dec 2017 11:38:20 -0800 Subject: [PATCH 13/16] Vendoring netlink library Signed-off-by: Flavio Crisciani Upstream-commit: 2e75d329773920713e63049ad1a00dfb64abf3c0 Component: engine --- components/engine/vendor.conf | 2 +- .../github.com/vishvananda/netlink/README.md | 10 +- .../vishvananda/netlink/addr_linux.go | 52 +++- .../vishvananda/netlink/bridge_linux.go | 115 +++++++ .../vishvananda/netlink/conntrack_linux.go | 45 ++- .../github.com/vishvananda/netlink/filter.go | 10 - .../vishvananda/netlink/filter_linux.go | 66 ++-- .../vishvananda/netlink/handle_linux.go | 41 ++- .../vishvananda/netlink/handle_unspecified.go | 4 + .../github.com/vishvananda/netlink/link.go | 26 ++ .../vishvananda/netlink/link_linux.go | 286 +++++++++++++++--- .../github.com/vishvananda/netlink/neigh.go | 1 + .../vishvananda/netlink/neigh_linux.go | 29 +- .../netlink/netlink_unspecified.go | 4 + .../vishvananda/netlink/nl/bridge_linux.go | 74 +++++ .../vishvananda/netlink/nl/conntrack_linux.go | 4 +- .../vishvananda/netlink/nl/link_linux.go | 36 ++- .../vishvananda/netlink/nl/nl_linux.go | 14 + .../vishvananda/netlink/qdisc_linux.go | 72 ++--- .../github.com/vishvananda/netlink/route.go | 62 +++- .../vishvananda/netlink/route_linux.go | 101 ++++++- .../github.com/vishvananda/netlink/rule.go | 1 + .../vishvananda/netlink/rule_linux.go | 3 + 23 files changed, 898 insertions(+), 160 deletions(-) create mode 100644 components/engine/vendor/github.com/vishvananda/netlink/bridge_linux.go create mode 100644 components/engine/vendor/github.com/vishvananda/netlink/nl/bridge_linux.go diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index f9cc8a7131..8c0b15a75f 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -42,7 +42,7 @@ github.com/hashicorp/go-multierror fcdddc395df1ddf4247c69bd436e84cfa0733f7e github.com/hashicorp/serf 598c54895cc5a7b1a24a398d635e8c0ea0959870 github.com/docker/libkv 1d8431073ae03cdaedb198a89722f3aab6d418ef github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25 -github.com/vishvananda/netlink bd6d5de5ccef2d66b0a26177928d0d8895d7f969 +github.com/vishvananda/netlink b2de5d10e38ecce8607e6b438b6d174f389a004e github.com/BurntSushi/toml f706d00e3de6abe700c994cdd545a1a4915af060 github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374 github.com/deckarep/golang-set ef32fa3046d9f249d399f98ebaf9be944430fd1d diff --git a/components/engine/vendor/github.com/vishvananda/netlink/README.md b/components/engine/vendor/github.com/vishvananda/netlink/README.md index 2367fae097..0b61be217e 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/README.md +++ b/components/engine/vendor/github.com/vishvananda/netlink/README.md @@ -38,15 +38,18 @@ Add a new bridge and add eth1 into it: package main import ( - "net" + "fmt" "github.com/vishvananda/netlink" ) func main() { la := netlink.NewLinkAttrs() la.Name = "foo" - mybridge := &netlink.Bridge{la}} - _ := netlink.LinkAdd(mybridge) + mybridge := &netlink.Bridge{LinkAttrs: la} + err := netlink.LinkAdd(mybridge) + if err != nil { + fmt.Printf("could not add %s: %v\n", la.Name, err) + } eth1, _ := netlink.LinkByName("eth1") netlink.LinkSetMaster(eth1, mybridge) } @@ -63,7 +66,6 @@ Add a new ip address to loopback: package main import ( - "net" "github.com/vishvananda/netlink" ) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/addr_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/addr_linux.go index f33242a7c2..8808b42d9b 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/addr_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/addr_linux.go @@ -2,7 +2,6 @@ package netlink import ( "fmt" - "log" "net" "strings" "syscall" @@ -65,7 +64,7 @@ func (h *Handle) addrHandle(link Link, addr *Addr, req *nl.NetlinkRequest) error msg := nl.NewIfAddrmsg(family) msg.Index = uint32(base.Index) msg.Scope = uint8(addr.Scope) - prefixlen, _ := addr.Mask.Size() + prefixlen, masklen := addr.Mask.Size() msg.Prefixlen = uint8(prefixlen) req.AddData(msg) @@ -103,9 +102,14 @@ func (h *Handle) addrHandle(link Link, addr *Addr, req *nl.NetlinkRequest) error } } - if addr.Broadcast != nil { - req.AddData(nl.NewRtAttr(syscall.IFA_BROADCAST, addr.Broadcast)) + if addr.Broadcast == nil { + calcBroadcast := make(net.IP, masklen/8) + for i := range localAddrData { + calcBroadcast[i] = localAddrData[i] | ^addr.Mask[i] + } + addr.Broadcast = calcBroadcast } + req.AddData(nl.NewRtAttr(syscall.IFA_BROADCAST, addr.Broadcast)) if addr.Label != "" { labelData := nl.NewRtAttr(syscall.IFA_LABEL, nl.ZeroTerminated(addr.Label)) @@ -232,16 +236,34 @@ type AddrUpdate struct { // AddrSubscribe takes a chan down which notifications will be sent // when addresses change. Close the 'done' chan to stop subscription. func AddrSubscribe(ch chan<- AddrUpdate, done <-chan struct{}) error { - return addrSubscribe(netns.None(), netns.None(), ch, done) + return addrSubscribeAt(netns.None(), netns.None(), ch, done, nil) } // AddrSubscribeAt works like AddrSubscribe plus it allows the caller // to choose the network namespace in which to subscribe (ns). func AddrSubscribeAt(ns netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}) error { - return addrSubscribe(ns, netns.None(), ch, done) + return addrSubscribeAt(ns, netns.None(), ch, done, nil) } -func addrSubscribe(newNs, curNs netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}) error { +// AddrSubscribeOptions contains a set of options to use with +// AddrSubscribeWithOptions. +type AddrSubscribeOptions struct { + Namespace *netns.NsHandle + ErrorCallback func(error) +} + +// AddrSubscribeWithOptions work like AddrSubscribe but enable to +// provide additional options to modify the behavior. Currently, the +// namespace can be provided as well as an error callback. +func AddrSubscribeWithOptions(ch chan<- AddrUpdate, done <-chan struct{}, options AddrSubscribeOptions) error { + if options.Namespace == nil { + none := netns.None() + options.Namespace = &none + } + return addrSubscribeAt(*options.Namespace, netns.None(), ch, done, options.ErrorCallback) +} + +func addrSubscribeAt(newNs, curNs netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}, cberr func(error)) error { s, err := nl.SubscribeAt(newNs, curNs, syscall.NETLINK_ROUTE, syscall.RTNLGRP_IPV4_IFADDR, syscall.RTNLGRP_IPV6_IFADDR) if err != nil { return err @@ -257,20 +279,26 @@ func addrSubscribe(newNs, curNs netns.NsHandle, ch chan<- AddrUpdate, done <-cha for { msgs, err := s.Receive() if err != nil { - log.Printf("netlink.AddrSubscribe: Receive() error: %v", err) + if cberr != nil { + cberr(err) + } return } for _, m := range msgs { msgType := m.Header.Type if msgType != syscall.RTM_NEWADDR && msgType != syscall.RTM_DELADDR { - log.Printf("netlink.AddrSubscribe: bad message type: %d", msgType) - continue + if cberr != nil { + cberr(fmt.Errorf("bad message type: %d", msgType)) + } + return } addr, _, ifindex, err := parseAddr(m.Data) if err != nil { - log.Printf("netlink.AddrSubscribe: could not parse address: %v", err) - continue + if cberr != nil { + cberr(fmt.Errorf("could not parse address: %v", err)) + } + return } ch <- AddrUpdate{LinkAddress: *addr.IPNet, diff --git a/components/engine/vendor/github.com/vishvananda/netlink/bridge_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/bridge_linux.go new file mode 100644 index 0000000000..a65d6a1319 --- /dev/null +++ b/components/engine/vendor/github.com/vishvananda/netlink/bridge_linux.go @@ -0,0 +1,115 @@ +package netlink + +import ( + "fmt" + "syscall" + + "github.com/vishvananda/netlink/nl" +) + +// BridgeVlanList gets a map of device id to bridge vlan infos. +// Equivalent to: `bridge vlan show` +func BridgeVlanList() (map[int32][]*nl.BridgeVlanInfo, error) { + return pkgHandle.BridgeVlanList() +} + +// BridgeVlanList gets a map of device id to bridge vlan infos. +// Equivalent to: `bridge vlan show` +func (h *Handle) BridgeVlanList() (map[int32][]*nl.BridgeVlanInfo, error) { + req := h.newNetlinkRequest(syscall.RTM_GETLINK, syscall.NLM_F_DUMP) + msg := nl.NewIfInfomsg(syscall.AF_BRIDGE) + req.AddData(msg) + req.AddData(nl.NewRtAttr(nl.IFLA_EXT_MASK, nl.Uint32Attr(uint32(nl.RTEXT_FILTER_BRVLAN)))) + + msgs, err := req.Execute(syscall.NETLINK_ROUTE, syscall.RTM_NEWLINK) + if err != nil { + return nil, err + } + ret := make(map[int32][]*nl.BridgeVlanInfo) + for _, m := range msgs { + msg := nl.DeserializeIfInfomsg(m) + + attrs, err := nl.ParseRouteAttr(m[msg.Len():]) + if err != nil { + return nil, err + } + for _, attr := range attrs { + switch attr.Attr.Type { + case nl.IFLA_AF_SPEC: + //nested attr + nestAttrs, err := nl.ParseRouteAttr(attr.Value) + if err != nil { + return nil, fmt.Errorf("failed to parse nested attr %v", err) + } + for _, nestAttr := range nestAttrs { + switch nestAttr.Attr.Type { + case nl.IFLA_BRIDGE_VLAN_INFO: + vlanInfo := nl.DeserializeBridgeVlanInfo(nestAttr.Value) + ret[msg.Index] = append(ret[msg.Index], vlanInfo) + } + } + } + } + } + return ret, nil +} + +// BridgeVlanAdd adds a new vlan filter entry +// Equivalent to: `bridge vlan add dev DEV vid VID [ pvid ] [ untagged ] [ self ] [ master ]` +func BridgeVlanAdd(link Link, vid uint16, pvid, untagged, self, master bool) error { + return pkgHandle.BridgeVlanAdd(link, vid, pvid, untagged, self, master) +} + +// BridgeVlanAdd adds a new vlan filter entry +// Equivalent to: `bridge vlan add dev DEV vid VID [ pvid ] [ untagged ] [ self ] [ master ]` +func (h *Handle) BridgeVlanAdd(link Link, vid uint16, pvid, untagged, self, master bool) error { + return h.bridgeVlanModify(syscall.RTM_SETLINK, link, vid, pvid, untagged, self, master) +} + +// BridgeVlanDel adds a new vlan filter entry +// Equivalent to: `bridge vlan del dev DEV vid VID [ pvid ] [ untagged ] [ self ] [ master ]` +func BridgeVlanDel(link Link, vid uint16, pvid, untagged, self, master bool) error { + return pkgHandle.BridgeVlanDel(link, vid, pvid, untagged, self, master) +} + +// BridgeVlanDel adds a new vlan filter entry +// Equivalent to: `bridge vlan del dev DEV vid VID [ pvid ] [ untagged ] [ self ] [ master ]` +func (h *Handle) BridgeVlanDel(link Link, vid uint16, pvid, untagged, self, master bool) error { + return h.bridgeVlanModify(syscall.RTM_DELLINK, link, vid, pvid, untagged, self, master) +} + +func (h *Handle) bridgeVlanModify(cmd int, link Link, vid uint16, pvid, untagged, self, master bool) error { + base := link.Attrs() + h.ensureIndex(base) + req := h.newNetlinkRequest(cmd, syscall.NLM_F_ACK) + + msg := nl.NewIfInfomsg(syscall.AF_BRIDGE) + msg.Index = int32(base.Index) + req.AddData(msg) + + br := nl.NewRtAttr(nl.IFLA_AF_SPEC, nil) + var flags uint16 + if self { + flags |= nl.BRIDGE_FLAGS_SELF + } + if master { + flags |= nl.BRIDGE_FLAGS_MASTER + } + if flags > 0 { + nl.NewRtAttrChild(br, nl.IFLA_BRIDGE_FLAGS, nl.Uint16Attr(flags)) + } + vlanInfo := &nl.BridgeVlanInfo{Vid: vid} + if pvid { + vlanInfo.Flags |= nl.BRIDGE_VLAN_INFO_PVID + } + if untagged { + vlanInfo.Flags |= nl.BRIDGE_VLAN_INFO_UNTAGGED + } + nl.NewRtAttrChild(br, nl.IFLA_BRIDGE_VLAN_INFO, vlanInfo.Serialize()) + req.AddData(br) + _, err := req.Execute(syscall.NETLINK_ROUTE, 0) + if err != nil { + return err + } + return nil +} diff --git a/components/engine/vendor/github.com/vishvananda/netlink/conntrack_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/conntrack_linux.go index 20df903006..ecf0445659 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/conntrack_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/conntrack_linux.go @@ -22,7 +22,11 @@ const ( // https://github.com/torvalds/linux/blob/master/include/uapi/linux/netfilter/nfnetlink.h -> #define NFNL_SUBSYS_CTNETLINK_EXP 2 ConntrackExpectTable = 2 ) - +const ( + // For Parsing Mark + TCP_PROTO = 6 + UDP_PROTO = 17 +) const ( // backward compatibility with golang 1.6 which does not have io.SeekCurrent seekCurrent = 1 @@ -56,7 +60,7 @@ func ConntrackTableFlush(table ConntrackTableType) error { // ConntrackDeleteFilter deletes entries on the specified table on the base of the filter // conntrack -D [table] parameters Delete conntrack or expectation -func ConntrackDeleteFilter(table ConntrackTableType, family InetFamily, filter *ConntrackFilter) (uint, error) { +func ConntrackDeleteFilter(table ConntrackTableType, family InetFamily, filter CustomConntrackFilter) (uint, error) { return pkgHandle.ConntrackDeleteFilter(table, family, filter) } @@ -88,7 +92,7 @@ func (h *Handle) ConntrackTableFlush(table ConntrackTableType) error { // ConntrackDeleteFilter deletes entries on the specified table on the base of the filter using the netlink handle passed // conntrack -D [table] parameters Delete conntrack or expectation -func (h *Handle) ConntrackDeleteFilter(table ConntrackTableType, family InetFamily, filter *ConntrackFilter) (uint, error) { +func (h *Handle) ConntrackDeleteFilter(table ConntrackTableType, family InetFamily, filter CustomConntrackFilter) (uint, error) { res, err := h.dumpConntrackTable(table, family) if err != nil { return 0, err @@ -142,15 +146,16 @@ type ConntrackFlow struct { FamilyType uint8 Forward ipTuple Reverse ipTuple + Mark uint32 } func (s *ConntrackFlow) String() string { // conntrack cmd output: - // udp 17 src=127.0.0.1 dst=127.0.0.1 sport=4001 dport=1234 [UNREPLIED] src=127.0.0.1 dst=127.0.0.1 sport=1234 dport=4001 - return fmt.Sprintf("%s\t%d src=%s dst=%s sport=%d dport=%d\tsrc=%s dst=%s sport=%d dport=%d", + // udp 17 src=127.0.0.1 dst=127.0.0.1 sport=4001 dport=1234 [UNREPLIED] src=127.0.0.1 dst=127.0.0.1 sport=1234 dport=4001 mark=0 + return fmt.Sprintf("%s\t%d src=%s dst=%s sport=%d dport=%d\tsrc=%s dst=%s sport=%d dport=%d mark=%d", nl.L4ProtoMap[s.Forward.Protocol], s.Forward.Protocol, s.Forward.SrcIP.String(), s.Forward.DstIP.String(), s.Forward.SrcPort, s.Forward.DstPort, - s.Reverse.SrcIP.String(), s.Reverse.DstIP.String(), s.Reverse.SrcPort, s.Reverse.DstPort) + s.Reverse.SrcIP.String(), s.Reverse.DstIP.String(), s.Reverse.SrcPort, s.Reverse.DstPort, s.Mark) } // This method parse the ip tuple structure @@ -160,7 +165,7 @@ func (s *ConntrackFlow) String() string { // // // -func parseIpTuple(reader *bytes.Reader, tpl *ipTuple) { +func parseIpTuple(reader *bytes.Reader, tpl *ipTuple) uint8 { for i := 0; i < 2; i++ { _, t, _, v := parseNfAttrTLV(reader) switch t { @@ -189,6 +194,7 @@ func parseIpTuple(reader *bytes.Reader, tpl *ipTuple) { // Skip some padding 2 byte reader.Seek(2, seekCurrent) } + return tpl.Protocol } func parseNfAttrTLV(r *bytes.Reader) (isNested bool, attrType, len uint16, value []byte) { @@ -216,6 +222,7 @@ func parseBERaw16(r *bytes.Reader, v *uint16) { func parseRawData(data []byte) *ConntrackFlow { s := &ConntrackFlow{} + var proto uint8 // First there is the Nfgenmsg header // consume only the family field reader := bytes.NewReader(data) @@ -234,7 +241,7 @@ func parseRawData(data []byte) *ConntrackFlow { nested, t, l := parseNfAttrTL(reader) if nested && t == nl.CTA_TUPLE_ORIG { if nested, t, _ = parseNfAttrTL(reader); nested && t == nl.CTA_TUPLE_IP { - parseIpTuple(reader, &s.Forward) + proto = parseIpTuple(reader, &s.Forward) } } else if nested && t == nl.CTA_TUPLE_REPLY { if nested, t, _ = parseNfAttrTL(reader); nested && t == nl.CTA_TUPLE_IP { @@ -248,7 +255,19 @@ func parseRawData(data []byte) *ConntrackFlow { } } } - + if proto == TCP_PROTO { + reader.Seek(64, seekCurrent) + _, t, _, v := parseNfAttrTLV(reader) + if t == nl.CTA_MARK { + s.Mark = uint32(v[3]) + } + } else if proto == UDP_PROTO { + reader.Seek(16, seekCurrent) + _, t, _, v := parseNfAttrTLV(reader) + if t == nl.CTA_MARK { + s.Mark = uint32(v[3]) + } + } return s } @@ -290,6 +309,12 @@ const ( ConntrackNatAnyIP // -any-nat ip Source or destination NAT ip ) +type CustomConntrackFilter interface { + // MatchConntrackFlow applies the filter to the flow and returns true if the flow matches + // the filter or false otherwise + MatchConntrackFlow(flow *ConntrackFlow) bool +} + type ConntrackFilter struct { ipFilter map[ConntrackFilterType]net.IP } @@ -342,3 +367,5 @@ func (f *ConntrackFilter) MatchConntrackFlow(flow *ConntrackFlow) bool { return match } + +var _ CustomConntrackFilter = (*ConntrackFilter)(nil) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/filter.go b/components/engine/vendor/github.com/vishvananda/netlink/filter.go index 938b28b0b0..1120c79d6a 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/filter.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/filter.go @@ -2,8 +2,6 @@ package netlink import ( "fmt" - - "github.com/vishvananda/netlink/nl" ) type Filter interface { @@ -184,14 +182,6 @@ func NewMirredAction(redirIndex int) *MirredAction { } } -// Constants used in TcU32Sel.Flags. -const ( - TC_U32_TERMINAL = nl.TC_U32_TERMINAL - TC_U32_OFFSET = nl.TC_U32_OFFSET - TC_U32_VAROFFSET = nl.TC_U32_VAROFFSET - TC_U32_EAT = nl.TC_U32_EAT -) - // Sel of the U32 filters that contains multiple TcU32Key. This is the copy // and the frontend representation of nl.TcU32Sel. It is serialized into canonical // nl.TcU32Sel with the appropriate endianness. diff --git a/components/engine/vendor/github.com/vishvananda/netlink/filter_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/filter_linux.go index dc0f90af88..5025bd56c1 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/filter_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/filter_linux.go @@ -11,6 +11,14 @@ import ( "github.com/vishvananda/netlink/nl" ) +// Constants used in TcU32Sel.Flags. +const ( + TC_U32_TERMINAL = nl.TC_U32_TERMINAL + TC_U32_OFFSET = nl.TC_U32_OFFSET + TC_U32_VAROFFSET = nl.TC_U32_VAROFFSET + TC_U32_EAT = nl.TC_U32_EAT +) + // Fw filter filters on firewall marks // NOTE: this is in filter_linux because it refers to nl.TcPolice which // is defined in nl/tc_linux.go @@ -128,9 +136,11 @@ func (h *Handle) FilterAdd(filter Filter) error { req.AddData(nl.NewRtAttr(nl.TCA_KIND, nl.ZeroTerminated(filter.Type()))) options := nl.NewRtAttr(nl.TCA_OPTIONS, nil) - if u32, ok := filter.(*U32); ok { + + switch filter := filter.(type) { + case *U32: // Convert TcU32Sel into nl.TcU32Sel as it is without copy. - sel := (*nl.TcU32Sel)(unsafe.Pointer(u32.Sel)) + sel := (*nl.TcU32Sel)(unsafe.Pointer(filter.Sel)) if sel == nil { // match all sel = &nl.TcU32Sel{ @@ -158,56 +168,56 @@ func (h *Handle) FilterAdd(filter Filter) error { } sel.Nkeys = uint8(len(sel.Keys)) nl.NewRtAttrChild(options, nl.TCA_U32_SEL, sel.Serialize()) - if u32.ClassId != 0 { - nl.NewRtAttrChild(options, nl.TCA_U32_CLASSID, nl.Uint32Attr(u32.ClassId)) + if filter.ClassId != 0 { + nl.NewRtAttrChild(options, nl.TCA_U32_CLASSID, nl.Uint32Attr(filter.ClassId)) } actionsAttr := nl.NewRtAttrChild(options, nl.TCA_U32_ACT, nil) // backwards compatibility - if u32.RedirIndex != 0 { - u32.Actions = append([]Action{NewMirredAction(u32.RedirIndex)}, u32.Actions...) + if filter.RedirIndex != 0 { + filter.Actions = append([]Action{NewMirredAction(filter.RedirIndex)}, filter.Actions...) } - if err := EncodeActions(actionsAttr, u32.Actions); err != nil { + if err := EncodeActions(actionsAttr, filter.Actions); err != nil { return err } - } else if fw, ok := filter.(*Fw); ok { - if fw.Mask != 0 { + case *Fw: + if filter.Mask != 0 { b := make([]byte, 4) - native.PutUint32(b, fw.Mask) + native.PutUint32(b, filter.Mask) nl.NewRtAttrChild(options, nl.TCA_FW_MASK, b) } - if fw.InDev != "" { - nl.NewRtAttrChild(options, nl.TCA_FW_INDEV, nl.ZeroTerminated(fw.InDev)) + if filter.InDev != "" { + nl.NewRtAttrChild(options, nl.TCA_FW_INDEV, nl.ZeroTerminated(filter.InDev)) } - if (fw.Police != nl.TcPolice{}) { + if (filter.Police != nl.TcPolice{}) { police := nl.NewRtAttrChild(options, nl.TCA_FW_POLICE, nil) - nl.NewRtAttrChild(police, nl.TCA_POLICE_TBF, fw.Police.Serialize()) - if (fw.Police.Rate != nl.TcRateSpec{}) { - payload := SerializeRtab(fw.Rtab) + nl.NewRtAttrChild(police, nl.TCA_POLICE_TBF, filter.Police.Serialize()) + if (filter.Police.Rate != nl.TcRateSpec{}) { + payload := SerializeRtab(filter.Rtab) nl.NewRtAttrChild(police, nl.TCA_POLICE_RATE, payload) } - if (fw.Police.PeakRate != nl.TcRateSpec{}) { - payload := SerializeRtab(fw.Ptab) + if (filter.Police.PeakRate != nl.TcRateSpec{}) { + payload := SerializeRtab(filter.Ptab) nl.NewRtAttrChild(police, nl.TCA_POLICE_PEAKRATE, payload) } } - if fw.ClassId != 0 { + if filter.ClassId != 0 { b := make([]byte, 4) - native.PutUint32(b, fw.ClassId) + native.PutUint32(b, filter.ClassId) nl.NewRtAttrChild(options, nl.TCA_FW_CLASSID, b) } - } else if bpf, ok := filter.(*BpfFilter); ok { + case *BpfFilter: var bpfFlags uint32 - if bpf.ClassId != 0 { - nl.NewRtAttrChild(options, nl.TCA_BPF_CLASSID, nl.Uint32Attr(bpf.ClassId)) + if filter.ClassId != 0 { + nl.NewRtAttrChild(options, nl.TCA_BPF_CLASSID, nl.Uint32Attr(filter.ClassId)) } - if bpf.Fd >= 0 { - nl.NewRtAttrChild(options, nl.TCA_BPF_FD, nl.Uint32Attr((uint32(bpf.Fd)))) + if filter.Fd >= 0 { + nl.NewRtAttrChild(options, nl.TCA_BPF_FD, nl.Uint32Attr((uint32(filter.Fd)))) } - if bpf.Name != "" { - nl.NewRtAttrChild(options, nl.TCA_BPF_NAME, nl.ZeroTerminated(bpf.Name)) + if filter.Name != "" { + nl.NewRtAttrChild(options, nl.TCA_BPF_NAME, nl.ZeroTerminated(filter.Name)) } - if bpf.DirectAction { + if filter.DirectAction { bpfFlags |= nl.TCA_BPF_FLAG_ACT_DIRECT } nl.NewRtAttrChild(options, nl.TCA_BPF_FLAGS, nl.Uint32Attr(bpfFlags)) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/handle_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/handle_linux.go index a04ceae6b6..d37b087c33 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/handle_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/handle_linux.go @@ -45,12 +45,27 @@ func (h *Handle) SetSocketTimeout(to time.Duration) error { } tv := syscall.NsecToTimeval(to.Nanoseconds()) for _, sh := range h.sockets { - fd := sh.Socket.GetFd() - err := syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, &tv) - if err != nil { + if err := sh.Socket.SetSendTimeout(&tv); err != nil { return err } - err = syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, syscall.SO_SNDTIMEO, &tv) + if err := sh.Socket.SetReceiveTimeout(&tv); err != nil { + return err + } + } + return nil +} + +// SetSocketReceiveBufferSize sets the receive buffer size for each +// socket in the netlink handle. The maximum value is capped by +// /proc/sys/net/core/rmem_max. +func (h *Handle) SetSocketReceiveBufferSize(size int, force bool) error { + opt := syscall.SO_RCVBUF + if force { + opt = syscall.SO_RCVBUFFORCE + } + for _, sh := range h.sockets { + fd := sh.Socket.GetFd() + err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, opt, size) if err != nil { return err } @@ -58,6 +73,24 @@ func (h *Handle) SetSocketTimeout(to time.Duration) error { return nil } +// GetSocketReceiveBufferSize gets the receiver buffer size for each +// socket in the netlink handle. The retrieved value should be the +// double to the one set for SetSocketReceiveBufferSize. +func (h *Handle) GetSocketReceiveBufferSize() ([]int, error) { + results := make([]int, len(h.sockets)) + i := 0 + for _, sh := range h.sockets { + fd := sh.Socket.GetFd() + size, err := syscall.GetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF) + if err != nil { + return nil, err + } + results[i] = size + i++ + } + return results, nil +} + // NewHandle returns a netlink handle on the network namespace // specified by ns. If ns=netns.None(), current network namespace // will be assumed diff --git a/components/engine/vendor/github.com/vishvananda/netlink/handle_unspecified.go b/components/engine/vendor/github.com/vishvananda/netlink/handle_unspecified.go index 32cf022732..7da21a6a18 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/handle_unspecified.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/handle_unspecified.go @@ -145,6 +145,10 @@ func (h *Handle) LinkSetFlood(link Link, mode bool) error { return ErrNotImplemented } +func (h *Handle) LinkSetTxQLen(link Link, qlen int) error { + return ErrNotImplemented +} + func (h *Handle) setProtinfoAttr(link Link, mode bool, attr int) error { return ErrNotImplemented } diff --git a/components/engine/vendor/github.com/vishvananda/netlink/link.go b/components/engine/vendor/github.com/vishvananda/netlink/link.go index 547e92ec12..5aa3a1790a 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/link.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/link.go @@ -37,6 +37,7 @@ type LinkAttrs struct { EncapType string Protinfo *Protinfo OperState LinkOperState + NetNsID int } // LinkOperState represents the values of the IFLA_OPERSTATE link @@ -171,6 +172,7 @@ type LinkXdp struct { Fd int Attached bool Flags uint32 + ProgId uint32 } // Device links cannot be created via netlink. These links @@ -339,6 +341,7 @@ type Vxlan struct { UDPCSum bool NoAge bool GBP bool + FlowBased bool Age int Limit int Port int @@ -684,6 +687,7 @@ type Gretap struct { EncapType uint16 EncapFlags uint16 Link uint32 + FlowBased bool } func (gretap *Gretap) Attrs() *LinkAttrs { @@ -729,6 +733,28 @@ func (iptun *Vti) Type() string { return "vti" } +type Gretun struct { + LinkAttrs + Link uint32 + IFlags uint16 + OFlags uint16 + IKey uint32 + OKey uint32 + Local net.IP + Remote net.IP + Ttl uint8 + Tos uint8 + PMtuDisc uint8 +} + +func (gretun *Gretun) Attrs() *LinkAttrs { + return &gretun.LinkAttrs +} + +func (gretun *Gretun) Type() string { + return "gre" +} + type Vrf struct { LinkAttrs Table uint32 diff --git a/components/engine/vendor/github.com/vishvananda/netlink/link_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/link_linux.go index 1c1bc52c4d..e94fd9766c 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/link_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/link_linux.go @@ -379,6 +379,74 @@ func (h *Handle) LinkSetVfTxRate(link Link, vf, rate int) error { return err } +// LinkSetVfSpoofchk enables/disables spoof check on a vf for the link. +// Equivalent to: `ip link set $link vf $vf spoofchk $check` +func LinkSetVfSpoofchk(link Link, vf int, check bool) error { + return pkgHandle.LinkSetVfSpoofchk(link, vf, check) +} + +// LinkSetVfSpookfchk enables/disables spoof check on a vf for the link. +// Equivalent to: `ip link set $link vf $vf spoofchk $check` +func (h *Handle) LinkSetVfSpoofchk(link Link, vf int, check bool) error { + var setting uint32 + base := link.Attrs() + h.ensureIndex(base) + req := h.newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) + + msg := nl.NewIfInfomsg(syscall.AF_UNSPEC) + msg.Index = int32(base.Index) + req.AddData(msg) + + data := nl.NewRtAttr(nl.IFLA_VFINFO_LIST, nil) + info := nl.NewRtAttrChild(data, nl.IFLA_VF_INFO, nil) + if check { + setting = 1 + } + vfmsg := nl.VfSpoofchk{ + Vf: uint32(vf), + Setting: setting, + } + nl.NewRtAttrChild(info, nl.IFLA_VF_SPOOFCHK, vfmsg.Serialize()) + req.AddData(data) + + _, err := req.Execute(syscall.NETLINK_ROUTE, 0) + return err +} + +// LinkSetVfTrust enables/disables trust state on a vf for the link. +// Equivalent to: `ip link set $link vf $vf trust $state` +func LinkSetVfTrust(link Link, vf int, state bool) error { + return pkgHandle.LinkSetVfTrust(link, vf, state) +} + +// LinkSetVfTrust enables/disables trust state on a vf for the link. +// Equivalent to: `ip link set $link vf $vf trust $state` +func (h *Handle) LinkSetVfTrust(link Link, vf int, state bool) error { + var setting uint32 + base := link.Attrs() + h.ensureIndex(base) + req := h.newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) + + msg := nl.NewIfInfomsg(syscall.AF_UNSPEC) + msg.Index = int32(base.Index) + req.AddData(msg) + + data := nl.NewRtAttr(nl.IFLA_VFINFO_LIST, nil) + info := nl.NewRtAttrChild(data, nl.IFLA_VF_INFO, nil) + if state { + setting = 1 + } + vfmsg := nl.VfTrust{ + Vf: uint32(vf), + Setting: setting, + } + nl.NewRtAttrChild(info, nl.IFLA_VF_TRUST, vfmsg.Serialize()) + req.AddData(data) + + _, err := req.Execute(syscall.NETLINK_ROUTE, 0) + return err +} + // LinkSetMaster sets the master of the link device. // Equivalent to: `ip link set $link master $master` func LinkSetMaster(link Link, master *Bridge) error { @@ -500,6 +568,12 @@ func (h *Handle) LinkSetNsFd(link Link, fd int) error { // LinkSetXdpFd adds a bpf function to the driver. The fd must be a bpf // program loaded with bpf(type=BPF_PROG_TYPE_XDP) func LinkSetXdpFd(link Link, fd int) error { + return LinkSetXdpFdWithFlags(link, fd, 0) +} + +// LinkSetXdpFdWithFlags adds a bpf function to the driver with the given +// options. The fd must be a bpf program loaded with bpf(type=BPF_PROG_TYPE_XDP) +func LinkSetXdpFdWithFlags(link Link, fd, flags int) error { base := link.Attrs() ensureIndex(base) req := nl.NewNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) @@ -508,7 +582,7 @@ func LinkSetXdpFd(link Link, fd int) error { msg.Index = int32(base.Index) req.AddData(msg) - addXdpAttrs(&LinkXdp{Fd: fd}, req) + addXdpAttrs(&LinkXdp{Fd: fd, Flags: uint32(flags)}, req) _, err := req.Execute(syscall.NETLINK_ROUTE, 0) return err @@ -528,7 +602,13 @@ type vxlanPortRange struct { func addVxlanAttrs(vxlan *Vxlan, linkInfo *nl.RtAttr) { data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) + + if vxlan.FlowBased { + vxlan.VxlanId = 0 + } + nl.NewRtAttrChild(data, nl.IFLA_VXLAN_ID, nl.Uint32Attr(uint32(vxlan.VxlanId))) + if vxlan.VtepDevIndex != 0 { nl.NewRtAttrChild(data, nl.IFLA_VXLAN_LINK, nl.Uint32Attr(uint32(vxlan.VtepDevIndex))) } @@ -569,6 +649,9 @@ func addVxlanAttrs(vxlan *Vxlan, linkInfo *nl.RtAttr) { if vxlan.GBP { nl.NewRtAttrChild(data, nl.IFLA_VXLAN_GBP, []byte{}) } + if vxlan.FlowBased { + nl.NewRtAttrChild(data, nl.IFLA_VXLAN_FLOWBASED, boolAttr(vxlan.FlowBased)) + } if vxlan.NoAge { nl.NewRtAttrChild(data, nl.IFLA_VXLAN_AGEING, nl.Uint32Attr(0)) } else if vxlan.Age > 0 { @@ -818,16 +901,17 @@ func (h *Handle) linkModify(link Link, flags int) error { linkInfo := nl.NewRtAttr(syscall.IFLA_LINKINFO, nil) nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_KIND, nl.NonZeroTerminated(link.Type())) - if vlan, ok := link.(*Vlan); ok { + switch link := link.(type) { + case *Vlan: b := make([]byte, 2) - native.PutUint16(b, uint16(vlan.VlanId)) + native.PutUint16(b, uint16(link.VlanId)) data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) nl.NewRtAttrChild(data, nl.IFLA_VLAN_ID, b) - } else if veth, ok := link.(*Veth); ok { + case *Veth: data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) peer := nl.NewRtAttrChild(data, nl.VETH_INFO_PEER, nil) nl.NewIfInfomsgChild(peer, syscall.AF_UNSPEC) - nl.NewRtAttrChild(peer, syscall.IFLA_IFNAME, nl.ZeroTerminated(veth.PeerName)) + nl.NewRtAttrChild(peer, syscall.IFLA_IFNAME, nl.ZeroTerminated(link.PeerName)) if base.TxQLen >= 0 { nl.NewRtAttrChild(peer, syscall.IFLA_TXQLEN, nl.Uint32Attr(uint32(base.TxQLen))) } @@ -835,35 +919,37 @@ func (h *Handle) linkModify(link Link, flags int) error { nl.NewRtAttrChild(peer, syscall.IFLA_MTU, nl.Uint32Attr(uint32(base.MTU))) } - } else if vxlan, ok := link.(*Vxlan); ok { - addVxlanAttrs(vxlan, linkInfo) - } else if bond, ok := link.(*Bond); ok { - addBondAttrs(bond, linkInfo) - } else if ipv, ok := link.(*IPVlan); ok { + case *Vxlan: + addVxlanAttrs(link, linkInfo) + case *Bond: + addBondAttrs(link, linkInfo) + case *IPVlan: data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) - nl.NewRtAttrChild(data, nl.IFLA_IPVLAN_MODE, nl.Uint16Attr(uint16(ipv.Mode))) - } else if macv, ok := link.(*Macvlan); ok { - if macv.Mode != MACVLAN_MODE_DEFAULT { + nl.NewRtAttrChild(data, nl.IFLA_IPVLAN_MODE, nl.Uint16Attr(uint16(link.Mode))) + case *Macvlan: + if link.Mode != MACVLAN_MODE_DEFAULT { data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) - nl.NewRtAttrChild(data, nl.IFLA_MACVLAN_MODE, nl.Uint32Attr(macvlanModes[macv.Mode])) + nl.NewRtAttrChild(data, nl.IFLA_MACVLAN_MODE, nl.Uint32Attr(macvlanModes[link.Mode])) } - } else if macv, ok := link.(*Macvtap); ok { - if macv.Mode != MACVLAN_MODE_DEFAULT { + case *Macvtap: + if link.Mode != MACVLAN_MODE_DEFAULT { data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) - nl.NewRtAttrChild(data, nl.IFLA_MACVLAN_MODE, nl.Uint32Attr(macvlanModes[macv.Mode])) + nl.NewRtAttrChild(data, nl.IFLA_MACVLAN_MODE, nl.Uint32Attr(macvlanModes[link.Mode])) } - } else if gretap, ok := link.(*Gretap); ok { - addGretapAttrs(gretap, linkInfo) - } else if iptun, ok := link.(*Iptun); ok { - addIptunAttrs(iptun, linkInfo) - } else if vti, ok := link.(*Vti); ok { - addVtiAttrs(vti, linkInfo) - } else if vrf, ok := link.(*Vrf); ok { - addVrfAttrs(vrf, linkInfo) - } else if bridge, ok := link.(*Bridge); ok { - addBridgeAttrs(bridge, linkInfo) - } else if gtp, ok := link.(*GTP); ok { - addGTPAttrs(gtp, linkInfo) + case *Gretap: + addGretapAttrs(link, linkInfo) + case *Iptun: + addIptunAttrs(link, linkInfo) + case *Gretun: + addGretunAttrs(link, linkInfo) + case *Vti: + addVtiAttrs(link, linkInfo) + case *Vrf: + addVrfAttrs(link, linkInfo) + case *Bridge: + addBridgeAttrs(link, linkInfo) + case *GTP: + addGTPAttrs(link, linkInfo) } req.AddData(linkInfo) @@ -1093,6 +1179,8 @@ func LinkDeserialize(hdr *syscall.NlMsghdr, m []byte) (Link, error) { link = &Gretap{} case "ipip": link = &Iptun{} + case "gre": + link = &Gretun{} case "vti": link = &Vti{} case "vrf": @@ -1124,6 +1212,8 @@ func LinkDeserialize(hdr *syscall.NlMsghdr, m []byte) (Link, error) { parseGretapData(link, data) case "ipip": parseIptunData(link, data) + case "gre": + parseGretunData(link, data) case "vti": parseVtiData(link, data) case "vrf": @@ -1178,6 +1268,8 @@ func LinkDeserialize(hdr *syscall.NlMsghdr, m []byte) (Link, error) { } case syscall.IFLA_OPERSTATE: base.OperState = LinkOperState(uint8(attr.Value[0])) + case nl.IFLA_LINK_NETNSID: + base.NetNsID = int(native.Uint32(attr.Value[0:4])) } } @@ -1239,16 +1331,34 @@ type LinkUpdate struct { // LinkSubscribe takes a chan down which notifications will be sent // when links change. Close the 'done' chan to stop subscription. func LinkSubscribe(ch chan<- LinkUpdate, done <-chan struct{}) error { - return linkSubscribe(netns.None(), netns.None(), ch, done) + return linkSubscribeAt(netns.None(), netns.None(), ch, done, nil) } // LinkSubscribeAt works like LinkSubscribe plus it allows the caller // to choose the network namespace in which to subscribe (ns). func LinkSubscribeAt(ns netns.NsHandle, ch chan<- LinkUpdate, done <-chan struct{}) error { - return linkSubscribe(ns, netns.None(), ch, done) + return linkSubscribeAt(ns, netns.None(), ch, done, nil) } -func linkSubscribe(newNs, curNs netns.NsHandle, ch chan<- LinkUpdate, done <-chan struct{}) error { +// LinkSubscribeOptions contains a set of options to use with +// LinkSubscribeWithOptions. +type LinkSubscribeOptions struct { + Namespace *netns.NsHandle + ErrorCallback func(error) +} + +// LinkSubscribeWithOptions work like LinkSubscribe but enable to +// provide additional options to modify the behavior. Currently, the +// namespace can be provided as well as an error callback. +func LinkSubscribeWithOptions(ch chan<- LinkUpdate, done <-chan struct{}, options LinkSubscribeOptions) error { + if options.Namespace == nil { + none := netns.None() + options.Namespace = &none + } + return linkSubscribeAt(*options.Namespace, netns.None(), ch, done, options.ErrorCallback) +} + +func linkSubscribeAt(newNs, curNs netns.NsHandle, ch chan<- LinkUpdate, done <-chan struct{}, cberr func(error)) error { s, err := nl.SubscribeAt(newNs, curNs, syscall.NETLINK_ROUTE, syscall.RTNLGRP_LINK) if err != nil { return err @@ -1264,12 +1374,18 @@ func linkSubscribe(newNs, curNs netns.NsHandle, ch chan<- LinkUpdate, done <-cha for { msgs, err := s.Receive() if err != nil { + if cberr != nil { + cberr(err) + } return } for _, m := range msgs { ifmsg := nl.DeserializeIfInfomsg(m.Data) link, err := LinkDeserialize(&m.Header, m.Data) if err != nil { + if cberr != nil { + cberr(err) + } return } ch <- LinkUpdate{IfInfomsg: *ifmsg, Header: m.Header, Link: link} @@ -1363,6 +1479,33 @@ func (h *Handle) setProtinfoAttr(link Link, mode bool, attr int) error { return nil } +// LinkSetTxQLen sets the transaction queue length for the link. +// Equivalent to: `ip link set $link txqlen $qlen` +func LinkSetTxQLen(link Link, qlen int) error { + return pkgHandle.LinkSetTxQLen(link, qlen) +} + +// LinkSetTxQLen sets the transaction queue length for the link. +// Equivalent to: `ip link set $link txqlen $qlen` +func (h *Handle) LinkSetTxQLen(link Link, qlen int) error { + base := link.Attrs() + h.ensureIndex(base) + req := h.newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) + + msg := nl.NewIfInfomsg(syscall.AF_UNSPEC) + msg.Index = int32(base.Index) + req.AddData(msg) + + b := make([]byte, 4) + native.PutUint32(b, uint32(qlen)) + + data := nl.NewRtAttr(syscall.IFLA_TXQLEN, b) + req.AddData(data) + + _, err := req.Execute(syscall.NETLINK_ROUTE, 0) + return err +} + func parseVlanData(link Link, data []syscall.NetlinkRouteAttr) { vlan := link.(*Vlan) for _, datum := range data { @@ -1407,6 +1550,8 @@ func parseVxlanData(link Link, data []syscall.NetlinkRouteAttr) { vxlan.UDPCSum = int8(datum.Value[0]) != 0 case nl.IFLA_VXLAN_GBP: vxlan.GBP = true + case nl.IFLA_VXLAN_FLOWBASED: + vxlan.FlowBased = int8(datum.Value[0]) != 0 case nl.IFLA_VXLAN_AGEING: vxlan.Age = int(native.Uint32(datum.Value[0:4])) vxlan.NoAge = vxlan.Age == 0 @@ -1547,6 +1692,12 @@ func linkFlags(rawFlags uint32) net.Flags { func addGretapAttrs(gretap *Gretap, linkInfo *nl.RtAttr) { data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) + if gretap.FlowBased { + // In flow based mode, no other attributes need to be configured + nl.NewRtAttrChild(data, nl.IFLA_GRE_COLLECT_METADATA, boolAttr(gretap.FlowBased)) + return + } + ip := gretap.Local.To4() if ip != nil { nl.NewRtAttrChild(data, nl.IFLA_GRE_LOCAL, []byte(ip)) @@ -1613,6 +1764,69 @@ func parseGretapData(link Link, data []syscall.NetlinkRouteAttr) { gre.EncapType = native.Uint16(datum.Value[0:2]) case nl.IFLA_GRE_ENCAP_FLAGS: gre.EncapFlags = native.Uint16(datum.Value[0:2]) + case nl.IFLA_GRE_COLLECT_METADATA: + gre.FlowBased = int8(datum.Value[0]) != 0 + } + } +} + +func addGretunAttrs(gre *Gretun, linkInfo *nl.RtAttr) { + data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil) + + ip := gre.Local.To4() + if ip != nil { + nl.NewRtAttrChild(data, nl.IFLA_GRE_LOCAL, []byte(ip)) + } + ip = gre.Remote.To4() + if ip != nil { + nl.NewRtAttrChild(data, nl.IFLA_GRE_REMOTE, []byte(ip)) + } + + if gre.IKey != 0 { + nl.NewRtAttrChild(data, nl.IFLA_GRE_IKEY, htonl(gre.IKey)) + gre.IFlags |= uint16(nl.GRE_KEY) + } + + if gre.OKey != 0 { + nl.NewRtAttrChild(data, nl.IFLA_GRE_OKEY, htonl(gre.OKey)) + gre.OFlags |= uint16(nl.GRE_KEY) + } + + nl.NewRtAttrChild(data, nl.IFLA_GRE_IFLAGS, htons(gre.IFlags)) + nl.NewRtAttrChild(data, nl.IFLA_GRE_OFLAGS, htons(gre.OFlags)) + + if gre.Link != 0 { + nl.NewRtAttrChild(data, nl.IFLA_GRE_LINK, nl.Uint32Attr(gre.Link)) + } + + nl.NewRtAttrChild(data, nl.IFLA_GRE_PMTUDISC, nl.Uint8Attr(gre.PMtuDisc)) + nl.NewRtAttrChild(data, nl.IFLA_GRE_TTL, nl.Uint8Attr(gre.Ttl)) + nl.NewRtAttrChild(data, nl.IFLA_GRE_TOS, nl.Uint8Attr(gre.Tos)) +} + +func parseGretunData(link Link, data []syscall.NetlinkRouteAttr) { + gre := link.(*Gretun) + for _, datum := range data { + switch datum.Attr.Type { + case nl.IFLA_GRE_OKEY: + gre.IKey = ntohl(datum.Value[0:4]) + case nl.IFLA_GRE_IKEY: + gre.OKey = ntohl(datum.Value[0:4]) + case nl.IFLA_GRE_LOCAL: + gre.Local = net.IP(datum.Value[0:4]) + case nl.IFLA_GRE_REMOTE: + gre.Remote = net.IP(datum.Value[0:4]) + case nl.IFLA_GRE_IFLAGS: + gre.IFlags = ntohs(datum.Value[0:2]) + case nl.IFLA_GRE_OFLAGS: + gre.OFlags = ntohs(datum.Value[0:2]) + + case nl.IFLA_GRE_TTL: + gre.Ttl = uint8(datum.Value[0]) + case nl.IFLA_GRE_TOS: + gre.Tos = uint8(datum.Value[0]) + case nl.IFLA_GRE_PMTUDISC: + gre.PMtuDisc = uint8(datum.Value[0]) } } } @@ -1630,8 +1844,10 @@ func addXdpAttrs(xdp *LinkXdp, req *nl.NetlinkRequest) { b := make([]byte, 4) native.PutUint32(b, uint32(xdp.Fd)) nl.NewRtAttrChild(attrs, nl.IFLA_XDP_FD, b) - native.PutUint32(b, xdp.Flags) - nl.NewRtAttrChild(attrs, nl.IFLA_XDP_FLAGS, b) + if xdp.Flags != 0 { + native.PutUint32(b, xdp.Flags) + nl.NewRtAttrChild(attrs, nl.IFLA_XDP_FLAGS, b) + } req.AddData(attrs) } @@ -1649,6 +1865,8 @@ func parseLinkXdp(data []byte) (*LinkXdp, error) { xdp.Attached = attr.Value[0] != 0 case nl.IFLA_XDP_FLAGS: xdp.Flags = native.Uint32(attr.Value[0:4]) + case nl.IFLA_XDP_PROG_ID: + xdp.ProgId = native.Uint32(attr.Value[0:4]) } } return xdp, nil diff --git a/components/engine/vendor/github.com/vishvananda/netlink/neigh.go b/components/engine/vendor/github.com/vishvananda/netlink/neigh.go index 0e5eb90c9e..6a6f71ce86 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/neigh.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/neigh.go @@ -14,6 +14,7 @@ type Neigh struct { Flags int IP net.IP HardwareAddr net.HardwareAddr + LLIPAddr net.IP //Used in the case of NHRP } // String returns $ip/$hwaddr $label diff --git a/components/engine/vendor/github.com/vishvananda/netlink/neigh_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/neigh_linux.go index f069db2573..5edc8b4125 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/neigh_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/neigh_linux.go @@ -128,6 +128,7 @@ func (h *Handle) NeighDel(neigh *Neigh) error { func neighHandle(neigh *Neigh, req *nl.NetlinkRequest) error { var family int + if neigh.Family > 0 { family = neigh.Family } else { @@ -151,7 +152,10 @@ func neighHandle(neigh *Neigh, req *nl.NetlinkRequest) error { dstData := nl.NewRtAttr(NDA_DST, ipData) req.AddData(dstData) - if neigh.Flags != NTF_PROXY || neigh.HardwareAddr != nil { + if neigh.LLIPAddr != nil { + llIPData := nl.NewRtAttr(NDA_LLADDR, neigh.LLIPAddr.To4()) + req.AddData(llIPData) + } else if neigh.Flags != NTF_PROXY || neigh.HardwareAddr != nil { hwData := nl.NewRtAttr(NDA_LLADDR, []byte(neigh.HardwareAddr)) req.AddData(hwData) } @@ -237,12 +241,33 @@ func NeighDeserialize(m []byte) (*Neigh, error) { return nil, err } + // This should be cached for perfomance + // once per table dump + link, err := LinkByIndex(neigh.LinkIndex) + if err != nil { + return nil, err + } + encapType := link.Attrs().EncapType + for _, attr := range attrs { switch attr.Attr.Type { case NDA_DST: neigh.IP = net.IP(attr.Value) case NDA_LLADDR: - neigh.HardwareAddr = net.HardwareAddr(attr.Value) + // BUG: Is this a bug in the netlink library? + // #define RTA_LENGTH(len) (RTA_ALIGN(sizeof(struct rtattr)) + (len)) + // #define RTA_PAYLOAD(rta) ((int)((rta)->rta_len) - RTA_LENGTH(0)) + attrLen := attr.Attr.Len - syscall.SizeofRtAttr + if attrLen == 4 && (encapType == "ipip" || + encapType == "sit" || + encapType == "gre") { + neigh.LLIPAddr = net.IP(attr.Value) + } else if attrLen == 16 && + encapType == "tunnel6" { + neigh.IP = net.IP(attr.Value) + } else { + neigh.HardwareAddr = net.HardwareAddr(attr.Value) + } } } diff --git a/components/engine/vendor/github.com/vishvananda/netlink/netlink_unspecified.go b/components/engine/vendor/github.com/vishvananda/netlink/netlink_unspecified.go index 2d57c16d74..86111b92ce 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/netlink_unspecified.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/netlink_unspecified.go @@ -108,6 +108,10 @@ func LinkSetFlood(link Link, mode bool) error { return ErrNotImplemented } +func LinkSetTxQLen(link Link, qlen int) error { + return ErrNotImplemented +} + func LinkAdd(link Link) error { return ErrNotImplemented } diff --git a/components/engine/vendor/github.com/vishvananda/netlink/nl/bridge_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/nl/bridge_linux.go new file mode 100644 index 0000000000..6c0d333387 --- /dev/null +++ b/components/engine/vendor/github.com/vishvananda/netlink/nl/bridge_linux.go @@ -0,0 +1,74 @@ +package nl + +import ( + "fmt" + "unsafe" +) + +const ( + SizeofBridgeVlanInfo = 0x04 +) + +/* Bridge Flags */ +const ( + BRIDGE_FLAGS_MASTER = iota /* Bridge command to/from master */ + BRIDGE_FLAGS_SELF /* Bridge command to/from lowerdev */ +) + +/* Bridge management nested attributes + * [IFLA_AF_SPEC] = { + * [IFLA_BRIDGE_FLAGS] + * [IFLA_BRIDGE_MODE] + * [IFLA_BRIDGE_VLAN_INFO] + * } + */ +const ( + IFLA_BRIDGE_FLAGS = iota + IFLA_BRIDGE_MODE + IFLA_BRIDGE_VLAN_INFO +) + +const ( + BRIDGE_VLAN_INFO_MASTER = 1 << iota + BRIDGE_VLAN_INFO_PVID + BRIDGE_VLAN_INFO_UNTAGGED + BRIDGE_VLAN_INFO_RANGE_BEGIN + BRIDGE_VLAN_INFO_RANGE_END +) + +// struct bridge_vlan_info { +// __u16 flags; +// __u16 vid; +// }; + +type BridgeVlanInfo struct { + Flags uint16 + Vid uint16 +} + +func (b *BridgeVlanInfo) Serialize() []byte { + return (*(*[SizeofBridgeVlanInfo]byte)(unsafe.Pointer(b)))[:] +} + +func DeserializeBridgeVlanInfo(b []byte) *BridgeVlanInfo { + return (*BridgeVlanInfo)(unsafe.Pointer(&b[0:SizeofBridgeVlanInfo][0])) +} + +func (b *BridgeVlanInfo) PortVID() bool { + return b.Flags&BRIDGE_VLAN_INFO_PVID > 0 +} + +func (b *BridgeVlanInfo) EngressUntag() bool { + return b.Flags&BRIDGE_VLAN_INFO_UNTAGGED > 0 +} + +func (b *BridgeVlanInfo) String() string { + return fmt.Sprintf("%+v", *b) +} + +/* New extended info filters for IFLA_EXT_MASK */ +const ( + RTEXT_FILTER_VF = 1 << iota + RTEXT_FILTER_BRVLAN + RTEXT_FILTER_BRVLAN_COMPRESSED +) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/nl/conntrack_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/nl/conntrack_linux.go index 6692b53ed9..380cc5967b 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/nl/conntrack_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/nl/conntrack_linux.go @@ -79,8 +79,8 @@ const ( CTA_TUPLE_ORIG = 1 CTA_TUPLE_REPLY = 2 CTA_STATUS = 3 - CTA_TIMEOUT = 8 - CTA_MARK = 9 + CTA_TIMEOUT = 7 + CTA_MARK = 8 CTA_PROTOINFO = 4 ) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/nl/link_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/nl/link_linux.go index f7b9575919..9ae65a12c2 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/nl/link_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/nl/link_linux.go @@ -231,7 +231,8 @@ const ( * on/off switch */ IFLA_VF_STATS /* network device statistics */ - IFLA_VF_MAX = IFLA_VF_STATS + IFLA_VF_TRUST /* Trust state of VF */ + IFLA_VF_MAX = IFLA_VF_TRUST ) const ( @@ -259,6 +260,7 @@ const ( SizeofVfSpoofchk = 0x08 SizeofVfLinkState = 0x08 SizeofVfRssQueryEn = 0x08 + SizeofVfTrust = 0x08 ) // struct ifla_vf_mac { @@ -419,12 +421,42 @@ func (msg *VfRssQueryEn) Serialize() []byte { return (*(*[SizeofVfRssQueryEn]byte)(unsafe.Pointer(msg)))[:] } +// struct ifla_vf_trust { +// __u32 vf; +// __u32 setting; +// }; + +type VfTrust struct { + Vf uint32 + Setting uint32 +} + +func (msg *VfTrust) Len() int { + return SizeofVfTrust +} + +func DeserializeVfTrust(b []byte) *VfTrust { + return (*VfTrust)(unsafe.Pointer(&b[0:SizeofVfTrust][0])) +} + +func (msg *VfTrust) Serialize() []byte { + return (*(*[SizeofVfTrust]byte)(unsafe.Pointer(msg)))[:] +} + +const ( + XDP_FLAGS_UPDATE_IF_NOEXIST = 1 << iota + XDP_FLAGS_SKB_MODE + XDP_FLAGS_DRV_MODE + XDP_FLAGS_MASK = XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_SKB_MODE | XDP_FLAGS_DRV_MODE +) + const ( IFLA_XDP_UNSPEC = iota IFLA_XDP_FD /* fd of xdp program to attach, or -1 to remove */ IFLA_XDP_ATTACHED /* read-only bool indicating if prog is attached */ IFLA_XDP_FLAGS /* xdp prog related flags */ - IFLA_XDP_MAX = IFLA_XDP_FLAGS + IFLA_XDP_PROG_ID /* xdp prog id */ + IFLA_XDP_MAX = IFLA_XDP_PROG_ID ) const ( diff --git a/components/engine/vendor/github.com/vishvananda/netlink/nl/nl_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/nl/nl_linux.go index 1329acd864..72f7f6af3c 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/nl/nl_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/nl/nl_linux.go @@ -621,6 +621,20 @@ func (s *NetlinkSocket) Receive() ([]syscall.NetlinkMessage, error) { return syscall.ParseNetlinkMessage(rb) } +// SetSendTimeout allows to set a send timeout on the socket +func (s *NetlinkSocket) SetSendTimeout(timeout *syscall.Timeval) error { + // Set a send timeout of SOCKET_SEND_TIMEOUT, this will allow the Send to periodically unblock and avoid that a routine + // remains stuck on a send on a closed fd + return syscall.SetsockoptTimeval(int(s.fd), syscall.SOL_SOCKET, syscall.SO_SNDTIMEO, timeout) +} + +// SetReceiveTimeout allows to set a receive timeout on the socket +func (s *NetlinkSocket) SetReceiveTimeout(timeout *syscall.Timeval) error { + // Set a read timeout of SOCKET_READ_TIMEOUT, this will allow the Read to periodically unblock and avoid that a routine + // remains stuck on a recvmsg on a closed fd + return syscall.SetsockoptTimeval(int(s.fd), syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, timeout) +} + func (s *NetlinkSocket) GetPid() (uint32, error) { fd := int(atomic.LoadInt32(&s.fd)) lsa, err := syscall.Getsockname(fd) diff --git a/components/engine/vendor/github.com/vishvananda/netlink/qdisc_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/qdisc_linux.go index 2c0deddb32..1123396e47 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/qdisc_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/qdisc_linux.go @@ -160,71 +160,73 @@ func qdiscPayload(req *nl.NetlinkRequest, qdisc Qdisc) error { req.AddData(nl.NewRtAttr(nl.TCA_KIND, nl.ZeroTerminated(qdisc.Type()))) options := nl.NewRtAttr(nl.TCA_OPTIONS, nil) - if prio, ok := qdisc.(*Prio); ok { + + switch qdisc := qdisc.(type) { + case *Prio: tcmap := nl.TcPrioMap{ - Bands: int32(prio.Bands), - Priomap: prio.PriorityMap, + Bands: int32(qdisc.Bands), + Priomap: qdisc.PriorityMap, } options = nl.NewRtAttr(nl.TCA_OPTIONS, tcmap.Serialize()) - } else if tbf, ok := qdisc.(*Tbf); ok { + case *Tbf: opt := nl.TcTbfQopt{} - opt.Rate.Rate = uint32(tbf.Rate) - opt.Peakrate.Rate = uint32(tbf.Peakrate) - opt.Limit = tbf.Limit - opt.Buffer = tbf.Buffer + opt.Rate.Rate = uint32(qdisc.Rate) + opt.Peakrate.Rate = uint32(qdisc.Peakrate) + opt.Limit = qdisc.Limit + opt.Buffer = qdisc.Buffer nl.NewRtAttrChild(options, nl.TCA_TBF_PARMS, opt.Serialize()) - if tbf.Rate >= uint64(1<<32) { - nl.NewRtAttrChild(options, nl.TCA_TBF_RATE64, nl.Uint64Attr(tbf.Rate)) + if qdisc.Rate >= uint64(1<<32) { + nl.NewRtAttrChild(options, nl.TCA_TBF_RATE64, nl.Uint64Attr(qdisc.Rate)) } - if tbf.Peakrate >= uint64(1<<32) { - nl.NewRtAttrChild(options, nl.TCA_TBF_PRATE64, nl.Uint64Attr(tbf.Peakrate)) + if qdisc.Peakrate >= uint64(1<<32) { + nl.NewRtAttrChild(options, nl.TCA_TBF_PRATE64, nl.Uint64Attr(qdisc.Peakrate)) } - if tbf.Peakrate > 0 { - nl.NewRtAttrChild(options, nl.TCA_TBF_PBURST, nl.Uint32Attr(tbf.Minburst)) + if qdisc.Peakrate > 0 { + nl.NewRtAttrChild(options, nl.TCA_TBF_PBURST, nl.Uint32Attr(qdisc.Minburst)) } - } else if htb, ok := qdisc.(*Htb); ok { + case *Htb: opt := nl.TcHtbGlob{} - opt.Version = htb.Version - opt.Rate2Quantum = htb.Rate2Quantum - opt.Defcls = htb.Defcls + opt.Version = qdisc.Version + opt.Rate2Quantum = qdisc.Rate2Quantum + opt.Defcls = qdisc.Defcls // TODO: Handle Debug properly. For now default to 0 - opt.Debug = htb.Debug - opt.DirectPkts = htb.DirectPkts + opt.Debug = qdisc.Debug + opt.DirectPkts = qdisc.DirectPkts nl.NewRtAttrChild(options, nl.TCA_HTB_INIT, opt.Serialize()) // nl.NewRtAttrChild(options, nl.TCA_HTB_DIRECT_QLEN, opt.Serialize()) - } else if netem, ok := qdisc.(*Netem); ok { + case *Netem: opt := nl.TcNetemQopt{} - opt.Latency = netem.Latency - opt.Limit = netem.Limit - opt.Loss = netem.Loss - opt.Gap = netem.Gap - opt.Duplicate = netem.Duplicate - opt.Jitter = netem.Jitter + opt.Latency = qdisc.Latency + opt.Limit = qdisc.Limit + opt.Loss = qdisc.Loss + opt.Gap = qdisc.Gap + opt.Duplicate = qdisc.Duplicate + opt.Jitter = qdisc.Jitter options = nl.NewRtAttr(nl.TCA_OPTIONS, opt.Serialize()) // Correlation corr := nl.TcNetemCorr{} - corr.DelayCorr = netem.DelayCorr - corr.LossCorr = netem.LossCorr - corr.DupCorr = netem.DuplicateCorr + corr.DelayCorr = qdisc.DelayCorr + corr.LossCorr = qdisc.LossCorr + corr.DupCorr = qdisc.DuplicateCorr if corr.DelayCorr > 0 || corr.LossCorr > 0 || corr.DupCorr > 0 { nl.NewRtAttrChild(options, nl.TCA_NETEM_CORR, corr.Serialize()) } // Corruption corruption := nl.TcNetemCorrupt{} - corruption.Probability = netem.CorruptProb - corruption.Correlation = netem.CorruptCorr + corruption.Probability = qdisc.CorruptProb + corruption.Correlation = qdisc.CorruptCorr if corruption.Probability > 0 { nl.NewRtAttrChild(options, nl.TCA_NETEM_CORRUPT, corruption.Serialize()) } // Reorder reorder := nl.TcNetemReorder{} - reorder.Probability = netem.ReorderProb - reorder.Correlation = netem.ReorderCorr + reorder.Probability = qdisc.ReorderProb + reorder.Correlation = qdisc.ReorderCorr if reorder.Probability > 0 { nl.NewRtAttrChild(options, nl.TCA_NETEM_REORDER, reorder.Serialize()) } - } else if _, ok := qdisc.(*Ingress); ok { + case *Ingress: // ingress filters must use the proper handle if qdisc.Attrs().Parent != HANDLE_INGRESS { return fmt.Errorf("Ingress filters must set Parent to HANDLE_INGRESS") diff --git a/components/engine/vendor/github.com/vishvananda/netlink/route.go b/components/engine/vendor/github.com/vishvananda/netlink/route.go index 03ac4b2391..68c6a2230d 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/route.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/route.go @@ -16,6 +16,7 @@ type Destination interface { Decode([]byte) error Encode() ([]byte, error) String() string + Equal(Destination) bool } type Encap interface { @@ -23,6 +24,7 @@ type Encap interface { Decode([]byte) error Encode() ([]byte, error) String() string + Equal(Encap) bool } // Route represents a netlink route. @@ -72,6 +74,25 @@ func (r Route) String() string { return fmt.Sprintf("{%s}", strings.Join(elems, " ")) } +func (r Route) Equal(x Route) bool { + return r.LinkIndex == x.LinkIndex && + r.ILinkIndex == x.ILinkIndex && + r.Scope == x.Scope && + ipNetEqual(r.Dst, x.Dst) && + r.Src.Equal(x.Src) && + r.Gw.Equal(x.Gw) && + nexthopInfoSlice(r.MultiPath).Equal(x.MultiPath) && + r.Protocol == x.Protocol && + r.Priority == x.Priority && + r.Table == x.Table && + r.Type == x.Type && + r.Tos == x.Tos && + r.Flags == x.Flags && + (r.MPLSDst == x.MPLSDst || (r.MPLSDst != nil && x.MPLSDst != nil && *r.MPLSDst == *x.MPLSDst)) && + (r.NewDst == x.NewDst || (r.NewDst != nil && r.NewDst.Equal(x.NewDst))) && + (r.Encap == x.Encap || (r.Encap != nil && r.Encap.Equal(x.Encap))) +} + func (r *Route) SetFlag(flag NextHopFlag) { r.Flags |= int(flag) } @@ -110,7 +131,46 @@ func (n *NexthopInfo) String() string { elems = append(elems, fmt.Sprintf("Encap: %s", n.Encap)) } elems = append(elems, fmt.Sprintf("Weight: %d", n.Hops+1)) - elems = append(elems, fmt.Sprintf("Gw: %d", n.Gw)) + elems = append(elems, fmt.Sprintf("Gw: %s", n.Gw)) elems = append(elems, fmt.Sprintf("Flags: %s", n.ListFlags())) return fmt.Sprintf("{%s}", strings.Join(elems, " ")) } + +func (n NexthopInfo) Equal(x NexthopInfo) bool { + return n.LinkIndex == x.LinkIndex && + n.Hops == x.Hops && + n.Gw.Equal(x.Gw) && + n.Flags == x.Flags && + (n.NewDst == x.NewDst || (n.NewDst != nil && n.NewDst.Equal(x.NewDst))) && + (n.Encap == x.Encap || (n.Encap != nil && n.Encap.Equal(x.Encap))) +} + +type nexthopInfoSlice []*NexthopInfo + +func (n nexthopInfoSlice) Equal(x []*NexthopInfo) bool { + if len(n) != len(x) { + return false + } + for i := range n { + if n[i] == nil || x[i] == nil { + return false + } + if !n[i].Equal(*x[i]) { + return false + } + } + return true +} + +// ipNetEqual returns true iff both IPNet are equal +func ipNetEqual(ipn1 *net.IPNet, ipn2 *net.IPNet) bool { + if ipn1 == ipn2 { + return true + } + if ipn1 == nil || ipn2 == nil { + return false + } + m1, _ := ipn1.Mask.Size() + m2, _ := ipn2.Mask.Size() + return m1 == m2 && ipn1.IP.Equal(ipn2.IP) +} diff --git a/components/engine/vendor/github.com/vishvananda/netlink/route_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/route_linux.go index cd739e7146..9234c6986d 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/route_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/route_linux.go @@ -86,6 +86,34 @@ func (d *MPLSDestination) String() string { return strings.Join(s, "/") } +func (d *MPLSDestination) Equal(x Destination) bool { + o, ok := x.(*MPLSDestination) + if !ok { + return false + } + if d == nil && o == nil { + return true + } + if d == nil || o == nil { + return false + } + if d.Labels == nil && o.Labels == nil { + return true + } + if d.Labels == nil || o.Labels == nil { + return false + } + if len(d.Labels) != len(o.Labels) { + return false + } + for i := range d.Labels { + if d.Labels[i] != o.Labels[i] { + return false + } + } + return true +} + type MPLSEncap struct { Labels []int } @@ -129,6 +157,34 @@ func (e *MPLSEncap) String() string { return strings.Join(s, "/") } +func (e *MPLSEncap) Equal(x Encap) bool { + o, ok := x.(*MPLSEncap) + if !ok { + return false + } + if e == nil && o == nil { + return true + } + if e == nil || o == nil { + return false + } + if e.Labels == nil && o.Labels == nil { + return true + } + if e.Labels == nil || o.Labels == nil { + return false + } + if len(e.Labels) != len(o.Labels) { + return false + } + for i := range e.Labels { + if e.Labels[i] != o.Labels[i] { + return false + } + } + return true +} + // RouteAdd will add a route to the system. // Equivalent to: `ip route add $route` func RouteAdd(route *Route) error { @@ -421,19 +477,8 @@ func (h *Handle) RouteListFiltered(family int, filter *Route, filterMask uint64) continue case filterMask&RT_FILTER_DST != 0: if filter.MPLSDst == nil || route.MPLSDst == nil || (*filter.MPLSDst) != (*route.MPLSDst) { - if filter.Dst == nil { - if route.Dst != nil { - continue - } - } else { - if route.Dst == nil { - continue - } - aMaskLen, aMaskBits := route.Dst.Mask.Size() - bMaskLen, bMaskBits := filter.Dst.Mask.Size() - if !(route.Dst.IP.Equal(filter.Dst.IP) && aMaskLen == bMaskLen && aMaskBits == bMaskBits) { - continue - } + if !ipNetEqual(route.Dst, filter.Dst) { + continue } } } @@ -633,16 +678,34 @@ func (h *Handle) RouteGet(destination net.IP) ([]Route, error) { // RouteSubscribe takes a chan down which notifications will be sent // when routes are added or deleted. Close the 'done' chan to stop subscription. func RouteSubscribe(ch chan<- RouteUpdate, done <-chan struct{}) error { - return routeSubscribeAt(netns.None(), netns.None(), ch, done) + return routeSubscribeAt(netns.None(), netns.None(), ch, done, nil) } // RouteSubscribeAt works like RouteSubscribe plus it allows the caller // to choose the network namespace in which to subscribe (ns). func RouteSubscribeAt(ns netns.NsHandle, ch chan<- RouteUpdate, done <-chan struct{}) error { - return routeSubscribeAt(ns, netns.None(), ch, done) + return routeSubscribeAt(ns, netns.None(), ch, done, nil) } -func routeSubscribeAt(newNs, curNs netns.NsHandle, ch chan<- RouteUpdate, done <-chan struct{}) error { +// RouteSubscribeOptions contains a set of options to use with +// RouteSubscribeWithOptions. +type RouteSubscribeOptions struct { + Namespace *netns.NsHandle + ErrorCallback func(error) +} + +// RouteSubscribeWithOptions work like RouteSubscribe but enable to +// provide additional options to modify the behavior. Currently, the +// namespace can be provided as well as an error callback. +func RouteSubscribeWithOptions(ch chan<- RouteUpdate, done <-chan struct{}, options RouteSubscribeOptions) error { + if options.Namespace == nil { + none := netns.None() + options.Namespace = &none + } + return routeSubscribeAt(*options.Namespace, netns.None(), ch, done, options.ErrorCallback) +} + +func routeSubscribeAt(newNs, curNs netns.NsHandle, ch chan<- RouteUpdate, done <-chan struct{}, cberr func(error)) error { s, err := nl.SubscribeAt(newNs, curNs, syscall.NETLINK_ROUTE, syscall.RTNLGRP_IPV4_ROUTE, syscall.RTNLGRP_IPV6_ROUTE) if err != nil { return err @@ -658,11 +721,17 @@ func routeSubscribeAt(newNs, curNs netns.NsHandle, ch chan<- RouteUpdate, done < for { msgs, err := s.Receive() if err != nil { + if cberr != nil { + cberr(err) + } return } for _, m := range msgs { route, err := deserializeRoute(m.Data) if err != nil { + if cberr != nil { + cberr(err) + } return } ch <- RouteUpdate{Type: m.Header.Type, Route: route} diff --git a/components/engine/vendor/github.com/vishvananda/netlink/rule.go b/components/engine/vendor/github.com/vishvananda/netlink/rule.go index f0243defd7..e4d9168d6c 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/rule.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/rule.go @@ -8,6 +8,7 @@ import ( // Rule represents a netlink rule. type Rule struct { Priority int + Family int Table int Mark int Mask int diff --git a/components/engine/vendor/github.com/vishvananda/netlink/rule_linux.go b/components/engine/vendor/github.com/vishvananda/netlink/rule_linux.go index f9cdc855f1..cbd91a56bb 100644 --- a/components/engine/vendor/github.com/vishvananda/netlink/rule_linux.go +++ b/components/engine/vendor/github.com/vishvananda/netlink/rule_linux.go @@ -37,6 +37,9 @@ func (h *Handle) RuleDel(rule *Rule) error { func ruleHandle(rule *Rule, req *nl.NetlinkRequest) error { msg := nl.NewRtMsg() msg.Family = syscall.AF_INET + if rule.Family != 0 { + msg.Family = uint8(rule.Family) + } var dstFamily uint8 var rtAttrs []*nl.RtAttr From 3916dafe17df45b34a6caf3cb33caef6db24936d Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 5 Dec 2017 15:29:37 +0100 Subject: [PATCH 14/16] api: generalize version information to any platform component This change adds a Platform struct with a Name field and a general Components field to the Version API type. This will allow API consumers to show version information for the whole platform and it will allow API providers to set the versions for the various components of the platform. All changes here are backwards compatible. Signed-off-by: Tibor Vass Upstream-commit: 9152e63290e4a4e586b811cce39082efc649b912 Component: engine --- components/engine/Makefile | 3 +- .../api/server/router/system/system_routes.go | 2 - components/engine/api/types/types.go | 12 +++++ components/engine/daemon/info.go | 46 ++++++++++++++----- .../engine/dockerversion/version_lib.go | 1 + components/engine/hack/make/.go-autogen | 1 + 6 files changed, 50 insertions(+), 15 deletions(-) diff --git a/components/engine/Makefile b/components/engine/Makefile index 32988150c7..6f5145a026 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -53,7 +53,8 @@ DOCKER_ENVS := \ -e http_proxy \ -e https_proxy \ -e no_proxy \ - -e VERSION + -e VERSION \ + -e PLATFORM # note: we _cannot_ add "-e DOCKER_BUILDTAGS" here because even if it's unset in the shell, that would shadow the "ENV DOCKER_BUILDTAGS" set in our Dockerfile, which is very important for our official builds # to allow `make BIND_DIR=. shell` or `make BIND_DIR= test` diff --git a/components/engine/api/server/router/system/system_routes.go b/components/engine/api/server/router/system/system_routes.go index 8f6aecd77a..535956d65f 100644 --- a/components/engine/api/server/router/system/system_routes.go +++ b/components/engine/api/server/router/system/system_routes.go @@ -6,7 +6,6 @@ import ( "net/http" "time" - "github.com/docker/docker/api" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/events" @@ -65,7 +64,6 @@ func (s *systemRouter) getInfo(ctx context.Context, w http.ResponseWriter, r *ht func (s *systemRouter) getVersion(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { info := s.backend.SystemVersion() - info.APIVersion = api.DefaultVersion return httputils.WriteJSON(w, http.StatusOK, info) } diff --git a/components/engine/api/types/types.go b/components/engine/api/types/types.go index f7ac772971..7814e6b934 100644 --- a/components/engine/api/types/types.go +++ b/components/engine/api/types/types.go @@ -107,9 +107,21 @@ type Ping struct { Experimental bool } +// ComponentVersion describes the version information for a specific component. +type ComponentVersion struct { + Name string + Version string + Details map[string]string `json:",omitempty"` +} + // Version contains response of Engine API: // GET "/version" type Version struct { + Platform struct{ Name string } `json:",omitempty"` + Components []ComponentVersion `json:",omitempty"` + + // The following fields are deprecated, they relate to the Engine component and are kept for backwards compatibility + Version string APIVersion string `json:"ApiVersion"` MinAPIVersion string `json:"MinAPIVersion,omitempty"` diff --git a/components/engine/daemon/info.go b/components/engine/daemon/info.go index b14e7ba809..bbb027ed54 100644 --- a/components/engine/daemon/info.go +++ b/components/engine/daemon/info.go @@ -154,24 +154,46 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { // SystemVersion returns version information about the daemon. func (daemon *Daemon) SystemVersion() types.Version { - v := types.Version{ - Version: dockerversion.Version, - GitCommit: dockerversion.GitCommit, - MinAPIVersion: api.MinVersion, - GoVersion: runtime.Version(), - Os: runtime.GOOS, - Arch: runtime.GOARCH, - BuildTime: dockerversion.BuildTime, - Experimental: daemon.configStore.Experimental, - } - kernelVersion := "" if kv, err := kernel.GetKernelVersion(); err != nil { logrus.Warnf("Could not get kernel version: %v", err) } else { kernelVersion = kv.String() } - v.KernelVersion = kernelVersion + + v := types.Version{ + Components: []types.ComponentVersion{ + { + Name: "Engine", + Version: dockerversion.Version, + Details: map[string]string{ + "GitCommit": dockerversion.GitCommit, + "ApiVersion": api.DefaultVersion, + "MinAPIVersion": api.MinVersion, + "GoVersion": runtime.Version(), + "Os": runtime.GOOS, + "Arch": runtime.GOARCH, + "BuildTime": dockerversion.BuildTime, + "KernelVersion": kernelVersion, + "Experimental": fmt.Sprintf("%t", daemon.configStore.Experimental), + }, + }, + }, + + // Populate deprecated fields for older clients + Version: dockerversion.Version, + GitCommit: dockerversion.GitCommit, + APIVersion: api.DefaultVersion, + MinAPIVersion: api.MinVersion, + GoVersion: runtime.Version(), + Os: runtime.GOOS, + Arch: runtime.GOARCH, + BuildTime: dockerversion.BuildTime, + KernelVersion: kernelVersion, + Experimental: daemon.configStore.Experimental, + } + + v.Platform.Name = dockerversion.PlatformName return v } diff --git a/components/engine/dockerversion/version_lib.go b/components/engine/dockerversion/version_lib.go index 33f77d3ce6..72f48939d6 100644 --- a/components/engine/dockerversion/version_lib.go +++ b/components/engine/dockerversion/version_lib.go @@ -13,4 +13,5 @@ const ( ContainerdCommitID string = "library-import" RuncCommitID string = "library-import" InitCommitID string = "library-import" + PlatformName string = "" ) diff --git a/components/engine/hack/make/.go-autogen b/components/engine/hack/make/.go-autogen index b68e3a7534..850c3ec9ac 100644 --- a/components/engine/hack/make/.go-autogen +++ b/components/engine/hack/make/.go-autogen @@ -18,6 +18,7 @@ const ( BuildTime string = "$BUILDTIME" IAmStatic string = "${IAMSTATIC:-true}" ContainerdCommitID string = "${CONTAINERD_COMMIT}" + PlatformName string = "${PLATFORM}" ) // AUTOGENERATED FILE; see /go/src/github.com/docker/docker/hack/make/.go-autogen From 947ed56f66143d7592c156bf0ef97e90664b3b0b Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Wed, 6 Dec 2017 21:29:22 +0100 Subject: [PATCH 15/16] Adjust swagger definitions Signed-off-by: Tibor Vass Upstream-commit: 87ebfa11f66d2109054ad3024217a5101b45aa8c Component: engine --- components/engine/api/swagger.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index 96386e0599..de45301b06 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -6764,6 +6764,28 @@ paths: schema: type: "object" properties: + Platform: + type: "object" + required: [Name] + properties: + Name: + type: "string" + Components: + type: "array" + items: + type: "object" + x-go-name: ComponentVersion + required: [Name, Version] + properties: + Name: + type: "string" + Version: + type: "string" + x-nullable: false + Details: + type: "object" + x-nullable: true + Version: type: "string" ApiVersion: From 00a55a319dd5b74c408da211850eddfaf3258ea3 Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Wed, 6 Dec 2017 22:02:30 +0100 Subject: [PATCH 16/16] Fix windows Signed-off-by: Tibor Vass Upstream-commit: cb8283a6e95f33a93fab0044ca4306cf468faf8c Component: engine --- components/engine/hack/make.ps1 | 2 +- components/engine/hack/make/.go-autogen.ps1 | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/components/engine/hack/make.ps1 b/components/engine/hack/make.ps1 index 3380a5b693..42a2b319fa 100644 --- a/components/engine/hack/make.ps1 +++ b/components/engine/hack/make.ps1 @@ -365,7 +365,7 @@ Try { # Run autogen if building binaries or running unit tests. if ($Client -or $Daemon -or $TestUnit) { Write-Host "INFO: Invoking autogen..." - Try { .\hack\make\.go-autogen.ps1 -CommitString $gitCommit -DockerVersion $dockerVersion } + Try { .\hack\make\.go-autogen.ps1 -CommitString $gitCommit -DockerVersion $dockerVersion -Platform "$env:PLATFORM" } Catch [Exception] { Throw $_ } } diff --git a/components/engine/hack/make/.go-autogen.ps1 b/components/engine/hack/make/.go-autogen.ps1 index 768badb6a5..cc14e9ee9d 100644 --- a/components/engine/hack/make/.go-autogen.ps1 +++ b/components/engine/hack/make/.go-autogen.ps1 @@ -14,7 +14,8 @@ param( [Parameter(Mandatory=$true)][string]$CommitString, - [Parameter(Mandatory=$true)][string]$DockerVersion + [Parameter(Mandatory=$true)][string]$DockerVersion, + [Parameter(Mandatory=$false)][string]$Platform ) $ErrorActionPreference = "Stop" @@ -43,6 +44,7 @@ const ( GitCommit string = "'+$CommitString+'" Version string = "'+$DockerVersion+'" BuildTime string = "'+$buildDateTime+'" + PlatformName string = "'+$Platform+'" ) // AUTOGENERATED FILE; see hack\make\.go-autogen.ps1