Merge component 'engine' from git@github.com:moby/moby master

This commit is contained in:
GordonTheTurtle
2017-12-07 02:18:48 +00:00
103 changed files with 3479 additions and 1268 deletions
+2 -1
View File
@@ -53,7 +53,8 @@ DOCKER_ENVS := \
-e http_proxy \
-e https_proxy \
-e no_proxy \
-e VERSION
-e VERSION \
-e PLATFORM
# note: we _cannot_ add "-e DOCKER_BUILDTAGS" here because even if it's unset in the shell, that would shadow the "ENV DOCKER_BUILDTAGS" set in our Dockerfile, which is very important for our official builds
# to allow `make BIND_DIR=. shell` or `make BIND_DIR= test`
@@ -6,7 +6,6 @@ import (
"net/http"
"time"
"github.com/docker/docker/api"
"github.com/docker/docker/api/server/httputils"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/events"
@@ -65,7 +64,6 @@ func (s *systemRouter) getInfo(ctx context.Context, w http.ResponseWriter, r *ht
func (s *systemRouter) getVersion(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
info := s.backend.SystemVersion()
info.APIVersion = api.DefaultVersion
return httputils.WriteJSON(w, http.StatusOK, info)
}
+25 -6
View File
@@ -254,6 +254,7 @@ definitions:
properties:
Propagation:
description: "A propagation mode with the value `[r]private`, `[r]shared`, or `[r]slave`."
type: "string"
enum:
- "private"
- "rprivate"
@@ -823,9 +824,7 @@ definitions:
type: "string"
Cmd:
description: "Command to run specified as a string or an array of strings."
type:
- "array"
- "string"
type: "array"
items:
type: "string"
Healthcheck:
@@ -853,9 +852,7 @@ definitions:
The entry point for the container as a string or an array of strings.
If the array consists of exactly one empty string (`[""]`) then the entry point is reset to system default (i.e., the entry point used by docker when there is no `ENTRYPOINT` instruction in the `Dockerfile`).
type:
- "array"
- "string"
type: "array"
items:
type: "string"
NetworkDisabled:
@@ -6764,6 +6761,28 @@ paths:
schema:
type: "object"
properties:
Platform:
type: "object"
required: [Name]
properties:
Name:
type: "string"
Components:
type: "array"
items:
type: "object"
x-go-name: ComponentVersion
required: [Name, Version]
properties:
Name:
type: "string"
Version:
type: "string"
x-nullable: false
Details:
type: "object"
x-nullable: true
Version:
type: "string"
ApiVersion:
+12
View File
@@ -107,9 +107,21 @@ type Ping struct {
Experimental bool
}
// ComponentVersion describes the version information for a specific component.
type ComponentVersion struct {
Name string
Version string
Details map[string]string `json:",omitempty"`
}
// Version contains response of Engine API:
// GET "/version"
type Version struct {
Platform struct{ Name string } `json:",omitempty"`
Components []ComponentVersion `json:",omitempty"`
// The following fields are deprecated, they relate to the Engine component and are kept for backwards compatibility
Version string
APIVersion string `json:"ApiVersion"`
MinAPIVersion string `json:"MinAPIVersion,omitempty"`
+2
View File
@@ -59,6 +59,8 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) {
flags.IntVar(&maxConcurrentDownloads, "max-concurrent-downloads", config.DefaultMaxConcurrentDownloads, "Set the max concurrent downloads for each pull")
flags.IntVar(&maxConcurrentUploads, "max-concurrent-uploads", config.DefaultMaxConcurrentUploads, "Set the max concurrent uploads for each push")
flags.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", defaultShutdownTimeout, "Set the default shutdown timeout")
flags.IntVar(&conf.NetworkDiagnosticPort, "network-diagnostic-port", 0, "TCP port number of the network diagnostic server")
flags.MarkHidden("network-diagnostic-port")
flags.StringVar(&conf.SwarmDefaultAdvertiseAddr, "swarm-default-advertise-addr", "", "Set default address or interface for swarm advertised address")
flags.BoolVar(&conf.Experimental, "experimental", false, "Enable experimental features")
@@ -228,6 +228,13 @@ func (c *Cluster) Update(version uint64, spec types.Spec, flags types.UpdateFlag
return err
}
// Validate spec name.
if spec.Annotations.Name == "" {
spec.Annotations.Name = "default"
} else if spec.Annotations.Name != "default" {
return validationError{errors.New(`swarm spec must be named "default"`)}
}
// In update, client should provide the complete spec of the swarm, including
// Name and Labels. If a field is specified with 0 or nil, then the default value
// will be used to swarmkit.
+21 -20
View File
@@ -85,26 +85,27 @@ type CommonTLSOptions struct {
// It includes json tags to deserialize configuration from a file
// using the same names that the flags in the command line use.
type CommonConfig struct {
AuthzMiddleware *authorization.Middleware `json:"-"`
AuthorizationPlugins []string `json:"authorization-plugins,omitempty"` // AuthorizationPlugins holds list of authorization plugins
AutoRestart bool `json:"-"`
Context map[string][]string `json:"-"`
DisableBridge bool `json:"-"`
DNS []string `json:"dns,omitempty"`
DNSOptions []string `json:"dns-opts,omitempty"`
DNSSearch []string `json:"dns-search,omitempty"`
ExecOptions []string `json:"exec-opts,omitempty"`
GraphDriver string `json:"storage-driver,omitempty"`
GraphOptions []string `json:"storage-opts,omitempty"`
Labels []string `json:"labels,omitempty"`
Mtu int `json:"mtu,omitempty"`
Pidfile string `json:"pidfile,omitempty"`
RawLogs bool `json:"raw-logs,omitempty"`
RootDeprecated string `json:"graph,omitempty"`
Root string `json:"data-root,omitempty"`
ExecRoot string `json:"exec-root,omitempty"`
SocketGroup string `json:"group,omitempty"`
CorsHeaders string `json:"api-cors-header,omitempty"`
AuthzMiddleware *authorization.Middleware `json:"-"`
AuthorizationPlugins []string `json:"authorization-plugins,omitempty"` // AuthorizationPlugins holds list of authorization plugins
AutoRestart bool `json:"-"`
Context map[string][]string `json:"-"`
DisableBridge bool `json:"-"`
DNS []string `json:"dns,omitempty"`
DNSOptions []string `json:"dns-opts,omitempty"`
DNSSearch []string `json:"dns-search,omitempty"`
ExecOptions []string `json:"exec-opts,omitempty"`
GraphDriver string `json:"storage-driver,omitempty"`
GraphOptions []string `json:"storage-opts,omitempty"`
Labels []string `json:"labels,omitempty"`
Mtu int `json:"mtu,omitempty"`
NetworkDiagnosticPort int `json:"network-diagnostic-port,omitempty"`
Pidfile string `json:"pidfile,omitempty"`
RawLogs bool `json:"raw-logs,omitempty"`
RootDeprecated string `json:"graph,omitempty"`
Root string `json:"data-root,omitempty"`
ExecRoot string `json:"exec-root,omitempty"`
SocketGroup string `json:"group,omitempty"`
CorsHeaders string `json:"api-cors-header,omitempty"`
// TrustKeyPath is used to generate the daemon ID and for signing schema 1 manifests
// when pushing to a registry which does not support schema 2. This field is marked as
@@ -89,7 +89,16 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
return nil, graphdriver.ErrNotSupported
}
fsMagic, err := graphdriver.GetFSMagic(root)
// Perform feature detection on /var/lib/docker/aufs if it's an existing directory.
// This covers situations where /var/lib/docker/aufs is a mount, and on a different
// filesystem than /var/lib/docker.
// If the path does not exist, fall back to using /var/lib/docker for feature detection.
testdir := root
if _, err := os.Stat(testdir); os.IsNotExist(err) {
testdir = filepath.Dir(testdir)
}
fsMagic, err := graphdriver.GetFSMagic(testdir)
if err != nil {
return nil, err
}
@@ -51,7 +51,16 @@ type btrfsOptions struct {
// An error is returned if BTRFS is not supported.
func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
fsMagic, err := graphdriver.GetFSMagic(home)
// Perform feature detection on /var/lib/docker/btrfs if it's an existing directory.
// This covers situations where /var/lib/docker/btrfs is a mount, and on a different
// filesystem than /var/lib/docker.
// If the path does not exist, fall back to using /var/lib/docker for feature detection.
testdir := home
if _, err := os.Stat(testdir); os.IsNotExist(err) {
testdir = filepath.Dir(testdir)
}
fsMagic, err := graphdriver.GetFSMagic(testdir)
if err != nil {
return nil, err
}
+14 -16
View File
@@ -1,7 +1,6 @@
package graphdriver
import (
"errors"
"fmt"
"io"
"os"
@@ -28,13 +27,6 @@ const (
var (
// All registered drivers
drivers map[string]InitFunc
// ErrNotSupported returned when driver is not supported.
ErrNotSupported = errors.New("driver not supported")
// ErrPrerequisites returned when driver does not meet prerequisites.
ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)")
// ErrIncompatibleFS returned when file system is not supported.
ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver")
)
//CreateOpts contains optional arguments for Create() and CreateReadWrite()
@@ -248,7 +240,7 @@ func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, err
for _, name := range list {
driver, err := getBuiltinDriver(name, config.Root, config.DriverOptions, config.UIDMaps, config.GIDMaps)
if err != nil {
if isDriverNotSupported(err) {
if IsDriverNotSupported(err) {
continue
}
return nil, err
@@ -260,7 +252,7 @@ func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, err
for name, initFunc := range drivers {
driver, err := initFunc(filepath.Join(config.Root, name), config.DriverOptions, config.UIDMaps, config.GIDMaps)
if err != nil {
if isDriverNotSupported(err) {
if IsDriverNotSupported(err) {
continue
}
return nil, err
@@ -270,12 +262,6 @@ func New(name string, pg plugingetter.PluginGetter, config Options) (Driver, err
return nil, fmt.Errorf("No supported storage backend found")
}
// isDriverNotSupported returns true if the error initializing
// the graph driver is a non-supported error.
func isDriverNotSupported(err error) bool {
return err == ErrNotSupported || err == ErrPrerequisites || err == ErrIncompatibleFS
}
// scanPriorDrivers returns an un-ordered scan of directories of prior storage drivers
func scanPriorDrivers(root string) map[string]bool {
driversMap := make(map[string]bool)
@@ -291,6 +277,18 @@ func scanPriorDrivers(root string) map[string]bool {
return driversMap
}
// IsInitialized checks if the driver's home-directory exists and is non-empty.
func IsInitialized(driverHome string) bool {
_, err := os.Stat(driverHome)
if os.IsNotExist(err) {
return false
}
if err != nil {
logrus.Warnf("graphdriver.IsInitialized: stat failed: %v", err)
}
return !isEmptyDir(driverHome)
}
// isEmptyDir checks if a directory is empty. It is used to check if prior
// storage-driver directories exist. If an error occurs, it also assumes the
// directory is not empty (which preserves the behavior _before_ this check
@@ -3,8 +3,6 @@
package graphdriver
import (
"path/filepath"
"github.com/docker/docker/pkg/mount"
"golang.org/x/sys/unix"
)
@@ -82,7 +80,7 @@ var (
// GetFSMagic returns the filesystem id given the path.
func GetFSMagic(rootpath string) (FsMagic, error) {
var buf unix.Statfs_t
if err := unix.Statfs(filepath.Dir(rootpath), &buf); err != nil {
if err := unix.Statfs(rootpath, &buf); err != nil {
return 0, err
}
return FsMagic(buf.Type), nil
@@ -0,0 +1,36 @@
package graphdriver
const (
// ErrNotSupported returned when driver is not supported.
ErrNotSupported NotSupportedError = "driver not supported"
// ErrPrerequisites returned when driver does not meet prerequisites.
ErrPrerequisites NotSupportedError = "prerequisites for driver not satisfied (wrong filesystem?)"
// ErrIncompatibleFS returned when file system is not supported.
ErrIncompatibleFS NotSupportedError = "backing file system is unsupported for this graph driver"
)
// ErrUnSupported signals that the graph-driver is not supported on the current configuration
type ErrUnSupported interface {
NotSupported()
}
// NotSupportedError signals that the graph-driver is not supported on the current configuration
type NotSupportedError string
func (e NotSupportedError) Error() string {
return string(e)
}
// NotSupported signals that a graph-driver is not supported.
func (e NotSupportedError) NotSupported() {}
// IsDriverNotSupported returns true if the error initializing
// the graph driver is a non-supported error.
func IsDriverNotSupported(err error) bool {
switch err.(type) {
case ErrUnSupported:
return true
default:
return false
}
}
@@ -42,7 +42,7 @@ func newDriver(t testing.TB, name string, options []string) *Driver {
d, err := graphdriver.GetDriver(name, nil, graphdriver.Options{DriverOptions: options, Root: root})
if err != nil {
t.Logf("graphdriver: %v\n", err)
if err == graphdriver.ErrNotSupported || err == graphdriver.ErrPrerequisites || err == graphdriver.ErrIncompatibleFS {
if graphdriver.IsDriverNotSupported(err) {
t.Skipf("Driver %s not supported", name)
}
t.Fatal(err)
@@ -10,6 +10,7 @@ import (
"os"
"os/exec"
"path"
"path/filepath"
"strconv"
"github.com/docker/docker/daemon/graphdriver"
@@ -119,7 +120,16 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
return nil, graphdriver.ErrNotSupported
}
fsMagic, err := graphdriver.GetFSMagic(home)
// Perform feature detection on /var/lib/docker/overlay if it's an existing directory.
// This covers situations where /var/lib/docker/overlay is a mount, and on a different
// filesystem than /var/lib/docker.
// If the path does not exist, fall back to using /var/lib/docker for feature detection.
testdir := home
if _, err := os.Stat(testdir); os.IsNotExist(err) {
testdir = filepath.Dir(testdir)
}
fsMagic, err := graphdriver.GetFSMagic(testdir)
if err != nil {
return nil, err
}
@@ -128,11 +138,23 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
}
switch fsMagic {
case graphdriver.FsMagicAufs, graphdriver.FsMagicBtrfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicZfs, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs:
case graphdriver.FsMagicAufs, graphdriver.FsMagicBtrfs, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs, graphdriver.FsMagicOverlay, graphdriver.FsMagicZfs:
logrus.Errorf("'overlay' is not supported over %s", backingFs)
return nil, graphdriver.ErrIncompatibleFS
}
supportsDType, err := fsutils.SupportsDType(testdir)
if err != nil {
return nil, err
}
if !supportsDType {
if !graphdriver.IsInitialized(home) {
return nil, overlayutils.ErrDTypeNotSupported("overlay", backingFs)
}
// allow running without d_type only for existing setups (#27443)
logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs))
}
rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
if err != nil {
return nil, err
@@ -146,15 +168,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
return nil, err
}
supportsDType, err := fsutils.SupportsDType(home)
if err != nil {
return nil, err
}
if !supportsDType {
// not a fatal error until v17.12 (#27443)
logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs))
}
d := &Driver{
home: home,
uidMaps: uidMaps,
@@ -136,7 +136,16 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
return nil, err
}
fsMagic, err := graphdriver.GetFSMagic(home)
// Perform feature detection on /var/lib/docker/overlay2 if it's an existing directory.
// This covers situations where /var/lib/docker/overlay2 is a mount, and on a different
// filesystem than /var/lib/docker.
// If the path does not exist, fall back to using /var/lib/docker for feature detection.
testdir := home
if _, err := os.Stat(testdir); os.IsNotExist(err) {
testdir = filepath.Dir(testdir)
}
fsMagic, err := graphdriver.GetFSMagic(testdir)
if err != nil {
return nil, err
}
@@ -144,9 +153,8 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
backingFs = fsName
}
// check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs
switch fsMagic {
case graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs:
case graphdriver.FsMagicAufs, graphdriver.FsMagicEcryptfs, graphdriver.FsMagicNfsFs, graphdriver.FsMagicOverlay, graphdriver.FsMagicZfs:
logrus.Errorf("'overlay2' is not supported over %s", backingFs)
return nil, graphdriver.ErrIncompatibleFS
case graphdriver.FsMagicBtrfs:
@@ -165,12 +173,23 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
if opts.overrideKernelCheck {
logrus.Warn("Using pre-4.0.0 kernel for overlay2, mount failures may require kernel update")
} else {
if err := supportsMultipleLowerDir(filepath.Dir(home)); err != nil {
if err := supportsMultipleLowerDir(testdir); err != nil {
logrus.Debugf("Multiple lower dirs not supported: %v", err)
return nil, graphdriver.ErrNotSupported
}
}
}
supportsDType, err := fsutils.SupportsDType(testdir)
if err != nil {
return nil, err
}
if !supportsDType {
if !graphdriver.IsInitialized(home) {
return nil, overlayutils.ErrDTypeNotSupported("overlay2", backingFs)
}
// allow running without d_type only for existing setups (#27443)
logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay2", backingFs))
}
rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
if err != nil {
@@ -185,15 +204,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
return nil, err
}
supportsDType, err := fsutils.SupportsDType(home)
if err != nil {
return nil, err
}
if !supportsDType {
// not a fatal error until v17.12 (#27443)
logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay2", backingFs))
}
d := &Driver{
home: home,
uidMaps: uidMaps,
@@ -3,8 +3,9 @@
package overlayutils
import (
"errors"
"fmt"
"github.com/docker/docker/daemon/graphdriver"
)
// ErrDTypeNotSupported denotes that the backing filesystem doesn't support d_type.
@@ -13,6 +14,7 @@ func ErrDTypeNotSupported(driver, backingFs string) error {
if backingFs == "xfs" {
msg += " Reformat the filesystem with ftype=1 to enable d_type support."
}
msg += " Running without d_type support will no longer be supported in Docker 17.12."
return errors.New(msg)
msg += " Backing filesystems without d_type support are not supported."
return graphdriver.NotSupportedError(msg)
}
+34 -12
View File
@@ -154,24 +154,46 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) {
// SystemVersion returns version information about the daemon.
func (daemon *Daemon) SystemVersion() types.Version {
v := types.Version{
Version: dockerversion.Version,
GitCommit: dockerversion.GitCommit,
MinAPIVersion: api.MinVersion,
GoVersion: runtime.Version(),
Os: runtime.GOOS,
Arch: runtime.GOARCH,
BuildTime: dockerversion.BuildTime,
Experimental: daemon.configStore.Experimental,
}
kernelVersion := "<unknown>"
if kv, err := kernel.GetKernelVersion(); err != nil {
logrus.Warnf("Could not get kernel version: %v", err)
} else {
kernelVersion = kv.String()
}
v.KernelVersion = kernelVersion
v := types.Version{
Components: []types.ComponentVersion{
{
Name: "Engine",
Version: dockerversion.Version,
Details: map[string]string{
"GitCommit": dockerversion.GitCommit,
"ApiVersion": api.DefaultVersion,
"MinAPIVersion": api.MinVersion,
"GoVersion": runtime.Version(),
"Os": runtime.GOOS,
"Arch": runtime.GOARCH,
"BuildTime": dockerversion.BuildTime,
"KernelVersion": kernelVersion,
"Experimental": fmt.Sprintf("%t", daemon.configStore.Experimental),
},
},
},
// Populate deprecated fields for older clients
Version: dockerversion.Version,
GitCommit: dockerversion.GitCommit,
APIVersion: api.DefaultVersion,
MinAPIVersion: api.MinVersion,
GoVersion: runtime.Version(),
Os: runtime.GOOS,
Arch: runtime.GOARCH,
BuildTime: dockerversion.BuildTime,
KernelVersion: kernelVersion,
Experimental: daemon.configStore.Experimental,
}
v.Platform.Name = dockerversion.PlatformName
return v
}
+18
View File
@@ -61,6 +61,9 @@ func (daemon *Daemon) Reload(conf *config.Config) (err error) {
if err := daemon.reloadLiveRestore(conf, attributes); err != nil {
return err
}
if err := daemon.reloadNetworkDiagnosticPort(conf, attributes); err != nil {
return err
}
return nil
}
@@ -308,3 +311,18 @@ func (daemon *Daemon) reloadLiveRestore(conf *config.Config, attributes map[stri
attributes["live-restore"] = fmt.Sprintf("%t", daemon.configStore.LiveRestoreEnabled)
return nil
}
// reloadNetworkDiagnosticPort updates the network controller starting the diagnose mode if the config is valid
func (daemon *Daemon) reloadNetworkDiagnosticPort(conf *config.Config, attributes map[string]string) error {
if conf == nil || daemon.netController == nil {
return nil
}
// Enable the network diagnose if the flag is set with a valid port withing the range
if conf.IsValueSet("network-diagnostic-port") && conf.NetworkDiagnosticPort > 0 && conf.NetworkDiagnosticPort < 65536 {
logrus.Warnf("Calling the diagnostic start with %d", conf.NetworkDiagnosticPort)
daemon.netController.StartDiagnose(conf.NetworkDiagnosticPort)
} else {
daemon.netController.StopDiagnose()
}
return nil
}
+69
View File
@@ -10,6 +10,7 @@ import (
"github.com/docker/docker/pkg/discovery"
_ "github.com/docker/docker/pkg/discovery/memory"
"github.com/docker/docker/registry"
"github.com/docker/libnetwork"
"github.com/stretchr/testify/assert"
)
@@ -479,3 +480,71 @@ func TestDaemonDiscoveryReloadOnlyClusterAdvertise(t *testing.T) {
t.Fatal(e)
}
}
func TestDaemonReloadNetworkDiagnosticPort(t *testing.T) {
daemon := &Daemon{}
daemon.configStore = &config.Config{}
valuesSet := make(map[string]interface{})
valuesSet["network-diagnostic-port"] = 2000
enableConfig := &config.Config{
CommonConfig: config.CommonConfig{
NetworkDiagnosticPort: 2000,
ValuesSet: valuesSet,
},
}
disableConfig := &config.Config{
CommonConfig: config.CommonConfig{},
}
netOptions, err := daemon.networkOptions(enableConfig, nil, nil)
if err != nil {
t.Fatal(err)
}
controller, err := libnetwork.New(netOptions...)
if err != nil {
t.Fatal(err)
}
daemon.netController = controller
// Enable/Disable the server for some iterations
for i := 0; i < 10; i++ {
enableConfig.CommonConfig.NetworkDiagnosticPort++
if err := daemon.Reload(enableConfig); err != nil {
t.Fatal(err)
}
// Check that the diagnose is enabled
if !daemon.netController.IsDiagnoseEnabled() {
t.Fatalf("diagnosed should be enable")
}
// Reload
if err := daemon.Reload(disableConfig); err != nil {
t.Fatal(err)
}
// Check that the diagnose is disabled
if daemon.netController.IsDiagnoseEnabled() {
t.Fatalf("diagnosed should be disable")
}
}
enableConfig.CommonConfig.NetworkDiagnosticPort++
// 2 times the enable should not create problems
if err := daemon.Reload(enableConfig); err != nil {
t.Fatal(err)
}
// Check that the diagnose is enabled
if !daemon.netController.IsDiagnoseEnabled() {
t.Fatalf("diagnosed should be enable")
}
// Check that another reload does not cause issues
if err := daemon.Reload(enableConfig); err != nil {
t.Fatal(err)
}
// Check that the diagnose is enable
if !daemon.netController.IsDiagnoseEnabled() {
t.Fatalf("diagnosed should be enable")
}
}
@@ -13,4 +13,5 @@ const (
ContainerdCommitID string = "library-import"
RuncCommitID string = "library-import"
InitCommitID string = "library-import"
PlatformName string = ""
)
@@ -10,8 +10,7 @@ Moby's development environment is itself, ultimately a Docker container.
You use the `moby/moby` repository and its `Dockerfile` to create a Docker image,
run a Docker container, and develop code in the container.
If you followed the procedures that <a href="/opensource/project/set-up-git/" target="_blank">
set up Git for contributing</a>, you should have a fork of the `moby/moby`
If you followed the procedures that [set up Git for contributing](./set-up-git.md), you should have a fork of the `moby/moby`
repository. You also created a branch called `dry-run-test`. In this section,
you continue working with your fork on this branch.
@@ -106,8 +105,7 @@ can take over 15 minutes to complete.
```
If you are following along with this guide, you created a `dry-run-test`
branch when you <a href="/opensource/project/set-up-git/" target="_blank">
set up Git for contributing</a>.
branch when you [set up Git for contributing](./set-up-git.md).
3. Ensure you are on your `dry-run-test` branch.
+1 -1
View File
@@ -365,7 +365,7 @@ Try {
# Run autogen if building binaries or running unit tests.
if ($Client -or $Daemon -or $TestUnit) {
Write-Host "INFO: Invoking autogen..."
Try { .\hack\make\.go-autogen.ps1 -CommitString $gitCommit -DockerVersion $dockerVersion }
Try { .\hack\make\.go-autogen.ps1 -CommitString $gitCommit -DockerVersion $dockerVersion -Platform "$env:PLATFORM" }
Catch [Exception] { Throw $_ }
}
+1
View File
@@ -18,6 +18,7 @@ const (
BuildTime string = "$BUILDTIME"
IAmStatic string = "${IAMSTATIC:-true}"
ContainerdCommitID string = "${CONTAINERD_COMMIT}"
PlatformName string = "${PLATFORM}"
)
// AUTOGENERATED FILE; see /go/src/github.com/docker/docker/hack/make/.go-autogen
+3 -1
View File
@@ -14,7 +14,8 @@
param(
[Parameter(Mandatory=$true)][string]$CommitString,
[Parameter(Mandatory=$true)][string]$DockerVersion
[Parameter(Mandatory=$true)][string]$DockerVersion,
[Parameter(Mandatory=$false)][string]$Platform
)
$ErrorActionPreference = "Stop"
@@ -43,6 +44,7 @@ const (
GitCommit string = "'+$CommitString+'"
Version string = "'+$DockerVersion+'"
BuildTime string = "'+$buildDateTime+'"
PlatformName string = "'+$Platform+'"
)
// AUTOGENERATED FILE; see hack\make\.go-autogen.ps1
@@ -849,7 +849,7 @@ func (s *DockerSwarmSuite) TestSwarmServiceTTY(c *check.C) {
waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1)
// We need to get the container id.
out, err = d.Cmd("ps", "-a", "-q", "--no-trunc")
out, err = d.Cmd("ps", "-q", "--no-trunc")
c.Assert(err, checker.IsNil)
id := strings.TrimSpace(out)
@@ -872,7 +872,7 @@ func (s *DockerSwarmSuite) TestSwarmServiceTTY(c *check.C) {
waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1)
// We need to get the container id.
out, err = d.Cmd("ps", "-a", "-q", "--no-trunc")
out, err = d.Cmd("ps", "-q", "--no-trunc")
c.Assert(err, checker.IsNil)
id = strings.TrimSpace(out)
@@ -66,10 +66,12 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T
cmd.Stderr = output
if err := cmd.Start(); err != nil {
w.Close()
return fmt.Errorf("Untar error on re-exec cmd: %v", err)
}
//write the options to the pipe for the untar exec to read
if err := json.NewEncoder(w).Encode(options); err != nil {
w.Close()
return fmt.Errorf("Untar json encode to pipe failed: %v", err)
}
w.Close()
+3 -3
View File
@@ -30,7 +30,7 @@ github.com/moby/buildkit aaff9d591ef128560018433fe61beb802e149de8
github.com/tonistiigi/fsutil dea3a0da73aee887fc02142d995be764106ac5e2
#get libnetwork packages
github.com/docker/libnetwork 64ae58878fc8f95e4a167499d654e13fa36abdc7
github.com/docker/libnetwork 9bca9a4a220b158cc94402e0f8c2c7714eb6f503
github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
@@ -42,7 +42,7 @@ github.com/hashicorp/go-multierror fcdddc395df1ddf4247c69bd436e84cfa0733f7e
github.com/hashicorp/serf 598c54895cc5a7b1a24a398d635e8c0ea0959870
github.com/docker/libkv 1d8431073ae03cdaedb198a89722f3aab6d418ef
github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25
github.com/vishvananda/netlink bd6d5de5ccef2d66b0a26177928d0d8895d7f969
github.com/vishvananda/netlink b2de5d10e38ecce8607e6b438b6d174f389a004e
github.com/BurntSushi/toml f706d00e3de6abe700c994cdd545a1a4915af060
github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374
github.com/deckarep/golang-set ef32fa3046d9f249d399f98ebaf9be944430fd1d
@@ -114,7 +114,7 @@ github.com/dmcgowan/go-tar go1.10
github.com/stevvooe/ttrpc 76e68349ad9ab4d03d764c713826d31216715e4f
# cluster
github.com/docker/swarmkit de950a7ed842c7b7e47e9451cde9bf8f96031894
github.com/docker/swarmkit 4429c763170d9ca96929249353c3270c19e7d39e
github.com/gogo/protobuf v0.4
github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a
github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b6506826e
+5 -3
View File
@@ -293,11 +293,13 @@ func (c *controller) agentInit(listenAddr, bindAddrOrInterface, advertiseAddr, d
c.Config().Daemon.NetworkControlPlaneMTU, netDBConf.PacketBufferSize)
}
nDB, err := networkdb.New(netDBConf)
if err != nil {
return err
}
// Register the diagnose handlers
c.DiagnoseServer.RegisterHandler(nDB, networkdb.NetDbPaths2Func)
var cancelList []func()
ch, cancel := nDB.Watch(libnetworkEPTable, "", "")
cancelList = append(cancelList, cancel)
@@ -436,7 +438,7 @@ func (n *network) Services() map[string]ServiceInfo {
for eid, value := range entries {
var epRec EndpointRecord
nid := n.ID()
if err := proto.Unmarshal(value.([]byte), &epRec); err != nil {
if err := proto.Unmarshal(value.Value, &epRec); err != nil {
logrus.Errorf("Unmarshal of libnetworkEPTable failed for endpoint %s in network %s, %v", eid, nid, err)
continue
}
@@ -461,7 +463,7 @@ func (n *network) Services() map[string]ServiceInfo {
}
entries := agent.networkDB.GetTableByNetwork(table.name, n.id)
for key, value := range entries {
epID, info := d.DecodeTableEntry(table.name, key, value.([]byte))
epID, info := d.DecodeTableEntry(table.name, key, value.Value)
if ep, ok := eps[epID]; !ok {
logrus.Errorf("Inconsistent driver and libnetwork state for endpoint %s", epID)
} else {
+36
View File
@@ -60,6 +60,7 @@ import (
"github.com/docker/libnetwork/cluster"
"github.com/docker/libnetwork/config"
"github.com/docker/libnetwork/datastore"
"github.com/docker/libnetwork/diagnose"
"github.com/docker/libnetwork/discoverapi"
"github.com/docker/libnetwork/driverapi"
"github.com/docker/libnetwork/drvregistry"
@@ -133,6 +134,13 @@ type NetworkController interface {
// SetKeys configures the encryption key for gossip and overlay data path
SetKeys(keys []*types.EncryptionKey) error
// StartDiagnose start the network diagnose mode
StartDiagnose(port int)
// StopDiagnose start the network diagnose mode
StopDiagnose()
// IsDiagnoseEnabled returns true if the diagnose is enabled
IsDiagnoseEnabled() bool
}
// NetworkWalker is a client provided function which will be used to walk the Networks.
@@ -167,6 +175,7 @@ type controller struct {
agentStopDone chan struct{}
keys []*types.EncryptionKey
clusterConfigAvailable bool
DiagnoseServer *diagnose.Server
sync.Mutex
}
@@ -185,7 +194,9 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
serviceBindings: make(map[serviceKey]*service),
agentInitDone: make(chan struct{}),
networkLocker: locker.New(),
DiagnoseServer: diagnose.New(),
}
c.DiagnoseServer.Init()
if err := c.initStores(); err != nil {
return nil, err
@@ -1291,3 +1302,28 @@ func (c *controller) Stop() {
c.stopExternalKeyListener()
osl.GC()
}
// StartDiagnose start the network diagnose mode
func (c *controller) StartDiagnose(port int) {
c.Lock()
if !c.DiagnoseServer.IsDebugEnable() {
c.DiagnoseServer.EnableDebug("127.0.0.1", port)
}
c.Unlock()
}
// StopDiagnose start the network diagnose mode
func (c *controller) StopDiagnose() {
c.Lock()
if c.DiagnoseServer.IsDebugEnable() {
c.DiagnoseServer.DisableDebug()
}
c.Unlock()
}
// IsDiagnoseEnabled returns true if the diagnose is enabled
func (c *controller) IsDiagnoseEnabled() bool {
c.Lock()
defer c.Unlock()
return c.DiagnoseServer.IsDebugEnable()
}
@@ -1,133 +0,0 @@
package diagnose
import (
"fmt"
"net"
"net/http"
"sync"
"github.com/sirupsen/logrus"
)
// HTTPHandlerFunc TODO
type HTTPHandlerFunc func(interface{}, http.ResponseWriter, *http.Request)
type httpHandlerCustom struct {
ctx interface{}
F func(interface{}, http.ResponseWriter, *http.Request)
}
// ServeHTTP TODO
func (h httpHandlerCustom) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h.F(h.ctx, w, r)
}
var diagPaths2Func = map[string]HTTPHandlerFunc{
"/": notImplemented,
"/help": help,
"/ready": ready,
}
// Server when the debug is enabled exposes a
// This data structure is protected by the Agent mutex so does not require and additional mutex here
type Server struct {
sk net.Listener
port int
mux *http.ServeMux
registeredHanders []string
sync.Mutex
}
// Init TODO
func (n *Server) Init() {
n.mux = http.NewServeMux()
// Register local handlers
n.RegisterHandler(n, diagPaths2Func)
}
// RegisterHandler TODO
func (n *Server) RegisterHandler(ctx interface{}, hdlrs map[string]HTTPHandlerFunc) {
n.Lock()
defer n.Unlock()
for path, fun := range hdlrs {
n.mux.Handle(path, httpHandlerCustom{ctx, fun})
n.registeredHanders = append(n.registeredHanders, path)
}
}
// EnableDebug opens a TCP socket to debug the passed network DB
func (n *Server) EnableDebug(ip string, port int) {
n.Lock()
defer n.Unlock()
n.port = port
logrus.SetLevel(logrus.DebugLevel)
if n.sk != nil {
logrus.Infof("The server is already up and running")
return
}
logrus.Infof("Starting the server listening on %d for commands", port)
// // Create the socket
// var err error
// n.sk, err = net.Listen("tcp", listeningAddr)
// if err != nil {
// log.Fatal(err)
// }
//
// go func() {
// http.Serve(n.sk, n.mux)
// }()
http.ListenAndServe(fmt.Sprintf(":%d", port), n.mux)
}
// DisableDebug stop the dubug and closes the tcp socket
func (n *Server) DisableDebug() {
n.Lock()
defer n.Unlock()
n.sk.Close()
n.sk = nil
}
// IsDebugEnable returns true when the debug is enabled
func (n *Server) IsDebugEnable() bool {
n.Lock()
defer n.Unlock()
return n.sk != nil
}
func notImplemented(ctx interface{}, w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "URL path: %s no method implemented check /help\n", r.URL.Path)
}
func help(ctx interface{}, w http.ResponseWriter, r *http.Request) {
n, ok := ctx.(*Server)
if ok {
for _, path := range n.registeredHanders {
fmt.Fprintf(w, "%s\n", path)
}
}
}
func ready(ctx interface{}, w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "OK\n")
}
// DebugHTTPForm TODO
func DebugHTTPForm(r *http.Request) {
r.ParseForm()
for k, v := range r.Form {
logrus.Debugf("Form[%q] = %q\n", k, v)
}
}
// HTTPReplyError TODO
func HTTPReplyError(w http.ResponseWriter, message, usage string) {
fmt.Fprintf(w, "%s\n", message)
if usage != "" {
fmt.Fprintf(w, "Usage: %s\n", usage)
}
}
@@ -0,0 +1,228 @@
package diagnose
import (
"context"
"encoding/json"
"fmt"
"net/http"
"sync"
"sync/atomic"
stackdump "github.com/docker/docker/pkg/signal"
"github.com/docker/libnetwork/common"
"github.com/sirupsen/logrus"
)
// HTTPHandlerFunc TODO
type HTTPHandlerFunc func(interface{}, http.ResponseWriter, *http.Request)
type httpHandlerCustom struct {
ctx interface{}
F func(interface{}, http.ResponseWriter, *http.Request)
}
// ServeHTTP TODO
func (h httpHandlerCustom) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h.F(h.ctx, w, r)
}
var diagPaths2Func = map[string]HTTPHandlerFunc{
"/": notImplemented,
"/help": help,
"/ready": ready,
"/stackdump": stackTrace,
}
// Server when the debug is enabled exposes a
// This data structure is protected by the Agent mutex so does not require and additional mutex here
type Server struct {
enable int32
srv *http.Server
port int
mux *http.ServeMux
registeredHanders map[string]bool
sync.Mutex
}
// New creates a new diagnose server
func New() *Server {
return &Server{
registeredHanders: make(map[string]bool),
}
}
// Init initialize the mux for the http handling and register the base hooks
func (s *Server) Init() {
s.mux = http.NewServeMux()
// Register local handlers
s.RegisterHandler(s, diagPaths2Func)
}
// RegisterHandler allows to register new handlers to the mux and to a specific path
func (s *Server) RegisterHandler(ctx interface{}, hdlrs map[string]HTTPHandlerFunc) {
s.Lock()
defer s.Unlock()
for path, fun := range hdlrs {
if _, ok := s.registeredHanders[path]; ok {
continue
}
s.mux.Handle(path, httpHandlerCustom{ctx, fun})
s.registeredHanders[path] = true
}
}
// ServeHTTP this is the method called bu the ListenAndServe, and is needed to allow us to
// use our custom mux
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.mux.ServeHTTP(w, r)
}
// EnableDebug opens a TCP socket to debug the passed network DB
func (s *Server) EnableDebug(ip string, port int) {
s.Lock()
defer s.Unlock()
s.port = port
if s.enable == 1 {
logrus.Info("The server is already up and running")
return
}
logrus.Infof("Starting the diagnose server listening on %d for commands", port)
srv := &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", port), Handler: s}
s.srv = srv
s.enable = 1
go func(n *Server) {
// Ingore ErrServerClosed that is returned on the Shutdown call
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
logrus.Errorf("ListenAndServe error: %s", err)
atomic.SwapInt32(&n.enable, 0)
}
}(s)
}
// DisableDebug stop the dubug and closes the tcp socket
func (s *Server) DisableDebug() {
s.Lock()
defer s.Unlock()
s.srv.Shutdown(context.Background())
s.srv = nil
s.enable = 0
logrus.Info("Disabling the diagnose server")
}
// IsDebugEnable returns true when the debug is enabled
func (s *Server) IsDebugEnable() bool {
s.Lock()
defer s.Unlock()
return s.enable == 1
}
func notImplemented(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
_, json := ParseHTTPFormOptions(r)
rsp := WrongCommand("not implemented", fmt.Sprintf("URL path: %s no method implemented check /help\n", r.URL.Path))
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("command not implemented done")
HTTPReply(w, rsp, json)
}
func help(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
_, json := ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("help done")
n, ok := ctx.(*Server)
var result string
if ok {
for path := range n.registeredHanders {
result += fmt.Sprintf("%s\n", path)
}
HTTPReply(w, CommandSucceed(&StringCmd{Info: result}), json)
}
}
func ready(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
_, json := ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("ready done")
HTTPReply(w, CommandSucceed(&StringCmd{Info: "OK"}), json)
}
func stackTrace(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
_, json := ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("stack trace")
path, err := stackdump.DumpStacks("/tmp/")
if err != nil {
log.WithError(err).Error("failed to write goroutines dump")
HTTPReply(w, FailCommand(err), json)
} else {
log.Info("stack trace done")
HTTPReply(w, CommandSucceed(&StringCmd{Info: fmt.Sprintf("goroutine stacks written to %s", path)}), json)
}
}
// DebugHTTPForm helper to print the form url parameters
func DebugHTTPForm(r *http.Request) {
for k, v := range r.Form {
logrus.Debugf("Form[%q] = %q\n", k, v)
}
}
// JSONOutput contains details on JSON output printing
type JSONOutput struct {
enable bool
prettyPrint bool
}
// ParseHTTPFormOptions easily parse the JSON printing options
func ParseHTTPFormOptions(r *http.Request) (bool, *JSONOutput) {
_, unsafe := r.Form["unsafe"]
v, json := r.Form["json"]
var pretty bool
if len(v) > 0 {
pretty = v[0] == "pretty"
}
return unsafe, &JSONOutput{enable: json, prettyPrint: pretty}
}
// HTTPReply helper function that takes care of sending the message out
func HTTPReply(w http.ResponseWriter, r *HTTPResult, j *JSONOutput) (int, error) {
var response []byte
if j.enable {
w.Header().Set("Content-Type", "application/json")
var err error
if j.prettyPrint {
response, err = json.MarshalIndent(r, "", " ")
if err != nil {
response, _ = json.MarshalIndent(FailCommand(err), "", " ")
}
} else {
response, err = json.Marshal(r)
if err != nil {
response, _ = json.Marshal(FailCommand(err))
}
}
} else {
response = []byte(r.String())
}
return fmt.Fprint(w, string(response))
}
+122
View File
@@ -0,0 +1,122 @@
package diagnose
import "fmt"
// StringInterface interface that has to be implemented by messages
type StringInterface interface {
String() string
}
// CommandSucceed creates a success message
func CommandSucceed(result StringInterface) *HTTPResult {
return &HTTPResult{
Message: "OK",
Details: result,
}
}
// FailCommand creates a failure message with error
func FailCommand(err error) *HTTPResult {
return &HTTPResult{
Message: "FAIL",
Details: &ErrorCmd{Error: err.Error()},
}
}
// WrongCommand creates a wrong command response
func WrongCommand(message, usage string) *HTTPResult {
return &HTTPResult{
Message: message,
Details: &UsageCmd{Usage: usage},
}
}
// HTTPResult Diagnose Server HTTP result operation
type HTTPResult struct {
Message string `json:"message"`
Details StringInterface `json:"details"`
}
func (h *HTTPResult) String() string {
rsp := h.Message
if h.Details != nil {
rsp += "\n" + h.Details.String()
}
return rsp
}
// UsageCmd command with usage field
type UsageCmd struct {
Usage string `json:"usage"`
}
func (u *UsageCmd) String() string {
return "Usage: " + u.Usage
}
// StringCmd command with info string
type StringCmd struct {
Info string `json:"info"`
}
func (s *StringCmd) String() string {
return s.Info
}
// ErrorCmd command with error
type ErrorCmd struct {
Error string `json:"error"`
}
func (e *ErrorCmd) String() string {
return "Error: " + e.Error
}
// TableObj network db table object
type TableObj struct {
Length int `json:"size"`
Elements []StringInterface `json:"entries"`
}
func (t *TableObj) String() string {
output := fmt.Sprintf("total entries: %d\n", t.Length)
for _, e := range t.Elements {
output += e.String()
}
return output
}
// PeerEntryObj entry in the networkdb peer table
type PeerEntryObj struct {
Index int `json:"-"`
Name string `json:"-=name"`
IP string `json:"ip"`
}
func (p *PeerEntryObj) String() string {
return fmt.Sprintf("%d) %s -> %s\n", p.Index, p.Name, p.IP)
}
// TableEntryObj network db table entry object
type TableEntryObj struct {
Index int `json:"-"`
Key string `json:"key"`
Value string `json:"value"`
Owner string `json:"owner"`
}
func (t *TableEntryObj) String() string {
return fmt.Sprintf("%d) k:`%s` -> v:`%s` owner:`%s`\n", t.Index, t.Key, t.Value, t.Owner)
}
// TableEndpointsResult fully typed message for proper unmarshaling on the client side
type TableEndpointsResult struct {
TableObj
Elements []TableEntryObj `json:"entries"`
}
// TablePeersResult fully typed message for proper unmarshaling on the client side
type TablePeersResult struct {
TableObj
Elements []PeerEntryObj `json:"entries"`
}
@@ -42,6 +42,14 @@ const (
DefaultGatewayV6AuxKey = "DefaultGatewayIPv6"
)
type defaultBridgeNetworkConflict struct {
ID string
}
func (d defaultBridgeNetworkConflict) Error() string {
return fmt.Sprintf("Stale default bridge network %s", d.ID)
}
type iptableCleanFunc func() error
type iptablesCleanFuncs []iptableCleanFunc
@@ -137,6 +145,7 @@ type driver struct {
networks map[string]*bridgeNetwork
store datastore.DataStore
nlh *netlink.Handle
configNetwork sync.Mutex
sync.Mutex
}
@@ -322,41 +331,6 @@ func (n *bridgeNetwork) isolateNetwork(others []*bridgeNetwork, enable bool) err
return nil
}
// Checks whether this network's configuration for the network with this id conflicts with any of the passed networks
func (c *networkConfiguration) conflictsWithNetworks(id string, others []*bridgeNetwork) error {
for _, nw := range others {
nw.Lock()
nwID := nw.id
nwConfig := nw.config
nwBridge := nw.bridge
nw.Unlock()
if nwID == id {
continue
}
// Verify the name (which may have been set by newInterface()) does not conflict with
// existing bridge interfaces. Ironically the system chosen name gets stored in the config...
// Basically we are checking if the two original configs were both empty.
if nwConfig.BridgeName == c.BridgeName {
return types.ForbiddenErrorf("conflicts with network %s (%s) by bridge name", nwID, nwConfig.BridgeName)
}
// If this network config specifies the AddressIPv4, we need
// to make sure it does not conflict with any previously allocated
// bridges. This could not be completely caught by the config conflict
// check, because networks which config does not specify the AddressIPv4
// get their address and subnet selected by the driver (see electBridgeIPv4())
if c.AddressIPv4 != nil && nwBridge.bridgeIPv4 != nil {
if nwBridge.bridgeIPv4.Contains(c.AddressIPv4.IP) ||
c.AddressIPv4.Contains(nwBridge.bridgeIPv4.IP) {
return types.ForbiddenErrorf("conflicts with network %s (%s) by ip network", nwID, nwConfig.BridgeName)
}
}
}
return nil
}
func (d *driver) configure(option map[string]interface{}) error {
var (
config *configuration
@@ -602,11 +576,27 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d
return err
}
err = config.processIPAM(id, ipV4Data, ipV6Data)
if err != nil {
if err = config.processIPAM(id, ipV4Data, ipV6Data); err != nil {
return err
}
// start the critical section, from this point onward we are dealing with the list of networks
// so to be consistent we cannot allow that the list changes
d.configNetwork.Lock()
defer d.configNetwork.Unlock()
// check network conflicts
if err = d.checkConflict(config); err != nil {
nerr, ok := err.(defaultBridgeNetworkConflict)
if !ok {
return err
}
// Got a conflict with a stale default network, clean that up and continue
logrus.Warn(nerr)
d.deleteNetwork(nerr.ID)
}
// there is no conflict, now create the network
if err = d.createNetwork(config); err != nil {
return err
}
@@ -614,33 +604,47 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d
return d.storeUpdate(config)
}
func (d *driver) checkConflict(config *networkConfiguration) error {
networkList := d.getNetworks()
for _, nw := range networkList {
nw.Lock()
nwConfig := nw.config
nw.Unlock()
if err := nwConfig.Conflicts(config); err != nil {
if config.DefaultBridge {
// We encountered and identified a stale default network
// We must delete it as libnetwork is the source of truth
// The default network being created must be the only one
// This can happen only from docker 1.12 on ward
logrus.Infof("Found stale default bridge network %s (%s)", nwConfig.ID, nwConfig.BridgeName)
return defaultBridgeNetworkConflict{nwConfig.ID}
}
return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s",
config.ID, config.BridgeName, nwConfig.ID, nwConfig.BridgeName, err.Error())
}
}
return nil
}
func (d *driver) createNetwork(config *networkConfiguration) error {
var err error
defer osl.InitOSContext()()
networkList := d.getNetworks()
for i, nw := range networkList {
nw.Lock()
nwConfig := nw.config
nw.Unlock()
if err := nwConfig.Conflicts(config); err != nil {
if config.DefaultBridge {
// We encountered and identified a stale default network
// We must delete it as libnetwork is the source of thruth
// The default network being created must be the only one
// This can happen only from docker 1.12 on ward
logrus.Infof("Removing stale default bridge network %s (%s)", nwConfig.ID, nwConfig.BridgeName)
if err := d.DeleteNetwork(nwConfig.ID); err != nil {
logrus.Warnf("Failed to remove stale default network: %s (%s): %v. Will remove from store.", nwConfig.ID, nwConfig.BridgeName, err)
d.storeDelete(nwConfig)
}
networkList = append(networkList[:i], networkList[i+1:]...)
} else {
return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s",
config.ID, config.BridgeName, nwConfig.ID, nwConfig.BridgeName, err.Error())
}
}
// Initialize handle when needed
d.Lock()
if d.nlh == nil {
d.nlh = ns.NlHandle()
}
d.Unlock()
// Create or retrieve the bridge L3 interface
bridgeIface, err := newInterface(d.nlh, config)
if err != nil {
return err
}
// Create and set network handler in driver
@@ -649,6 +653,7 @@ func (d *driver) createNetwork(config *networkConfiguration) error {
endpoints: make(map[string]*bridgeEndpoint),
config: config,
portMapper: portmapper.New(d.config.UserlandProxyPath),
bridge: bridgeIface,
driver: d,
}
@@ -665,35 +670,15 @@ func (d *driver) createNetwork(config *networkConfiguration) error {
}
}()
// Initialize handle when needed
d.Lock()
if d.nlh == nil {
d.nlh = ns.NlHandle()
}
d.Unlock()
// Create or retrieve the bridge L3 interface
bridgeIface, err := newInterface(d.nlh, config)
if err != nil {
return err
}
network.bridge = bridgeIface
// Verify the network configuration does not conflict with previously installed
// networks. This step is needed now because driver might have now set the bridge
// name on this config struct. And because we need to check for possible address
// conflicts, so we need to check against operationa lnetworks.
if err = config.conflictsWithNetworks(config.ID, networkList); err != nil {
return err
}
// Add inter-network communication rules.
setupNetworkIsolationRules := func(config *networkConfiguration, i *bridgeInterface) error {
if err := network.isolateNetwork(networkList, true); err != nil {
if err := network.isolateNetwork(networkList, false); err != nil {
if err = network.isolateNetwork(networkList, false); err != nil {
logrus.Warnf("Failed on removing the inter-network iptables rules on cleanup: %v", err)
}
return err
}
// register the cleanup function
network.registerIptCleanFunc(func() error {
nwList := d.getNetworks()
return network.isolateNetwork(nwList, false)
@@ -767,10 +752,17 @@ func (d *driver) createNetwork(config *networkConfiguration) error {
}
func (d *driver) DeleteNetwork(nid string) error {
d.configNetwork.Lock()
defer d.configNetwork.Unlock()
return d.deleteNetwork(nid)
}
func (d *driver) deleteNetwork(nid string) error {
var err error
defer osl.InitOSContext()()
// Get network handler and remove it from driver
d.Lock()
n, ok := d.networks[nid]
@@ -814,12 +806,6 @@ func (d *driver) DeleteNetwork(nid string) error {
}
}()
// Sanity check
if n == nil {
err = driverapi.ErrNoNetwork(nid)
return err
}
switch config.BridgeIfaceCreator {
case ifaceCreatedByLibnetwork, ifaceCreatorUnknown:
// We only delete the bridge if it was created by the bridge driver and
@@ -9,7 +9,7 @@ func (n *bridgeNetwork) setupFirewalld(config *networkConfiguration, i *bridgeIn
d.Unlock()
// Sanity check.
if driverConfig.EnableIPTables == false {
if !driverConfig.EnableIPTables {
return IPTableCfgError(config.BridgeName)
}
@@ -696,6 +696,12 @@ func (n *network) initSandbox(restore bool) error {
var nlSock *nl.NetlinkSocket
sbox.InvokeFunc(func() {
nlSock, err = nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH)
if err != nil {
return
}
// set the receive timeout to not remain stuck on the RecvFrom if the fd gets closed
tv := syscall.NsecToTimeval(soTimeout.Nanoseconds())
err = nlSock.SetReceiveTimeout(&tv)
})
n.setNetlinkSocket(nlSock)
@@ -721,6 +727,11 @@ func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {
// The netlink socket got closed, simply exit to not leak this goroutine
return
}
// When the receive timeout expires the receive will return EAGAIN
if err == syscall.EAGAIN {
// we continue here to avoid spam for timeouts
continue
}
logrus.Errorf("Failed to receive from netlink: %v ", err)
continue
}
+16
View File
@@ -5,12 +5,19 @@ package ipvs
import (
"net"
"syscall"
"time"
"fmt"
"github.com/vishvananda/netlink/nl"
"github.com/vishvananda/netns"
)
const (
netlinkRecvSocketsTimeout = 3 * time.Second
netlinkSendSocketTimeout = 30 * time.Second
)
// Service defines an IPVS service in its entirety.
type Service struct {
// Virtual service address.
@@ -82,6 +89,15 @@ func New(path string) (*Handle, error) {
if err != nil {
return nil, err
}
// Add operation timeout to avoid deadlocks
tv := syscall.NsecToTimeval(netlinkSendSocketTimeout.Nanoseconds())
if err := sock.SetSendTimeout(&tv); err != nil {
return nil, err
}
tv = syscall.NsecToTimeval(netlinkRecvSocketsTimeout.Nanoseconds())
if err := sock.SetReceiveTimeout(&tv); err != nil {
return nil, err
}
return &Handle{sock: sock}, nil
}
+7 -4
View File
@@ -203,10 +203,6 @@ func newGenlRequest(familyID int, cmd uint8) *nl.NetlinkRequest {
}
func execute(s *nl.NetlinkSocket, req *nl.NetlinkRequest, resType uint16) ([][]byte, error) {
var (
err error
)
if err := s.Send(req); err != nil {
return nil, err
}
@@ -222,6 +218,13 @@ done:
for {
msgs, err := s.Receive()
if err != nil {
if s.GetFd() == -1 {
return nil, fmt.Errorf("Socket got closed on receive")
}
if err == syscall.EAGAIN {
// timeout fired
continue
}
return nil, err
}
for _, m := range msgs {
@@ -72,7 +72,7 @@ func (e *eventDelegate) NotifyLeave(mn *memberlist.Node) {
// If the node instead left because was going down, then it makes sense to just delete all its state
e.nDB.Lock()
defer e.nDB.Unlock()
e.nDB.deleteNetworkEntriesForNode(mn.Name)
e.nDB.deleteNodeFromNetworks(mn.Name)
e.nDB.deleteNodeTableEntries(mn.Name)
if n, ok := e.nDB.nodes[mn.Name]; ok {
delete(e.nDB.nodes, mn.Name)
@@ -401,17 +401,23 @@ func (nDB *NetworkDB) UpdateEntry(tname, nid, key string, value []byte) error {
return nil
}
// TableElem elem
type TableElem struct {
Value []byte
owner string
}
// GetTableByNetwork walks the networkdb by the give table and network id and
// returns a map of keys and values
func (nDB *NetworkDB) GetTableByNetwork(tname, nid string) map[string]interface{} {
entries := make(map[string]interface{})
func (nDB *NetworkDB) GetTableByNetwork(tname, nid string) map[string]*TableElem {
entries := make(map[string]*TableElem)
nDB.indexes[byTable].WalkPrefix(fmt.Sprintf("/%s/%s", tname, nid), func(k string, v interface{}) bool {
entry := v.(*entry)
if entry.deleting {
return false
}
key := k[strings.LastIndex(k, "/")+1:]
entries[key] = entry.value
entries[key] = &TableElem{Value: entry.value, owner: entry.node}
return false
})
return entries
@@ -445,7 +451,7 @@ func (nDB *NetworkDB) DeleteEntry(tname, nid, key string) error {
return nil
}
func (nDB *NetworkDB) deleteNetworkEntriesForNode(deletedNode string) {
func (nDB *NetworkDB) deleteNodeFromNetworks(deletedNode string) {
for nid, nodes := range nDB.networkNodes {
updatedNodes := make([]string, 0, len(nodes))
for _, node := range nodes {
@@ -547,7 +553,9 @@ func (nDB *NetworkDB) deleteNodeTableEntries(node string) {
nDB.deleteEntry(nid, tname, key)
nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, oldEntry.value))
if !oldEntry.deleting {
nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, oldEntry.value))
}
return false
})
}
@@ -1,17 +1,19 @@
package networkdb
import (
"encoding/base64"
"fmt"
"net/http"
"strings"
stackdump "github.com/docker/docker/pkg/signal"
"github.com/docker/libnetwork/common"
"github.com/docker/libnetwork/diagnose"
"github.com/sirupsen/logrus"
)
const (
missingParameter = "missing parameter"
dbNotAvailable = "database not available"
)
// NetDbPaths2Func TODO
@@ -26,14 +28,21 @@ var NetDbPaths2Func = map[string]diagnose.HTTPHandlerFunc{
"/deleteentry": dbDeleteEntry,
"/getentry": dbGetEntry,
"/gettable": dbGetTable,
"/dump": dbStackTrace,
}
func dbJoin(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
diagnose.DebugHTTPForm(r)
_, json := diagnose.ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("join cluster")
if len(r.Form["members"]) < 1 {
diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?members=ip1,ip2,...", r.URL.Path))
rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?members=ip1,ip2,...", r.URL.Path))
log.Error("join cluster failed, wrong input")
diagnose.HTTPReply(w, rsp, json)
return
}
@@ -41,51 +50,88 @@ func dbJoin(ctx interface{}, w http.ResponseWriter, r *http.Request) {
if ok {
err := nDB.Join(strings.Split(r.Form["members"][0], ","))
if err != nil {
fmt.Fprintf(w, "%s error in the DB join %s\n", r.URL.Path, err)
rsp := diagnose.FailCommand(fmt.Errorf("%s error in the DB join %s", r.URL.Path, err))
log.WithError(err).Error("join cluster failed")
diagnose.HTTPReply(w, rsp, json)
return
}
fmt.Fprintf(w, "OK\n")
log.Info("join cluster done")
diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json)
return
}
diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json)
}
func dbPeers(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
diagnose.DebugHTTPForm(r)
_, json := diagnose.ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("network peers")
if len(r.Form["nid"]) < 1 {
diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?nid=test", r.URL.Path))
rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?nid=test", r.URL.Path))
log.Error("network peers failed, wrong input")
diagnose.HTTPReply(w, rsp, json)
return
}
nDB, ok := ctx.(*NetworkDB)
if ok {
peers := nDB.Peers(r.Form["nid"][0])
fmt.Fprintf(w, "Network:%s Total peers: %d\n", r.Form["nid"], len(peers))
rsp := &diagnose.TableObj{Length: len(peers)}
for i, peerInfo := range peers {
fmt.Fprintf(w, "%d) %s -> %s\n", i, peerInfo.Name, peerInfo.IP)
rsp.Elements = append(rsp.Elements, &diagnose.PeerEntryObj{Index: i, Name: peerInfo.Name, IP: peerInfo.IP})
}
log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("network peers done")
diagnose.HTTPReply(w, diagnose.CommandSucceed(rsp), json)
return
}
diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json)
}
func dbClusterPeers(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
diagnose.DebugHTTPForm(r)
_, json := diagnose.ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("cluster peers")
nDB, ok := ctx.(*NetworkDB)
if ok {
peers := nDB.ClusterPeers()
fmt.Fprintf(w, "Total peers: %d\n", len(peers))
rsp := &diagnose.TableObj{Length: len(peers)}
for i, peerInfo := range peers {
fmt.Fprintf(w, "%d) %s -> %s\n", i, peerInfo.Name, peerInfo.IP)
rsp.Elements = append(rsp.Elements, &diagnose.PeerEntryObj{Index: i, Name: peerInfo.Name, IP: peerInfo.IP})
}
log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("cluster peers done")
diagnose.HTTPReply(w, diagnose.CommandSucceed(rsp), json)
return
}
diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json)
}
func dbCreateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
diagnose.DebugHTTPForm(r)
unsafe, json := diagnose.ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("create entry")
if len(r.Form["tname"]) < 1 ||
len(r.Form["nid"]) < 1 ||
len(r.Form["key"]) < 1 ||
len(r.Form["value"]) < 1 {
diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path))
rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path))
log.Error("create entry failed, wrong input")
diagnose.HTTPReply(w, rsp, json)
return
}
@@ -93,25 +139,48 @@ func dbCreateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) {
nid := r.Form["nid"][0]
key := r.Form["key"][0]
value := r.Form["value"][0]
decodedValue := []byte(value)
if !unsafe {
var err error
decodedValue, err = base64.StdEncoding.DecodeString(value)
if err != nil {
log.WithError(err).Error("create entry failed")
diagnose.HTTPReply(w, diagnose.FailCommand(err), json)
return
}
}
nDB, ok := ctx.(*NetworkDB)
if ok {
if err := nDB.CreateEntry(tname, nid, key, []byte(value)); err != nil {
diagnose.HTTPReplyError(w, err.Error(), "")
if err := nDB.CreateEntry(tname, nid, key, decodedValue); err != nil {
rsp := diagnose.FailCommand(err)
diagnose.HTTPReply(w, rsp, json)
log.WithError(err).Error("create entry failed")
return
}
fmt.Fprintf(w, "OK\n")
log.Info("create entry done")
diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json)
return
}
diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json)
}
func dbUpdateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
diagnose.DebugHTTPForm(r)
unsafe, json := diagnose.ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("update entry")
if len(r.Form["tname"]) < 1 ||
len(r.Form["nid"]) < 1 ||
len(r.Form["key"]) < 1 ||
len(r.Form["value"]) < 1 {
diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path))
rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k&value=v", r.URL.Path))
log.Error("update entry failed, wrong input")
diagnose.HTTPReply(w, rsp, json)
return
}
@@ -119,24 +188,46 @@ func dbUpdateEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) {
nid := r.Form["nid"][0]
key := r.Form["key"][0]
value := r.Form["value"][0]
decodedValue := []byte(value)
if !unsafe {
var err error
decodedValue, err = base64.StdEncoding.DecodeString(value)
if err != nil {
log.WithError(err).Error("update entry failed")
diagnose.HTTPReply(w, diagnose.FailCommand(err), json)
return
}
}
nDB, ok := ctx.(*NetworkDB)
if ok {
if err := nDB.UpdateEntry(tname, nid, key, []byte(value)); err != nil {
diagnose.HTTPReplyError(w, err.Error(), "")
if err := nDB.UpdateEntry(tname, nid, key, decodedValue); err != nil {
log.WithError(err).Error("update entry failed")
diagnose.HTTPReply(w, diagnose.FailCommand(err), json)
return
}
fmt.Fprintf(w, "OK\n")
log.Info("update entry done")
diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json)
return
}
diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json)
}
func dbDeleteEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
diagnose.DebugHTTPForm(r)
_, json := diagnose.ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("delete entry")
if len(r.Form["tname"]) < 1 ||
len(r.Form["nid"]) < 1 ||
len(r.Form["key"]) < 1 {
diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path))
rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path))
log.Error("delete entry failed, wrong input")
diagnose.HTTPReply(w, rsp, json)
return
}
@@ -148,20 +239,32 @@ func dbDeleteEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) {
if ok {
err := nDB.DeleteEntry(tname, nid, key)
if err != nil {
diagnose.HTTPReplyError(w, err.Error(), "")
log.WithError(err).Error("delete entry failed")
diagnose.HTTPReply(w, diagnose.FailCommand(err), json)
return
}
fmt.Fprintf(w, "OK\n")
log.Info("delete entry done")
diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json)
return
}
diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json)
}
func dbGetEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
diagnose.DebugHTTPForm(r)
unsafe, json := diagnose.ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("get entry")
if len(r.Form["tname"]) < 1 ||
len(r.Form["nid"]) < 1 ||
len(r.Form["key"]) < 1 {
diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path))
rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id&key=k", r.URL.Path))
log.Error("get entry failed, wrong input")
diagnose.HTTPReply(w, rsp, json)
return
}
@@ -173,18 +276,39 @@ func dbGetEntry(ctx interface{}, w http.ResponseWriter, r *http.Request) {
if ok {
value, err := nDB.GetEntry(tname, nid, key)
if err != nil {
diagnose.HTTPReplyError(w, err.Error(), "")
log.WithError(err).Error("get entry failed")
diagnose.HTTPReply(w, diagnose.FailCommand(err), json)
return
}
fmt.Fprintf(w, "key:`%s` value:`%s`\n", key, string(value))
var encodedValue string
if unsafe {
encodedValue = string(value)
} else {
encodedValue = base64.StdEncoding.EncodeToString(value)
}
rsp := &diagnose.TableEntryObj{Key: key, Value: encodedValue}
log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("update entry done")
diagnose.HTTPReply(w, diagnose.CommandSucceed(rsp), json)
return
}
diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json)
}
func dbJoinNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
diagnose.DebugHTTPForm(r)
_, json := diagnose.ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("join network")
if len(r.Form["nid"]) < 1 {
diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path))
rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path))
log.Error("join network failed, wrong input")
diagnose.HTTPReply(w, rsp, json)
return
}
@@ -193,18 +317,30 @@ func dbJoinNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) {
nDB, ok := ctx.(*NetworkDB)
if ok {
if err := nDB.JoinNetwork(nid); err != nil {
diagnose.HTTPReplyError(w, err.Error(), "")
log.WithError(err).Error("join network failed")
diagnose.HTTPReply(w, diagnose.FailCommand(err), json)
return
}
fmt.Fprintf(w, "OK\n")
log.Info("join network done")
diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json)
return
}
diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json)
}
func dbLeaveNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
diagnose.DebugHTTPForm(r)
_, json := diagnose.ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("leave network")
if len(r.Form["nid"]) < 1 {
diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path))
rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?nid=network_id", r.URL.Path))
log.Error("leave network failed, wrong input")
diagnose.HTTPReply(w, rsp, json)
return
}
@@ -213,19 +349,31 @@ func dbLeaveNetwork(ctx interface{}, w http.ResponseWriter, r *http.Request) {
nDB, ok := ctx.(*NetworkDB)
if ok {
if err := nDB.LeaveNetwork(nid); err != nil {
diagnose.HTTPReplyError(w, err.Error(), "")
log.WithError(err).Error("leave network failed")
diagnose.HTTPReply(w, diagnose.FailCommand(err), json)
return
}
fmt.Fprintf(w, "OK\n")
log.Info("leave network done")
diagnose.HTTPReply(w, diagnose.CommandSucceed(nil), json)
return
}
diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json)
}
func dbGetTable(ctx interface{}, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
diagnose.DebugHTTPForm(r)
unsafe, json := diagnose.ParseHTTPFormOptions(r)
// audit logs
log := logrus.WithFields(logrus.Fields{"component": "diagnose", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})
log.Info("get table")
if len(r.Form["tname"]) < 1 ||
len(r.Form["nid"]) < 1 {
diagnose.HTTPReplyError(w, missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id", r.URL.Path))
rsp := diagnose.WrongCommand(missingParameter, fmt.Sprintf("%s?tname=table_name&nid=network_id", r.URL.Path))
log.Error("get table failed, wrong input")
diagnose.HTTPReply(w, rsp, json)
return
}
@@ -235,20 +383,26 @@ func dbGetTable(ctx interface{}, w http.ResponseWriter, r *http.Request) {
nDB, ok := ctx.(*NetworkDB)
if ok {
table := nDB.GetTableByNetwork(tname, nid)
fmt.Fprintf(w, "total elements: %d\n", len(table))
i := 0
rsp := &diagnose.TableObj{Length: len(table)}
var i = 0
for k, v := range table {
fmt.Fprintf(w, "%d) k:`%s` -> v:`%s`\n", i, k, string(v.([]byte)))
i++
var encodedValue string
if unsafe {
encodedValue = string(v.Value)
} else {
encodedValue = base64.StdEncoding.EncodeToString(v.Value)
}
rsp.Elements = append(rsp.Elements,
&diagnose.TableEntryObj{
Index: i,
Key: k,
Value: encodedValue,
Owner: v.owner,
})
}
log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("get table done")
diagnose.HTTPReply(w, diagnose.CommandSucceed(rsp), json)
return
}
}
func dbStackTrace(ctx interface{}, w http.ResponseWriter, r *http.Request) {
path, err := stackdump.DumpStacks("/tmp/")
if err != nil {
logrus.WithError(err).Error("failed to write goroutines dump")
} else {
fmt.Fprintf(w, "goroutine stacks written to %s", path)
}
diagnose.HTTPReply(w, diagnose.FailCommand(fmt.Errorf("%s", dbNotAvailable)), json)
}
+1 -1
View File
@@ -45,7 +45,7 @@ github.com/sirupsen/logrus v1.0.3
github.com/stretchr/testify dab07ac62d4905d3e48d17dc549c684ac3b7c15a
github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16
github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065
github.com/vishvananda/netlink bd6d5de5ccef2d66b0a26177928d0d8895d7f969
github.com/vishvananda/netlink b2de5d10e38ecce8607e6b438b6d174f389a004e
github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25
golang.org/x/crypto 558b6879de74bc843225cde5686419267ff707ca
golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6
@@ -288,7 +288,9 @@ func Do(ctx context.Context, task *api.Task, ctlr Controller) (*api.TaskStatus,
status.PortStatus = portStatus
}()
if task.DesiredState == api.TaskStateShutdown {
// this branch bounds the largest state achievable in the agent as SHUTDOWN, which
// is exactly the correct behavior for the agent.
if task.DesiredState >= api.TaskStateShutdown {
if status.State >= api.TaskStateCompleted {
return noop()
}
+7 -4
View File
@@ -131,6 +131,8 @@
LeaveResponse
ProcessRaftMessageRequest
ProcessRaftMessageResponse
StreamRaftMessageRequest
StreamRaftMessageResponse
ResolveAddressRequest
ResolveAddressResponse
InternalRaftRequest
@@ -235,6 +237,7 @@ import (
import raftselector "github.com/docker/swarmkit/manager/raftselector"
import codes "google.golang.org/grpc/codes"
import status "google.golang.org/grpc/status"
import metadata "google.golang.org/grpc/metadata"
import transport "google.golang.org/grpc/transport"
import rafttime "time"
@@ -984,12 +987,12 @@ func NewRaftProxyCAServer(local CAServer, connSelector raftselector.ConnProvider
redirectChecker := func(ctx context.Context) (context.Context, error) {
s, ok := transport.StreamFromContext(ctx)
if !ok {
return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context")
return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context")
}
addr := s.ServerTransport().RemoteAddr().String()
md, ok := metadata.FromContext(ctx)
if ok && len(md["redirect"]) != 0 {
return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
}
if !ok {
md = metadata.New(map[string]string{})
@@ -1126,12 +1129,12 @@ func NewRaftProxyNodeCAServer(local NodeCAServer, connSelector raftselector.Conn
redirectChecker := func(ctx context.Context) (context.Context, error) {
s, ok := transport.StreamFromContext(ctx)
if !ok {
return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context")
return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context")
}
addr := s.ServerTransport().RemoteAddr().String()
md, ok := metadata.FromContext(ctx)
if ok && len(md["redirect"]) != 0 {
return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
}
if !ok {
md = metadata.New(map[string]string{})
+3 -2
View File
@@ -19,6 +19,7 @@ import (
import raftselector "github.com/docker/swarmkit/manager/raftselector"
import codes "google.golang.org/grpc/codes"
import status "google.golang.org/grpc/status"
import metadata "google.golang.org/grpc/metadata"
import transport "google.golang.org/grpc/transport"
import rafttime "time"
@@ -5859,12 +5860,12 @@ func NewRaftProxyControlServer(local ControlServer, connSelector raftselector.Co
redirectChecker := func(ctx context.Context) (context.Context, error) {
s, ok := transport.StreamFromContext(ctx)
if !ok {
return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context")
return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context")
}
addr := s.ServerTransport().RemoteAddr().String()
md, ok := metadata.FromContext(ctx)
if ok && len(md["redirect"]) != 0 {
return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
}
if !ok {
md = metadata.New(map[string]string{})
+3 -2
View File
@@ -24,6 +24,7 @@ import github_com_gogo_protobuf_types "github.com/gogo/protobuf/types"
import raftselector "github.com/docker/swarmkit/manager/raftselector"
import codes "google.golang.org/grpc/codes"
import status "google.golang.org/grpc/status"
import metadata "google.golang.org/grpc/metadata"
import transport "google.golang.org/grpc/transport"
import rafttime "time"
@@ -1602,12 +1603,12 @@ func NewRaftProxyDispatcherServer(local DispatcherServer, connSelector raftselec
redirectChecker := func(ctx context.Context) (context.Context, error) {
s, ok := transport.StreamFromContext(ctx)
if !ok {
return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context")
return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context")
}
addr := s.ServerTransport().RemoteAddr().String()
md, ok := metadata.FromContext(ctx)
if ok && len(md["redirect"]) != 0 {
return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
}
if !ok {
md = metadata.New(map[string]string{})
+3 -2
View File
@@ -17,6 +17,7 @@ import (
import raftselector "github.com/docker/swarmkit/manager/raftselector"
import codes "google.golang.org/grpc/codes"
import status "google.golang.org/grpc/status"
import metadata "google.golang.org/grpc/metadata"
import transport "google.golang.org/grpc/transport"
import rafttime "time"
@@ -286,12 +287,12 @@ func NewRaftProxyHealthServer(local HealthServer, connSelector raftselector.Conn
redirectChecker := func(ctx context.Context) (context.Context, error) {
s, ok := transport.StreamFromContext(ctx)
if !ok {
return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context")
return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context")
}
addr := s.ServerTransport().RemoteAddr().String()
md, ok := metadata.FromContext(ctx)
if ok && len(md["redirect"]) != 0 {
return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
}
if !ok {
md = metadata.New(map[string]string{})
+5 -4
View File
@@ -20,6 +20,7 @@ import (
import raftselector "github.com/docker/swarmkit/manager/raftselector"
import codes "google.golang.org/grpc/codes"
import status "google.golang.org/grpc/status"
import metadata "google.golang.org/grpc/metadata"
import transport "google.golang.org/grpc/transport"
import rafttime "time"
@@ -1273,12 +1274,12 @@ func NewRaftProxyLogsServer(local LogsServer, connSelector raftselector.ConnProv
redirectChecker := func(ctx context.Context) (context.Context, error) {
s, ok := transport.StreamFromContext(ctx)
if !ok {
return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context")
return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context")
}
addr := s.ServerTransport().RemoteAddr().String()
md, ok := metadata.FromContext(ctx)
if ok && len(md["redirect"]) != 0 {
return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
}
if !ok {
md = metadata.New(map[string]string{})
@@ -1396,12 +1397,12 @@ func NewRaftProxyLogBrokerServer(local LogBrokerServer, connSelector raftselecto
redirectChecker := func(ctx context.Context) (context.Context, error) {
s, ok := transport.StreamFromContext(ctx)
if !ok {
return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context")
return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context")
}
addr := s.ServerTransport().RemoteAddr().String()
md, ok := metadata.FromContext(ctx)
if ok && len(md["redirect"]) != 0 {
return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
}
if !ok {
md = metadata.New(map[string]string{})
+481 -91
View File
@@ -21,6 +21,7 @@ import (
import raftselector "github.com/docker/swarmkit/manager/raftselector"
import codes "google.golang.org/grpc/codes"
import status "google.golang.org/grpc/status"
import metadata "google.golang.org/grpc/metadata"
import transport "google.golang.org/grpc/transport"
import rafttime "time"
@@ -133,6 +134,23 @@ func (m *ProcessRaftMessageResponse) Reset() { *m = ProcessRa
func (*ProcessRaftMessageResponse) ProtoMessage() {}
func (*ProcessRaftMessageResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{6} }
// Raft message streaming request.
type StreamRaftMessageRequest struct {
Message *raftpb.Message `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"`
}
func (m *StreamRaftMessageRequest) Reset() { *m = StreamRaftMessageRequest{} }
func (*StreamRaftMessageRequest) ProtoMessage() {}
func (*StreamRaftMessageRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{7} }
// Raft message streaming response.
type StreamRaftMessageResponse struct {
}
func (m *StreamRaftMessageResponse) Reset() { *m = StreamRaftMessageResponse{} }
func (*StreamRaftMessageResponse) ProtoMessage() {}
func (*StreamRaftMessageResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{8} }
type ResolveAddressRequest struct {
// raft_id is the ID to resolve to an address.
RaftID uint64 `protobuf:"varint,1,opt,name=raft_id,json=raftId,proto3" json:"raft_id,omitempty"`
@@ -140,7 +158,7 @@ type ResolveAddressRequest struct {
func (m *ResolveAddressRequest) Reset() { *m = ResolveAddressRequest{} }
func (*ResolveAddressRequest) ProtoMessage() {}
func (*ResolveAddressRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{7} }
func (*ResolveAddressRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{9} }
type ResolveAddressResponse struct {
// Addr specifies the address of the member
@@ -149,7 +167,7 @@ type ResolveAddressResponse struct {
func (m *ResolveAddressResponse) Reset() { *m = ResolveAddressResponse{} }
func (*ResolveAddressResponse) ProtoMessage() {}
func (*ResolveAddressResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{8} }
func (*ResolveAddressResponse) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{10} }
// Contains one of many protobuf encoded objects to replicate
// over the raft backend with a request ID to track when the
@@ -161,7 +179,7 @@ type InternalRaftRequest struct {
func (m *InternalRaftRequest) Reset() { *m = InternalRaftRequest{} }
func (*InternalRaftRequest) ProtoMessage() {}
func (*InternalRaftRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{9} }
func (*InternalRaftRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{11} }
// StoreAction defines a target and operation to apply on the storage system.
type StoreAction struct {
@@ -181,7 +199,7 @@ type StoreAction struct {
func (m *StoreAction) Reset() { *m = StoreAction{} }
func (*StoreAction) ProtoMessage() {}
func (*StoreAction) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{10} }
func (*StoreAction) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{12} }
type isStoreAction_Target interface {
isStoreAction_Target()
@@ -512,6 +530,8 @@ func init() {
proto.RegisterType((*LeaveResponse)(nil), "docker.swarmkit.v1.LeaveResponse")
proto.RegisterType((*ProcessRaftMessageRequest)(nil), "docker.swarmkit.v1.ProcessRaftMessageRequest")
proto.RegisterType((*ProcessRaftMessageResponse)(nil), "docker.swarmkit.v1.ProcessRaftMessageResponse")
proto.RegisterType((*StreamRaftMessageRequest)(nil), "docker.swarmkit.v1.StreamRaftMessageRequest")
proto.RegisterType((*StreamRaftMessageResponse)(nil), "docker.swarmkit.v1.StreamRaftMessageResponse")
proto.RegisterType((*ResolveAddressRequest)(nil), "docker.swarmkit.v1.ResolveAddressRequest")
proto.RegisterType((*ResolveAddressResponse)(nil), "docker.swarmkit.v1.ResolveAddressResponse")
proto.RegisterType((*InternalRaftRequest)(nil), "docker.swarmkit.v1.InternalRaftRequest")
@@ -539,6 +559,14 @@ func (p *authenticatedWrapperRaftServer) ProcessRaftMessage(ctx context.Context,
return p.local.ProcessRaftMessage(ctx, r)
}
func (p *authenticatedWrapperRaftServer) StreamRaftMessage(stream Raft_StreamRaftMessageServer) error {
if err := p.authorize(stream.Context(), []string{"swarm-manager"}); err != nil {
return err
}
return p.local.StreamRaftMessage(stream)
}
func (p *authenticatedWrapperRaftServer) ResolveAddress(ctx context.Context, r *ResolveAddressRequest) (*ResolveAddressResponse, error) {
if err := p.authorize(ctx, []string{"swarm-manager"}); err != nil {
@@ -673,6 +701,16 @@ func (m *ProcessRaftMessageResponse) Copy() *ProcessRaftMessageResponse {
}
func (m *ProcessRaftMessageResponse) CopyFrom(src interface{}) {}
func (m *StreamRaftMessageResponse) Copy() *StreamRaftMessageResponse {
if m == nil {
return nil
}
o := &StreamRaftMessageResponse{}
o.CopyFrom(m)
return o
}
func (m *StreamRaftMessageResponse) CopyFrom(src interface{}) {}
func (m *ResolveAddressRequest) Copy() *ResolveAddressRequest {
if m == nil {
return nil
@@ -813,6 +851,12 @@ type RaftClient interface {
// ProcessRaftMessage sends a raft message to be processed on a raft member, it is
// called from the RaftMember willing to send a message to its destination ('To' field)
ProcessRaftMessage(ctx context.Context, in *ProcessRaftMessageRequest, opts ...grpc.CallOption) (*ProcessRaftMessageResponse, error)
// StreamRaftMessage accepts a stream of raft messages of type StreamRaftMessageRequest
// to be processed on a raft member, returning a StreamRaftMessageResponse
// when processing of the streamed messages is complete. A single stream corresponds
// to a single raft message, which may be disassembled and streamed as individual messages.
// It is called from the Raft leader, which uses it to stream messages to a raft member.
StreamRaftMessage(ctx context.Context, opts ...grpc.CallOption) (Raft_StreamRaftMessageClient, error)
// ResolveAddress returns the address where the node with the given ID can be reached.
ResolveAddress(ctx context.Context, in *ResolveAddressRequest, opts ...grpc.CallOption) (*ResolveAddressResponse, error)
}
@@ -834,6 +878,40 @@ func (c *raftClient) ProcessRaftMessage(ctx context.Context, in *ProcessRaftMess
return out, nil
}
func (c *raftClient) StreamRaftMessage(ctx context.Context, opts ...grpc.CallOption) (Raft_StreamRaftMessageClient, error) {
stream, err := grpc.NewClientStream(ctx, &_Raft_serviceDesc.Streams[0], c.cc, "/docker.swarmkit.v1.Raft/StreamRaftMessage", opts...)
if err != nil {
return nil, err
}
x := &raftStreamRaftMessageClient{stream}
return x, nil
}
type Raft_StreamRaftMessageClient interface {
Send(*StreamRaftMessageRequest) error
CloseAndRecv() (*StreamRaftMessageResponse, error)
grpc.ClientStream
}
type raftStreamRaftMessageClient struct {
grpc.ClientStream
}
func (x *raftStreamRaftMessageClient) Send(m *StreamRaftMessageRequest) error {
return x.ClientStream.SendMsg(m)
}
func (x *raftStreamRaftMessageClient) CloseAndRecv() (*StreamRaftMessageResponse, error) {
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
m := new(StreamRaftMessageResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
func (c *raftClient) ResolveAddress(ctx context.Context, in *ResolveAddressRequest, opts ...grpc.CallOption) (*ResolveAddressResponse, error) {
out := new(ResolveAddressResponse)
err := grpc.Invoke(ctx, "/docker.swarmkit.v1.Raft/ResolveAddress", in, out, c.cc, opts...)
@@ -849,6 +927,12 @@ type RaftServer interface {
// ProcessRaftMessage sends a raft message to be processed on a raft member, it is
// called from the RaftMember willing to send a message to its destination ('To' field)
ProcessRaftMessage(context.Context, *ProcessRaftMessageRequest) (*ProcessRaftMessageResponse, error)
// StreamRaftMessage accepts a stream of raft messages of type StreamRaftMessageRequest
// to be processed on a raft member, returning a StreamRaftMessageResponse
// when processing of the streamed messages is complete. A single stream corresponds
// to a single raft message, which may be disassembled and streamed as individual messages.
// It is called from the Raft leader, which uses it to stream messages to a raft member.
StreamRaftMessage(Raft_StreamRaftMessageServer) error
// ResolveAddress returns the address where the node with the given ID can be reached.
ResolveAddress(context.Context, *ResolveAddressRequest) (*ResolveAddressResponse, error)
}
@@ -875,6 +959,32 @@ func _Raft_ProcessRaftMessage_Handler(srv interface{}, ctx context.Context, dec
return interceptor(ctx, in, info, handler)
}
func _Raft_StreamRaftMessage_Handler(srv interface{}, stream grpc.ServerStream) error {
return srv.(RaftServer).StreamRaftMessage(&raftStreamRaftMessageServer{stream})
}
type Raft_StreamRaftMessageServer interface {
SendAndClose(*StreamRaftMessageResponse) error
Recv() (*StreamRaftMessageRequest, error)
grpc.ServerStream
}
type raftStreamRaftMessageServer struct {
grpc.ServerStream
}
func (x *raftStreamRaftMessageServer) SendAndClose(m *StreamRaftMessageResponse) error {
return x.ServerStream.SendMsg(m)
}
func (x *raftStreamRaftMessageServer) Recv() (*StreamRaftMessageRequest, error) {
m := new(StreamRaftMessageRequest)
if err := x.ServerStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
func _Raft_ResolveAddress_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ResolveAddressRequest)
if err := dec(in); err != nil {
@@ -906,7 +1016,13 @@ var _Raft_serviceDesc = grpc.ServiceDesc{
Handler: _Raft_ResolveAddress_Handler,
},
},
Streams: []grpc.StreamDesc{},
Streams: []grpc.StreamDesc{
{
StreamName: "StreamRaftMessage",
Handler: _Raft_StreamRaftMessage_Handler,
ClientStreams: true,
},
},
Metadata: "github.com/docker/swarmkit/api/raft.proto",
}
@@ -1212,6 +1328,52 @@ func (m *ProcessRaftMessageResponse) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *StreamRaftMessageRequest) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *StreamRaftMessageRequest) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if m.Message != nil {
dAtA[i] = 0xa
i++
i = encodeVarintRaft(dAtA, i, uint64(m.Message.Size()))
n4, err := m.Message.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n4
}
return i, nil
}
func (m *StreamRaftMessageResponse) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *StreamRaftMessageResponse) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
return i, nil
}
func (m *ResolveAddressRequest) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
@@ -1315,11 +1477,11 @@ func (m *StoreAction) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintRaft(dAtA, i, uint64(m.Action))
}
if m.Target != nil {
nn4, err := m.Target.MarshalTo(dAtA[i:])
nn5, err := m.Target.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += nn4
i += nn5
}
return i, nil
}
@@ -1330,11 +1492,11 @@ func (m *StoreAction_Node) MarshalTo(dAtA []byte) (int, error) {
dAtA[i] = 0x12
i++
i = encodeVarintRaft(dAtA, i, uint64(m.Node.Size()))
n5, err := m.Node.MarshalTo(dAtA[i:])
n6, err := m.Node.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n5
i += n6
}
return i, nil
}
@@ -1344,11 +1506,11 @@ func (m *StoreAction_Service) MarshalTo(dAtA []byte) (int, error) {
dAtA[i] = 0x1a
i++
i = encodeVarintRaft(dAtA, i, uint64(m.Service.Size()))
n6, err := m.Service.MarshalTo(dAtA[i:])
n7, err := m.Service.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n6
i += n7
}
return i, nil
}
@@ -1358,11 +1520,11 @@ func (m *StoreAction_Task) MarshalTo(dAtA []byte) (int, error) {
dAtA[i] = 0x22
i++
i = encodeVarintRaft(dAtA, i, uint64(m.Task.Size()))
n7, err := m.Task.MarshalTo(dAtA[i:])
n8, err := m.Task.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n7
i += n8
}
return i, nil
}
@@ -1372,11 +1534,11 @@ func (m *StoreAction_Network) MarshalTo(dAtA []byte) (int, error) {
dAtA[i] = 0x2a
i++
i = encodeVarintRaft(dAtA, i, uint64(m.Network.Size()))
n8, err := m.Network.MarshalTo(dAtA[i:])
n9, err := m.Network.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n8
i += n9
}
return i, nil
}
@@ -1386,11 +1548,11 @@ func (m *StoreAction_Cluster) MarshalTo(dAtA []byte) (int, error) {
dAtA[i] = 0x32
i++
i = encodeVarintRaft(dAtA, i, uint64(m.Cluster.Size()))
n9, err := m.Cluster.MarshalTo(dAtA[i:])
n10, err := m.Cluster.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n9
i += n10
}
return i, nil
}
@@ -1400,11 +1562,11 @@ func (m *StoreAction_Secret) MarshalTo(dAtA []byte) (int, error) {
dAtA[i] = 0x3a
i++
i = encodeVarintRaft(dAtA, i, uint64(m.Secret.Size()))
n10, err := m.Secret.MarshalTo(dAtA[i:])
n11, err := m.Secret.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n10
i += n11
}
return i, nil
}
@@ -1414,11 +1576,11 @@ func (m *StoreAction_Resource) MarshalTo(dAtA []byte) (int, error) {
dAtA[i] = 0x42
i++
i = encodeVarintRaft(dAtA, i, uint64(m.Resource.Size()))
n11, err := m.Resource.MarshalTo(dAtA[i:])
n12, err := m.Resource.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n11
i += n12
}
return i, nil
}
@@ -1428,11 +1590,11 @@ func (m *StoreAction_Extension) MarshalTo(dAtA []byte) (int, error) {
dAtA[i] = 0x4a
i++
i = encodeVarintRaft(dAtA, i, uint64(m.Extension.Size()))
n12, err := m.Extension.MarshalTo(dAtA[i:])
n13, err := m.Extension.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n12
i += n13
}
return i, nil
}
@@ -1442,11 +1604,11 @@ func (m *StoreAction_Config) MarshalTo(dAtA []byte) (int, error) {
dAtA[i] = 0x52
i++
i = encodeVarintRaft(dAtA, i, uint64(m.Config.Size()))
n13, err := m.Config.MarshalTo(dAtA[i:])
n14, err := m.Config.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n13
i += n14
}
return i, nil
}
@@ -1488,12 +1650,12 @@ func NewRaftProxyRaftServer(local RaftServer, connSelector raftselector.ConnProv
redirectChecker := func(ctx context.Context) (context.Context, error) {
s, ok := transport.StreamFromContext(ctx)
if !ok {
return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context")
return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context")
}
addr := s.ServerTransport().RemoteAddr().String()
md, ok := metadata.FromContext(ctx)
if ok && len(md["redirect"]) != 0 {
return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
}
if !ok {
md = metadata.New(map[string]string{})
@@ -1585,6 +1747,63 @@ func (p *raftProxyRaftServer) ProcessRaftMessage(ctx context.Context, r *Process
return resp, err
}
type Raft_StreamRaftMessageServerWrapper struct {
Raft_StreamRaftMessageServer
ctx context.Context
}
func (s Raft_StreamRaftMessageServerWrapper) Context() context.Context {
return s.ctx
}
func (p *raftProxyRaftServer) StreamRaftMessage(stream Raft_StreamRaftMessageServer) error {
ctx := stream.Context()
conn, err := p.connSelector.LeaderConn(ctx)
if err != nil {
if err == raftselector.ErrIsLeader {
ctx, err = p.runCtxMods(ctx, p.localCtxMods)
if err != nil {
return err
}
streamWrapper := Raft_StreamRaftMessageServerWrapper{
Raft_StreamRaftMessageServer: stream,
ctx: ctx,
}
return p.local.StreamRaftMessage(streamWrapper)
}
return err
}
ctx, err = p.runCtxMods(ctx, p.remoteCtxMods)
if err != nil {
return err
}
clientStream, err := NewRaftClient(conn).StreamRaftMessage(ctx)
if err != nil {
return err
}
for {
msg, err := stream.Recv()
if err == io.EOF {
break
}
if err != nil {
return err
}
if err := clientStream.Send(msg); err != nil {
return err
}
}
reply, err := clientStream.CloseAndRecv()
if err != nil {
return err
}
return stream.SendAndClose(reply)
}
func (p *raftProxyRaftServer) ResolveAddress(ctx context.Context, r *ResolveAddressRequest) (*ResolveAddressResponse, error) {
conn, err := p.connSelector.LeaderConn(ctx)
@@ -1630,12 +1849,12 @@ func NewRaftProxyRaftMembershipServer(local RaftMembershipServer, connSelector r
redirectChecker := func(ctx context.Context) (context.Context, error) {
s, ok := transport.StreamFromContext(ctx)
if !ok {
return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context")
return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context")
}
addr := s.ServerTransport().RemoteAddr().String()
md, ok := metadata.FromContext(ctx)
if ok && len(md["redirect"]) != 0 {
return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
}
if !ok {
md = metadata.New(map[string]string{})
@@ -1843,6 +2062,22 @@ func (m *ProcessRaftMessageResponse) Size() (n int) {
return n
}
func (m *StreamRaftMessageRequest) Size() (n int) {
var l int
_ = l
if m.Message != nil {
l = m.Message.Size()
n += 1 + l + sovRaft(uint64(l))
}
return n
}
func (m *StreamRaftMessageResponse) Size() (n int) {
var l int
_ = l
return n
}
func (m *ResolveAddressRequest) Size() (n int) {
var l int
_ = l
@@ -2057,6 +2292,25 @@ func (this *ProcessRaftMessageResponse) String() string {
}, "")
return s
}
func (this *StreamRaftMessageRequest) String() string {
if this == nil {
return "nil"
}
s := strings.Join([]string{`&StreamRaftMessageRequest{`,
`Message:` + strings.Replace(fmt.Sprintf("%v", this.Message), "Message", "raftpb.Message", 1) + `,`,
`}`,
}, "")
return s
}
func (this *StreamRaftMessageResponse) String() string {
if this == nil {
return "nil"
}
s := strings.Join([]string{`&StreamRaftMessageResponse{`,
`}`,
}, "")
return s
}
func (this *ResolveAddressRequest) String() string {
if this == nil {
return "nil"
@@ -2861,6 +3115,139 @@ func (m *ProcessRaftMessageResponse) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *StreamRaftMessageRequest) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRaft
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: StreamRaftMessageRequest: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: StreamRaftMessageRequest: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRaft
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthRaft
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.Message == nil {
m.Message = &raftpb.Message{}
}
if err := m.Message.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipRaft(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthRaft
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *StreamRaftMessageResponse) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRaft
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: StreamRaftMessageResponse: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: StreamRaftMessageResponse: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
default:
iNdEx = preIndex
skippy, err := skipRaft(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthRaft
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *ResolveAddressRequest) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
@@ -3574,66 +3961,69 @@ var (
func init() { proto.RegisterFile("github.com/docker/swarmkit/api/raft.proto", fileDescriptorRaft) }
var fileDescriptorRaft = []byte{
// 974 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x96, 0x4f, 0x6f, 0x1b, 0x45,
0x18, 0xc6, 0x77, 0xd7, 0x5b, 0x27, 0x79, 0xd3, 0x26, 0xd1, 0x94, 0x84, 0xed, 0x52, 0x1c, 0x77,
0x8b, 0x84, 0x13, 0x9a, 0xb5, 0x30, 0x48, 0x45, 0x85, 0x1e, 0x62, 0xc7, 0x92, 0x4d, 0x5b, 0xa7,
0xda, 0x24, 0xd0, 0x5b, 0x58, 0xef, 0x4e, 0xdc, 0xc5, 0xf6, 0x8e, 0x99, 0x19, 0x3b, 0x70, 0x41,
0x3d, 0xa2, 0x5c, 0x39, 0x80, 0x90, 0x7a, 0x82, 0x73, 0x3f, 0x00, 0x1f, 0x00, 0x45, 0x9c, 0xb8,
0xc1, 0x29, 0xa2, 0xfe, 0x00, 0xf0, 0x15, 0xd0, 0xcc, 0xee, 0x3a, 0xc6, 0x59, 0x3b, 0xb9, 0x24,
0xa3, 0x9d, 0xdf, 0xf3, 0x3e, 0xef, 0x3b, 0x7f, 0xde, 0x31, 0x6c, 0xb4, 0x02, 0xfe, 0xbc, 0xdf,
0xb4, 0x3d, 0xd2, 0x2d, 0xfa, 0xc4, 0x6b, 0x63, 0x5a, 0x64, 0xc7, 0x2e, 0xed, 0xb6, 0x03, 0x5e,
0x74, 0x7b, 0x41, 0x91, 0xba, 0x47, 0xdc, 0xee, 0x51, 0xc2, 0x09, 0x42, 0xd1, 0xbc, 0x9d, 0xcc,
0xdb, 0x83, 0xf7, 0xcd, 0x7b, 0x97, 0xc8, 0x49, 0xf3, 0x4b, 0xec, 0x71, 0x16, 0x45, 0x30, 0x37,
0x2f, 0xa1, 0xf9, 0x37, 0x3d, 0x9c, 0xb0, 0x5b, 0x63, 0xac, 0x47, 0x28, 0x26, 0xac, 0x88, 0xb9,
0xe7, 0xcb, 0x84, 0xe4, 0x9f, 0x5e, 0x73, 0x2c, 0x39, 0xf3, 0x8d, 0x16, 0x69, 0x11, 0x39, 0x2c,
0x8a, 0x51, 0xfc, 0xf5, 0xfe, 0x0c, 0x43, 0x49, 0x34, 0xfb, 0x47, 0xc5, 0x5e, 0xa7, 0xdf, 0x0a,
0xc2, 0xf8, 0x5f, 0x24, 0xb4, 0x5e, 0xa9, 0x00, 0x8e, 0x7b, 0xc4, 0x9f, 0xe0, 0x6e, 0x13, 0x53,
0x74, 0x17, 0xe6, 0x84, 0xd7, 0x61, 0xe0, 0x1b, 0x6a, 0x5e, 0x2d, 0xe8, 0x65, 0x18, 0x9e, 0xad,
0x67, 0x05, 0x50, 0xdf, 0x71, 0xb2, 0x62, 0xaa, 0xee, 0x0b, 0x28, 0x24, 0x3e, 0x16, 0x90, 0x96,
0x57, 0x0b, 0x0b, 0x11, 0xd4, 0x20, 0x3e, 0x16, 0x90, 0x98, 0xaa, 0xfb, 0x08, 0x81, 0xee, 0xfa,
0x3e, 0x35, 0x32, 0x82, 0x70, 0xe4, 0x18, 0x95, 0x21, 0xcb, 0xb8, 0xcb, 0xfb, 0xcc, 0xd0, 0xf3,
0x6a, 0x61, 0xb1, 0xf4, 0x8e, 0x7d, 0x71, 0xa5, 0xed, 0xf3, 0x6c, 0xf6, 0x24, 0x5b, 0xd6, 0x4f,
0xcf, 0xd6, 0x15, 0x27, 0x56, 0x5a, 0x77, 0x60, 0xf1, 0x53, 0x12, 0x84, 0x0e, 0xfe, 0xaa, 0x8f,
0x19, 0x1f, 0xd9, 0xa8, 0xe7, 0x36, 0xd6, 0x4f, 0x2a, 0x5c, 0x8f, 0x18, 0xd6, 0x23, 0x21, 0xc3,
0x57, 0xab, 0xea, 0x23, 0x98, 0xeb, 0x4a, 0x5b, 0x66, 0x68, 0xf9, 0x4c, 0x61, 0xb1, 0x94, 0x9b,
0x9d, 0x9d, 0x93, 0xe0, 0xe8, 0x3d, 0x58, 0xa6, 0xb8, 0x4b, 0x06, 0xd8, 0x3f, 0x4c, 0x22, 0x64,
0xf2, 0x99, 0x82, 0x5e, 0xd6, 0x56, 0x14, 0x67, 0x29, 0x9e, 0x8a, 0x44, 0xcc, 0x2a, 0xc3, 0xf5,
0xc7, 0xd8, 0x1d, 0xe0, 0xa4, 0x80, 0x12, 0xe8, 0x62, 0xc5, 0x64, 0x62, 0x97, 0x7b, 0x4a, 0xd6,
0x5a, 0x86, 0x1b, 0x71, 0x8c, 0xa8, 0x40, 0xeb, 0x31, 0xdc, 0x7a, 0x4a, 0x89, 0x87, 0x19, 0x8b,
0x58, 0xc6, 0xdc, 0xd6, 0xc8, 0x61, 0x43, 0x14, 0x26, 0xbf, 0xc4, 0x26, 0xcb, 0x76, 0x74, 0xac,
0xec, 0x04, 0x4c, 0xe6, 0x1f, 0xe8, 0x2f, 0x7e, 0xb0, 0x14, 0xeb, 0x36, 0x98, 0x69, 0xd1, 0x62,
0xaf, 0x4f, 0x60, 0xd5, 0xc1, 0x8c, 0x74, 0x06, 0x78, 0xdb, 0xf7, 0xa9, 0x80, 0x62, 0x9f, 0xab,
0xac, 0xb2, 0x75, 0x0f, 0xd6, 0x26, 0xd5, 0xf1, 0x26, 0xa5, 0xed, 0x64, 0x07, 0x6e, 0xd6, 0x43,
0x8e, 0x69, 0xe8, 0x76, 0x44, 0x9c, 0xc4, 0x69, 0x0d, 0xb4, 0x91, 0x49, 0x76, 0x78, 0xb6, 0xae,
0xd5, 0x77, 0x1c, 0x2d, 0xf0, 0xd1, 0x43, 0xc8, 0xba, 0x1e, 0x0f, 0x48, 0x18, 0xef, 0xe0, 0x7a,
0xda, 0x6a, 0xee, 0x71, 0x42, 0xf1, 0xb6, 0xc4, 0x92, 0xa3, 0x15, 0x89, 0xac, 0xdf, 0x74, 0x58,
0x1c, 0x9b, 0x45, 0x1f, 0x8f, 0xc2, 0x09, 0xab, 0xa5, 0xd2, 0xdd, 0x4b, 0xc2, 0x3d, 0x0a, 0x42,
0x3f, 0x09, 0x86, 0xec, 0x78, 0x5f, 0x35, 0xb9, 0xe4, 0x46, 0x9a, 0x54, 0xdc, 0x98, 0x9a, 0x12,
0xed, 0x29, 0xba, 0x0f, 0x73, 0x0c, 0xd3, 0x41, 0xe0, 0x61, 0x79, 0x65, 0x16, 0x4b, 0x6f, 0xa5,
0xba, 0x45, 0x48, 0x4d, 0x71, 0x12, 0x5a, 0x18, 0x71, 0x97, 0xb5, 0xe3, 0x2b, 0x95, 0x6a, 0xb4,
0xef, 0xb2, 0xb6, 0x30, 0x12, 0x9c, 0x30, 0x0a, 0x31, 0x3f, 0x26, 0xb4, 0x6d, 0x5c, 0x9b, 0x6e,
0xd4, 0x88, 0x10, 0x61, 0x14, 0xd3, 0x42, 0xe8, 0x75, 0xfa, 0x8c, 0x63, 0x6a, 0x64, 0xa7, 0x0b,
0x2b, 0x11, 0x22, 0x84, 0x31, 0x8d, 0x3e, 0x84, 0x2c, 0xc3, 0x1e, 0xc5, 0xdc, 0x98, 0x93, 0x3a,
0x33, 0xbd, 0x32, 0x41, 0xd4, 0xc4, 0x45, 0x97, 0x23, 0xf4, 0x00, 0xe6, 0x29, 0x66, 0xa4, 0x4f,
0x3d, 0x6c, 0xcc, 0x4b, 0xdd, 0xed, 0xd4, 0xcb, 0x11, 0x33, 0x35, 0xc5, 0x19, 0xf1, 0xe8, 0x21,
0x2c, 0xe0, 0xaf, 0x39, 0x0e, 0x99, 0xd8, 0xbc, 0x05, 0x29, 0x7e, 0x3b, 0x4d, 0x5c, 0x4d, 0xa0,
0x9a, 0xe2, 0x9c, 0x2b, 0x44, 0xc2, 0x1e, 0x09, 0x8f, 0x82, 0x96, 0x01, 0xd3, 0x13, 0xae, 0x48,
0x42, 0x24, 0x1c, 0xb1, 0xe5, 0x79, 0xc8, 0x72, 0x97, 0xb6, 0x30, 0xdf, 0xfc, 0x57, 0x85, 0xe5,
0x89, 0x73, 0x81, 0xde, 0x85, 0xb9, 0x83, 0xc6, 0xa3, 0xc6, 0xee, 0xe7, 0x8d, 0x15, 0xc5, 0x34,
0x4f, 0x5e, 0xe6, 0xd7, 0x26, 0x88, 0x83, 0xb0, 0x1d, 0x92, 0xe3, 0x10, 0x95, 0xe0, 0xe6, 0xde,
0xfe, 0xae, 0x53, 0x3d, 0xdc, 0xae, 0xec, 0xd7, 0x77, 0x1b, 0x87, 0x15, 0xa7, 0xba, 0xbd, 0x5f,
0x5d, 0x51, 0xcd, 0x5b, 0x27, 0x2f, 0xf3, 0xab, 0x13, 0xa2, 0x0a, 0xc5, 0x2e, 0xc7, 0x17, 0x34,
0x07, 0x4f, 0x77, 0x84, 0x46, 0x4b, 0xd5, 0x1c, 0xf4, 0xfc, 0x34, 0x8d, 0x53, 0x7d, 0xb2, 0xfb,
0x59, 0x75, 0x25, 0x93, 0xaa, 0x71, 0x64, 0x13, 0x33, 0xdf, 0xfc, 0xee, 0xe7, 0x9c, 0xf2, 0xeb,
0x2f, 0xb9, 0xc9, 0xea, 0x4a, 0xdf, 0x6b, 0xa0, 0x8b, 0x1b, 0x8a, 0x4e, 0x54, 0x40, 0x17, 0x9b,
0x07, 0xda, 0x4a, 0x5b, 0xc1, 0xa9, 0x2d, 0xcb, 0xb4, 0xaf, 0x8a, 0xc7, 0x3d, 0x69, 0xf5, 0xf7,
0x57, 0xff, 0xfc, 0xa8, 0x2d, 0xc3, 0x0d, 0xc9, 0x6f, 0x75, 0xdd, 0xd0, 0x6d, 0x61, 0x8a, 0xbe,
0x85, 0xa5, 0xff, 0x37, 0x1b, 0xb4, 0x31, 0xed, 0x08, 0x5d, 0x68, 0x67, 0xe6, 0xe6, 0x55, 0xd0,
0x99, 0xfe, 0xa5, 0x3f, 0x55, 0x58, 0x3a, 0x6f, 0xde, 0xec, 0x79, 0xd0, 0x43, 0x5f, 0x80, 0x2e,
0x9e, 0x26, 0x94, 0xda, 0x9a, 0xc6, 0x1e, 0x36, 0x33, 0x3f, 0x1d, 0x98, 0x5d, 0xb4, 0x07, 0xd7,
0xe4, 0xe3, 0x80, 0x52, 0x23, 0x8c, 0xbf, 0x3d, 0xe6, 0x9d, 0x19, 0xc4, 0x4c, 0x93, 0xb2, 0x71,
0xfa, 0x3a, 0xa7, 0xfc, 0xf5, 0x3a, 0xa7, 0xbc, 0x18, 0xe6, 0xd4, 0xd3, 0x61, 0x4e, 0xfd, 0x63,
0x98, 0x53, 0xff, 0x1e, 0xe6, 0xd4, 0x67, 0x99, 0x67, 0x7a, 0x33, 0x2b, 0x7f, 0x5b, 0x7c, 0xf0,
0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x64, 0x01, 0xa3, 0x4f, 0x74, 0x09, 0x00, 0x00,
// 1015 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x96, 0xc1, 0x6e, 0x1b, 0x45,
0x18, 0xc7, 0x77, 0xed, 0xad, 0xd3, 0x7c, 0x69, 0x93, 0x30, 0x25, 0x61, 0xb3, 0x2d, 0x8e, 0xbb,
0x45, 0xc2, 0x09, 0xc9, 0x5a, 0x18, 0xa4, 0xa2, 0x42, 0x0f, 0x71, 0x62, 0x29, 0x26, 0xad, 0x53,
0x6d, 0x12, 0xe8, 0x2d, 0xac, 0x77, 0x27, 0xee, 0x62, 0x7b, 0xc7, 0xcc, 0x8c, 0x1d, 0xb8, 0xa0,
0x1e, 0x21, 0x2f, 0x00, 0x42, 0xaa, 0x38, 0xc0, 0xb9, 0x0f, 0xc0, 0x03, 0xa0, 0x88, 0x13, 0x37,
0x38, 0x45, 0xd4, 0x0f, 0x00, 0xaf, 0x80, 0x66, 0x76, 0xd7, 0x31, 0xf6, 0xda, 0xf1, 0x81, 0x4b,
0x32, 0xda, 0xf9, 0xfd, 0xbf, 0xff, 0x37, 0x33, 0xdf, 0x7c, 0x63, 0x58, 0xab, 0xfb, 0xfc, 0x59,
0xa7, 0x66, 0xb9, 0xa4, 0x55, 0xf0, 0x88, 0xdb, 0xc0, 0xb4, 0xc0, 0x4e, 0x1d, 0xda, 0x6a, 0xf8,
0xbc, 0xe0, 0xb4, 0xfd, 0x02, 0x75, 0x4e, 0xb8, 0xd5, 0xa6, 0x84, 0x13, 0x84, 0xc2, 0x79, 0x2b,
0x9e, 0xb7, 0xba, 0xef, 0x1a, 0x1b, 0x57, 0xc8, 0x49, 0xed, 0x73, 0xec, 0x72, 0x16, 0x46, 0x30,
0xd6, 0xaf, 0xa0, 0xf9, 0x57, 0x6d, 0x1c, 0xb3, 0x9b, 0x03, 0xac, 0x4b, 0x28, 0x26, 0xac, 0x80,
0xb9, 0xeb, 0xc9, 0x84, 0xe4, 0x9f, 0x76, 0x6d, 0x20, 0x39, 0xe3, 0xf5, 0x3a, 0xa9, 0x13, 0x39,
0x2c, 0x88, 0x51, 0xf4, 0xf5, 0xfe, 0x04, 0x43, 0x49, 0xd4, 0x3a, 0x27, 0x85, 0x76, 0xb3, 0x53,
0xf7, 0x83, 0xe8, 0x5f, 0x28, 0x34, 0x5f, 0xaa, 0x00, 0xb6, 0x73, 0xc2, 0x1f, 0xe3, 0x56, 0x0d,
0x53, 0x74, 0x0f, 0x66, 0x84, 0xd7, 0xb1, 0xef, 0xe9, 0x6a, 0x4e, 0xcd, 0x6b, 0x25, 0xe8, 0x5d,
0xac, 0x66, 0x04, 0x50, 0xd9, 0xb1, 0x33, 0x62, 0xaa, 0xe2, 0x09, 0x28, 0x20, 0x1e, 0x16, 0x50,
0x2a, 0xa7, 0xe6, 0x67, 0x43, 0xa8, 0x4a, 0x3c, 0x2c, 0x20, 0x31, 0x55, 0xf1, 0x10, 0x02, 0xcd,
0xf1, 0x3c, 0xaa, 0xa7, 0x05, 0x61, 0xcb, 0x31, 0x2a, 0x41, 0x86, 0x71, 0x87, 0x77, 0x98, 0xae,
0xe5, 0xd4, 0xfc, 0x5c, 0xf1, 0x2d, 0x6b, 0x74, 0xa7, 0xad, 0xcb, 0x6c, 0x0e, 0x24, 0x5b, 0xd2,
0xce, 0x2f, 0x56, 0x15, 0x3b, 0x52, 0x9a, 0x77, 0x61, 0xee, 0x63, 0xe2, 0x07, 0x36, 0xfe, 0xa2,
0x83, 0x19, 0xef, 0xdb, 0xa8, 0x97, 0x36, 0xe6, 0x0f, 0x2a, 0xdc, 0x08, 0x19, 0xd6, 0x26, 0x01,
0xc3, 0xd3, 0xad, 0xea, 0x03, 0x98, 0x69, 0x49, 0x5b, 0xa6, 0xa7, 0x72, 0xe9, 0xfc, 0x5c, 0x31,
0x3b, 0x39, 0x3b, 0x3b, 0xc6, 0xd1, 0x3b, 0xb0, 0x40, 0x71, 0x8b, 0x74, 0xb1, 0x77, 0x1c, 0x47,
0x48, 0xe7, 0xd2, 0x79, 0xad, 0x94, 0x5a, 0x54, 0xec, 0xf9, 0x68, 0x2a, 0x14, 0x31, 0xb3, 0x04,
0x37, 0x1e, 0x61, 0xa7, 0x8b, 0xe3, 0x05, 0x14, 0x41, 0x13, 0x3b, 0x26, 0x13, 0xbb, 0xda, 0x53,
0xb2, 0xe6, 0x02, 0xdc, 0x8c, 0x62, 0x84, 0x0b, 0x34, 0x1f, 0xc1, 0xca, 0x13, 0x4a, 0x5c, 0xcc,
0x58, 0xc8, 0x32, 0xe6, 0xd4, 0xfb, 0x0e, 0x6b, 0x62, 0x61, 0xf2, 0x4b, 0x64, 0xb2, 0x60, 0x85,
0x65, 0x65, 0xc5, 0x60, 0x3c, 0xff, 0x40, 0x7b, 0xfe, 0x9d, 0xa9, 0x98, 0x77, 0xc0, 0x48, 0x8a,
0x16, 0x79, 0xed, 0x81, 0x7e, 0xc0, 0x29, 0x76, 0x5a, 0xff, 0x87, 0xd5, 0x6d, 0x58, 0x49, 0x08,
0x16, 0x39, 0x7d, 0x04, 0x4b, 0x36, 0x66, 0xa4, 0xd9, 0xc5, 0x5b, 0x9e, 0x47, 0x45, 0x3a, 0x91,
0xcd, 0x34, 0xe7, 0x69, 0x6e, 0xc0, 0xf2, 0xb0, 0x3a, 0x2a, 0x87, 0xa4, 0x9a, 0x69, 0xc2, 0xad,
0x4a, 0xc0, 0x31, 0x0d, 0x9c, 0xa6, 0x88, 0x13, 0x3b, 0x2d, 0x43, 0xaa, 0x6f, 0x92, 0xe9, 0x5d,
0xac, 0xa6, 0x2a, 0x3b, 0x76, 0xca, 0xf7, 0xd0, 0x43, 0xc8, 0x38, 0x2e, 0xf7, 0x49, 0x10, 0xd5,
0xca, 0x6a, 0xd2, 0xb9, 0x1d, 0x70, 0x42, 0xf1, 0x96, 0xc4, 0xe2, 0x22, 0x0e, 0x45, 0xe6, 0xaf,
0x1a, 0xcc, 0x0d, 0xcc, 0xa2, 0x0f, 0xfb, 0xe1, 0x84, 0xd5, 0x7c, 0xf1, 0xde, 0x15, 0xe1, 0xf6,
0xfc, 0xc0, 0x8b, 0x83, 0x21, 0x2b, 0xaa, 0xa0, 0x94, 0xdc, 0x71, 0x3d, 0x49, 0x2a, 0xee, 0xe6,
0xae, 0x12, 0x56, 0x0f, 0xba, 0x0f, 0x33, 0x0c, 0xd3, 0xae, 0xef, 0x62, 0x79, 0x39, 0xe7, 0x8a,
0xb7, 0x13, 0xdd, 0x42, 0x64, 0x57, 0xb1, 0x63, 0x5a, 0x18, 0x71, 0x87, 0x35, 0xa2, 0xcb, 0x9b,
0x68, 0x74, 0xe8, 0xb0, 0x86, 0x30, 0x12, 0x9c, 0x30, 0x0a, 0x30, 0x3f, 0x25, 0xb4, 0xa1, 0x5f,
0x1b, 0x6f, 0x54, 0x0d, 0x11, 0x61, 0x14, 0xd1, 0x42, 0xe8, 0x36, 0x3b, 0x8c, 0x63, 0xaa, 0x67,
0xc6, 0x0b, 0xb7, 0x43, 0x44, 0x08, 0x23, 0x1a, 0xbd, 0x0f, 0x19, 0x86, 0x5d, 0x8a, 0xb9, 0x3e,
0x23, 0x75, 0x46, 0xf2, 0xca, 0x04, 0xb1, 0x2b, 0x5a, 0x8a, 0x1c, 0xa1, 0x07, 0x70, 0x9d, 0x62,
0x46, 0x3a, 0xd4, 0xc5, 0xfa, 0x75, 0xa9, 0xbb, 0x93, 0x78, 0x0d, 0x23, 0x66, 0x57, 0xb1, 0xfb,
0x3c, 0x7a, 0x08, 0xb3, 0xf8, 0x4b, 0x8e, 0x03, 0x26, 0x0e, 0x6f, 0x56, 0x8a, 0xdf, 0x4c, 0x12,
0x97, 0x63, 0x68, 0x57, 0xb1, 0x2f, 0x15, 0x22, 0x61, 0x97, 0x04, 0x27, 0x7e, 0x5d, 0x87, 0xf1,
0x09, 0x6f, 0x4b, 0x42, 0x24, 0x1c, 0xb2, 0xa5, 0xeb, 0x90, 0xe1, 0x0e, 0xad, 0x63, 0xbe, 0xfe,
0x8f, 0x0a, 0x0b, 0x43, 0x75, 0x81, 0xde, 0x86, 0x99, 0xa3, 0xea, 0x5e, 0x75, 0xff, 0xd3, 0xea,
0xa2, 0x62, 0x18, 0x67, 0x2f, 0x72, 0xcb, 0x43, 0xc4, 0x51, 0xd0, 0x08, 0xc8, 0x69, 0x80, 0x8a,
0x70, 0xeb, 0xe0, 0x70, 0xdf, 0x2e, 0x1f, 0x6f, 0x6d, 0x1f, 0x56, 0xf6, 0xab, 0xc7, 0xdb, 0x76,
0x79, 0xeb, 0xb0, 0xbc, 0xa8, 0x1a, 0x2b, 0x67, 0x2f, 0x72, 0x4b, 0x43, 0xa2, 0x6d, 0x8a, 0x1d,
0x8e, 0x47, 0x34, 0x47, 0x4f, 0x76, 0x84, 0x26, 0x95, 0xa8, 0x39, 0x6a, 0x7b, 0x49, 0x1a, 0xbb,
0xfc, 0x78, 0xff, 0x93, 0xf2, 0x62, 0x3a, 0x51, 0x63, 0xcb, 0x76, 0x69, 0xbc, 0xf1, 0xcd, 0x4f,
0x59, 0xe5, 0x97, 0x9f, 0xb3, 0xc3, 0xab, 0x2b, 0xfe, 0x98, 0x06, 0x4d, 0xdc, 0x50, 0x74, 0xa6,
0x02, 0x1a, 0x6d, 0x53, 0x68, 0x33, 0x69, 0x07, 0xc7, 0x36, 0x47, 0xc3, 0x9a, 0x16, 0x8f, 0x7a,
0xd2, 0xd2, 0x6f, 0x2f, 0xff, 0xfe, 0x3e, 0xb5, 0x00, 0x37, 0x25, 0xbf, 0xd9, 0x72, 0x02, 0xa7,
0x8e, 0x29, 0xfa, 0x56, 0x85, 0xd7, 0x46, 0x1a, 0x19, 0xda, 0x48, 0xbe, 0xc6, 0xc9, 0xcd, 0xd3,
0xd8, 0x9c, 0x92, 0x9e, 0x98, 0x49, 0x5e, 0x45, 0x5f, 0xc3, 0xfc, 0x7f, 0x1b, 0x1f, 0x5a, 0x1b,
0x57, 0xce, 0x23, 0xad, 0xd5, 0x58, 0x9f, 0x06, 0x9d, 0x98, 0x41, 0xf1, 0x0f, 0x15, 0xe6, 0x2f,
0x9f, 0x2c, 0xf6, 0xcc, 0x6f, 0xa3, 0xcf, 0x40, 0x13, 0x0f, 0x32, 0x4a, 0x6c, 0x93, 0x03, 0xcf,
0xb9, 0x91, 0x1b, 0x0f, 0x4c, 0x3e, 0x00, 0x17, 0xae, 0xc9, 0x27, 0x11, 0x25, 0x46, 0x18, 0x7c,
0x71, 0x8d, 0xbb, 0x13, 0x88, 0x89, 0x26, 0x25, 0xfd, 0xfc, 0x55, 0x56, 0xf9, 0xf3, 0x55, 0x56,
0x79, 0xde, 0xcb, 0xaa, 0xe7, 0xbd, 0xac, 0xfa, 0x7b, 0x2f, 0xab, 0xfe, 0xd5, 0xcb, 0xaa, 0x4f,
0xd3, 0x4f, 0xb5, 0x5a, 0x46, 0xfe, 0xa2, 0x7a, 0xef, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x3e,
0x7a, 0x8b, 0xe7, 0x6a, 0x0a, 0x00, 0x00,
}
+18
View File
@@ -16,6 +16,15 @@ service Raft {
option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" };
};
// StreamRaftMessage accepts a stream of raft messages of type StreamRaftMessageRequest
// to be processed on a raft member, returning a StreamRaftMessageResponse
// when processing of the streamed messages is complete. A single stream corresponds
// to a single raft message, which may be disassembled and streamed as individual messages.
// It is called from the Raft leader, which uses it to stream messages to a raft member.
rpc StreamRaftMessage(stream StreamRaftMessageRequest) returns (StreamRaftMessageResponse) {
option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" };
};
// ResolveAddress returns the address where the node with the given ID can be reached.
rpc ResolveAddress(ResolveAddressRequest) returns (ResolveAddressResponse) {
option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" };
@@ -82,6 +91,15 @@ message ProcessRaftMessageRequest {
message ProcessRaftMessageResponse {}
// Raft message streaming request.
message StreamRaftMessageRequest {
option (docker.protobuf.plugin.deepcopy) = false;
raftpb.Message message = 1;
}
// Raft message streaming response.
message StreamRaftMessageResponse {}
message ResolveAddressRequest {
// raft_id is the ID to resolve to an address.
uint64 raft_id = 1;
+3 -2
View File
@@ -19,6 +19,7 @@ import (
import raftselector "github.com/docker/swarmkit/manager/raftselector"
import codes "google.golang.org/grpc/codes"
import status "google.golang.org/grpc/status"
import metadata "google.golang.org/grpc/metadata"
import transport "google.golang.org/grpc/transport"
import rafttime "time"
@@ -403,12 +404,12 @@ func NewRaftProxyResourceAllocatorServer(local ResourceAllocatorServer, connSele
redirectChecker := func(ctx context.Context) (context.Context, error) {
s, ok := transport.StreamFromContext(ctx)
if !ok {
return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context")
return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context")
}
addr := s.ServerTransport().RemoteAddr().String()
md, ok := metadata.FromContext(ctx)
if ok && len(md["redirect"]) != 0 {
return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
}
if !ok {
md = metadata.New(map[string]string{})
+329 -316
View File
@@ -72,7 +72,16 @@ const (
TaskStateShutdown TaskState = 640
TaskStateFailed TaskState = 704
TaskStateRejected TaskState = 768
TaskStateOrphaned TaskState = 832
// TaskStateRemove is used to correctly handle service deletions and scale
// downs. This allows us to keep track of tasks that have been marked for
// deletion, but can't yet be removed because the agent is in the process of
// shutting them down. Once the agent has shut down tasks with desired state
// REMOVE, the task reaper is responsible for removing them.
TaskStateRemove TaskState = 800
// TaskStateOrphaned is used to free up resources associated with service
// tasks on unresponsive nodes without having to delete those tasks. This
// state is directly assigned to the task by the orchestrator.
TaskStateOrphaned TaskState = 832
)
var TaskState_name = map[int32]string{
@@ -88,6 +97,7 @@ var TaskState_name = map[int32]string{
640: "SHUTDOWN",
704: "FAILED",
768: "REJECTED",
800: "REMOVE",
832: "ORPHANED",
}
var TaskState_value = map[string]int32{
@@ -103,6 +113,7 @@ var TaskState_value = map[string]int32{
"SHUTDOWN": 640,
"FAILED": 704,
"REJECTED": 768,
"REMOVE": 800,
"ORPHANED": 832,
}
@@ -17029,319 +17040,321 @@ var (
func init() { proto.RegisterFile("github.com/docker/swarmkit/api/types.proto", fileDescriptorTypes) }
var fileDescriptorTypes = []byte{
// 5020 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x5a, 0x4d, 0x6c, 0x24, 0x49,
0x56, 0x76, 0xfd, 0xba, 0xea, 0x55, 0xd9, 0x4e, 0x47, 0x7b, 0x7b, 0xdc, 0xb5, 0xdd, 0x76, 0x4d,
0xce, 0xf4, 0xce, 0x6c, 0x6f, 0x53, 0xfd, 0xb7, 0xbb, 0xea, 0x99, 0x61, 0x77, 0xa6, 0xfe, 0x6c,
0xd7, 0xb6, 0x5d, 0x55, 0x8a, 0x2a, 0x77, 0xef, 0x22, 0x41, 0x2a, 0x9d, 0x19, 0x2e, 0xe7, 0x38,
0x2b, 0xa3, 0xc8, 0xcc, 0xb2, 0xbb, 0x58, 0x10, 0x2d, 0x0e, 0x80, 0x7c, 0x82, 0xdb, 0x22, 0x64,
0x2e, 0x70, 0x42, 0x48, 0x1c, 0x40, 0x42, 0x70, 0x1a, 0x24, 0x0e, 0x7b, 0x83, 0x05, 0x09, 0xad,
0x40, 0x32, 0xac, 0x0f, 0xdc, 0x56, 0x70, 0x59, 0x71, 0x01, 0x09, 0xc5, 0x4f, 0x66, 0xa5, 0xab,
0xd3, 0x76, 0x0f, 0xb3, 0x17, 0xbb, 0xe2, 0xbd, 0xef, 0xbd, 0x78, 0xf1, 0x22, 0xe2, 0xc5, 0x7b,
0x11, 0x09, 0xf7, 0x06, 0x96, 0x7f, 0x30, 0xde, 0xab, 0x18, 0x74, 0xf8, 0xc0, 0xa4, 0xc6, 0x21,
0x71, 0x1f, 0x78, 0xc7, 0xba, 0x3b, 0x3c, 0xb4, 0xfc, 0x07, 0xfa, 0xc8, 0x7a, 0xe0, 0x4f, 0x46,
0xc4, 0xab, 0x8c, 0x5c, 0xea, 0x53, 0x84, 0x04, 0xa0, 0x12, 0x00, 0x2a, 0x47, 0x8f, 0x4a, 0xeb,
0x03, 0x4a, 0x07, 0x36, 0x79, 0xc0, 0x11, 0x7b, 0xe3, 0xfd, 0x07, 0xbe, 0x35, 0x24, 0x9e, 0xaf,
0x0f, 0x47, 0x42, 0xa8, 0xb4, 0x36, 0x0b, 0x30, 0xc7, 0xae, 0xee, 0x5b, 0xd4, 0x91, 0xfc, 0x95,
0x01, 0x1d, 0x50, 0xfe, 0xf3, 0x01, 0xfb, 0x25, 0xa8, 0xea, 0x3a, 0xcc, 0x3f, 0x27, 0xae, 0x67,
0x51, 0x07, 0xad, 0x40, 0xc6, 0x72, 0x4c, 0xf2, 0x72, 0x35, 0x51, 0x4e, 0xbc, 0x9f, 0xc6, 0xa2,
0xa1, 0x3e, 0x04, 0x68, 0xb1, 0x1f, 0x4d, 0xc7, 0x77, 0x27, 0x48, 0x81, 0xd4, 0x21, 0x99, 0x70,
0x44, 0x1e, 0xb3, 0x9f, 0x8c, 0x72, 0xa4, 0xdb, 0xab, 0x49, 0x41, 0x39, 0xd2, 0x6d, 0xf5, 0x27,
0x09, 0x28, 0x54, 0x1d, 0x87, 0xfa, 0xbc, 0x77, 0x0f, 0x21, 0x48, 0x3b, 0xfa, 0x90, 0x48, 0x21,
0xfe, 0x1b, 0xd5, 0x21, 0x6b, 0xeb, 0x7b, 0xc4, 0xf6, 0x56, 0x93, 0xe5, 0xd4, 0xfb, 0x85, 0xc7,
0x5f, 0xab, 0xbc, 0x3e, 0xe4, 0x4a, 0x44, 0x49, 0x65, 0x9b, 0xa3, 0xb9, 0x11, 0x58, 0x8a, 0xa2,
0x6f, 0xc3, 0xbc, 0xe5, 0x98, 0x96, 0x41, 0xbc, 0xd5, 0x34, 0xd7, 0xb2, 0x16, 0xa7, 0x65, 0x6a,
0x7d, 0x2d, 0xfd, 0xc3, 0xb3, 0xf5, 0x39, 0x1c, 0x08, 0x95, 0x3e, 0x80, 0x42, 0x44, 0x6d, 0xcc,
0xd8, 0x56, 0x20, 0x73, 0xa4, 0xdb, 0x63, 0x22, 0x47, 0x27, 0x1a, 0x1f, 0x26, 0x9f, 0x26, 0xd4,
0x4f, 0x60, 0xa5, 0xad, 0x0f, 0x89, 0xb9, 0x49, 0x1c, 0xe2, 0x5a, 0x06, 0x26, 0x1e, 0x1d, 0xbb,
0x06, 0x61, 0x63, 0x3d, 0xb4, 0x1c, 0x33, 0x18, 0x2b, 0xfb, 0x1d, 0xaf, 0x45, 0xad, 0xc3, 0x5b,
0x0d, 0xcb, 0x33, 0x5c, 0xe2, 0x93, 0xcf, 0xad, 0x24, 0x15, 0x28, 0x39, 0x4b, 0xc0, 0xd2, 0xac,
0xf4, 0x2f, 0xc1, 0x0d, 0xe6, 0x62, 0x53, 0x73, 0x25, 0x45, 0xf3, 0x46, 0xc4, 0xe0, 0xca, 0x0a,
0x8f, 0xdf, 0x8f, 0xf3, 0x50, 0xdc, 0x48, 0xb6, 0xe6, 0xf0, 0x32, 0x57, 0x13, 0x10, 0x7a, 0x23,
0x62, 0x20, 0x03, 0x6e, 0x9a, 0xd2, 0xe8, 0x19, 0xf5, 0x49, 0xae, 0x3e, 0x76, 0x1a, 0x2f, 0x19,
0xe6, 0xd6, 0x1c, 0x5e, 0x09, 0x94, 0x45, 0x3b, 0xa9, 0x01, 0xe4, 0x02, 0xdd, 0xea, 0x0f, 0x12,
0x90, 0x0f, 0x98, 0x1e, 0xfa, 0x2a, 0xe4, 0x1d, 0xdd, 0xa1, 0x9a, 0x31, 0x1a, 0x7b, 0x7c, 0x40,
0xa9, 0x5a, 0xf1, 0xfc, 0x6c, 0x3d, 0xd7, 0xd6, 0x1d, 0x5a, 0xef, 0xee, 0x7a, 0x38, 0xc7, 0xd8,
0xf5, 0xd1, 0xd8, 0x43, 0x6f, 0x43, 0x71, 0x48, 0x86, 0xd4, 0x9d, 0x68, 0x7b, 0x13, 0x9f, 0x78,
0xd2, 0x6d, 0x05, 0x41, 0xab, 0x31, 0x12, 0xfa, 0x16, 0xcc, 0x0f, 0x84, 0x49, 0xab, 0x29, 0xbe,
0x7c, 0xde, 0x89, 0xb3, 0x7e, 0xc6, 0x6a, 0x1c, 0xc8, 0xa8, 0xbf, 0x97, 0x80, 0x95, 0x90, 0x4a,
0x7e, 0x75, 0x6c, 0xb9, 0x64, 0x48, 0x1c, 0xdf, 0x43, 0xdf, 0x80, 0xac, 0x6d, 0x0d, 0x2d, 0xdf,
0x93, 0x3e, 0xbf, 0x13, 0xa7, 0x36, 0x1c, 0x14, 0x96, 0x60, 0x54, 0x85, 0xa2, 0x4b, 0x3c, 0xe2,
0x1e, 0x89, 0x15, 0x2f, 0x3d, 0x7a, 0x8d, 0xf0, 0x05, 0x11, 0x75, 0x03, 0x72, 0x5d, 0x5b, 0xf7,
0xf7, 0xa9, 0x3b, 0x44, 0x2a, 0x14, 0x75, 0xd7, 0x38, 0xb0, 0x7c, 0x62, 0xf8, 0x63, 0x37, 0xd8,
0x7d, 0x17, 0x68, 0xe8, 0x26, 0x24, 0xa9, 0xe8, 0x28, 0x5f, 0xcb, 0x9e, 0x9f, 0xad, 0x27, 0x3b,
0x3d, 0x9c, 0xa4, 0x9e, 0xfa, 0x11, 0x2c, 0x77, 0xed, 0xf1, 0xc0, 0x72, 0x1a, 0xc4, 0x33, 0x5c,
0x6b, 0xc4, 0xb4, 0xb3, 0x55, 0xc9, 0x62, 0x54, 0xb0, 0x2a, 0xd9, 0xef, 0x70, 0x6b, 0x27, 0xa7,
0x5b, 0x5b, 0xfd, 0x9d, 0x24, 0x2c, 0x37, 0x9d, 0x81, 0xe5, 0x90, 0xa8, 0xf4, 0x5d, 0x58, 0x24,
0x9c, 0xa8, 0x1d, 0x89, 0x70, 0x23, 0xf5, 0x2c, 0x08, 0x6a, 0x10, 0x83, 0x5a, 0x33, 0x71, 0xe1,
0x51, 0xdc, 0xf0, 0x5f, 0xd3, 0x1e, 0x1b, 0x1d, 0x9a, 0x30, 0x3f, 0xe2, 0x83, 0xf0, 0xe4, 0xf4,
0xde, 0x8d, 0xd3, 0xf5, 0xda, 0x38, 0x83, 0x20, 0x21, 0x65, 0xbf, 0x48, 0x90, 0xf8, 0xb3, 0x24,
0x2c, 0xb5, 0xa9, 0x79, 0xc1, 0x0f, 0x25, 0xc8, 0x1d, 0x50, 0xcf, 0x8f, 0x04, 0xc4, 0xb0, 0x8d,
0x9e, 0x42, 0x6e, 0x24, 0xa7, 0x4f, 0xce, 0xfe, 0xed, 0x78, 0x93, 0x05, 0x06, 0x87, 0x68, 0xf4,
0x11, 0xe4, 0x83, 0x2d, 0xc3, 0x46, 0xfb, 0x06, 0x0b, 0x67, 0x8a, 0x47, 0xdf, 0x82, 0xac, 0x98,
0x84, 0xd5, 0x34, 0x97, 0xbc, 0xfb, 0x46, 0x3e, 0xc7, 0x52, 0x08, 0x6d, 0x42, 0xce, 0xb7, 0x3d,
0xcd, 0x72, 0xf6, 0xe9, 0x6a, 0x86, 0x2b, 0x58, 0x8f, 0x0d, 0x32, 0xd4, 0x24, 0xfd, 0xed, 0x5e,
0xcb, 0xd9, 0xa7, 0xb5, 0xc2, 0xf9, 0xd9, 0xfa, 0xbc, 0x6c, 0xe0, 0x79, 0xdf, 0xf6, 0xd8, 0x0f,
0xf5, 0xf7, 0x13, 0x50, 0x88, 0xa0, 0xd0, 0x1d, 0x00, 0xdf, 0x1d, 0x7b, 0xbe, 0xe6, 0x52, 0xea,
0x73, 0x67, 0x15, 0x71, 0x9e, 0x53, 0x30, 0xa5, 0x3e, 0xaa, 0xc0, 0x0d, 0x83, 0xb8, 0xbe, 0x66,
0x79, 0xde, 0x98, 0xb8, 0x9a, 0x37, 0xde, 0xfb, 0x94, 0x18, 0x3e, 0x77, 0x5c, 0x11, 0x2f, 0x33,
0x56, 0x8b, 0x73, 0x7a, 0x82, 0x81, 0x9e, 0xc0, 0xcd, 0x28, 0x7e, 0x34, 0xde, 0xb3, 0x2d, 0x43,
0x63, 0x93, 0x99, 0xe2, 0x22, 0x37, 0xa6, 0x22, 0x5d, 0xce, 0x7b, 0x46, 0x26, 0xea, 0x8f, 0x13,
0xa0, 0x60, 0x7d, 0xdf, 0xdf, 0x21, 0xc3, 0x3d, 0xe2, 0xf6, 0x7c, 0xdd, 0x1f, 0x7b, 0xe8, 0x26,
0x64, 0x6d, 0xa2, 0x9b, 0xc4, 0xe5, 0x46, 0xe5, 0xb0, 0x6c, 0xa1, 0x5d, 0xb6, 0x83, 0x75, 0xe3,
0x40, 0xdf, 0xb3, 0x6c, 0xcb, 0x9f, 0x70, 0x53, 0x16, 0xe3, 0x97, 0xf0, 0xac, 0xce, 0x0a, 0x8e,
0x08, 0xe2, 0x0b, 0x6a, 0xd0, 0x2a, 0xcc, 0x0f, 0x89, 0xe7, 0xe9, 0x03, 0xc2, 0x2d, 0xcd, 0xe3,
0xa0, 0xa9, 0x7e, 0x04, 0xc5, 0xa8, 0x1c, 0x2a, 0xc0, 0xfc, 0x6e, 0xfb, 0x59, 0xbb, 0xf3, 0xa2,
0xad, 0xcc, 0xa1, 0x25, 0x28, 0xec, 0xb6, 0x71, 0xb3, 0x5a, 0xdf, 0xaa, 0xd6, 0xb6, 0x9b, 0x4a,
0x02, 0x2d, 0x40, 0x7e, 0xda, 0x4c, 0xaa, 0x7f, 0x91, 0x00, 0x60, 0xee, 0x96, 0x83, 0xfa, 0x10,
0x32, 0x9e, 0xaf, 0xfb, 0x62, 0x55, 0x2e, 0x3e, 0x7e, 0xf7, 0xb2, 0x39, 0x94, 0xf6, 0xb2, 0x7f,
0x04, 0x0b, 0x91, 0xa8, 0x85, 0xc9, 0x0b, 0x16, 0xb2, 0x00, 0xa1, 0x9b, 0xa6, 0x2b, 0x0d, 0xe7,
0xbf, 0xd5, 0x8f, 0x20, 0xc3, 0xa5, 0x2f, 0x9a, 0x9b, 0x83, 0x74, 0x83, 0xfd, 0x4a, 0xa0, 0x3c,
0x64, 0x70, 0xb3, 0xda, 0xf8, 0x9e, 0x92, 0x44, 0x0a, 0x14, 0x1b, 0xad, 0x5e, 0xbd, 0xd3, 0x6e,
0x37, 0xeb, 0xfd, 0x66, 0x43, 0x49, 0xa9, 0x77, 0x21, 0xd3, 0x1a, 0x32, 0xcd, 0xb7, 0xd9, 0x92,
0xdf, 0x27, 0x2e, 0x71, 0x8c, 0x60, 0x27, 0x4d, 0x09, 0xea, 0x4f, 0x0b, 0x90, 0xd9, 0xa1, 0x63,
0xc7, 0x47, 0x8f, 0x23, 0x61, 0x6b, 0x31, 0x3e, 0x43, 0xe0, 0xc0, 0x4a, 0x7f, 0x32, 0x22, 0x32,
0xac, 0xdd, 0x84, 0xac, 0xd8, 0x1c, 0x72, 0x38, 0xb2, 0xc5, 0xe8, 0xbe, 0xee, 0x0e, 0x88, 0x2f,
0xc7, 0x23, 0x5b, 0xe8, 0x7d, 0x76, 0x62, 0xe9, 0x26, 0x75, 0xec, 0x09, 0xdf, 0x43, 0x39, 0x71,
0x2c, 0x61, 0xa2, 0x9b, 0x1d, 0xc7, 0x9e, 0xe0, 0x90, 0x8b, 0xb6, 0xa0, 0xb8, 0x67, 0x39, 0xa6,
0x46, 0x47, 0x22, 0xc8, 0x67, 0x2e, 0xdf, 0x71, 0xc2, 0xaa, 0x9a, 0xe5, 0x98, 0x1d, 0x01, 0xc6,
0x85, 0xbd, 0x69, 0x03, 0xb5, 0x61, 0xf1, 0x88, 0xda, 0xe3, 0x21, 0x09, 0x75, 0x65, 0xb9, 0xae,
0xf7, 0x2e, 0xd7, 0xf5, 0x9c, 0xe3, 0x03, 0x6d, 0x0b, 0x47, 0xd1, 0x26, 0x7a, 0x06, 0x0b, 0xfe,
0x70, 0xb4, 0xef, 0x85, 0xea, 0xe6, 0xb9, 0xba, 0xaf, 0x5c, 0xe1, 0x30, 0x06, 0x0f, 0xb4, 0x15,
0xfd, 0x48, 0x0b, 0x6d, 0x42, 0xc1, 0xa0, 0x8e, 0x67, 0x79, 0x3e, 0x71, 0x8c, 0xc9, 0x6a, 0x8e,
0xfb, 0xfe, 0x8a, 0x51, 0xd6, 0xa7, 0x60, 0x1c, 0x95, 0x2c, 0xfd, 0x56, 0x0a, 0x0a, 0x11, 0x17,
0xa0, 0x1e, 0x14, 0x46, 0x2e, 0x1d, 0xe9, 0x03, 0x7e, 0xe2, 0xc9, 0x49, 0x7d, 0xf4, 0x46, 0xee,
0xab, 0x74, 0xa7, 0x82, 0x38, 0xaa, 0x45, 0x3d, 0x4d, 0x42, 0x21, 0xc2, 0x44, 0xf7, 0x20, 0x87,
0xbb, 0xb8, 0xf5, 0xbc, 0xda, 0x6f, 0x2a, 0x73, 0xa5, 0xdb, 0x27, 0xa7, 0xe5, 0x55, 0xae, 0x2d,
0xaa, 0xa0, 0xeb, 0x5a, 0x47, 0x6c, 0x0d, 0xbf, 0x0f, 0xf3, 0x01, 0x34, 0x51, 0xfa, 0xf2, 0xc9,
0x69, 0xf9, 0xad, 0x59, 0x68, 0x04, 0x89, 0x7b, 0x5b, 0x55, 0xdc, 0x6c, 0x28, 0xc9, 0x78, 0x24,
0xee, 0x1d, 0xe8, 0x2e, 0x31, 0xd1, 0x57, 0x20, 0x2b, 0x81, 0xa9, 0x52, 0xe9, 0xe4, 0xb4, 0x7c,
0x73, 0x16, 0x38, 0xc5, 0xe1, 0xde, 0x76, 0xf5, 0x79, 0x53, 0x49, 0xc7, 0xe3, 0x70, 0xcf, 0xd6,
0x8f, 0x08, 0x7a, 0x17, 0x32, 0x02, 0x96, 0x29, 0xdd, 0x3a, 0x39, 0x2d, 0x7f, 0xe9, 0x35, 0x75,
0x0c, 0x55, 0x5a, 0xfd, 0xdd, 0x3f, 0x5e, 0x9b, 0xfb, 0x9b, 0x3f, 0x59, 0x53, 0x66, 0xd9, 0xa5,
0xff, 0x49, 0xc0, 0xc2, 0x85, 0xb5, 0x83, 0x54, 0xc8, 0x3a, 0xd4, 0xa0, 0x23, 0x71, 0x10, 0xe6,
0x6a, 0x70, 0x7e, 0xb6, 0x9e, 0x6d, 0xd3, 0x3a, 0x1d, 0x4d, 0xb0, 0xe4, 0xa0, 0x67, 0x33, 0x47,
0xf9, 0x93, 0x37, 0x5c, 0x98, 0xb1, 0x87, 0xf9, 0xc7, 0xb0, 0x60, 0xba, 0xd6, 0x11, 0x71, 0x35,
0x83, 0x3a, 0xfb, 0xd6, 0x40, 0x1e, 0x72, 0xa5, 0xd8, 0x7c, 0x93, 0x03, 0x71, 0x51, 0x08, 0xd4,
0x39, 0xfe, 0x0b, 0x1c, 0xe3, 0xa5, 0xe7, 0x50, 0x8c, 0x2e, 0x75, 0x76, 0x2e, 0x79, 0xd6, 0xaf,
0x11, 0x99, 0x58, 0xf2, 0x34, 0x14, 0xe7, 0x19, 0x45, 0xa4, 0x95, 0xef, 0x41, 0x7a, 0x48, 0x4d,
0xa1, 0x67, 0xa1, 0x76, 0x83, 0x65, 0x13, 0xff, 0x72, 0xb6, 0x5e, 0xa0, 0x5e, 0x65, 0xc3, 0xb2,
0xc9, 0x0e, 0x35, 0x09, 0xe6, 0x00, 0xf5, 0x08, 0xd2, 0x2c, 0xe6, 0xa0, 0x2f, 0x43, 0xba, 0xd6,
0x6a, 0x37, 0x94, 0xb9, 0xd2, 0xf2, 0xc9, 0x69, 0x79, 0x81, 0xbb, 0x84, 0x31, 0xd8, 0xda, 0x45,
0xeb, 0x90, 0x7d, 0xde, 0xd9, 0xde, 0xdd, 0x61, 0xcb, 0xeb, 0xc6, 0xc9, 0x69, 0x79, 0x29, 0x64,
0x0b, 0xa7, 0xa1, 0x3b, 0x90, 0xe9, 0xef, 0x74, 0x37, 0x7a, 0x4a, 0xb2, 0x84, 0x4e, 0x4e, 0xcb,
0x8b, 0x21, 0x9f, 0xdb, 0x5c, 0x5a, 0x96, 0xb3, 0x9a, 0x0f, 0xe9, 0xea, 0x8f, 0x12, 0x50, 0x88,
0x6c, 0x38, 0xb6, 0x30, 0x1b, 0xcd, 0x8d, 0xea, 0xee, 0x76, 0x5f, 0x99, 0x8b, 0x2c, 0xcc, 0x08,
0xa4, 0x41, 0xf6, 0xf5, 0xb1, 0xcd, 0xe2, 0x1c, 0xd4, 0x3b, 0xed, 0x5e, 0xab, 0xd7, 0x6f, 0xb6,
0xfb, 0x4a, 0xa2, 0xb4, 0x7a, 0x72, 0x5a, 0x5e, 0x99, 0x05, 0x6f, 0x8c, 0x6d, 0x9b, 0x2d, 0xcd,
0x7a, 0xb5, 0xbe, 0xc5, 0xd7, 0xfa, 0x74, 0x69, 0x46, 0x50, 0x75, 0xdd, 0x38, 0x20, 0x26, 0xba,
0x0f, 0xf9, 0x46, 0x73, 0xbb, 0xb9, 0x59, 0xe5, 0xd1, 0xbd, 0x74, 0xe7, 0xe4, 0xb4, 0x7c, 0xeb,
0xf5, 0xde, 0x6d, 0x32, 0xd0, 0x7d, 0x62, 0xce, 0x2c, 0xd1, 0x08, 0x44, 0xfd, 0x59, 0x12, 0x16,
0x30, 0x2b, 0x87, 0x5d, 0xbf, 0x4b, 0x6d, 0xcb, 0x98, 0xa0, 0x2e, 0xe4, 0x0d, 0xea, 0x98, 0x56,
0x24, 0x4e, 0x3c, 0xbe, 0x24, 0x25, 0x9a, 0x4a, 0x05, 0xad, 0x7a, 0x20, 0x89, 0xa7, 0x4a, 0xd0,
0x03, 0xc8, 0x98, 0xc4, 0xd6, 0x27, 0x32, 0x37, 0xbb, 0x55, 0x11, 0x05, 0x77, 0x25, 0x28, 0xb8,
0x2b, 0x0d, 0x59, 0x70, 0x63, 0x81, 0xe3, 0x35, 0x88, 0xfe, 0x52, 0xd3, 0x7d, 0x9f, 0x0c, 0x47,
0xbe, 0x48, 0xcc, 0xd2, 0xb8, 0x30, 0xd4, 0x5f, 0x56, 0x25, 0x09, 0x3d, 0x82, 0xec, 0xb1, 0xe5,
0x98, 0xf4, 0x58, 0xe6, 0x5e, 0x57, 0x28, 0x95, 0x40, 0xf5, 0x84, 0xa5, 0x24, 0x33, 0x66, 0xb2,
0x35, 0xd4, 0xee, 0xb4, 0x9b, 0xc1, 0x1a, 0x92, 0xfc, 0x8e, 0xd3, 0xa6, 0x0e, 0xdb, 0xff, 0xd0,
0x69, 0x6b, 0x1b, 0xd5, 0xd6, 0xf6, 0x2e, 0x66, 0xeb, 0x68, 0xe5, 0xe4, 0xb4, 0xac, 0x84, 0x90,
0x0d, 0xdd, 0xb2, 0x59, 0x31, 0x70, 0x0b, 0x52, 0xd5, 0xf6, 0xf7, 0x94, 0x64, 0x49, 0x39, 0x39,
0x2d, 0x17, 0x43, 0x76, 0xd5, 0x99, 0x4c, 0xfd, 0x3e, 0xdb, 0xaf, 0xfa, 0xf7, 0x29, 0x28, 0xee,
0x8e, 0x4c, 0xdd, 0x27, 0x62, 0x9f, 0xa1, 0x32, 0x14, 0x46, 0xba, 0xab, 0xdb, 0x36, 0xb1, 0x2d,
0x6f, 0x28, 0xaf, 0x12, 0xa2, 0x24, 0xf4, 0xc1, 0x9b, 0xba, 0xb1, 0x96, 0x63, 0x7b, 0xe7, 0x07,
0xff, 0xb6, 0x9e, 0x08, 0x1c, 0xba, 0x0b, 0x8b, 0xfb, 0xc2, 0x5a, 0x4d, 0x37, 0xf8, 0xc4, 0xa6,
0xf8, 0xc4, 0x56, 0xe2, 0x26, 0x36, 0x6a, 0x56, 0x45, 0x0e, 0xb2, 0xca, 0xa5, 0xf0, 0xc2, 0x7e,
0xb4, 0x89, 0x9e, 0xc0, 0xfc, 0x90, 0x3a, 0x96, 0x4f, 0xdd, 0xeb, 0x67, 0x21, 0x40, 0xa2, 0x7b,
0xb0, 0xcc, 0x26, 0x37, 0xb0, 0x87, 0xb3, 0xf9, 0x71, 0x9e, 0xc4, 0x4b, 0x43, 0xfd, 0xa5, 0xec,
0x10, 0x33, 0x32, 0xaa, 0x41, 0x86, 0xba, 0x2c, 0x5f, 0xcc, 0x72, 0x73, 0xef, 0x5f, 0x6b, 0xae,
0x68, 0x74, 0x98, 0x0c, 0x16, 0xa2, 0xea, 0x37, 0x61, 0xe1, 0xc2, 0x20, 0x58, 0x9a, 0xd4, 0xad,
0xee, 0xf6, 0x9a, 0xca, 0x1c, 0x2a, 0x42, 0xae, 0xde, 0x69, 0xf7, 0x5b, 0xed, 0x5d, 0x96, 0xe7,
0x15, 0x21, 0x87, 0x3b, 0xdb, 0xdb, 0xb5, 0x6a, 0xfd, 0x99, 0x92, 0x54, 0x2b, 0x50, 0x88, 0x68,
0x43, 0x8b, 0x00, 0xbd, 0x7e, 0xa7, 0xab, 0x6d, 0xb4, 0x70, 0xaf, 0x2f, 0xb2, 0xc4, 0x5e, 0xbf,
0x8a, 0xfb, 0x92, 0x90, 0x50, 0xff, 0x33, 0x19, 0xcc, 0xa8, 0x4c, 0x0c, 0x6b, 0x17, 0x13, 0xc3,
0x2b, 0x8c, 0x97, 0xa9, 0xe1, 0xb4, 0x11, 0x26, 0x88, 0x1f, 0x00, 0xf0, 0x85, 0x43, 0x4c, 0x4d,
0xf7, 0xe5, 0xc4, 0x97, 0x5e, 0x73, 0x72, 0x3f, 0xb8, 0xd1, 0xc2, 0x79, 0x89, 0xae, 0xfa, 0xe8,
0x5b, 0x50, 0x34, 0xe8, 0x70, 0x64, 0x13, 0x29, 0x9c, 0xba, 0x56, 0xb8, 0x10, 0xe2, 0xab, 0x7e,
0x34, 0x35, 0x4d, 0x5f, 0x4c, 0x9e, 0x7f, 0x3b, 0x11, 0x78, 0x26, 0x26, 0x1b, 0x2d, 0x42, 0x6e,
0xb7, 0xdb, 0xa8, 0xf6, 0x5b, 0xed, 0x4d, 0x25, 0x81, 0x00, 0xb2, 0xdc, 0xd5, 0x0d, 0x25, 0xc9,
0xb2, 0xe8, 0x7a, 0x67, 0xa7, 0xbb, 0xdd, 0xe4, 0x11, 0x0b, 0xad, 0x80, 0x12, 0x38, 0x5b, 0xe3,
0x8e, 0x6c, 0x36, 0x94, 0x34, 0xba, 0x01, 0x4b, 0x21, 0x55, 0x4a, 0x66, 0xd0, 0x4d, 0x40, 0x21,
0x71, 0xaa, 0x22, 0xab, 0xfe, 0x06, 0x2c, 0xd5, 0xa9, 0xe3, 0xeb, 0x96, 0x13, 0x56, 0x18, 0x8f,
0xd9, 0xa0, 0x25, 0x49, 0xb3, 0xe4, 0x4d, 0x50, 0x6d, 0xe9, 0xfc, 0x6c, 0xbd, 0x10, 0x42, 0x5b,
0x0d, 0x9e, 0x2a, 0xc9, 0x86, 0xc9, 0xf6, 0xef, 0xc8, 0x32, 0xb9, 0x73, 0x33, 0xb5, 0xf9, 0xf3,
0xb3, 0xf5, 0x54, 0xb7, 0xd5, 0xc0, 0x8c, 0x86, 0xbe, 0x0c, 0x79, 0xf2, 0xd2, 0xf2, 0x35, 0x83,
0x9d, 0x4b, 0xcc, 0x81, 0x19, 0x9c, 0x63, 0x84, 0x3a, 0x3b, 0x86, 0x6a, 0x00, 0x5d, 0xea, 0xfa,
0xb2, 0xe7, 0xaf, 0x43, 0x66, 0x44, 0x5d, 0x7e, 0x77, 0x71, 0xe9, 0x8d, 0x1a, 0x83, 0x8b, 0x85,
0x8a, 0x05, 0x58, 0xfd, 0x83, 0x14, 0x40, 0x5f, 0xf7, 0x0e, 0xa5, 0x92, 0xa7, 0x90, 0x0f, 0x6f,
0x27, 0xe5, 0x25, 0xc8, 0x95, 0xb3, 0x1d, 0x82, 0xd1, 0x93, 0x60, 0xb1, 0x89, 0xda, 0x29, 0xb6,
0x88, 0x0d, 0x3a, 0x8a, 0x2b, 0x3f, 0x2e, 0x16, 0x48, 0xec, 0x98, 0x27, 0xae, 0x2b, 0x67, 0x9e,
0xfd, 0x44, 0x75, 0x7e, 0x2c, 0x08, 0xa7, 0xc9, 0xec, 0x3b, 0xf6, 0xda, 0x67, 0x66, 0x46, 0xb6,
0xe6, 0xf0, 0x54, 0x0e, 0x7d, 0x0c, 0x05, 0x36, 0x6e, 0xcd, 0xe3, 0x3c, 0x99, 0x78, 0x5f, 0xea,
0x2a, 0xa1, 0x01, 0xc3, 0x68, 0xea, 0xe5, 0x3b, 0x00, 0xfa, 0x68, 0x64, 0x5b, 0xc4, 0xd4, 0xf6,
0x26, 0x3c, 0xd3, 0xce, 0xe3, 0xbc, 0xa4, 0xd4, 0x26, 0x6c, 0xbb, 0x04, 0x6c, 0xdd, 0xe7, 0xd9,
0xf3, 0x35, 0x0e, 0x94, 0xe8, 0xaa, 0x5f, 0x53, 0x60, 0xd1, 0x1d, 0x3b, 0xcc, 0xa1, 0xd2, 0x3a,
0xf5, 0xcf, 0x93, 0xf0, 0x56, 0x9b, 0xf8, 0xc7, 0xd4, 0x3d, 0xac, 0xfa, 0xbe, 0x6e, 0x1c, 0x0c,
0x89, 0x23, 0xa7, 0x2f, 0x52, 0xd0, 0x24, 0x2e, 0x14, 0x34, 0xab, 0x30, 0xaf, 0xdb, 0x96, 0xee,
0x11, 0x91, 0xbc, 0xe5, 0x71, 0xd0, 0x64, 0x65, 0x17, 0x2b, 0xe2, 0x88, 0xe7, 0x11, 0x71, 0xaf,
0xc2, 0x0c, 0x0f, 0x08, 0xe8, 0xfb, 0x70, 0x53, 0xa6, 0x69, 0x7a, 0xd8, 0x15, 0x2b, 0x28, 0x82,
0x0b, 0xda, 0x66, 0x6c, 0x55, 0x19, 0x6f, 0x9c, 0xcc, 0xe3, 0xa6, 0xe4, 0xce, 0xc8, 0x97, 0x59,
0xe1, 0x8a, 0x19, 0xc3, 0x2a, 0x6d, 0xc2, 0xad, 0x4b, 0x45, 0x3e, 0xd7, 0xbd, 0xcd, 0x3f, 0x25,
0x01, 0x5a, 0xdd, 0xea, 0x8e, 0x74, 0x52, 0x03, 0xb2, 0xfb, 0xfa, 0xd0, 0xb2, 0x27, 0x57, 0x45,
0xc0, 0x29, 0xbe, 0x52, 0x15, 0xee, 0xd8, 0xe0, 0x32, 0x58, 0xca, 0xf2, 0x9a, 0x72, 0xbc, 0xe7,
0x10, 0x3f, 0xac, 0x29, 0x79, 0x8b, 0x99, 0xe1, 0xea, 0x4e, 0xb8, 0x74, 0x45, 0x83, 0x4d, 0x00,
0x4b, 0x79, 0x8e, 0xf5, 0x49, 0x10, 0xb6, 0x64, 0x13, 0x6d, 0xf1, 0xdb, 0x51, 0xe2, 0x1e, 0x11,
0x73, 0x35, 0xc3, 0x9d, 0x7a, 0x9d, 0x3d, 0x58, 0xc2, 0x85, 0xef, 0x42, 0xe9, 0xd2, 0x47, 0x3c,
0x65, 0x9a, 0xb2, 0x3e, 0x97, 0x8f, 0x1e, 0xc2, 0xc2, 0x85, 0x71, 0xbe, 0x56, 0xcc, 0xb7, 0xba,
0xcf, 0xbf, 0xae, 0xa4, 0xe5, 0xaf, 0x6f, 0x2a, 0x59, 0xf5, 0x4f, 0x53, 0x22, 0xd0, 0x48, 0xaf,
0xc6, 0xbf, 0x0a, 0xe4, 0xf8, 0xea, 0x36, 0xa8, 0x2d, 0x03, 0xc0, 0x7b, 0x57, 0xc7, 0x1f, 0x56,
0xd3, 0x71, 0x38, 0x0e, 0x05, 0xd1, 0x3a, 0x14, 0xc4, 0x2a, 0xd6, 0xd8, 0x86, 0xe3, 0x6e, 0x5d,
0xc0, 0x20, 0x48, 0x4c, 0x12, 0xdd, 0x85, 0x45, 0x7e, 0xf9, 0xe3, 0x1d, 0x10, 0x53, 0x60, 0xd2,
0x1c, 0xb3, 0x10, 0x52, 0x39, 0x6c, 0x07, 0x8a, 0x92, 0xa0, 0xf1, 0x7c, 0x3e, 0xc3, 0x0d, 0xba,
0x77, 0x9d, 0x41, 0x42, 0x84, 0xa7, 0xf9, 0x85, 0xd1, 0xb4, 0xa1, 0x36, 0x20, 0x17, 0x18, 0x8b,
0x56, 0x21, 0xd5, 0xaf, 0x77, 0x95, 0xb9, 0xd2, 0xd2, 0xc9, 0x69, 0xb9, 0x10, 0x90, 0xfb, 0xf5,
0x2e, 0xe3, 0xec, 0x36, 0xba, 0x4a, 0xe2, 0x22, 0x67, 0xb7, 0xd1, 0x2d, 0xa5, 0x59, 0x0e, 0xa6,
0xee, 0x43, 0x21, 0xd2, 0x03, 0x7a, 0x07, 0xe6, 0x5b, 0xed, 0x4d, 0xdc, 0xec, 0xf5, 0x94, 0xb9,
0xd2, 0xcd, 0x93, 0xd3, 0x32, 0x8a, 0x70, 0x5b, 0xce, 0x80, 0xcd, 0x0f, 0xba, 0x03, 0xe9, 0xad,
0x0e, 0x3b, 0xdb, 0x45, 0x01, 0x11, 0x41, 0x6c, 0x51, 0xcf, 0x2f, 0xdd, 0x90, 0xc9, 0x5d, 0x54,
0xb1, 0xfa, 0x87, 0x09, 0xc8, 0x8a, 0xcd, 0x14, 0x3b, 0x51, 0x55, 0x98, 0x0f, 0xae, 0x09, 0x44,
0x71, 0xf7, 0xde, 0xe5, 0x85, 0x58, 0x45, 0xd6, 0x4d, 0x62, 0xf9, 0x05, 0x72, 0xa5, 0x0f, 0xa1,
0x18, 0x65, 0x7c, 0xae, 0xc5, 0xf7, 0x7d, 0x28, 0xb0, 0xf5, 0x1d, 0x14, 0x64, 0x8f, 0x21, 0x2b,
0x02, 0x42, 0x78, 0xd6, 0x5c, 0x5e, 0x15, 0x4a, 0x24, 0x7a, 0x0a, 0xf3, 0xa2, 0x92, 0x0c, 0x6e,
0x87, 0xd7, 0xae, 0xde, 0x45, 0x38, 0x80, 0xab, 0x1f, 0x43, 0xba, 0x4b, 0x88, 0xcb, 0x7c, 0xef,
0x50, 0x93, 0x4c, 0x8f, 0x67, 0x59, 0x04, 0x9b, 0xa4, 0xd5, 0x60, 0x45, 0xb0, 0x49, 0x5a, 0x66,
0x78, 0xff, 0x95, 0x8c, 0xdc, 0x7f, 0xf5, 0xa1, 0xf8, 0x82, 0x58, 0x83, 0x03, 0x9f, 0x98, 0x5c,
0xd1, 0x7d, 0x48, 0x8f, 0x48, 0x68, 0xfc, 0x6a, 0xec, 0x02, 0x23, 0xc4, 0xc5, 0x1c, 0xc5, 0xe2,
0xc8, 0x31, 0x97, 0x96, 0x4f, 0x1a, 0xb2, 0xa5, 0xfe, 0x63, 0x12, 0x16, 0x5b, 0x9e, 0x37, 0xd6,
0x1d, 0x23, 0xc8, 0xdc, 0xbe, 0x7d, 0x31, 0x73, 0x8b, 0x7d, 0xfb, 0xb9, 0x28, 0x72, 0xf1, 0x5a,
0x4f, 0x9e, 0x9e, 0xc9, 0xf0, 0xf4, 0x54, 0x7f, 0x9a, 0x08, 0xee, 0xee, 0xee, 0x46, 0xb6, 0xbb,
0xa8, 0x03, 0xa3, 0x9a, 0xc8, 0xae, 0x73, 0xe8, 0xd0, 0x63, 0x07, 0xbd, 0x0d, 0x19, 0xdc, 0x6c,
0x37, 0x5f, 0x28, 0x09, 0xb1, 0x3c, 0x2f, 0x80, 0x30, 0x71, 0xc8, 0x31, 0xd3, 0xd4, 0x6d, 0xb6,
0x1b, 0x2c, 0xd3, 0x4a, 0xc6, 0x68, 0xea, 0x12, 0xc7, 0xb4, 0x9c, 0x01, 0x7a, 0x07, 0xb2, 0xad,
0x5e, 0x6f, 0x97, 0x97, 0x89, 0x6f, 0x9d, 0x9c, 0x96, 0x6f, 0x5c, 0x40, 0xf1, 0x7b, 0x5b, 0x93,
0x81, 0x58, 0x99, 0xc3, 0x72, 0xb0, 0x18, 0x10, 0xcb, 0x9f, 0x05, 0x08, 0x77, 0xfa, 0xd5, 0x7e,
0x53, 0xc9, 0xc4, 0x80, 0x30, 0x65, 0x7f, 0xe5, 0x76, 0xfb, 0xd7, 0x24, 0x28, 0x55, 0xc3, 0x20,
0x23, 0x9f, 0xf1, 0x65, 0x65, 0xd9, 0x87, 0xdc, 0x88, 0xfd, 0xb2, 0x48, 0x90, 0x25, 0x3d, 0x8d,
0x7d, 0xbd, 0x9c, 0x91, 0xab, 0x60, 0x6a, 0x93, 0xaa, 0x39, 0xb4, 0x3c, 0xcf, 0xa2, 0x8e, 0xa0,
0xe1, 0x50, 0x53, 0xe9, 0xbf, 0x12, 0x70, 0x23, 0x06, 0x81, 0x1e, 0x42, 0xda, 0xa5, 0x76, 0x30,
0x87, 0xb7, 0x2f, 0xbb, 0x96, 0x65, 0xa2, 0x98, 0x23, 0xd1, 0x1a, 0x80, 0x3e, 0xf6, 0xa9, 0xce,
0xfb, 0xe7, 0xb3, 0x97, 0xc3, 0x11, 0x0a, 0x7a, 0x01, 0x59, 0x8f, 0x18, 0x2e, 0x09, 0x72, 0xe9,
0x8f, 0xff, 0xbf, 0xd6, 0x57, 0x7a, 0x5c, 0x0d, 0x96, 0xea, 0x4a, 0x15, 0xc8, 0x0a, 0x0a, 0x5b,
0xf6, 0xa6, 0xee, 0xeb, 0xf2, 0xd2, 0x9e, 0xff, 0x66, 0xab, 0x49, 0xb7, 0x07, 0xc1, 0x6a, 0xd2,
0xed, 0x81, 0xfa, 0x77, 0x49, 0x80, 0xe6, 0x4b, 0x9f, 0xb8, 0x8e, 0x6e, 0xd7, 0xab, 0xa8, 0x19,
0x89, 0xfe, 0x62, 0xb4, 0x5f, 0x8d, 0x7d, 0x89, 0x08, 0x25, 0x2a, 0xf5, 0x6a, 0x4c, 0xfc, 0xbf,
0x05, 0xa9, 0xb1, 0x2b, 0x1f, 0xa4, 0x45, 0x1e, 0xbc, 0x8b, 0xb7, 0x31, 0xa3, 0xa1, 0xe6, 0x34,
0x6c, 0xa5, 0x2e, 0x7f, 0x76, 0x8e, 0x74, 0x10, 0x1b, 0xba, 0xd8, 0xce, 0x37, 0x74, 0xcd, 0x20,
0xf2, 0xe4, 0x28, 0x8a, 0x9d, 0x5f, 0xaf, 0xd6, 0x89, 0xeb, 0xe3, 0xac, 0xa1, 0xb3, 0xff, 0x5f,
0x28, 0xbe, 0xdd, 0x07, 0x98, 0x0e, 0x0d, 0xad, 0x41, 0xa6, 0xbe, 0xd1, 0xeb, 0x6d, 0x2b, 0x73,
0x22, 0x80, 0x4f, 0x59, 0x9c, 0xac, 0xfe, 0x75, 0x12, 0x72, 0xf5, 0xaa, 0x3c, 0x56, 0xeb, 0xa0,
0xf0, 0xa8, 0xc4, 0x9f, 0x3a, 0xc8, 0xcb, 0x91, 0xe5, 0x4e, 0x64, 0x60, 0xb9, 0xa2, 0xa8, 0x5d,
0x64, 0x22, 0xcc, 0xea, 0x26, 0x17, 0x40, 0x18, 0x8a, 0x44, 0x3a, 0x41, 0x33, 0xf4, 0x20, 0xc6,
0xaf, 0x5d, 0xed, 0x2c, 0x51, 0x9e, 0x4c, 0xdb, 0x1e, 0x2e, 0x04, 0x4a, 0xea, 0xba, 0x87, 0x3e,
0x80, 0x25, 0xcf, 0x1a, 0x38, 0x96, 0x33, 0xd0, 0x02, 0xe7, 0xf1, 0x77, 0x97, 0xda, 0xf2, 0xf9,
0xd9, 0xfa, 0x42, 0x4f, 0xb0, 0xa4, 0x0f, 0x17, 0x24, 0xb2, 0xce, 0x5d, 0x89, 0xbe, 0x09, 0x8b,
0x11, 0x51, 0xe6, 0x45, 0xe1, 0x76, 0xe5, 0xfc, 0x6c, 0xbd, 0x18, 0x4a, 0x3e, 0x23, 0x13, 0x5c,
0x0c, 0x05, 0x9f, 0x11, 0x7e, 0xff, 0xb2, 0x4f, 0x5d, 0x83, 0x68, 0x2e, 0xdf, 0xd3, 0xfc, 0x04,
0x4f, 0xe3, 0x02, 0xa7, 0x89, 0x6d, 0xae, 0x3e, 0x87, 0x1b, 0x1d, 0xd7, 0x38, 0x20, 0x9e, 0x2f,
0x5c, 0x21, 0xbd, 0xf8, 0x31, 0xdc, 0xf6, 0x75, 0xef, 0x50, 0x3b, 0xb0, 0x3c, 0x9f, 0xba, 0x13,
0xcd, 0x25, 0x3e, 0x71, 0x18, 0x5f, 0xe3, 0x8f, 0xb5, 0xf2, 0xd2, 0xef, 0x16, 0xc3, 0x6c, 0x09,
0x08, 0x0e, 0x10, 0xdb, 0x0c, 0xa0, 0xb6, 0xa0, 0xc8, 0xca, 0x14, 0x79, 0x71, 0xc6, 0x46, 0x0f,
0x36, 0x1d, 0x68, 0x6f, 0x7c, 0x4c, 0xe5, 0x6d, 0x3a, 0x10, 0x3f, 0xd5, 0xef, 0x82, 0xd2, 0xb0,
0xbc, 0x91, 0xee, 0x1b, 0x07, 0xc1, 0x6d, 0x26, 0x6a, 0x80, 0x72, 0x40, 0x74, 0xd7, 0xdf, 0x23,
0xba, 0xaf, 0x8d, 0x88, 0x6b, 0x51, 0xf3, 0xfa, 0x59, 0x5e, 0x0a, 0x45, 0xba, 0x5c, 0x42, 0xfd,
0xef, 0x04, 0x00, 0xd6, 0xf7, 0x83, 0x8c, 0xec, 0x6b, 0xb0, 0xec, 0x39, 0xfa, 0xc8, 0x3b, 0xa0,
0xbe, 0x66, 0x39, 0x3e, 0x71, 0x8f, 0x74, 0x5b, 0x5e, 0xe0, 0x28, 0x01, 0xa3, 0x25, 0xe9, 0xe8,
0x3e, 0xa0, 0x43, 0x42, 0x46, 0x1a, 0xb5, 0x4d, 0x2d, 0x60, 0x8a, 0xa7, 0xe4, 0x34, 0x56, 0x18,
0xa7, 0x63, 0x9b, 0xbd, 0x80, 0x8e, 0x6a, 0xb0, 0xc6, 0x86, 0x4f, 0x1c, 0xdf, 0xb5, 0x88, 0xa7,
0xed, 0x53, 0x57, 0xf3, 0x6c, 0x7a, 0xac, 0xed, 0x53, 0xdb, 0xa6, 0xc7, 0xc4, 0x0d, 0xee, 0xc6,
0x4a, 0x36, 0x1d, 0x34, 0x05, 0x68, 0x83, 0xba, 0x3d, 0x9b, 0x1e, 0x6f, 0x04, 0x08, 0x96, 0xb6,
0x4d, 0xc7, 0xec, 0x5b, 0xc6, 0x61, 0x90, 0xb6, 0x85, 0xd4, 0xbe, 0x65, 0x1c, 0xa2, 0x77, 0x60,
0x81, 0xd8, 0x84, 0x5f, 0x91, 0x08, 0x54, 0x86, 0xa3, 0x8a, 0x01, 0x91, 0x81, 0xd4, 0x4f, 0x40,
0x69, 0x3a, 0x86, 0x3b, 0x19, 0x45, 0xe6, 0xfc, 0x3e, 0x20, 0x16, 0x24, 0x35, 0x9b, 0x1a, 0x87,
0xda, 0x50, 0x77, 0xf4, 0x01, 0xb3, 0x4b, 0xbc, 0xf0, 0x29, 0x8c, 0xb3, 0x4d, 0x8d, 0xc3, 0x1d,
0x49, 0x57, 0x3f, 0x00, 0xe8, 0x8d, 0x5c, 0xa2, 0x9b, 0x1d, 0x96, 0x4d, 0x30, 0xd7, 0xf1, 0x96,
0x66, 0xca, 0x17, 0x52, 0xea, 0xca, 0xad, 0xae, 0x08, 0x46, 0x23, 0xa4, 0xab, 0xbf, 0x0c, 0x37,
0xba, 0xb6, 0x6e, 0xf0, 0xaf, 0x05, 0xba, 0xe1, 0x93, 0x15, 0x7a, 0x0a, 0x59, 0x01, 0x95, 0x33,
0x19, 0xbb, 0xdd, 0xa6, 0x7d, 0x6e, 0xcd, 0x61, 0x89, 0xaf, 0x15, 0x01, 0xa6, 0x7a, 0xd4, 0xbf,
0x4c, 0x40, 0x3e, 0xd4, 0x8f, 0xca, 0xe2, 0x25, 0xc6, 0x77, 0x75, 0xcb, 0x91, 0x55, 0x7d, 0x1e,
0x47, 0x49, 0xa8, 0x05, 0x85, 0x51, 0x28, 0x7d, 0x65, 0x3e, 0x17, 0x63, 0x35, 0x8e, 0xca, 0xa2,
0x0f, 0x21, 0x1f, 0x3c, 0x49, 0x07, 0x11, 0xf6, 0xea, 0x17, 0xec, 0x29, 0x5c, 0xfd, 0x36, 0xc0,
0x77, 0xa8, 0xe5, 0xf4, 0xe9, 0x21, 0x71, 0xf8, 0x13, 0x2b, 0xab, 0x09, 0x49, 0xe0, 0x45, 0xd9,
0xe2, 0xa5, 0xbe, 0x98, 0x82, 0xf0, 0xa5, 0x51, 0x34, 0xd5, 0xbf, 0x4d, 0x42, 0x16, 0x53, 0xea,
0xd7, 0xab, 0xa8, 0x0c, 0x59, 0x19, 0x27, 0xf8, 0xf9, 0x53, 0xcb, 0x9f, 0x9f, 0xad, 0x67, 0x44,
0x80, 0xc8, 0x18, 0x3c, 0x32, 0x44, 0x22, 0x78, 0xf2, 0xb2, 0x08, 0x8e, 0x1e, 0x42, 0x51, 0x82,
0xb4, 0x03, 0xdd, 0x3b, 0x10, 0x05, 0x5a, 0x6d, 0xf1, 0xfc, 0x6c, 0x1d, 0x04, 0x72, 0x4b, 0xf7,
0x0e, 0x30, 0x08, 0x34, 0xfb, 0x8d, 0x9a, 0x50, 0xf8, 0x94, 0x5a, 0x8e, 0xe6, 0xf3, 0x41, 0xc8,
0xcb, 0xc4, 0xd8, 0x79, 0x9c, 0x0e, 0x55, 0x7e, 0x6f, 0x00, 0x9f, 0x4e, 0x07, 0xdf, 0x84, 0x05,
0x97, 0x52, 0x5f, 0x84, 0x2d, 0x8b, 0x3a, 0xf2, 0x9e, 0xa2, 0x1c, 0x7b, 0x7d, 0x4d, 0xa9, 0x8f,
0x25, 0x0e, 0x17, 0xdd, 0x48, 0x0b, 0x3d, 0x84, 0x15, 0x5b, 0xf7, 0x7c, 0x8d, 0xc7, 0x3b, 0x73,
0xaa, 0x2d, 0xcb, 0xb7, 0x1a, 0x62, 0xbc, 0x0d, 0xce, 0x0a, 0x24, 0xd4, 0x7f, 0x4e, 0x40, 0x81,
0x0d, 0xc6, 0xda, 0xb7, 0x0c, 0x96, 0xe4, 0x7d, 0xfe, 0xdc, 0xe3, 0x16, 0xa4, 0x0c, 0xcf, 0x95,
0x4e, 0xe5, 0x87, 0x6f, 0xbd, 0x87, 0x31, 0xa3, 0xa1, 0x4f, 0x20, 0x2b, 0xef, 0x4b, 0x44, 0xda,
0xa1, 0x5e, 0x9f, 0x8e, 0x4a, 0xdf, 0x48, 0x39, 0xbe, 0x96, 0xa7, 0xd6, 0x89, 0x43, 0x00, 0x47,
0x49, 0xe8, 0x26, 0x24, 0x0d, 0xe1, 0x2e, 0xf9, 0x41, 0x4b, 0xbd, 0x8d, 0x93, 0x86, 0xa3, 0xfe,
0x28, 0x01, 0x0b, 0xd3, 0x0d, 0xcf, 0x56, 0xc0, 0x6d, 0xc8, 0x7b, 0xe3, 0x3d, 0x6f, 0xe2, 0xf9,
0x64, 0x18, 0x3c, 0x1f, 0x87, 0x04, 0xd4, 0x82, 0xbc, 0x6e, 0x0f, 0xa8, 0x6b, 0xf9, 0x07, 0x43,
0x59, 0x89, 0xc6, 0xa7, 0x0a, 0x51, 0x9d, 0x95, 0x6a, 0x20, 0x82, 0xa7, 0xd2, 0xc1, 0xb9, 0x2f,
0xbe, 0x31, 0xe0, 0xe7, 0xfe, 0xdb, 0x50, 0xb4, 0xf5, 0x21, 0xbf, 0x40, 0xf2, 0xad, 0xa1, 0x18,
0x47, 0x1a, 0x17, 0x24, 0xad, 0x6f, 0x0d, 0x89, 0xaa, 0x42, 0x3e, 0x54, 0x86, 0x96, 0xa0, 0x50,
0x6d, 0xf6, 0xb4, 0x47, 0x8f, 0x9f, 0x6a, 0x9b, 0xf5, 0x1d, 0x65, 0x4e, 0xe6, 0xa6, 0x7f, 0x95,
0x80, 0x05, 0x19, 0x8e, 0x64, 0xbe, 0xff, 0x0e, 0xcc, 0xbb, 0xfa, 0xbe, 0x1f, 0x54, 0x24, 0x69,
0xb1, 0xaa, 0x59, 0x84, 0x67, 0x15, 0x09, 0x63, 0xc5, 0x57, 0x24, 0x91, 0x0f, 0x1a, 0x52, 0x57,
0x7e, 0xd0, 0x90, 0xfe, 0xb9, 0x7c, 0xd0, 0xa0, 0xfe, 0x26, 0xc0, 0x86, 0x65, 0x93, 0xbe, 0xb8,
0x6b, 0x8a, 0xab, 0x2f, 0x59, 0x0e, 0x27, 0xef, 0x32, 0x83, 0x1c, 0xae, 0xd5, 0xc0, 0x8c, 0xc6,
0x58, 0x03, 0xcb, 0x94, 0x9b, 0x91, 0xb3, 0x36, 0x19, 0x6b, 0x60, 0x99, 0xe1, 0xcb, 0x5b, 0xfa,
0xba, 0x97, 0xb7, 0xd3, 0x04, 0x2c, 0xc9, 0xdc, 0x35, 0x0c, 0xbf, 0x5f, 0x85, 0xbc, 0x48, 0x63,
0xa7, 0x05, 0x1d, 0x7f, 0xc4, 0x17, 0xb8, 0x56, 0x03, 0xe7, 0x04, 0xbb, 0x65, 0xa2, 0x75, 0x28,
0x48, 0x68, 0xe4, 0xe3, 0x27, 0x10, 0xa4, 0x36, 0x33, 0xff, 0xeb, 0x90, 0xde, 0xb7, 0x6c, 0x22,
0x17, 0x7a, 0x6c, 0x00, 0x98, 0x3a, 0x60, 0x6b, 0x0e, 0x73, 0x74, 0x2d, 0x17, 0x5c, 0xc6, 0x71,
0xfb, 0x64, 0xd9, 0x19, 0xb5, 0x4f, 0x54, 0xa0, 0x33, 0xf6, 0x09, 0x1c, 0xb3, 0x4f, 0xb0, 0x85,
0x7d, 0x12, 0x1a, 0xb5, 0x4f, 0x90, 0x7e, 0x2e, 0xf6, 0x6d, 0xc3, 0xcd, 0x9a, 0xad, 0x1b, 0x87,
0xb6, 0xe5, 0xf9, 0xc4, 0x8c, 0x46, 0x8c, 0xc7, 0x90, 0xbd, 0x90, 0x74, 0x5e, 0x75, 0x6b, 0x29,
0x91, 0xea, 0x7f, 0x24, 0xa0, 0xb8, 0x45, 0x74, 0xdb, 0x3f, 0x98, 0x5e, 0x0d, 0xf9, 0xc4, 0xf3,
0xe5, 0x61, 0xc5, 0x7f, 0xa3, 0x6f, 0x40, 0x2e, 0xcc, 0x49, 0xae, 0x7d, 0x7f, 0x0b, 0xa1, 0xe8,
0x09, 0xcc, 0xb3, 0x3d, 0x46, 0xc7, 0x41, 0xb1, 0x73, 0xd5, 0xd3, 0x8e, 0x44, 0xb2, 0x43, 0xc6,
0x25, 0x3c, 0x09, 0xe1, 0x4b, 0x29, 0x83, 0x83, 0x26, 0xfa, 0x45, 0x28, 0xf2, 0x97, 0x89, 0x20,
0xe7, 0xca, 0x5c, 0xa7, 0xb3, 0x20, 0x1e, 0x17, 0x45, 0xbe, 0xf5, 0xbf, 0x09, 0x58, 0xd9, 0xd1,
0x27, 0x7b, 0x44, 0x86, 0x0d, 0x62, 0x62, 0x62, 0x50, 0xd7, 0x44, 0xdd, 0x68, 0xb8, 0xb9, 0xe2,
0xad, 0x32, 0x4e, 0x38, 0x3e, 0xea, 0x04, 0x05, 0x58, 0x32, 0x52, 0x80, 0xad, 0x40, 0xc6, 0xa1,
0x8e, 0x41, 0x64, 0x2c, 0x12, 0x0d, 0xd5, 0x8a, 0x86, 0x9a, 0x52, 0xf8, 0x8c, 0xc8, 0x1f, 0x01,
0xdb, 0xd4, 0x0f, 0x7b, 0x43, 0x9f, 0x40, 0xa9, 0xd7, 0xac, 0xe3, 0x66, 0xbf, 0xd6, 0xf9, 0xae,
0xd6, 0xab, 0x6e, 0xf7, 0xaa, 0x8f, 0x1f, 0x6a, 0xdd, 0xce, 0xf6, 0xf7, 0x1e, 0x3d, 0x79, 0xf8,
0x0d, 0x25, 0x51, 0x2a, 0x9f, 0x9c, 0x96, 0x6f, 0xb7, 0xab, 0xf5, 0x6d, 0xb1, 0x63, 0xf6, 0xe8,
0xcb, 0x9e, 0x6e, 0x7b, 0xfa, 0xe3, 0x87, 0x5d, 0x6a, 0x4f, 0x18, 0x86, 0x2d, 0xeb, 0x62, 0xf4,
0xbc, 0x8a, 0x1e, 0xc3, 0x89, 0x4b, 0x8f, 0xe1, 0xe9, 0x69, 0x9e, 0xbc, 0xe4, 0x34, 0xdf, 0x80,
0x15, 0xc3, 0xa5, 0x9e, 0xa7, 0xb1, 0xec, 0x9f, 0x98, 0x33, 0xf5, 0xc5, 0x97, 0xce, 0xcf, 0xd6,
0x97, 0xeb, 0x8c, 0xdf, 0xe3, 0x6c, 0xa9, 0x7e, 0xd9, 0x88, 0x90, 0x78, 0x4f, 0xea, 0x1f, 0xa5,
0x58, 0x22, 0x65, 0x1d, 0x59, 0x36, 0x19, 0x10, 0x0f, 0x3d, 0x87, 0x25, 0xc3, 0x25, 0x26, 0x4b,
0xeb, 0x75, 0x3b, 0xfa, 0x11, 0xed, 0x2f, 0xc4, 0xe6, 0x34, 0xa1, 0x60, 0xa5, 0x1e, 0x4a, 0xf5,
0x46, 0xc4, 0xc0, 0x8b, 0xc6, 0x85, 0x36, 0xfa, 0x14, 0x96, 0x3c, 0x62, 0x5b, 0xce, 0xf8, 0xa5,
0x66, 0x50, 0xc7, 0x27, 0x2f, 0x83, 0x17, 0xb1, 0xeb, 0xf4, 0xf6, 0x9a, 0xdb, 0x4c, 0xaa, 0x2e,
0x84, 0x6a, 0xe8, 0xfc, 0x6c, 0x7d, 0xf1, 0x22, 0x0d, 0x2f, 0x4a, 0xcd, 0xb2, 0x5d, 0x6a, 0xc3,
0xe2, 0x45, 0x6b, 0xd0, 0x8a, 0xdc, 0xfb, 0x3c, 0x84, 0x04, 0x7b, 0x1b, 0xdd, 0x86, 0x9c, 0x4b,
0x06, 0x96, 0xe7, 0xbb, 0xc2, 0xcd, 0x8c, 0x13, 0x52, 0xd8, 0xce, 0x17, 0x5f, 0x40, 0x95, 0x7e,
0x1d, 0x66, 0x7a, 0x64, 0x9b, 0xc5, 0xb4, 0x3c, 0x7d, 0x4f, 0xaa, 0xcc, 0xe1, 0xa0, 0xc9, 0xd6,
0xe0, 0xd8, 0x0b, 0x13, 0x35, 0xfe, 0x9b, 0xd1, 0x78, 0x46, 0x21, 0xbf, 0x07, 0xe3, 0x39, 0x43,
0xf0, 0x61, 0x69, 0x3a, 0xf2, 0x61, 0xe9, 0x0a, 0x64, 0x6c, 0x72, 0x44, 0x6c, 0x71, 0x96, 0x63,
0xd1, 0xb8, 0xf7, 0x10, 0x8a, 0xc1, 0x17, 0x8c, 0xfc, 0xcb, 0x89, 0x1c, 0xa4, 0xfb, 0xd5, 0xde,
0x33, 0x65, 0x0e, 0x01, 0x64, 0xc5, 0xe2, 0x14, 0xaf, 0x75, 0xf5, 0x4e, 0x7b, 0xa3, 0xb5, 0xa9,
0x24, 0xef, 0xfd, 0x2c, 0x05, 0xf9, 0xf0, 0xbd, 0x88, 0x9d, 0x1d, 0xed, 0xe6, 0x8b, 0x60, 0x75,
0x87, 0xf4, 0x36, 0x39, 0x46, 0x6f, 0x4f, 0x6f, 0xa1, 0x3e, 0x11, 0x0f, 0xe4, 0x21, 0x3b, 0xb8,
0x81, 0x7a, 0x17, 0x72, 0xd5, 0x5e, 0xaf, 0xb5, 0xd9, 0x6e, 0x36, 0x94, 0xcf, 0x12, 0xa5, 0x2f,
0x9d, 0x9c, 0x96, 0x97, 0x43, 0x50, 0xd5, 0x13, 0x8b, 0x8f, 0xa3, 0xea, 0xf5, 0x66, 0xb7, 0xdf,
0x6c, 0x28, 0xaf, 0x92, 0xb3, 0x28, 0x7e, 0xab, 0xc2, 0x3f, 0xdd, 0xc9, 0x77, 0x71, 0xb3, 0x5b,
0xc5, 0xac, 0xc3, 0xcf, 0x92, 0xe2, 0x72, 0x6c, 0xda, 0xa3, 0x4b, 0x46, 0xba, 0xcb, 0xfa, 0x5c,
0x0b, 0xbe, 0x85, 0x7b, 0x95, 0x12, 0x9f, 0x77, 0x4c, 0x1f, 0xbf, 0x88, 0x6e, 0x4e, 0x58, 0x6f,
0xfc, 0xd5, 0x91, 0xab, 0x49, 0xcd, 0xf4, 0xd6, 0x63, 0xb1, 0x87, 0x69, 0x51, 0x61, 0x1e, 0xef,
0xb6, 0xdb, 0x0c, 0xf4, 0x2a, 0x3d, 0x33, 0x3a, 0x3c, 0x76, 0x58, 0xc5, 0x8c, 0xee, 0x42, 0x2e,
0x78, 0x94, 0x54, 0x3e, 0x4b, 0xcf, 0x18, 0x54, 0x0f, 0x5e, 0x54, 0x79, 0x87, 0x5b, 0xbb, 0x7d,
0xfe, 0xa9, 0xde, 0xab, 0xcc, 0x6c, 0x87, 0x07, 0x63, 0xdf, 0xa4, 0xc7, 0x0e, 0xdb, 0xb3, 0xf2,
0x1e, 0xee, 0xb3, 0x8c, 0xb8, 0xb4, 0x08, 0x31, 0xf2, 0x12, 0xee, 0x5d, 0xc8, 0xe1, 0xe6, 0x77,
0xc4, 0x57, 0x7d, 0xaf, 0xb2, 0x33, 0x7a, 0x30, 0xf9, 0x94, 0x18, 0xb2, 0xb7, 0x0e, 0xee, 0x6e,
0x55, 0xb9, 0xcb, 0x67, 0x51, 0x1d, 0x77, 0x74, 0xa0, 0x3b, 0xc4, 0x9c, 0x7e, 0xe3, 0x12, 0xb2,
0xee, 0xfd, 0x0a, 0xe4, 0x82, 0xcc, 0x14, 0xad, 0x41, 0xf6, 0x45, 0x07, 0x3f, 0x6b, 0x62, 0x65,
0x4e, 0xf8, 0x30, 0xe0, 0xbc, 0x10, 0x35, 0x45, 0x19, 0xe6, 0x77, 0xaa, 0xed, 0xea, 0x66, 0x13,
0x07, 0x57, 0xe4, 0x01, 0x40, 0xa6, 0x57, 0x25, 0x45, 0x76, 0x10, 0xea, 0xac, 0xad, 0xfe, 0xf0,
0x27, 0x6b, 0x73, 0x3f, 0xfe, 0xc9, 0xda, 0xdc, 0xab, 0xf3, 0xb5, 0xc4, 0x0f, 0xcf, 0xd7, 0x12,
0xff, 0x70, 0xbe, 0x96, 0xf8, 0xf7, 0xf3, 0xb5, 0xc4, 0x5e, 0x96, 0x1f, 0x02, 0x4f, 0xfe, 0x2f,
0x00, 0x00, 0xff, 0xff, 0x4b, 0xdb, 0xdc, 0xec, 0xf0, 0x31, 0x00, 0x00,
// 5043 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x7a, 0x4d, 0x6c, 0x24, 0x49,
0x56, 0xbf, 0xeb, 0xd3, 0x55, 0xaf, 0xca, 0x76, 0x3a, 0xda, 0xdb, 0xe3, 0xae, 0xed, 0xb6, 0x6b,
0x72, 0xa6, 0x77, 0x66, 0x7b, 0xfb, 0x5f, 0xfd, 0xb5, 0xbb, 0xea, 0x99, 0xf9, 0xef, 0xce, 0xd4,
0x97, 0xed, 0xda, 0xb6, 0xab, 0x4a, 0x51, 0xe5, 0xee, 0x5d, 0x24, 0x48, 0xa5, 0x33, 0xc3, 0xe5,
0x1c, 0x67, 0x65, 0x14, 0x99, 0x59, 0x76, 0x17, 0x0b, 0xa2, 0xc5, 0x01, 0x90, 0x4f, 0x70, 0x62,
0x11, 0x32, 0x42, 0x82, 0x13, 0x42, 0xe2, 0x00, 0x12, 0x82, 0xd3, 0x20, 0x71, 0xd8, 0x1b, 0x2c,
0x48, 0x68, 0x05, 0x92, 0x61, 0x7d, 0xe0, 0xb6, 0x82, 0x0b, 0xe2, 0x02, 0x12, 0x8a, 0x8f, 0xcc,
0x4a, 0x57, 0xa7, 0xed, 0x1e, 0x66, 0x2f, 0x76, 0xc5, 0x7b, 0xbf, 0xf7, 0xe2, 0xc5, 0x8b, 0x88,
0x17, 0xef, 0x45, 0x24, 0xdc, 0x1b, 0x58, 0xfe, 0xc1, 0x78, 0xaf, 0x62, 0xd0, 0xe1, 0x03, 0x93,
0x1a, 0x87, 0xc4, 0x7d, 0xe0, 0x1d, 0xeb, 0xee, 0xf0, 0xd0, 0xf2, 0x1f, 0xe8, 0x23, 0xeb, 0x81,
0x3f, 0x19, 0x11, 0xaf, 0x32, 0x72, 0xa9, 0x4f, 0x11, 0x12, 0x80, 0x4a, 0x00, 0xa8, 0x1c, 0x3d,
0x2a, 0xad, 0x0f, 0x28, 0x1d, 0xd8, 0xe4, 0x01, 0x47, 0xec, 0x8d, 0xf7, 0x1f, 0xf8, 0xd6, 0x90,
0x78, 0xbe, 0x3e, 0x1c, 0x09, 0xa1, 0xd2, 0xda, 0x2c, 0xc0, 0x1c, 0xbb, 0xba, 0x6f, 0x51, 0x47,
0xf2, 0x57, 0x06, 0x74, 0x40, 0xf9, 0xcf, 0x07, 0xec, 0x97, 0xa0, 0xaa, 0xeb, 0x30, 0xff, 0x9c,
0xb8, 0x9e, 0x45, 0x1d, 0xb4, 0x02, 0x19, 0xcb, 0x31, 0xc9, 0xcb, 0xd5, 0x44, 0x39, 0xf1, 0x7e,
0x1a, 0x8b, 0x86, 0xfa, 0x10, 0xa0, 0xc5, 0x7e, 0x34, 0x1d, 0xdf, 0x9d, 0x20, 0x05, 0x52, 0x87,
0x64, 0xc2, 0x11, 0x79, 0xcc, 0x7e, 0x32, 0xca, 0x91, 0x6e, 0xaf, 0x26, 0x05, 0xe5, 0x48, 0xb7,
0xd5, 0x9f, 0x24, 0xa0, 0x50, 0x75, 0x1c, 0xea, 0xf3, 0xde, 0x3d, 0x84, 0x20, 0xed, 0xe8, 0x43,
0x22, 0x85, 0xf8, 0x6f, 0x54, 0x87, 0xac, 0xad, 0xef, 0x11, 0xdb, 0x5b, 0x4d, 0x96, 0x53, 0xef,
0x17, 0x1e, 0x7f, 0xad, 0xf2, 0xfa, 0x90, 0x2b, 0x11, 0x25, 0x95, 0x6d, 0x8e, 0xe6, 0x46, 0x60,
0x29, 0x8a, 0xbe, 0x0d, 0xf3, 0x96, 0x63, 0x5a, 0x06, 0xf1, 0x56, 0xd3, 0x5c, 0xcb, 0x5a, 0x9c,
0x96, 0xa9, 0xf5, 0xb5, 0xf4, 0x0f, 0xcf, 0xd6, 0xe7, 0x70, 0x20, 0x54, 0xfa, 0x00, 0x0a, 0x11,
0xb5, 0x31, 0x63, 0x5b, 0x81, 0xcc, 0x91, 0x6e, 0x8f, 0x89, 0x1c, 0x9d, 0x68, 0x7c, 0x98, 0x7c,
0x9a, 0x50, 0x3f, 0x81, 0x95, 0xb6, 0x3e, 0x24, 0xe6, 0x26, 0x71, 0x88, 0x6b, 0x19, 0x98, 0x78,
0x74, 0xec, 0x1a, 0x84, 0x8d, 0xf5, 0xd0, 0x72, 0xcc, 0x60, 0xac, 0xec, 0x77, 0xbc, 0x16, 0xb5,
0x0e, 0x6f, 0x35, 0x2c, 0xcf, 0x70, 0x89, 0x4f, 0x3e, 0xb7, 0x92, 0x54, 0xa0, 0xe4, 0x2c, 0x01,
0x4b, 0xb3, 0xd2, 0x3f, 0x07, 0x37, 0x98, 0x8b, 0x4d, 0xcd, 0x95, 0x14, 0xcd, 0x1b, 0x11, 0x83,
0x2b, 0x2b, 0x3c, 0x7e, 0x3f, 0xce, 0x43, 0x71, 0x23, 0xd9, 0x9a, 0xc3, 0xcb, 0x5c, 0x4d, 0x40,
0xe8, 0x8d, 0x88, 0x81, 0x0c, 0xb8, 0x69, 0x4a, 0xa3, 0x67, 0xd4, 0x27, 0xb9, 0xfa, 0xd8, 0x69,
0xbc, 0x64, 0x98, 0x5b, 0x73, 0x78, 0x25, 0x50, 0x16, 0xed, 0xa4, 0x06, 0x90, 0x0b, 0x74, 0xab,
0x3f, 0x48, 0x40, 0x3e, 0x60, 0x7a, 0xe8, 0xab, 0x90, 0x77, 0x74, 0x87, 0x6a, 0xc6, 0x68, 0xec,
0xf1, 0x01, 0xa5, 0x6a, 0xc5, 0xf3, 0xb3, 0xf5, 0x5c, 0x5b, 0x77, 0x68, 0xbd, 0xbb, 0xeb, 0xe1,
0x1c, 0x63, 0xd7, 0x47, 0x63, 0x0f, 0xbd, 0x0d, 0xc5, 0x21, 0x19, 0x52, 0x77, 0xa2, 0xed, 0x4d,
0x7c, 0xe2, 0x49, 0xb7, 0x15, 0x04, 0xad, 0xc6, 0x48, 0xe8, 0x5b, 0x30, 0x3f, 0x10, 0x26, 0xad,
0xa6, 0xf8, 0xf2, 0x79, 0x27, 0xce, 0xfa, 0x19, 0xab, 0x71, 0x20, 0xa3, 0xfe, 0x56, 0x02, 0x56,
0x42, 0x2a, 0xf9, 0xc5, 0xb1, 0xe5, 0x92, 0x21, 0x71, 0x7c, 0x0f, 0x7d, 0x03, 0xb2, 0xb6, 0x35,
0xb4, 0x7c, 0x4f, 0xfa, 0xfc, 0x4e, 0x9c, 0xda, 0x70, 0x50, 0x58, 0x82, 0x51, 0x15, 0x8a, 0x2e,
0xf1, 0x88, 0x7b, 0x24, 0x56, 0xbc, 0xf4, 0xe8, 0x35, 0xc2, 0x17, 0x44, 0xd4, 0x0d, 0xc8, 0x75,
0x6d, 0xdd, 0xdf, 0xa7, 0xee, 0x10, 0xa9, 0x50, 0xd4, 0x5d, 0xe3, 0xc0, 0xf2, 0x89, 0xe1, 0x8f,
0xdd, 0x60, 0xf7, 0x5d, 0xa0, 0xa1, 0x9b, 0x90, 0xa4, 0xa2, 0xa3, 0x7c, 0x2d, 0x7b, 0x7e, 0xb6,
0x9e, 0xec, 0xf4, 0x70, 0x92, 0x7a, 0xea, 0x47, 0xb0, 0xdc, 0xb5, 0xc7, 0x03, 0xcb, 0x69, 0x10,
0xcf, 0x70, 0xad, 0x11, 0xd3, 0xce, 0x56, 0x25, 0x8b, 0x51, 0xc1, 0xaa, 0x64, 0xbf, 0xc3, 0xad,
0x9d, 0x9c, 0x6e, 0x6d, 0xf5, 0x37, 0x92, 0xb0, 0xdc, 0x74, 0x06, 0x96, 0x43, 0xa2, 0xd2, 0x77,
0x61, 0x91, 0x70, 0xa2, 0x76, 0x24, 0xc2, 0x8d, 0xd4, 0xb3, 0x20, 0xa8, 0x41, 0x0c, 0x6a, 0xcd,
0xc4, 0x85, 0x47, 0x71, 0xc3, 0x7f, 0x4d, 0x7b, 0x6c, 0x74, 0x68, 0xc2, 0xfc, 0x88, 0x0f, 0xc2,
0x93, 0xd3, 0x7b, 0x37, 0x4e, 0xd7, 0x6b, 0xe3, 0x0c, 0x82, 0x84, 0x94, 0xfd, 0x22, 0x41, 0xe2,
0x4f, 0x92, 0xb0, 0xd4, 0xa6, 0xe6, 0x05, 0x3f, 0x94, 0x20, 0x77, 0x40, 0x3d, 0x3f, 0x12, 0x10,
0xc3, 0x36, 0x7a, 0x0a, 0xb9, 0x91, 0x9c, 0x3e, 0x39, 0xfb, 0xb7, 0xe3, 0x4d, 0x16, 0x18, 0x1c,
0xa2, 0xd1, 0x47, 0x90, 0x0f, 0xb6, 0x0c, 0x1b, 0xed, 0x1b, 0x2c, 0x9c, 0x29, 0x1e, 0x7d, 0x0b,
0xb2, 0x62, 0x12, 0x56, 0xd3, 0x5c, 0xf2, 0xee, 0x1b, 0xf9, 0x1c, 0x4b, 0x21, 0xb4, 0x09, 0x39,
0xdf, 0xf6, 0x34, 0xcb, 0xd9, 0xa7, 0xab, 0x19, 0xae, 0x60, 0x3d, 0x36, 0xc8, 0x50, 0x93, 0xf4,
0xb7, 0x7b, 0x2d, 0x67, 0x9f, 0xd6, 0x0a, 0xe7, 0x67, 0xeb, 0xf3, 0xb2, 0x81, 0xe7, 0x7d, 0xdb,
0x63, 0x3f, 0xd4, 0xdf, 0x4e, 0x40, 0x21, 0x82, 0x42, 0x77, 0x00, 0x7c, 0x77, 0xec, 0xf9, 0x9a,
0x4b, 0xa9, 0xcf, 0x9d, 0x55, 0xc4, 0x79, 0x4e, 0xc1, 0x94, 0xfa, 0xa8, 0x02, 0x37, 0x0c, 0xe2,
0xfa, 0x9a, 0xe5, 0x79, 0x63, 0xe2, 0x6a, 0xde, 0x78, 0xef, 0x53, 0x62, 0xf8, 0xdc, 0x71, 0x45,
0xbc, 0xcc, 0x58, 0x2d, 0xce, 0xe9, 0x09, 0x06, 0x7a, 0x02, 0x37, 0xa3, 0xf8, 0xd1, 0x78, 0xcf,
0xb6, 0x0c, 0x8d, 0x4d, 0x66, 0x8a, 0x8b, 0xdc, 0x98, 0x8a, 0x74, 0x39, 0xef, 0x19, 0x99, 0xa8,
0x3f, 0x4e, 0x80, 0x82, 0xf5, 0x7d, 0x7f, 0x87, 0x0c, 0xf7, 0x88, 0xdb, 0xf3, 0x75, 0x7f, 0xec,
0xa1, 0x9b, 0x90, 0xb5, 0x89, 0x6e, 0x12, 0x97, 0x1b, 0x95, 0xc3, 0xb2, 0x85, 0x76, 0xd9, 0x0e,
0xd6, 0x8d, 0x03, 0x7d, 0xcf, 0xb2, 0x2d, 0x7f, 0xc2, 0x4d, 0x59, 0x8c, 0x5f, 0xc2, 0xb3, 0x3a,
0x2b, 0x38, 0x22, 0x88, 0x2f, 0xa8, 0x41, 0xab, 0x30, 0x3f, 0x24, 0x9e, 0xa7, 0x0f, 0x08, 0xb7,
0x34, 0x8f, 0x83, 0xa6, 0xfa, 0x11, 0x14, 0xa3, 0x72, 0xa8, 0x00, 0xf3, 0xbb, 0xed, 0x67, 0xed,
0xce, 0x8b, 0xb6, 0x32, 0x87, 0x96, 0xa0, 0xb0, 0xdb, 0xc6, 0xcd, 0x6a, 0x7d, 0xab, 0x5a, 0xdb,
0x6e, 0x2a, 0x09, 0xb4, 0x00, 0xf9, 0x69, 0x33, 0xa9, 0xfe, 0x59, 0x02, 0x80, 0xb9, 0x5b, 0x0e,
0xea, 0x43, 0xc8, 0x78, 0xbe, 0xee, 0x8b, 0x55, 0xb9, 0xf8, 0xf8, 0xdd, 0xcb, 0xe6, 0x50, 0xda,
0xcb, 0xfe, 0x11, 0x2c, 0x44, 0xa2, 0x16, 0x26, 0x2f, 0x58, 0xc8, 0x02, 0x84, 0x6e, 0x9a, 0xae,
0x34, 0x9c, 0xff, 0x56, 0x3f, 0x82, 0x0c, 0x97, 0xbe, 0x68, 0x6e, 0x0e, 0xd2, 0x0d, 0xf6, 0x2b,
0x81, 0xf2, 0x90, 0xc1, 0xcd, 0x6a, 0xe3, 0x7b, 0x4a, 0x12, 0x29, 0x50, 0x6c, 0xb4, 0x7a, 0xf5,
0x4e, 0xbb, 0xdd, 0xac, 0xf7, 0x9b, 0x0d, 0x25, 0xa5, 0xde, 0x85, 0x4c, 0x6b, 0xc8, 0x34, 0xdf,
0x66, 0x4b, 0x7e, 0x9f, 0xb8, 0xc4, 0x31, 0x82, 0x9d, 0x34, 0x25, 0xa8, 0x3f, 0x2d, 0x40, 0x66,
0x87, 0x8e, 0x1d, 0x1f, 0x3d, 0x8e, 0x84, 0xad, 0xc5, 0xf8, 0x0c, 0x81, 0x03, 0x2b, 0xfd, 0xc9,
0x88, 0xc8, 0xb0, 0x76, 0x13, 0xb2, 0x62, 0x73, 0xc8, 0xe1, 0xc8, 0x16, 0xa3, 0xfb, 0xba, 0x3b,
0x20, 0xbe, 0x1c, 0x8f, 0x6c, 0xa1, 0xf7, 0xd9, 0x89, 0xa5, 0x9b, 0xd4, 0xb1, 0x27, 0x7c, 0x0f,
0xe5, 0xc4, 0xb1, 0x84, 0x89, 0x6e, 0x76, 0x1c, 0x7b, 0x82, 0x43, 0x2e, 0xda, 0x82, 0xe2, 0x9e,
0xe5, 0x98, 0x1a, 0x1d, 0x89, 0x20, 0x9f, 0xb9, 0x7c, 0xc7, 0x09, 0xab, 0x6a, 0x96, 0x63, 0x76,
0x04, 0x18, 0x17, 0xf6, 0xa6, 0x0d, 0xd4, 0x86, 0xc5, 0x23, 0x6a, 0x8f, 0x87, 0x24, 0xd4, 0x95,
0xe5, 0xba, 0xde, 0xbb, 0x5c, 0xd7, 0x73, 0x8e, 0x0f, 0xb4, 0x2d, 0x1c, 0x45, 0x9b, 0xe8, 0x19,
0x2c, 0xf8, 0xc3, 0xd1, 0xbe, 0x17, 0xaa, 0x9b, 0xe7, 0xea, 0xbe, 0x72, 0x85, 0xc3, 0x18, 0x3c,
0xd0, 0x56, 0xf4, 0x23, 0x2d, 0xb4, 0x09, 0x05, 0x83, 0x3a, 0x9e, 0xe5, 0xf9, 0xc4, 0x31, 0x26,
0xab, 0x39, 0xee, 0xfb, 0x2b, 0x46, 0x59, 0x9f, 0x82, 0x71, 0x54, 0xb2, 0xf4, 0x6b, 0x29, 0x28,
0x44, 0x5c, 0x80, 0x7a, 0x50, 0x18, 0xb9, 0x74, 0xa4, 0x0f, 0xf8, 0x89, 0x27, 0x27, 0xf5, 0xd1,
0x1b, 0xb9, 0xaf, 0xd2, 0x9d, 0x0a, 0xe2, 0xa8, 0x16, 0xf5, 0x34, 0x09, 0x85, 0x08, 0x13, 0xdd,
0x83, 0x1c, 0xee, 0xe2, 0xd6, 0xf3, 0x6a, 0xbf, 0xa9, 0xcc, 0x95, 0x6e, 0x9f, 0x9c, 0x96, 0x57,
0xb9, 0xb6, 0xa8, 0x82, 0xae, 0x6b, 0x1d, 0xb1, 0x35, 0xfc, 0x3e, 0xcc, 0x07, 0xd0, 0x44, 0xe9,
0xcb, 0x27, 0xa7, 0xe5, 0xb7, 0x66, 0xa1, 0x11, 0x24, 0xee, 0x6d, 0x55, 0x71, 0xb3, 0xa1, 0x24,
0xe3, 0x91, 0xb8, 0x77, 0xa0, 0xbb, 0xc4, 0x44, 0x5f, 0x81, 0xac, 0x04, 0xa6, 0x4a, 0xa5, 0x93,
0xd3, 0xf2, 0xcd, 0x59, 0xe0, 0x14, 0x87, 0x7b, 0xdb, 0xd5, 0xe7, 0x4d, 0x25, 0x1d, 0x8f, 0xc3,
0x3d, 0x5b, 0x3f, 0x22, 0xe8, 0x5d, 0xc8, 0x08, 0x58, 0xa6, 0x74, 0xeb, 0xe4, 0xb4, 0xfc, 0xa5,
0xd7, 0xd4, 0x31, 0x54, 0x69, 0xf5, 0x37, 0xff, 0x70, 0x6d, 0xee, 0xaf, 0xfe, 0x68, 0x4d, 0x99,
0x65, 0x97, 0xfe, 0x3b, 0x01, 0x0b, 0x17, 0xd6, 0x0e, 0x52, 0x21, 0xeb, 0x50, 0x83, 0x8e, 0xc4,
0x41, 0x98, 0xab, 0xc1, 0xf9, 0xd9, 0x7a, 0xb6, 0x4d, 0xeb, 0x74, 0x34, 0xc1, 0x92, 0x83, 0x9e,
0xcd, 0x1c, 0xe5, 0x4f, 0xde, 0x70, 0x61, 0xc6, 0x1e, 0xe6, 0x1f, 0xc3, 0x82, 0xe9, 0x5a, 0x47,
0xc4, 0xd5, 0x0c, 0xea, 0xec, 0x5b, 0x03, 0x79, 0xc8, 0x95, 0x62, 0xf3, 0x4d, 0x0e, 0xc4, 0x45,
0x21, 0x50, 0xe7, 0xf8, 0x2f, 0x70, 0x8c, 0x97, 0x9e, 0x43, 0x31, 0xba, 0xd4, 0xd9, 0xb9, 0xe4,
0x59, 0xbf, 0x44, 0x64, 0x62, 0xc9, 0xd3, 0x50, 0x9c, 0x67, 0x14, 0x91, 0x56, 0xbe, 0x07, 0xe9,
0x21, 0x35, 0x85, 0x9e, 0x85, 0xda, 0x0d, 0x96, 0x4d, 0xfc, 0xd3, 0xd9, 0x7a, 0x81, 0x7a, 0x95,
0x0d, 0xcb, 0x26, 0x3b, 0xd4, 0x24, 0x98, 0x03, 0xd4, 0x23, 0x48, 0xb3, 0x98, 0x83, 0xbe, 0x0c,
0xe9, 0x5a, 0xab, 0xdd, 0x50, 0xe6, 0x4a, 0xcb, 0x27, 0xa7, 0xe5, 0x05, 0xee, 0x12, 0xc6, 0x60,
0x6b, 0x17, 0xad, 0x43, 0xf6, 0x79, 0x67, 0x7b, 0x77, 0x87, 0x2d, 0xaf, 0x1b, 0x27, 0xa7, 0xe5,
0xa5, 0x90, 0x2d, 0x9c, 0x86, 0xee, 0x40, 0xa6, 0xbf, 0xd3, 0xdd, 0xe8, 0x29, 0xc9, 0x12, 0x3a,
0x39, 0x2d, 0x2f, 0x86, 0x7c, 0x6e, 0x73, 0x69, 0x59, 0xce, 0x6a, 0x3e, 0xa4, 0xab, 0x3f, 0x4a,
0x40, 0x21, 0xb2, 0xe1, 0xd8, 0xc2, 0x6c, 0x34, 0x37, 0xaa, 0xbb, 0xdb, 0x7d, 0x65, 0x2e, 0xb2,
0x30, 0x23, 0x90, 0x06, 0xd9, 0xd7, 0xc7, 0x36, 0x8b, 0x73, 0x50, 0xef, 0xb4, 0x7b, 0xad, 0x5e,
0xbf, 0xd9, 0xee, 0x2b, 0x89, 0xd2, 0xea, 0xc9, 0x69, 0x79, 0x65, 0x16, 0xbc, 0x31, 0xb6, 0x6d,
0xb6, 0x34, 0xeb, 0xd5, 0xfa, 0x16, 0x5f, 0xeb, 0xd3, 0xa5, 0x19, 0x41, 0xd5, 0x75, 0xe3, 0x80,
0x98, 0xe8, 0x3e, 0xe4, 0x1b, 0xcd, 0xed, 0xe6, 0x66, 0x95, 0x47, 0xf7, 0xd2, 0x9d, 0x93, 0xd3,
0xf2, 0xad, 0xd7, 0x7b, 0xb7, 0xc9, 0x40, 0xf7, 0x89, 0x39, 0xb3, 0x44, 0x23, 0x10, 0xf5, 0x3f,
0x93, 0xb0, 0x80, 0x59, 0x39, 0xec, 0xfa, 0x5d, 0x6a, 0x5b, 0xc6, 0x04, 0x75, 0x21, 0x6f, 0x50,
0xc7, 0xb4, 0x22, 0x71, 0xe2, 0xf1, 0x25, 0x29, 0xd1, 0x54, 0x2a, 0x68, 0xd5, 0x03, 0x49, 0x3c,
0x55, 0x82, 0x1e, 0x40, 0xc6, 0x24, 0xb6, 0x3e, 0x91, 0xb9, 0xd9, 0xad, 0x8a, 0x28, 0xb8, 0x2b,
0x41, 0xc1, 0x5d, 0x69, 0xc8, 0x82, 0x1b, 0x0b, 0x1c, 0xaf, 0x41, 0xf4, 0x97, 0x9a, 0xee, 0xfb,
0x64, 0x38, 0xf2, 0x45, 0x62, 0x96, 0xc6, 0x85, 0xa1, 0xfe, 0xb2, 0x2a, 0x49, 0xe8, 0x11, 0x64,
0x8f, 0x2d, 0xc7, 0xa4, 0xc7, 0x32, 0xf7, 0xba, 0x42, 0xa9, 0x04, 0xaa, 0x27, 0x2c, 0x25, 0x99,
0x31, 0x93, 0xad, 0xa1, 0x76, 0xa7, 0xdd, 0x0c, 0xd6, 0x90, 0xe4, 0x77, 0x9c, 0x36, 0x75, 0xd8,
0xfe, 0x87, 0x4e, 0x5b, 0xdb, 0xa8, 0xb6, 0xb6, 0x77, 0x31, 0x5b, 0x47, 0x2b, 0x27, 0xa7, 0x65,
0x25, 0x84, 0x6c, 0xe8, 0x96, 0xcd, 0x8a, 0x81, 0x5b, 0x90, 0xaa, 0xb6, 0xbf, 0xa7, 0x24, 0x4b,
0xca, 0xc9, 0x69, 0xb9, 0x18, 0xb2, 0xab, 0xce, 0x64, 0xea, 0xf7, 0xd9, 0x7e, 0xd5, 0xbf, 0x4d,
0x41, 0x71, 0x77, 0x64, 0xea, 0x3e, 0x11, 0xfb, 0x0c, 0x95, 0xa1, 0x30, 0xd2, 0x5d, 0xdd, 0xb6,
0x89, 0x6d, 0x79, 0x43, 0x79, 0x95, 0x10, 0x25, 0xa1, 0x0f, 0xde, 0xd4, 0x8d, 0xb5, 0x1c, 0xdb,
0x3b, 0x3f, 0xf8, 0x97, 0xf5, 0x44, 0xe0, 0xd0, 0x5d, 0x58, 0xdc, 0x17, 0xd6, 0x6a, 0xba, 0xc1,
0x27, 0x36, 0xc5, 0x27, 0xb6, 0x12, 0x37, 0xb1, 0x51, 0xb3, 0x2a, 0x72, 0x90, 0x55, 0x2e, 0x85,
0x17, 0xf6, 0xa3, 0x4d, 0xf4, 0x04, 0xe6, 0x87, 0xd4, 0xb1, 0x7c, 0xea, 0x5e, 0x3f, 0x0b, 0x01,
0x12, 0xdd, 0x83, 0x65, 0x36, 0xb9, 0x81, 0x3d, 0x9c, 0xcd, 0x8f, 0xf3, 0x24, 0x5e, 0x1a, 0xea,
0x2f, 0x65, 0x87, 0x98, 0x91, 0x51, 0x0d, 0x32, 0xd4, 0x65, 0xf9, 0x62, 0x96, 0x9b, 0x7b, 0xff,
0x5a, 0x73, 0x45, 0xa3, 0xc3, 0x64, 0xb0, 0x10, 0x55, 0xbf, 0x09, 0x0b, 0x17, 0x06, 0xc1, 0xd2,
0xa4, 0x6e, 0x75, 0xb7, 0xd7, 0x54, 0xe6, 0x50, 0x11, 0x72, 0xf5, 0x4e, 0xbb, 0xdf, 0x6a, 0xef,
0xb2, 0x3c, 0xaf, 0x08, 0x39, 0xdc, 0xd9, 0xde, 0xae, 0x55, 0xeb, 0xcf, 0x94, 0xa4, 0x5a, 0x81,
0x42, 0x44, 0x1b, 0x5a, 0x04, 0xe8, 0xf5, 0x3b, 0x5d, 0x6d, 0xa3, 0x85, 0x7b, 0x7d, 0x91, 0x25,
0xf6, 0xfa, 0x55, 0xdc, 0x97, 0x84, 0x84, 0xfa, 0xef, 0xc9, 0x60, 0x46, 0x65, 0x62, 0x58, 0xbb,
0x98, 0x18, 0x5e, 0x61, 0xbc, 0x4c, 0x0d, 0xa7, 0x8d, 0x30, 0x41, 0xfc, 0x00, 0x80, 0x2f, 0x1c,
0x62, 0x6a, 0xba, 0x2f, 0x27, 0xbe, 0xf4, 0x9a, 0x93, 0xfb, 0xc1, 0x8d, 0x16, 0xce, 0x4b, 0x74,
0xd5, 0x47, 0xdf, 0x82, 0xa2, 0x41, 0x87, 0x23, 0x9b, 0x48, 0xe1, 0xd4, 0xb5, 0xc2, 0x85, 0x10,
0x5f, 0xf5, 0xa3, 0xa9, 0x69, 0xfa, 0x62, 0xf2, 0xfc, 0xeb, 0x89, 0xc0, 0x33, 0x31, 0xd9, 0x68,
0x11, 0x72, 0xbb, 0xdd, 0x46, 0xb5, 0xdf, 0x6a, 0x6f, 0x2a, 0x09, 0x04, 0x90, 0xe5, 0xae, 0x6e,
0x28, 0x49, 0x96, 0x45, 0xd7, 0x3b, 0x3b, 0xdd, 0xed, 0x26, 0x8f, 0x58, 0x68, 0x05, 0x94, 0xc0,
0xd9, 0x1a, 0x77, 0x64, 0xb3, 0xa1, 0xa4, 0xd1, 0x0d, 0x58, 0x0a, 0xa9, 0x52, 0x32, 0x83, 0x6e,
0x02, 0x0a, 0x89, 0x53, 0x15, 0x59, 0xf5, 0x57, 0x60, 0xa9, 0x4e, 0x1d, 0x5f, 0xb7, 0x9c, 0xb0,
0xc2, 0x78, 0xcc, 0x06, 0x2d, 0x49, 0x9a, 0x25, 0x6f, 0x82, 0x6a, 0x4b, 0xe7, 0x67, 0xeb, 0x85,
0x10, 0xda, 0x6a, 0xf0, 0x54, 0x49, 0x36, 0x4c, 0xb6, 0x7f, 0x47, 0x96, 0xc9, 0x9d, 0x9b, 0xa9,
0xcd, 0x9f, 0x9f, 0xad, 0xa7, 0xba, 0xad, 0x06, 0x66, 0x34, 0xf4, 0x65, 0xc8, 0x93, 0x97, 0x96,
0xaf, 0x19, 0xec, 0x5c, 0x62, 0x0e, 0xcc, 0xe0, 0x1c, 0x23, 0xd4, 0xd9, 0x31, 0x54, 0x03, 0xe8,
0x52, 0xd7, 0x97, 0x3d, 0x7f, 0x1d, 0x32, 0x23, 0xea, 0xf2, 0xbb, 0x8b, 0x4b, 0x6f, 0xd4, 0x18,
0x5c, 0x2c, 0x54, 0x2c, 0xc0, 0xea, 0xef, 0xa6, 0x00, 0xfa, 0xba, 0x77, 0x28, 0x95, 0x3c, 0x85,
0x7c, 0x78, 0x3b, 0x29, 0x2f, 0x41, 0xae, 0x9c, 0xed, 0x10, 0x8c, 0x9e, 0x04, 0x8b, 0x4d, 0xd4,
0x4e, 0xb1, 0x45, 0x6c, 0xd0, 0x51, 0x5c, 0xf9, 0x71, 0xb1, 0x40, 0x62, 0xc7, 0x3c, 0x71, 0x5d,
0x39, 0xf3, 0xec, 0x27, 0xaa, 0xf3, 0x63, 0x41, 0x38, 0x4d, 0x66, 0xdf, 0xb1, 0xd7, 0x3e, 0x33,
0x33, 0xb2, 0x35, 0x87, 0xa7, 0x72, 0xe8, 0x63, 0x28, 0xb0, 0x71, 0x6b, 0x1e, 0xe7, 0xc9, 0xc4,
0xfb, 0x52, 0x57, 0x09, 0x0d, 0x18, 0x46, 0x53, 0x2f, 0xdf, 0x01, 0xd0, 0x47, 0x23, 0xdb, 0x22,
0xa6, 0xb6, 0x37, 0xe1, 0x99, 0x76, 0x1e, 0xe7, 0x25, 0xa5, 0x36, 0x61, 0xdb, 0x25, 0x60, 0xeb,
0x3e, 0xcf, 0x9e, 0xaf, 0x71, 0xa0, 0x44, 0x57, 0xfd, 0x9a, 0x02, 0x8b, 0xee, 0xd8, 0x61, 0x0e,
0x95, 0xd6, 0xa9, 0x7f, 0x9a, 0x84, 0xb7, 0xda, 0xc4, 0x3f, 0xa6, 0xee, 0x61, 0xd5, 0xf7, 0x75,
0xe3, 0x60, 0x48, 0x1c, 0x39, 0x7d, 0x91, 0x82, 0x26, 0x71, 0xa1, 0xa0, 0x59, 0x85, 0x79, 0xdd,
0xb6, 0x74, 0x8f, 0x88, 0xe4, 0x2d, 0x8f, 0x83, 0x26, 0x2b, 0xbb, 0x58, 0x11, 0x47, 0x3c, 0x8f,
0x88, 0x7b, 0x15, 0x66, 0x78, 0x40, 0x40, 0xdf, 0x87, 0x9b, 0x32, 0x4d, 0xd3, 0xc3, 0xae, 0x58,
0x41, 0x11, 0x5c, 0xd0, 0x36, 0x63, 0xab, 0xca, 0x78, 0xe3, 0x64, 0x1e, 0x37, 0x25, 0x77, 0x46,
0xbe, 0xcc, 0x0a, 0x57, 0xcc, 0x18, 0x56, 0x69, 0x13, 0x6e, 0x5d, 0x2a, 0xf2, 0xb9, 0xee, 0x6d,
0xfe, 0x21, 0x09, 0xd0, 0xea, 0x56, 0x77, 0xa4, 0x93, 0x1a, 0x90, 0xdd, 0xd7, 0x87, 0x96, 0x3d,
0xb9, 0x2a, 0x02, 0x4e, 0xf1, 0x95, 0xaa, 0x70, 0xc7, 0x06, 0x97, 0xc1, 0x52, 0x96, 0xd7, 0x94,
0xe3, 0x3d, 0x87, 0xf8, 0x61, 0x4d, 0xc9, 0x5b, 0xcc, 0x0c, 0x57, 0x77, 0xc2, 0xa5, 0x2b, 0x1a,
0x6c, 0x02, 0x58, 0xca, 0x73, 0xac, 0x4f, 0x82, 0xb0, 0x25, 0x9b, 0x68, 0x8b, 0xdf, 0x8e, 0x12,
0xf7, 0x88, 0x98, 0xab, 0x19, 0xee, 0xd4, 0xeb, 0xec, 0xc1, 0x12, 0x2e, 0x7c, 0x17, 0x4a, 0x97,
0x3e, 0xe2, 0x29, 0xd3, 0x94, 0xf5, 0xb9, 0x7c, 0xf4, 0x10, 0x16, 0x2e, 0x8c, 0xf3, 0xb5, 0x62,
0xbe, 0xd5, 0x7d, 0xfe, 0x75, 0x25, 0x2d, 0x7f, 0x7d, 0x53, 0xc9, 0xaa, 0x7f, 0x9c, 0x12, 0x81,
0x46, 0x7a, 0x35, 0xfe, 0x55, 0x20, 0xc7, 0x57, 0xb7, 0x41, 0x6d, 0x19, 0x00, 0xde, 0xbb, 0x3a,
0xfe, 0xb0, 0x9a, 0x8e, 0xc3, 0x71, 0x28, 0x88, 0xd6, 0xa1, 0x20, 0x56, 0xb1, 0xc6, 0x36, 0x1c,
0x77, 0xeb, 0x02, 0x06, 0x41, 0x62, 0x92, 0xe8, 0x2e, 0x2c, 0xf2, 0xcb, 0x1f, 0xef, 0x80, 0x98,
0x02, 0x93, 0xe6, 0x98, 0x85, 0x90, 0xca, 0x61, 0x3b, 0x50, 0x94, 0x04, 0x8d, 0xe7, 0xf3, 0x19,
0x6e, 0xd0, 0xbd, 0xeb, 0x0c, 0x12, 0x22, 0x3c, 0xcd, 0x2f, 0x8c, 0xa6, 0x0d, 0xb5, 0x01, 0xb9,
0xc0, 0x58, 0xb4, 0x0a, 0xa9, 0x7e, 0xbd, 0xab, 0xcc, 0x95, 0x96, 0x4e, 0x4e, 0xcb, 0x85, 0x80,
0xdc, 0xaf, 0x77, 0x19, 0x67, 0xb7, 0xd1, 0x55, 0x12, 0x17, 0x39, 0xbb, 0x8d, 0x6e, 0x29, 0xcd,
0x72, 0x30, 0x75, 0x1f, 0x0a, 0x91, 0x1e, 0xd0, 0x3b, 0x30, 0xdf, 0x6a, 0x6f, 0xe2, 0x66, 0xaf,
0xa7, 0xcc, 0x95, 0x6e, 0x9e, 0x9c, 0x96, 0x51, 0x84, 0xdb, 0x72, 0x06, 0x6c, 0x7e, 0xd0, 0x1d,
0x48, 0x6f, 0x75, 0xd8, 0xd9, 0x2e, 0x0a, 0x88, 0x08, 0x62, 0x8b, 0x7a, 0x7e, 0xe9, 0x86, 0x4c,
0xee, 0xa2, 0x8a, 0xd5, 0xdf, 0x4b, 0x40, 0x56, 0x6c, 0xa6, 0xd8, 0x89, 0xaa, 0xc2, 0x7c, 0x70,
0x4d, 0x20, 0x8a, 0xbb, 0xf7, 0x2e, 0x2f, 0xc4, 0x2a, 0xb2, 0x6e, 0x12, 0xcb, 0x2f, 0x90, 0x2b,
0x7d, 0x08, 0xc5, 0x28, 0xe3, 0x73, 0x2d, 0xbe, 0xef, 0x43, 0x81, 0xad, 0xef, 0xa0, 0x20, 0x7b,
0x0c, 0x59, 0x11, 0x10, 0xc2, 0xb3, 0xe6, 0xf2, 0xaa, 0x50, 0x22, 0xd1, 0x53, 0x98, 0x17, 0x95,
0x64, 0x70, 0x3b, 0xbc, 0x76, 0xf5, 0x2e, 0xc2, 0x01, 0x5c, 0xfd, 0x18, 0xd2, 0x5d, 0x42, 0x5c,
0xe6, 0x7b, 0x87, 0x9a, 0x64, 0x7a, 0x3c, 0xcb, 0x22, 0xd8, 0x24, 0xad, 0x06, 0x2b, 0x82, 0x4d,
0xd2, 0x32, 0xc3, 0xfb, 0xaf, 0x64, 0xe4, 0xfe, 0xab, 0x0f, 0xc5, 0x17, 0xc4, 0x1a, 0x1c, 0xf8,
0xc4, 0xe4, 0x8a, 0xee, 0x43, 0x7a, 0x44, 0x42, 0xe3, 0x57, 0x63, 0x17, 0x18, 0x21, 0x2e, 0xe6,
0x28, 0x16, 0x47, 0x8e, 0xb9, 0xb4, 0x7c, 0xd2, 0x90, 0x2d, 0xf5, 0xef, 0x93, 0xb0, 0xd8, 0xf2,
0xbc, 0xb1, 0xee, 0x18, 0x41, 0xe6, 0xf6, 0xed, 0x8b, 0x99, 0x5b, 0xec, 0xdb, 0xcf, 0x45, 0x91,
0x8b, 0xd7, 0x7a, 0xf2, 0xf4, 0x4c, 0x86, 0xa7, 0xa7, 0xfa, 0xd3, 0x44, 0x70, 0x77, 0x77, 0x37,
0xb2, 0xdd, 0x45, 0x1d, 0x18, 0xd5, 0x44, 0x76, 0x9d, 0x43, 0x87, 0x1e, 0x3b, 0xe8, 0x6d, 0xc8,
0xe0, 0x66, 0xbb, 0xf9, 0x42, 0x49, 0x88, 0xe5, 0x79, 0x01, 0x84, 0x89, 0x43, 0x8e, 0x99, 0xa6,
0x6e, 0xb3, 0xdd, 0x60, 0x99, 0x56, 0x32, 0x46, 0x53, 0x97, 0x38, 0xa6, 0xe5, 0x0c, 0xd0, 0x3b,
0x90, 0x6d, 0xf5, 0x7a, 0xbb, 0xbc, 0x4c, 0x7c, 0xeb, 0xe4, 0xb4, 0x7c, 0xe3, 0x02, 0x8a, 0xdf,
0xdb, 0x9a, 0x0c, 0xc4, 0xca, 0x1c, 0x96, 0x83, 0xc5, 0x80, 0x58, 0xfe, 0x2c, 0x40, 0xb8, 0xd3,
0xaf, 0xf6, 0x9b, 0x4a, 0x26, 0x06, 0x84, 0x29, 0xfb, 0x2b, 0xb7, 0xdb, 0x3f, 0x27, 0x41, 0xa9,
0x1a, 0x06, 0x19, 0xf9, 0x8c, 0x2f, 0x2b, 0xcb, 0x3e, 0xe4, 0x46, 0xec, 0x97, 0x45, 0x82, 0x2c,
0xe9, 0x69, 0xec, 0xeb, 0xe5, 0x8c, 0x5c, 0x05, 0x53, 0x9b, 0x54, 0xcd, 0xa1, 0xe5, 0x79, 0x16,
0x75, 0x04, 0x0d, 0x87, 0x9a, 0x4a, 0xff, 0x91, 0x80, 0x1b, 0x31, 0x08, 0xf4, 0x10, 0xd2, 0x2e,
0xb5, 0x83, 0x39, 0xbc, 0x7d, 0xd9, 0xb5, 0x2c, 0x13, 0xc5, 0x1c, 0x89, 0xd6, 0x00, 0xf4, 0xb1,
0x4f, 0x75, 0xde, 0x3f, 0x9f, 0xbd, 0x1c, 0x8e, 0x50, 0xd0, 0x0b, 0xc8, 0x7a, 0xc4, 0x70, 0x49,
0x90, 0x4b, 0x7f, 0xfc, 0x7f, 0xb5, 0xbe, 0xd2, 0xe3, 0x6a, 0xb0, 0x54, 0x57, 0xaa, 0x40, 0x56,
0x50, 0xd8, 0xb2, 0x37, 0x75, 0x5f, 0x97, 0x97, 0xf6, 0xfc, 0x37, 0x5b, 0x4d, 0xba, 0x3d, 0x08,
0x56, 0x93, 0x6e, 0x0f, 0xd4, 0xbf, 0x49, 0x02, 0x34, 0x5f, 0xfa, 0xc4, 0x75, 0x74, 0xbb, 0x5e,
0x45, 0xcd, 0x48, 0xf4, 0x17, 0xa3, 0xfd, 0x6a, 0xec, 0x4b, 0x44, 0x28, 0x51, 0xa9, 0x57, 0x63,
0xe2, 0xff, 0x2d, 0x48, 0x8d, 0x5d, 0xf9, 0x20, 0x2d, 0xf2, 0xe0, 0x5d, 0xbc, 0x8d, 0x19, 0x0d,
0x35, 0xa7, 0x61, 0x2b, 0x75, 0xf9, 0xb3, 0x73, 0xa4, 0x83, 0xd8, 0xd0, 0xc5, 0x76, 0xbe, 0xa1,
0x6b, 0x06, 0x91, 0x27, 0x47, 0x51, 0xec, 0xfc, 0x7a, 0xb5, 0x4e, 0x5c, 0x1f, 0x67, 0x0d, 0x9d,
0xfd, 0xff, 0x42, 0xf1, 0xed, 0x3e, 0xc0, 0x74, 0x68, 0x68, 0x0d, 0x32, 0xf5, 0x8d, 0x5e, 0x6f,
0x5b, 0x99, 0x13, 0x01, 0x7c, 0xca, 0xe2, 0x64, 0xf5, 0x2f, 0x93, 0x90, 0xab, 0x57, 0xe5, 0xb1,
0x5a, 0x07, 0x85, 0x47, 0x25, 0xfe, 0xd4, 0x41, 0x5e, 0x8e, 0x2c, 0x77, 0x22, 0x03, 0xcb, 0x15,
0x45, 0xed, 0x22, 0x13, 0x61, 0x56, 0x37, 0xb9, 0x00, 0xc2, 0x50, 0x24, 0xd2, 0x09, 0x9a, 0xa1,
0x07, 0x31, 0x7e, 0xed, 0x6a, 0x67, 0x89, 0xf2, 0x64, 0xda, 0xf6, 0x70, 0x21, 0x50, 0x52, 0xd7,
0x3d, 0xf4, 0x01, 0x2c, 0x79, 0xd6, 0xc0, 0xb1, 0x9c, 0x81, 0x16, 0x38, 0x8f, 0xbf, 0xbb, 0xd4,
0x96, 0xcf, 0xcf, 0xd6, 0x17, 0x7a, 0x82, 0x25, 0x7d, 0xb8, 0x20, 0x91, 0x75, 0xee, 0x4a, 0xf4,
0x4d, 0x58, 0x8c, 0x88, 0x32, 0x2f, 0x0a, 0xb7, 0x2b, 0xe7, 0x67, 0xeb, 0xc5, 0x50, 0xf2, 0x19,
0x99, 0xe0, 0x62, 0x28, 0xf8, 0x8c, 0xf0, 0xfb, 0x97, 0x7d, 0xea, 0x1a, 0x44, 0x73, 0xf9, 0x9e,
0xe6, 0x27, 0x78, 0x1a, 0x17, 0x38, 0x4d, 0x6c, 0x73, 0xf5, 0x39, 0xdc, 0xe8, 0xb8, 0xc6, 0x01,
0xf1, 0x7c, 0xe1, 0x0a, 0xe9, 0xc5, 0x8f, 0xe1, 0xb6, 0xaf, 0x7b, 0x87, 0xda, 0x81, 0xe5, 0xf9,
0xd4, 0x9d, 0x68, 0x2e, 0xf1, 0x89, 0xc3, 0xf8, 0x1a, 0x7f, 0xac, 0x95, 0x97, 0x7e, 0xb7, 0x18,
0x66, 0x4b, 0x40, 0x70, 0x80, 0xd8, 0x66, 0x00, 0xb5, 0x05, 0x45, 0x56, 0xa6, 0xc8, 0x8b, 0x33,
0x36, 0x7a, 0xb0, 0xe9, 0x40, 0x7b, 0xe3, 0x63, 0x2a, 0x6f, 0xd3, 0x81, 0xf8, 0xa9, 0x7e, 0x17,
0x94, 0x86, 0xe5, 0x8d, 0x74, 0xdf, 0x38, 0x08, 0x6e, 0x33, 0x51, 0x03, 0x94, 0x03, 0xa2, 0xbb,
0xfe, 0x1e, 0xd1, 0x7d, 0x6d, 0x44, 0x5c, 0x8b, 0x9a, 0xd7, 0xcf, 0xf2, 0x52, 0x28, 0xd2, 0xe5,
0x12, 0xea, 0x7f, 0x25, 0x00, 0xb0, 0xbe, 0x1f, 0x64, 0x64, 0x5f, 0x83, 0x65, 0xcf, 0xd1, 0x47,
0xde, 0x01, 0xf5, 0x35, 0xcb, 0xf1, 0x89, 0x7b, 0xa4, 0xdb, 0xf2, 0x02, 0x47, 0x09, 0x18, 0x2d,
0x49, 0x47, 0xf7, 0x01, 0x1d, 0x12, 0x32, 0xd2, 0xa8, 0x6d, 0x6a, 0x01, 0x53, 0x3c, 0x25, 0xa7,
0xb1, 0xc2, 0x38, 0x1d, 0xdb, 0xec, 0x05, 0x74, 0x54, 0x83, 0x35, 0x36, 0x7c, 0xe2, 0xf8, 0xae,
0x45, 0x3c, 0x6d, 0x9f, 0xba, 0x9a, 0x67, 0xd3, 0x63, 0x6d, 0x9f, 0xda, 0x36, 0x3d, 0x26, 0x6e,
0x70, 0x37, 0x56, 0xb2, 0xe9, 0xa0, 0x29, 0x40, 0x1b, 0xd4, 0xed, 0xd9, 0xf4, 0x78, 0x23, 0x40,
0xb0, 0xb4, 0x6d, 0x3a, 0x66, 0xdf, 0x32, 0x0e, 0x83, 0xb4, 0x2d, 0xa4, 0xf6, 0x2d, 0xe3, 0x10,
0xbd, 0x03, 0x0b, 0xc4, 0x26, 0xfc, 0x8a, 0x44, 0xa0, 0x32, 0x1c, 0x55, 0x0c, 0x88, 0x0c, 0xa4,
0x7e, 0x02, 0x4a, 0xd3, 0x31, 0xdc, 0xc9, 0x28, 0x32, 0xe7, 0xf7, 0x01, 0xb1, 0x20, 0xa9, 0xd9,
0xd4, 0x38, 0xd4, 0x86, 0xba, 0xa3, 0x0f, 0x98, 0x5d, 0xe2, 0x85, 0x4f, 0x61, 0x9c, 0x6d, 0x6a,
0x1c, 0xee, 0x48, 0xba, 0xfa, 0x01, 0x40, 0x6f, 0xe4, 0x12, 0xdd, 0xec, 0xb0, 0x6c, 0x82, 0xb9,
0x8e, 0xb7, 0x34, 0x53, 0xbe, 0x90, 0x52, 0x57, 0x6e, 0x75, 0x45, 0x30, 0x1a, 0x21, 0x5d, 0xfd,
0x79, 0xb8, 0xd1, 0xb5, 0x75, 0x83, 0x7f, 0x2d, 0xd0, 0x0d, 0x9f, 0xac, 0xd0, 0x53, 0xc8, 0x0a,
0xa8, 0x9c, 0xc9, 0xd8, 0xed, 0x36, 0xed, 0x73, 0x6b, 0x0e, 0x4b, 0x7c, 0xad, 0x08, 0x30, 0xd5,
0xa3, 0xfe, 0x79, 0x02, 0xf2, 0xa1, 0x7e, 0x54, 0x16, 0x2f, 0x31, 0xbe, 0xab, 0x5b, 0x8e, 0xac,
0xea, 0xf3, 0x38, 0x4a, 0x42, 0x2d, 0x28, 0x8c, 0x42, 0xe9, 0x2b, 0xf3, 0xb9, 0x18, 0xab, 0x71,
0x54, 0x16, 0x7d, 0x08, 0xf9, 0xe0, 0x49, 0x3a, 0x88, 0xb0, 0x57, 0xbf, 0x60, 0x4f, 0xe1, 0xea,
0xb7, 0x01, 0xbe, 0x43, 0x2d, 0xa7, 0x4f, 0x0f, 0x89, 0xc3, 0x9f, 0x58, 0x59, 0x4d, 0x48, 0x02,
0x2f, 0xca, 0x16, 0x2f, 0xf5, 0xc5, 0x14, 0x84, 0x2f, 0x8d, 0xa2, 0xa9, 0xfe, 0x75, 0x12, 0xb2,
0x98, 0x52, 0xbf, 0x5e, 0x45, 0x65, 0xc8, 0xca, 0x38, 0xc1, 0xcf, 0x9f, 0x5a, 0xfe, 0xfc, 0x6c,
0x3d, 0x23, 0x02, 0x44, 0xc6, 0xe0, 0x91, 0x21, 0x12, 0xc1, 0x93, 0x97, 0x45, 0x70, 0xf4, 0x10,
0x8a, 0x12, 0xa4, 0x1d, 0xe8, 0xde, 0x81, 0x28, 0xd0, 0x6a, 0x8b, 0xe7, 0x67, 0xeb, 0x20, 0x90,
0x5b, 0xba, 0x77, 0x80, 0x41, 0xa0, 0xd9, 0x6f, 0xd4, 0x84, 0xc2, 0xa7, 0xd4, 0x72, 0x34, 0x9f,
0x0f, 0x42, 0x5e, 0x26, 0xc6, 0xce, 0xe3, 0x74, 0xa8, 0xf2, 0x7b, 0x03, 0xf8, 0x74, 0x3a, 0xf8,
0x26, 0x2c, 0xb8, 0x94, 0xfa, 0x22, 0x6c, 0x59, 0xd4, 0x91, 0xf7, 0x14, 0xe5, 0xd8, 0xeb, 0x6b,
0x4a, 0x7d, 0x2c, 0x71, 0xb8, 0xe8, 0x46, 0x5a, 0xe8, 0x21, 0xac, 0xd8, 0xba, 0xe7, 0x6b, 0x3c,
0xde, 0x99, 0x53, 0x6d, 0x59, 0xbe, 0xd5, 0x10, 0xe3, 0x6d, 0x70, 0x56, 0x20, 0xa1, 0xfe, 0x63,
0x02, 0x0a, 0x6c, 0x30, 0xd6, 0xbe, 0x65, 0xb0, 0x24, 0xef, 0xf3, 0xe7, 0x1e, 0xb7, 0x20, 0x65,
0x78, 0xae, 0x74, 0x2a, 0x3f, 0x7c, 0xeb, 0x3d, 0x8c, 0x19, 0x0d, 0x7d, 0x02, 0x59, 0x79, 0x5f,
0x22, 0xd2, 0x0e, 0xf5, 0xfa, 0x74, 0x54, 0xfa, 0x46, 0xca, 0xf1, 0xb5, 0x3c, 0xb5, 0x4e, 0x1c,
0x02, 0x38, 0x4a, 0x42, 0x37, 0x21, 0x69, 0x08, 0x77, 0xc9, 0x0f, 0x5a, 0xea, 0x6d, 0x9c, 0x34,
0x1c, 0xf5, 0x47, 0x09, 0x58, 0x98, 0x6e, 0x78, 0xb6, 0x02, 0x6e, 0x43, 0xde, 0x1b, 0xef, 0x79,
0x13, 0xcf, 0x27, 0xc3, 0xe0, 0xf9, 0x38, 0x24, 0xa0, 0x16, 0xe4, 0x75, 0x7b, 0x40, 0x5d, 0xcb,
0x3f, 0x18, 0xca, 0x4a, 0x34, 0x3e, 0x55, 0x88, 0xea, 0xac, 0x54, 0x03, 0x11, 0x3c, 0x95, 0x0e,
0xce, 0x7d, 0xf1, 0x8d, 0x01, 0x3f, 0xf7, 0xdf, 0x86, 0xa2, 0xad, 0x0f, 0xf9, 0x05, 0x92, 0x6f,
0x0d, 0xc5, 0x38, 0xd2, 0xb8, 0x20, 0x69, 0x7d, 0x6b, 0x48, 0x54, 0x15, 0xf2, 0xa1, 0x32, 0xb4,
0x04, 0x85, 0x6a, 0xb3, 0xa7, 0x3d, 0x7a, 0xfc, 0x54, 0xdb, 0xac, 0xef, 0x28, 0x73, 0x32, 0x37,
0xfd, 0x8b, 0x04, 0x2c, 0xc8, 0x70, 0x24, 0xf3, 0xfd, 0x77, 0x60, 0xde, 0xd5, 0xf7, 0xfd, 0xa0,
0x22, 0x49, 0x8b, 0x55, 0xcd, 0x22, 0x3c, 0xab, 0x48, 0x18, 0x2b, 0xbe, 0x22, 0x89, 0x7c, 0xd0,
0x90, 0xba, 0xf2, 0x83, 0x86, 0xf4, 0xcf, 0xe4, 0x83, 0x06, 0xf5, 0x57, 0x01, 0x36, 0x2c, 0x9b,
0xf4, 0xc5, 0x5d, 0x53, 0x5c, 0x7d, 0xc9, 0x72, 0x38, 0x79, 0x97, 0x19, 0xe4, 0x70, 0xad, 0x06,
0x66, 0x34, 0xc6, 0x1a, 0x58, 0xa6, 0xdc, 0x8c, 0x9c, 0xb5, 0xc9, 0x58, 0x03, 0xcb, 0x0c, 0x5f,
0xde, 0xd2, 0xd7, 0xbd, 0xbc, 0x9d, 0x26, 0x60, 0x49, 0xe6, 0xae, 0x61, 0xf8, 0xfd, 0x2a, 0xe4,
0x45, 0x1a, 0x3b, 0x2d, 0xe8, 0xf8, 0x23, 0xbe, 0xc0, 0xb5, 0x1a, 0x38, 0x27, 0xd8, 0x2d, 0x13,
0xad, 0x43, 0x41, 0x42, 0x23, 0x1f, 0x3f, 0x81, 0x20, 0xb5, 0x99, 0xf9, 0x5f, 0x87, 0xf4, 0xbe,
0x65, 0x13, 0xb9, 0xd0, 0x63, 0x03, 0xc0, 0xd4, 0x01, 0x5b, 0x73, 0x98, 0xa3, 0x6b, 0xb9, 0xe0,
0x32, 0x8e, 0xdb, 0x27, 0xcb, 0xce, 0xa8, 0x7d, 0xa2, 0x02, 0x9d, 0xb1, 0x4f, 0xe0, 0x98, 0x7d,
0x82, 0x2d, 0xec, 0x93, 0xd0, 0xa8, 0x7d, 0x82, 0xf4, 0x33, 0xb1, 0x6f, 0x1b, 0x6e, 0xd6, 0x6c,
0xdd, 0x38, 0xb4, 0x2d, 0xcf, 0x27, 0x66, 0x34, 0x62, 0x3c, 0x86, 0xec, 0x85, 0xa4, 0xf3, 0xaa,
0x5b, 0x4b, 0x89, 0x54, 0xff, 0x2d, 0x01, 0xc5, 0x2d, 0xa2, 0xdb, 0xfe, 0xc1, 0xf4, 0x6a, 0xc8,
0x27, 0x9e, 0x2f, 0x0f, 0x2b, 0xfe, 0x1b, 0x7d, 0x03, 0x72, 0x61, 0x4e, 0x72, 0xed, 0xfb, 0x5b,
0x08, 0x45, 0x4f, 0x60, 0x9e, 0xed, 0x31, 0x3a, 0x0e, 0x8a, 0x9d, 0xab, 0x9e, 0x76, 0x24, 0x92,
0x1d, 0x32, 0x2e, 0xe1, 0x49, 0x08, 0x5f, 0x4a, 0x19, 0x1c, 0x34, 0xd1, 0xff, 0x87, 0x22, 0x7f,
0x99, 0x08, 0x72, 0xae, 0xcc, 0x75, 0x3a, 0x0b, 0xe2, 0x71, 0x51, 0xe4, 0x5b, 0xff, 0x93, 0x80,
0x95, 0x1d, 0x7d, 0xb2, 0x47, 0x64, 0xd8, 0x20, 0x26, 0x26, 0x06, 0x75, 0x4d, 0xd4, 0x8d, 0x86,
0x9b, 0x2b, 0xde, 0x2a, 0xe3, 0x84, 0xe3, 0xa3, 0x4e, 0x50, 0x80, 0x25, 0x23, 0x05, 0xd8, 0x0a,
0x64, 0x1c, 0xea, 0x18, 0x44, 0xc6, 0x22, 0xd1, 0x50, 0xad, 0x68, 0xa8, 0x29, 0x85, 0xcf, 0x88,
0xfc, 0x11, 0xb0, 0x4d, 0xfd, 0xb0, 0x37, 0xf4, 0x09, 0x94, 0x7a, 0xcd, 0x3a, 0x6e, 0xf6, 0x6b,
0x9d, 0xef, 0x6a, 0xbd, 0xea, 0x76, 0xaf, 0xfa, 0xf8, 0xa1, 0xd6, 0xed, 0x6c, 0x7f, 0xef, 0xd1,
0x93, 0x87, 0xdf, 0x50, 0x12, 0xa5, 0xf2, 0xc9, 0x69, 0xf9, 0x76, 0xbb, 0x5a, 0xdf, 0x16, 0x3b,
0x66, 0x8f, 0xbe, 0xec, 0xe9, 0xb6, 0xa7, 0x3f, 0x7e, 0xd8, 0xa5, 0xf6, 0x84, 0x61, 0xd8, 0xb2,
0x2e, 0x46, 0xcf, 0xab, 0xe8, 0x31, 0x9c, 0xb8, 0xf4, 0x18, 0x9e, 0x9e, 0xe6, 0xc9, 0x4b, 0x4e,
0xf3, 0x0d, 0x58, 0x31, 0x5c, 0xea, 0x79, 0x1a, 0xcb, 0xfe, 0x89, 0x39, 0x53, 0x5f, 0x7c, 0xe9,
0xfc, 0x6c, 0x7d, 0xb9, 0xce, 0xf8, 0x3d, 0xce, 0x96, 0xea, 0x97, 0x8d, 0x08, 0x89, 0xf7, 0xa4,
0xfe, 0x7e, 0x8a, 0x25, 0x52, 0xd6, 0x91, 0x65, 0x93, 0x01, 0xf1, 0xd0, 0x73, 0x58, 0x32, 0x5c,
0x62, 0xb2, 0xb4, 0x5e, 0xb7, 0xa3, 0x1f, 0xd1, 0xfe, 0xbf, 0xd8, 0x9c, 0x26, 0x14, 0xac, 0xd4,
0x43, 0xa9, 0xde, 0x88, 0x18, 0x78, 0xd1, 0xb8, 0xd0, 0x46, 0x9f, 0xc2, 0x92, 0x47, 0x6c, 0xcb,
0x19, 0xbf, 0xd4, 0x0c, 0xea, 0xf8, 0xe4, 0x65, 0xf0, 0x22, 0x76, 0x9d, 0xde, 0x5e, 0x73, 0x9b,
0x49, 0xd5, 0x85, 0x50, 0x0d, 0x9d, 0x9f, 0xad, 0x2f, 0x5e, 0xa4, 0xe1, 0x45, 0xa9, 0x59, 0xb6,
0x4b, 0x6d, 0x58, 0xbc, 0x68, 0x0d, 0x5a, 0x91, 0x7b, 0x9f, 0x87, 0x90, 0x60, 0x6f, 0xa3, 0xdb,
0x90, 0x73, 0xc9, 0xc0, 0xf2, 0x7c, 0x57, 0xb8, 0x99, 0x71, 0x42, 0x0a, 0xdb, 0xf9, 0xe2, 0x0b,
0xa8, 0xd2, 0x2f, 0xc3, 0x4c, 0x8f, 0x6c, 0xb3, 0x98, 0x96, 0xa7, 0xef, 0x49, 0x95, 0x39, 0x1c,
0x34, 0xd9, 0x1a, 0x1c, 0x7b, 0x61, 0xa2, 0xc6, 0x7f, 0x33, 0x1a, 0xcf, 0x28, 0xe4, 0xf7, 0x60,
0x3c, 0x67, 0x08, 0x3e, 0x2c, 0x4d, 0x47, 0x3e, 0x2c, 0x5d, 0x81, 0x8c, 0x4d, 0x8e, 0x88, 0x2d,
0xce, 0x72, 0x2c, 0x1a, 0xf7, 0x1e, 0x42, 0x31, 0xf8, 0x82, 0x91, 0x7f, 0x39, 0x91, 0x83, 0x74,
0xbf, 0xda, 0x7b, 0xa6, 0xcc, 0x21, 0x80, 0xac, 0x58, 0x9c, 0xe2, 0xb5, 0xae, 0xde, 0x69, 0x6f,
0xb4, 0x36, 0x95, 0xe4, 0xbd, 0xdf, 0x49, 0x43, 0x3e, 0x7c, 0x2f, 0x62, 0x67, 0x47, 0xbb, 0xf9,
0x22, 0x58, 0xdd, 0x21, 0xbd, 0x4d, 0x8e, 0xd1, 0xdb, 0xd3, 0x5b, 0xa8, 0x4f, 0xc4, 0x03, 0x79,
0xc8, 0x0e, 0x6e, 0xa0, 0xde, 0x85, 0x5c, 0xb5, 0xd7, 0x6b, 0x6d, 0xb6, 0x9b, 0x0d, 0xe5, 0xb3,
0x44, 0xe9, 0x4b, 0x27, 0xa7, 0xe5, 0xe5, 0x10, 0x54, 0xf5, 0xc4, 0xe2, 0xe3, 0xa8, 0x7a, 0xbd,
0xd9, 0xed, 0x37, 0x1b, 0xca, 0xab, 0xe4, 0x2c, 0x8a, 0xdf, 0xaa, 0xf0, 0x4f, 0x77, 0xf2, 0x5d,
0xdc, 0xec, 0x56, 0x31, 0xeb, 0xf0, 0xb3, 0xa4, 0xb8, 0x1c, 0x9b, 0xf6, 0xe8, 0x92, 0x91, 0xee,
0xb2, 0x3e, 0xd7, 0x82, 0x6f, 0xe1, 0x5e, 0xa5, 0xc4, 0xe7, 0x1d, 0xd3, 0xc7, 0x2f, 0xa2, 0x9b,
0x13, 0xd6, 0x1b, 0x7f, 0x75, 0xe4, 0x6a, 0x52, 0x33, 0xbd, 0xf5, 0x58, 0xec, 0x61, 0x5a, 0x54,
0x98, 0xc7, 0xbb, 0xed, 0x36, 0x03, 0xbd, 0x4a, 0xcf, 0x8c, 0x0e, 0x8f, 0x1d, 0x56, 0x31, 0xa3,
0xbb, 0x90, 0x0b, 0x1e, 0x25, 0x95, 0xcf, 0xd2, 0x33, 0x06, 0xd5, 0x83, 0x17, 0x55, 0xde, 0xe1,
0xd6, 0x6e, 0x9f, 0x7f, 0xaa, 0xf7, 0x2a, 0x33, 0xdb, 0xe1, 0xc1, 0xd8, 0x37, 0xe9, 0xb1, 0xc3,
0xf6, 0xac, 0xbc, 0x87, 0xfb, 0x2c, 0x23, 0x2e, 0x2d, 0x42, 0x8c, 0xbc, 0x84, 0x7b, 0x17, 0x72,
0xb8, 0xf9, 0x1d, 0xf1, 0x55, 0xdf, 0xab, 0xec, 0x8c, 0x1e, 0x4c, 0x3e, 0x25, 0x06, 0xeb, 0xad,
0x0c, 0x59, 0xdc, 0xdc, 0xe9, 0x3c, 0x6f, 0x2a, 0x7f, 0x90, 0x9d, 0xd1, 0x83, 0xc9, 0x90, 0xf2,
0x6f, 0x9b, 0x72, 0x1d, 0xdc, 0xdd, 0xaa, 0xf2, 0x49, 0x99, 0xd5, 0xd3, 0x71, 0x47, 0x07, 0xba,
0x43, 0xcc, 0xe9, 0x57, 0x30, 0x21, 0xeb, 0xde, 0x2f, 0x40, 0x2e, 0xc8, 0x5d, 0xd1, 0x1a, 0x64,
0x5f, 0x74, 0xf0, 0xb3, 0x26, 0x56, 0xe6, 0x84, 0x97, 0x03, 0xce, 0x0b, 0x51, 0x75, 0x94, 0x61,
0x7e, 0xa7, 0xda, 0xae, 0x6e, 0x36, 0x71, 0x70, 0x89, 0x1e, 0x00, 0x64, 0x02, 0x56, 0x52, 0x64,
0x07, 0xa1, 0xce, 0xda, 0xea, 0x0f, 0x7f, 0xb2, 0x36, 0xf7, 0xe3, 0x9f, 0xac, 0xcd, 0xbd, 0x3a,
0x5f, 0x4b, 0xfc, 0xf0, 0x7c, 0x2d, 0xf1, 0x77, 0xe7, 0x6b, 0x89, 0x7f, 0x3d, 0x5f, 0x4b, 0xec,
0x65, 0xf9, 0x31, 0xf1, 0xe4, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x08, 0x6c, 0xc9, 0x88, 0x12,
0x32, 0x00, 0x00,
}
+10 -1
View File
@@ -458,7 +458,16 @@ enum TaskState {
SHUTDOWN = 640 [(gogoproto.enumvalue_customname)="TaskStateShutdown"]; // orchestrator requested shutdown
FAILED = 704 [(gogoproto.enumvalue_customname)="TaskStateFailed"]; // task execution failed with error
REJECTED = 768 [(gogoproto.enumvalue_customname)="TaskStateRejected"]; // task could not be executed here.
ORPHANED = 832 [(gogoproto.enumvalue_customname)="TaskStateOrphaned"]; // The node on which this task is scheduled is Down for too long
// TaskStateRemove is used to correctly handle service deletions and scale
// downs. This allows us to keep track of tasks that have been marked for
// deletion, but can't yet be removed because the agent is in the process of
// shutting them down. Once the agent has shut down tasks with desired state
// REMOVE, the task reaper is responsible for removing them.
REMOVE = 800 [(gogoproto.enumvalue_customname)="TaskStateRemove"];
// TaskStateOrphaned is used to free up resources associated with service
// tasks on unresponsive nodes without having to delete those tasks. This
// state is directly assigned to the task by the orchestrator.
ORPHANED = 832 [(gogoproto.enumvalue_customname)="TaskStateOrphaned"];
// NOTE(stevvooe): The state of a task is actually a lamport clock, in that
// given two observations, the greater of the two can be considered
+3 -2
View File
@@ -19,6 +19,7 @@ import (
import raftselector "github.com/docker/swarmkit/manager/raftselector"
import codes "google.golang.org/grpc/codes"
import status "google.golang.org/grpc/status"
import metadata "google.golang.org/grpc/metadata"
import transport "google.golang.org/grpc/transport"
import rafttime "time"
@@ -2043,12 +2044,12 @@ func NewRaftProxyWatchServer(local WatchServer, connSelector raftselector.ConnPr
redirectChecker := func(ctx context.Context) (context.Context, error) {
s, ok := transport.StreamFromContext(ctx)
if !ok {
return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context")
return ctx, status.Errorf(codes.InvalidArgument, "remote addr is not found in context")
}
addr := s.ServerTransport().RemoteAddr().String()
md, ok := metadata.FromContext(ctx)
if ok && len(md["redirect"]) != 0 {
return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
return ctx, status.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"])
}
if !ok {
md = metadata.New(map[string]string{})
+16 -16
View File
@@ -10,10 +10,10 @@ import (
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/log"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
)
type localRequestKeyType struct{}
@@ -52,13 +52,13 @@ func LogTLSState(ctx context.Context, tlsState *tls.ConnectionState) {
// getCertificateSubject extracts the subject from a verified client certificate
func getCertificateSubject(tlsState *tls.ConnectionState) (pkix.Name, error) {
if tlsState == nil {
return pkix.Name{}, grpc.Errorf(codes.PermissionDenied, "request is not using TLS")
return pkix.Name{}, status.Errorf(codes.PermissionDenied, "request is not using TLS")
}
if len(tlsState.PeerCertificates) == 0 {
return pkix.Name{}, grpc.Errorf(codes.PermissionDenied, "no client certificates in request")
return pkix.Name{}, status.Errorf(codes.PermissionDenied, "no client certificates in request")
}
if len(tlsState.VerifiedChains) == 0 {
return pkix.Name{}, grpc.Errorf(codes.PermissionDenied, "no verified chains for remote certificate")
return pkix.Name{}, status.Errorf(codes.PermissionDenied, "no verified chains for remote certificate")
}
return tlsState.VerifiedChains[0][0].Subject, nil
@@ -67,11 +67,11 @@ func getCertificateSubject(tlsState *tls.ConnectionState) (pkix.Name, error) {
func tlsConnStateFromContext(ctx context.Context) (*tls.ConnectionState, error) {
peer, ok := peer.FromContext(ctx)
if !ok {
return nil, grpc.Errorf(codes.PermissionDenied, "Permission denied: no peer info")
return nil, status.Errorf(codes.PermissionDenied, "Permission denied: no peer info")
}
tlsInfo, ok := peer.AuthInfo.(credentials.TLSInfo)
if !ok {
return nil, grpc.Errorf(codes.PermissionDenied, "Permission denied: peer didn't not present valid peer certificate")
return nil, status.Errorf(codes.PermissionDenied, "Permission denied: peer didn't not present valid peer certificate")
}
return &tlsInfo.State, nil
}
@@ -98,21 +98,21 @@ func AuthorizeOrgAndRole(ctx context.Context, org string, blacklistedCerts map[s
return authorizeOrg(certSubj, org, blacklistedCerts)
}
return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of OUs: %v", ou)
return "", status.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of OUs: %v", ou)
}
// authorizeOrg takes in a certificate subject and an organization, and returns
// the Node ID of the node.
func authorizeOrg(certSubj pkix.Name, org string, blacklistedCerts map[string]*api.BlacklistedCertificate) (string, error) {
if _, ok := blacklistedCerts[certSubj.CommonName]; ok {
return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: node %s was removed from swarm", certSubj.CommonName)
return "", status.Errorf(codes.PermissionDenied, "Permission denied: node %s was removed from swarm", certSubj.CommonName)
}
if len(certSubj.Organization) > 0 && certSubj.Organization[0] == org {
return certSubj.CommonName, nil
}
return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of organization: %s", org)
return "", status.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of organization: %s", org)
}
// AuthorizeForwardedRoleAndOrg checks for proper roles and organization of caller. The RPC may have
@@ -123,7 +123,7 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde
if isForwardedRequest(ctx) {
_, err := AuthorizeOrgAndRole(ctx, org, blacklistedCerts, forwarderRoles...)
if err != nil {
return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarder role: %v", err)
return "", status.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarder role: %v", err)
}
// This was a forwarded request. Authorize the forwarder, and
@@ -132,15 +132,15 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde
_, forwardedID, forwardedOrg, forwardedOUs := forwardedTLSInfoFromContext(ctx)
if len(forwardedOUs) == 0 || forwardedID == "" || forwardedOrg == "" {
return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request")
return "", status.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request")
}
if !intersectArrays(forwardedOUs, authorizedRoles) {
return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarded role, expecting: %v", authorizedRoles)
return "", status.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarded role, expecting: %v", authorizedRoles)
}
if forwardedOrg != org {
return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: organization mismatch, expecting: %s", org)
return "", status.Errorf(codes.PermissionDenied, "Permission denied: organization mismatch, expecting: %s", org)
}
return forwardedID, nil
@@ -152,7 +152,7 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde
return nodeID, nil
}
return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized peer role: %v", err)
return "", status.Errorf(codes.PermissionDenied, "Permission denied: unauthorized peer role: %v", err)
}
// intersectArrays returns true when there is at least one element in common
@@ -219,7 +219,7 @@ func RemoteNode(ctx context.Context) (RemoteNodeInfo, error) {
peer, ok := peer.FromContext(ctx)
if !ok {
return RemoteNodeInfo{}, grpc.Errorf(codes.PermissionDenied, "Permission denied: no peer info")
return RemoteNodeInfo{}, status.Errorf(codes.PermissionDenied, "Permission denied: no peer info")
}
directInfo := RemoteNodeInfo{
@@ -232,7 +232,7 @@ func RemoteNode(ctx context.Context) (RemoteNodeInfo, error) {
if isForwardedRequest(ctx) {
remoteAddr, cn, org, ous := forwardedTLSInfoFromContext(ctx)
if len(ous) == 0 || cn == "" || org == "" {
return RemoteNodeInfo{}, grpc.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request")
return RemoteNodeInfo{}, status.Errorf(codes.PermissionDenied, "Permission denied: missing information in forwarded request")
}
return RemoteNodeInfo{
Roles: ous,
+9 -9
View File
@@ -16,8 +16,8 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
@@ -149,7 +149,7 @@ func (s *Server) GetUnlockKey(ctx context.Context, request *api.GetUnlockKeyRequ
// NodeCertificateStatus returns the current issuance status of an issuance request identified by the nodeID
func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCertificateStatusRequest) (*api.NodeCertificateStatusResponse, error) {
if request.NodeID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, codes.InvalidArgument.String())
return nil, status.Errorf(codes.InvalidArgument, codes.InvalidArgument.String())
}
serverCtx, err := s.isRunningLocked()
@@ -180,7 +180,7 @@ func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCer
// This node ID doesn't exist
if node == nil {
return nil, grpc.Errorf(codes.NotFound, codes.NotFound.String())
return nil, status.Errorf(codes.NotFound, codes.NotFound.String())
}
log.G(ctx).WithFields(logrus.Fields{
@@ -236,7 +236,7 @@ func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCer
func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNodeCertificateRequest) (*api.IssueNodeCertificateResponse, error) {
// First, let's see if the remote node is presenting a non-empty CSR
if len(request.CSR) == 0 {
return nil, grpc.Errorf(codes.InvalidArgument, codes.InvalidArgument.String())
return nil, status.Errorf(codes.InvalidArgument, codes.InvalidArgument.String())
}
if err := s.isReadyLocked(); err != nil {
@@ -295,7 +295,7 @@ func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNod
s.mu.Unlock()
if role < 0 {
return nil, grpc.Errorf(codes.InvalidArgument, "A valid join token is necessary to join this cluster")
return nil, status.Errorf(codes.InvalidArgument, "A valid join token is necessary to join this cluster")
}
// Max number of collisions of ID or CN to tolerate before giving up
@@ -369,7 +369,7 @@ func (s *Server) issueRenewCertificate(ctx context.Context, nodeID string, csr [
"method": "issueRenewCertificate",
}).Warnf("node does not exist")
// If this node doesn't exist, we shouldn't be renewing a certificate for it
return grpc.Errorf(codes.NotFound, "node %s not found when attempting to renew certificate", nodeID)
return status.Errorf(codes.NotFound, "node %s not found when attempting to renew certificate", nodeID)
}
// Create a new Certificate entry for this node with the new CSR and a RENEW state
@@ -594,7 +594,7 @@ func (s *Server) isRunningLocked() (context.Context, error) {
s.mu.Lock()
if !s.isRunning() {
s.mu.Unlock()
return nil, grpc.Errorf(codes.Aborted, "CA signer is stopped")
return nil, status.Errorf(codes.Aborted, "CA signer is stopped")
}
ctx := s.ctx
s.mu.Unlock()
@@ -605,10 +605,10 @@ func (s *Server) isReadyLocked() error {
s.mu.Lock()
defer s.mu.Unlock()
if !s.isRunning() {
return grpc.Errorf(codes.Aborted, "CA signer is stopped")
return status.Errorf(codes.Aborted, "CA signer is stopped")
}
if s.joinTokens == nil {
return grpc.Errorf(codes.Aborted, "CA signer is still starting")
return status.Errorf(codes.Aborted, "CA signer is still starting")
}
return nil
}
@@ -10,13 +10,12 @@ import (
"net/url"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"github.com/cloudflare/cfssl/helpers"
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/ca"
"github.com/docker/swarmkit/log"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
var minRootExpiration = 1 * helpers.OneYear
@@ -60,7 +59,7 @@ func newRootRotationObject(ctx context.Context, securityConfig *ca.SecurityConfi
crossSignedCert, err = oldRootCA.CrossSignCACertificate(rootCert)
}
case !newRootHasSigner: // the original CA and the new CA both require external CAs
return nil, grpc.Errorf(codes.InvalidArgument, "rotating from one external CA to a different external CA is not supported")
return nil, status.Errorf(codes.InvalidArgument, "rotating from one external CA to a different external CA is not supported")
default:
// We need the same credentials but to connect to the original URLs (in case we are in the middle of a root rotation already)
var urls []string
@@ -70,7 +69,7 @@ func newRootRotationObject(ctx context.Context, securityConfig *ca.SecurityConfi
}
}
if len(urls) == 0 {
return nil, grpc.Errorf(codes.InvalidArgument,
return nil, status.Errorf(codes.InvalidArgument,
"must provide an external CA for the current external root CA to generate a cross-signed certificate")
}
rootPool := x509.NewCertPool()
@@ -83,7 +82,7 @@ func newRootRotationObject(ctx context.Context, securityConfig *ca.SecurityConfi
if err != nil {
log.G(ctx).WithError(err).Error("unable to generate a cross-signed certificate for root rotation")
return nil, grpc.Errorf(codes.Internal, "unable to generate a cross-signed certificate for root rotation")
return nil, status.Errorf(codes.Internal, "unable to generate a cross-signed certificate for root rotation")
}
copied := apiRootCA.Copy()
@@ -146,7 +145,7 @@ func validateHasAtLeastOneExternalCA(ctx context.Context, externalCAs map[string
}
}
}
return nil, grpc.Errorf(codes.InvalidArgument, "there must be at least one valid, reachable external CA corresponding to the %s CA certificate", desc)
return nil, status.Errorf(codes.InvalidArgument, "there must be at least one valid, reachable external CA corresponding to the %s CA certificate", desc)
}
// validates that the list of external CAs have valid certs associated with them, and produce a mapping of subject/pubkey:external
@@ -193,7 +192,7 @@ func validateCAConfig(ctx context.Context, securityConfig *ca.SecurityConfig, cl
newConfig.SigningCACert = ca.NormalizePEMs(newConfig.SigningCACert) // ensure this is normalized before we use it
if len(newConfig.SigningCAKey) > 0 && len(newConfig.SigningCACert) == 0 {
return nil, grpc.Errorf(codes.InvalidArgument, "if a signing CA key is provided, the signing CA cert must also be provided")
return nil, status.Errorf(codes.InvalidArgument, "if a signing CA key is provided, the signing CA cert must also be provided")
}
normalizedRootCA := ca.NormalizePEMs(cluster.RootCA.CACert)
@@ -216,7 +215,7 @@ func validateCAConfig(ctx context.Context, securityConfig *ca.SecurityConfig, cl
if cluster.RootCA.LastForcedRotation != newConfig.ForceRotate {
newRootCA, err := ca.CreateRootCA(ca.DefaultRootCN)
if err != nil {
return nil, grpc.Errorf(codes.Internal, err.Error())
return nil, status.Errorf(codes.Internal, err.Error())
}
return newRootRotationObject(ctx, securityConfig, &cluster.RootCA, newRootCA, oldCertExtCAs, newConfig.ForceRotate)
}
@@ -240,21 +239,21 @@ func validateCAConfig(ctx context.Context, securityConfig *ca.SecurityConfig, cl
}
newRootCA, err := ca.NewRootCA(newConfig.SigningCACert, signingCert, newConfig.SigningCAKey, ca.DefaultNodeCertExpiration, nil)
if err != nil {
return nil, grpc.Errorf(codes.InvalidArgument, err.Error())
return nil, status.Errorf(codes.InvalidArgument, err.Error())
}
if len(newRootCA.Pool.Subjects()) != 1 {
return nil, grpc.Errorf(codes.InvalidArgument, "the desired CA certificate cannot contain multiple certificates")
return nil, status.Errorf(codes.InvalidArgument, "the desired CA certificate cannot contain multiple certificates")
}
parsedCert, err := helpers.ParseCertificatePEM(newConfig.SigningCACert)
if err != nil {
return nil, grpc.Errorf(codes.InvalidArgument, "could not parse the desired CA certificate")
return nil, status.Errorf(codes.InvalidArgument, "could not parse the desired CA certificate")
}
// The new certificate's expiry must be at least one year away
if parsedCert.NotAfter.Before(time.Now().Add(minRootExpiration)) {
return nil, grpc.Errorf(codes.InvalidArgument, "CA certificate expires too soon")
return nil, status.Errorf(codes.InvalidArgument, "CA certificate expires too soon")
}
if !hasSigningKey(newConfig) {
@@ -11,8 +11,8 @@ import (
"github.com/docker/swarmkit/manager/state/store"
gogotypes "github.com/gogo/protobuf/types"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
@@ -23,17 +23,17 @@ const (
func validateClusterSpec(spec *api.ClusterSpec) error {
if spec == nil {
return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
// Validate that expiry time being provided is valid, and over our minimum
if spec.CAConfig.NodeCertExpiry != nil {
expiry, err := gogotypes.DurationFromProto(spec.CAConfig.NodeCertExpiry)
if err != nil {
return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if expiry < ca.MinNodeCertExpiration {
return grpc.Errorf(codes.InvalidArgument, "minimum certificate expiry time is: %s", ca.MinNodeCertExpiration)
return status.Errorf(codes.InvalidArgument, "minimum certificate expiry time is: %s", ca.MinNodeCertExpiration)
}
}
@@ -42,7 +42,7 @@ func validateClusterSpec(spec *api.ClusterSpec) error {
if len(spec.AcceptancePolicy.Policies) > 0 {
for _, policy := range spec.AcceptancePolicy.Policies {
if policy.Secret != nil && strings.ToLower(policy.Secret.Alg) != "bcrypt" {
return grpc.Errorf(codes.InvalidArgument, "hashing algorithm is not supported: %s", policy.Secret.Alg)
return status.Errorf(codes.InvalidArgument, "hashing algorithm is not supported: %s", policy.Secret.Alg)
}
}
}
@@ -51,13 +51,17 @@ func validateClusterSpec(spec *api.ClusterSpec) error {
if spec.Dispatcher.HeartbeatPeriod != nil {
heartbeatPeriod, err := gogotypes.DurationFromProto(spec.Dispatcher.HeartbeatPeriod)
if err != nil {
return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if heartbeatPeriod < 0 {
return grpc.Errorf(codes.InvalidArgument, "heartbeat time period cannot be a negative duration")
return status.Errorf(codes.InvalidArgument, "heartbeat time period cannot be a negative duration")
}
}
if spec.Annotations.Name != store.DefaultClusterName {
return status.Errorf(codes.InvalidArgument, "modification of cluster name is not allowed")
}
return nil
}
@@ -66,7 +70,7 @@ func validateClusterSpec(spec *api.ClusterSpec) error {
// - Returns `NotFound` if the Cluster is not found.
func (s *Server) GetCluster(ctx context.Context, request *api.GetClusterRequest) (*api.GetClusterResponse, error) {
if request.ClusterID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
var cluster *api.Cluster
@@ -74,7 +78,7 @@ func (s *Server) GetCluster(ctx context.Context, request *api.GetClusterRequest)
cluster = store.GetCluster(tx, request.ClusterID)
})
if cluster == nil {
return nil, grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID)
return nil, status.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID)
}
redactedClusters := redactClusters([]*api.Cluster{cluster})
@@ -92,7 +96,7 @@ func (s *Server) GetCluster(ctx context.Context, request *api.GetClusterRequest)
// - Returns an error if the update fails.
func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRequest) (*api.UpdateClusterResponse, error) {
if request.ClusterID == "" || request.ClusterVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateClusterSpec(request.Spec); err != nil {
return nil, err
@@ -102,7 +106,7 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe
err := s.store.Update(func(tx store.Tx) error {
cluster = store.GetCluster(tx, request.ClusterID)
if cluster == nil {
return grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID)
return status.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID)
}
// This ensures that we have the current rootCA with which to generate tokens (expiration doesn't matter
// for generating the tokens)
@@ -110,7 +114,7 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe
if err != nil {
log.G(ctx).WithField(
"method", "(*controlapi.Server).UpdateCluster").WithError(err).Error("invalid cluster root CA")
return grpc.Errorf(codes.Internal, "error loading cluster rootCA for update")
return status.Errorf(codes.Internal, "error loading cluster rootCA for update")
}
cluster.Meta.Version = *request.ClusterVersion
@@ -10,8 +10,8 @@ import (
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/manager/allocator"
"github.com/docker/swarmkit/manager/state/store"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
var isValidDNSName = regexp.MustCompile(`^[a-zA-Z0-9](?:[-_]*[A-Za-z0-9]+)*$`)
@@ -70,25 +70,25 @@ func filterMatchLabels(match map[string]string, candidates map[string]string) bo
func validateAnnotations(m api.Annotations) error {
if m.Name == "" {
return grpc.Errorf(codes.InvalidArgument, "meta: name must be provided")
return status.Errorf(codes.InvalidArgument, "meta: name must be provided")
}
if !isValidDNSName.MatchString(m.Name) {
// if the name doesn't match the regex
return grpc.Errorf(codes.InvalidArgument, "name must be valid as a DNS name component")
return status.Errorf(codes.InvalidArgument, "name must be valid as a DNS name component")
}
if len(m.Name) > 63 {
// DNS labels are limited to 63 characters
return grpc.Errorf(codes.InvalidArgument, "name must be 63 characters or fewer")
return status.Errorf(codes.InvalidArgument, "name must be 63 characters or fewer")
}
return nil
}
func validateConfigOrSecretAnnotations(m api.Annotations) error {
if m.Name == "" {
return grpc.Errorf(codes.InvalidArgument, "name must be provided")
return status.Errorf(codes.InvalidArgument, "name must be provided")
} else if len(m.Name) > 64 || !isValidConfigOrSecretName.MatchString(m.Name) {
// if the name doesn't match the regex
return grpc.Errorf(codes.InvalidArgument,
return status.Errorf(codes.InvalidArgument,
"invalid name, only 64 [a-zA-Z0-9-_.] characters allowed, and the start and end character must be [a-zA-Z0-9]")
}
return nil
@@ -102,7 +102,7 @@ func validateDriver(driver *api.Driver, pg plugingetter.PluginGetter, pluginType
}
if driver.Name == "" {
return grpc.Errorf(codes.InvalidArgument, "driver name: if driver is specified name is required")
return status.Errorf(codes.InvalidArgument, "driver name: if driver is specified name is required")
}
// First check against the known drivers
@@ -119,16 +119,16 @@ func validateDriver(driver *api.Driver, pg plugingetter.PluginGetter, pluginType
}
if pg == nil {
return grpc.Errorf(codes.InvalidArgument, "plugin %s not supported", driver.Name)
return status.Errorf(codes.InvalidArgument, "plugin %s not supported", driver.Name)
}
p, err := pg.Get(driver.Name, pluginType, plugingetter.Lookup)
if err != nil {
return grpc.Errorf(codes.InvalidArgument, "error during lookup of plugin %s", driver.Name)
return status.Errorf(codes.InvalidArgument, "error during lookup of plugin %s", driver.Name)
}
if p.IsV1() {
return grpc.Errorf(codes.InvalidArgument, "legacy plugin %s of type %s is not supported in swarm mode", driver.Name, pluginType)
return status.Errorf(codes.InvalidArgument, "legacy plugin %s of type %s is not supported in swarm mode", driver.Name, pluginType)
}
return nil
@@ -10,8 +10,8 @@ import (
"github.com/docker/swarmkit/manager/state/store"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// MaxConfigSize is the maximum byte length of the `Config.Spec.Data` field.
@@ -32,7 +32,7 @@ func configFromConfigSpec(spec *api.ConfigSpec) *api.Config {
// - Returns an error if getting fails.
func (s *Server) GetConfig(ctx context.Context, request *api.GetConfigRequest) (*api.GetConfigResponse, error) {
if request.ConfigID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, "config ID must be provided")
return nil, status.Errorf(codes.InvalidArgument, "config ID must be provided")
}
var config *api.Config
@@ -41,7 +41,7 @@ func (s *Server) GetConfig(ctx context.Context, request *api.GetConfigRequest) (
})
if config == nil {
return nil, grpc.Errorf(codes.NotFound, "config %s not found", request.ConfigID)
return nil, status.Errorf(codes.NotFound, "config %s not found", request.ConfigID)
}
return &api.GetConfigResponse{Config: config}, nil
@@ -53,21 +53,21 @@ func (s *Server) GetConfig(ctx context.Context, request *api.GetConfigRequest) (
// - Returns an error if the update fails.
func (s *Server) UpdateConfig(ctx context.Context, request *api.UpdateConfigRequest) (*api.UpdateConfigResponse, error) {
if request.ConfigID == "" || request.ConfigVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
var config *api.Config
err := s.store.Update(func(tx store.Tx) error {
config = store.GetConfig(tx, request.ConfigID)
if config == nil {
return grpc.Errorf(codes.NotFound, "config %s not found", request.ConfigID)
return status.Errorf(codes.NotFound, "config %s not found", request.ConfigID)
}
// Check if the Name is different than the current name, or the config is non-nil and different
// than the current config
if config.Spec.Annotations.Name != request.Spec.Annotations.Name ||
(request.Spec.Data != nil && !bytes.Equal(request.Spec.Data, config.Spec.Data)) {
return grpc.Errorf(codes.InvalidArgument, "only updates to Labels are allowed")
return status.Errorf(codes.InvalidArgument, "only updates to Labels are allowed")
}
// We only allow updating Labels
@@ -164,7 +164,7 @@ func (s *Server) CreateConfig(ctx context.Context, request *api.CreateConfigRequ
switch err {
case store.ErrNameConflict:
return nil, grpc.Errorf(codes.AlreadyExists, "config %s already exists", request.Spec.Annotations.Name)
return nil, status.Errorf(codes.AlreadyExists, "config %s already exists", request.Spec.Annotations.Name)
case nil:
log.G(ctx).WithFields(logrus.Fields{
"config.Name": request.Spec.Annotations.Name,
@@ -184,20 +184,20 @@ func (s *Server) CreateConfig(ctx context.Context, request *api.CreateConfigRequ
// - Returns an error if the deletion fails.
func (s *Server) RemoveConfig(ctx context.Context, request *api.RemoveConfigRequest) (*api.RemoveConfigResponse, error) {
if request.ConfigID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, "config ID must be provided")
return nil, status.Errorf(codes.InvalidArgument, "config ID must be provided")
}
err := s.store.Update(func(tx store.Tx) error {
// Check if the config exists
config := store.GetConfig(tx, request.ConfigID)
if config == nil {
return grpc.Errorf(codes.NotFound, "could not find config %s", request.ConfigID)
return status.Errorf(codes.NotFound, "could not find config %s", request.ConfigID)
}
// Check if any services currently reference this config, return error if so
services, err := store.FindServices(tx, store.ByReferencedConfigID(request.ConfigID))
if err != nil {
return grpc.Errorf(codes.Internal, "could not find services using config %s: %v", request.ConfigID, err)
return status.Errorf(codes.Internal, "could not find services using config %s: %v", request.ConfigID, err)
}
if len(services) != 0 {
@@ -213,14 +213,14 @@ func (s *Server) RemoveConfig(ctx context.Context, request *api.RemoveConfigRequ
serviceStr = "service"
}
return grpc.Errorf(codes.InvalidArgument, "config '%s' is in use by the following %s: %v", configName, serviceStr, serviceNameStr)
return status.Errorf(codes.InvalidArgument, "config '%s' is in use by the following %s: %v", configName, serviceStr, serviceNameStr)
}
return store.DeleteConfig(tx, request.ConfigID)
})
switch err {
case store.ErrNotExist:
return nil, grpc.Errorf(codes.NotFound, "config %s not found", request.ConfigID)
return nil, status.Errorf(codes.NotFound, "config %s not found", request.ConfigID)
case nil:
log.G(ctx).WithFields(logrus.Fields{
"config.ID": request.ConfigID,
@@ -235,14 +235,14 @@ func (s *Server) RemoveConfig(ctx context.Context, request *api.RemoveConfigRequ
func validateConfigSpec(spec *api.ConfigSpec) error {
if spec == nil {
return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateConfigOrSecretAnnotations(spec.Annotations); err != nil {
return err
}
if len(spec.Data) >= MaxConfigSize || len(spec.Data) < 1 {
return grpc.Errorf(codes.InvalidArgument, "config data must be larger than 0 and less than %d bytes", MaxConfigSize)
return status.Errorf(codes.InvalidArgument, "config data must be larger than 0 and less than %d bytes", MaxConfigSize)
}
return nil
}
@@ -12,39 +12,39 @@ import (
"github.com/docker/swarmkit/manager/allocator/networkallocator"
"github.com/docker/swarmkit/manager/state/store"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
func validateIPAMConfiguration(ipamConf *api.IPAMConfig) error {
if ipamConf == nil {
return grpc.Errorf(codes.InvalidArgument, "ipam configuration: cannot be empty")
return status.Errorf(codes.InvalidArgument, "ipam configuration: cannot be empty")
}
_, subnet, err := net.ParseCIDR(ipamConf.Subnet)
if err != nil {
return grpc.Errorf(codes.InvalidArgument, "ipam configuration: invalid subnet %s", ipamConf.Subnet)
return status.Errorf(codes.InvalidArgument, "ipam configuration: invalid subnet %s", ipamConf.Subnet)
}
if ipamConf.Range != "" {
ip, _, err := net.ParseCIDR(ipamConf.Range)
if err != nil {
return grpc.Errorf(codes.InvalidArgument, "ipam configuration: invalid range %s", ipamConf.Range)
return status.Errorf(codes.InvalidArgument, "ipam configuration: invalid range %s", ipamConf.Range)
}
if !subnet.Contains(ip) {
return grpc.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain range %s", ipamConf.Subnet, ipamConf.Range)
return status.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain range %s", ipamConf.Subnet, ipamConf.Range)
}
}
if ipamConf.Gateway != "" {
ip := net.ParseIP(ipamConf.Gateway)
if ip == nil {
return grpc.Errorf(codes.InvalidArgument, "ipam configuration: invalid gateway %s", ipamConf.Gateway)
return status.Errorf(codes.InvalidArgument, "ipam configuration: invalid gateway %s", ipamConf.Gateway)
}
if !subnet.Contains(ip) {
return grpc.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain gateway %s", ipamConf.Subnet, ipamConf.Gateway)
return status.Errorf(codes.InvalidArgument, "ipam configuration: subnet %s does not contain gateway %s", ipamConf.Subnet, ipamConf.Gateway)
}
}
@@ -73,15 +73,15 @@ func validateIPAM(ipam *api.IPAMOptions, pg plugingetter.PluginGetter) error {
func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) error {
if spec == nil {
return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if spec.Ingress && spec.DriverConfig != nil && spec.DriverConfig.Name != "overlay" {
return grpc.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network")
return status.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network")
}
if spec.Attachable && spec.Ingress {
return grpc.Errorf(codes.InvalidArgument, "ingress network cannot be attachable")
return status.Errorf(codes.InvalidArgument, "ingress network cannot be attachable")
}
if err := validateAnnotations(spec.Annotations); err != nil {
@@ -89,7 +89,7 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er
}
if _, ok := spec.Annotations.Labels[networkallocator.PredefinedLabel]; ok {
return grpc.Errorf(codes.PermissionDenied, "label %s is for internally created predefined networks and cannot be applied by users",
return status.Errorf(codes.PermissionDenied, "label %s is for internally created predefined networks and cannot be applied by users",
networkallocator.PredefinedLabel)
}
if err := validateDriver(spec.DriverConfig, pg, driverapi.NetworkPluginEndpointType); err != nil {
@@ -117,9 +117,9 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe
err := s.store.Update(func(tx store.Tx) error {
if request.Spec.Ingress {
if n, err := allocator.GetIngressNetwork(s.store); err == nil {
return grpc.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID)
return status.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID)
} else if err != allocator.ErrNoIngress {
return grpc.Errorf(codes.Internal, "failed ingress network presence check: %v", err)
return status.Errorf(codes.Internal, "failed ingress network presence check: %v", err)
}
}
return store.CreateNetwork(tx, n)
@@ -138,7 +138,7 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe
// - Returns `NotFound` if the Network is not found.
func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest) (*api.GetNetworkResponse, error) {
if request.NetworkID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
var n *api.Network
@@ -146,7 +146,7 @@ func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest)
n = store.GetNetwork(tx, request.NetworkID)
})
if n == nil {
return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
return nil, status.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
}
return &api.GetNetworkResponse{
Network: n,
@@ -159,7 +159,7 @@ func (s *Server) GetNetwork(ctx context.Context, request *api.GetNetworkRequest)
// - Returns an error if the deletion fails.
func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRequest) (*api.RemoveNetworkResponse, error) {
if request.NetworkID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
var (
@@ -171,7 +171,7 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe
n = store.GetNetwork(tx, request.NetworkID)
})
if n == nil {
return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
return nil, status.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
}
if allocator.IsIngressNetwork(n) {
@@ -179,13 +179,13 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe
}
if v, ok := n.Spec.Annotations.Labels[networkallocator.PredefinedLabel]; ok && v == "true" {
return nil, grpc.Errorf(codes.FailedPrecondition, "network %s (%s) is a swarm predefined network and cannot be removed",
return nil, status.Errorf(codes.FailedPrecondition, "network %s (%s) is a swarm predefined network and cannot be removed",
request.NetworkID, n.Spec.Annotations.Name)
}
if err := rm(n.ID); err != nil {
if err == store.ErrNotExist {
return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
return nil, status.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
}
return nil, err
}
@@ -196,21 +196,21 @@ func (s *Server) removeNetwork(id string) error {
return s.store.Update(func(tx store.Tx) error {
services, err := store.FindServices(tx, store.ByReferencedNetworkID(id))
if err != nil {
return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err)
return status.Errorf(codes.Internal, "could not find services using network %s: %v", id, err)
}
if len(services) != 0 {
return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID)
return status.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID)
}
tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(id))
if err != nil {
return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err)
return status.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err)
}
for _, t := range tasks {
if t.DesiredState <= api.TaskStateRunning && t.Status.State <= api.TaskStateRunning {
return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID)
return status.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID)
}
}
@@ -222,11 +222,11 @@ func (s *Server) removeIngressNetwork(id string) error {
return s.store.Update(func(tx store.Tx) error {
services, err := store.FindServices(tx, store.All)
if err != nil {
return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err)
return status.Errorf(codes.Internal, "could not find services using network %s: %v", id, err)
}
for _, srv := range services {
if allocator.IsIngressNetworkNeeded(srv) {
return grpc.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID)
return status.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID)
}
}
return store.DeleteNetwork(tx, id)
@@ -9,13 +9,13 @@ import (
"github.com/docker/swarmkit/manager/state/store"
gogotypes "github.com/gogo/protobuf/types"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
func validateNodeSpec(spec *api.NodeSpec) error {
if spec == nil {
return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
return nil
}
@@ -25,7 +25,7 @@ func validateNodeSpec(spec *api.NodeSpec) error {
// - Returns `NotFound` if the Node is not found.
func (s *Server) GetNode(ctx context.Context, request *api.GetNodeRequest) (*api.GetNodeResponse, error) {
if request.NodeID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
var node *api.Node
@@ -33,7 +33,7 @@ func (s *Server) GetNode(ctx context.Context, request *api.GetNodeRequest) (*api
node = store.GetNode(tx, request.NodeID)
})
if node == nil {
return nil, grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID)
return nil, status.Errorf(codes.NotFound, "node %s not found", request.NodeID)
}
if s.raft != nil {
@@ -196,7 +196,7 @@ func (s *Server) ListNodes(ctx context.Context, request *api.ListNodesRequest) (
// - Returns an error if the update fails.
func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest) (*api.UpdateNodeResponse, error) {
if request.NodeID == "" || request.NodeVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateNodeSpec(request.Spec); err != nil {
return nil, err
@@ -210,7 +210,7 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest)
err := s.store.Update(func(tx store.Tx) error {
node = store.GetNode(tx, request.NodeID)
if node == nil {
return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID)
return status.Errorf(codes.NotFound, "node %s not found", request.NodeID)
}
// Demotion sanity checks.
@@ -218,20 +218,20 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest)
// Check for manager entries in Store.
managers, err := store.FindNodes(tx, store.ByRole(api.NodeRoleManager))
if err != nil {
return grpc.Errorf(codes.Internal, "internal store error: %v", err)
return status.Errorf(codes.Internal, "internal store error: %v", err)
}
if len(managers) == 1 && managers[0].ID == node.ID {
return grpc.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm")
return status.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm")
}
// Check for node in memberlist
if member = s.raft.GetMemberByNodeID(request.NodeID); member == nil {
return grpc.Errorf(codes.NotFound, "can't find manager in raft memberlist")
return status.Errorf(codes.NotFound, "can't find manager in raft memberlist")
}
// Quorum safeguard
if !s.raft.CanRemoveMember(member.RaftID) {
return grpc.Errorf(codes.FailedPrecondition, "can't remove member from the raft: this would result in a loss of quorum")
return status.Errorf(codes.FailedPrecondition, "can't remove member from the raft: this would result in a loss of quorum")
}
}
@@ -278,33 +278,33 @@ func removeNodeAttachments(tx store.Tx, nodeID string) error {
// - Returns an error if the delete fails.
func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest) (*api.RemoveNodeResponse, error) {
if request.NodeID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
err := s.store.Update(func(tx store.Tx) error {
node := store.GetNode(tx, request.NodeID)
if node == nil {
return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID)
return status.Errorf(codes.NotFound, "node %s not found", request.NodeID)
}
if node.Spec.DesiredRole == api.NodeRoleManager {
if s.raft == nil {
return grpc.Errorf(codes.FailedPrecondition, "node %s is a manager but cannot access node information from the raft memberlist", request.NodeID)
return status.Errorf(codes.FailedPrecondition, "node %s is a manager but cannot access node information from the raft memberlist", request.NodeID)
}
if member := s.raft.GetMemberByNodeID(request.NodeID); member != nil {
return grpc.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID)
return status.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID)
}
}
if !request.Force && node.Status.State == api.NodeStatus_READY {
return grpc.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID)
return status.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID)
}
// lookup the cluster
clusters, err := store.FindClusters(tx, store.ByName("default"))
clusters, err := store.FindClusters(tx, store.ByName(store.DefaultClusterName))
if err != nil {
return err
}
if len(clusters) != 1 {
return grpc.Errorf(codes.Internal, "could not fetch cluster object")
return status.Errorf(codes.Internal, "could not fetch cluster object")
}
cluster := clusters[0]
@@ -11,8 +11,8 @@ import (
"github.com/docker/swarmkit/manager/state/store"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// assumes spec is not nil
@@ -30,7 +30,7 @@ func secretFromSecretSpec(spec *api.SecretSpec) *api.Secret {
// - Returns an error if getting fails.
func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) (*api.GetSecretResponse, error) {
if request.SecretID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, "secret ID must be provided")
return nil, status.Errorf(codes.InvalidArgument, "secret ID must be provided")
}
var secret *api.Secret
@@ -39,7 +39,7 @@ func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) (
})
if secret == nil {
return nil, grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID)
return nil, status.Errorf(codes.NotFound, "secret %s not found", request.SecretID)
}
secret.Spec.Data = nil // clean the actual secret data so it's never returned
@@ -52,20 +52,20 @@ func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) (
// - Returns an error if the update fails.
func (s *Server) UpdateSecret(ctx context.Context, request *api.UpdateSecretRequest) (*api.UpdateSecretResponse, error) {
if request.SecretID == "" || request.SecretVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
var secret *api.Secret
err := s.store.Update(func(tx store.Tx) error {
secret = store.GetSecret(tx, request.SecretID)
if secret == nil {
return grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID)
return status.Errorf(codes.NotFound, "secret %s not found", request.SecretID)
}
// Check if the Name is different than the current name, or the secret is non-nil and different
// than the current secret
if secret.Spec.Annotations.Name != request.Spec.Annotations.Name ||
(request.Spec.Data != nil && subtle.ConstantTimeCompare(request.Spec.Data, secret.Spec.Data) == 0) {
return grpc.Errorf(codes.InvalidArgument, "only updates to Labels are allowed")
return status.Errorf(codes.InvalidArgument, "only updates to Labels are allowed")
}
// We only allow updating Labels
@@ -171,7 +171,7 @@ func (s *Server) CreateSecret(ctx context.Context, request *api.CreateSecretRequ
switch err {
case store.ErrNameConflict:
return nil, grpc.Errorf(codes.AlreadyExists, "secret %s already exists", request.Spec.Annotations.Name)
return nil, status.Errorf(codes.AlreadyExists, "secret %s already exists", request.Spec.Annotations.Name)
case nil:
secret.Spec.Data = nil // clean the actual secret data so it's never returned
log.G(ctx).WithFields(logrus.Fields{
@@ -192,20 +192,20 @@ func (s *Server) CreateSecret(ctx context.Context, request *api.CreateSecretRequ
// - Returns an error if the deletion fails.
func (s *Server) RemoveSecret(ctx context.Context, request *api.RemoveSecretRequest) (*api.RemoveSecretResponse, error) {
if request.SecretID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, "secret ID must be provided")
return nil, status.Errorf(codes.InvalidArgument, "secret ID must be provided")
}
err := s.store.Update(func(tx store.Tx) error {
// Check if the secret exists
secret := store.GetSecret(tx, request.SecretID)
if secret == nil {
return grpc.Errorf(codes.NotFound, "could not find secret %s", request.SecretID)
return status.Errorf(codes.NotFound, "could not find secret %s", request.SecretID)
}
// Check if any services currently reference this secret, return error if so
services, err := store.FindServices(tx, store.ByReferencedSecretID(request.SecretID))
if err != nil {
return grpc.Errorf(codes.Internal, "could not find services using secret %s: %v", request.SecretID, err)
return status.Errorf(codes.Internal, "could not find services using secret %s: %v", request.SecretID, err)
}
if len(services) != 0 {
@@ -221,14 +221,14 @@ func (s *Server) RemoveSecret(ctx context.Context, request *api.RemoveSecretRequ
serviceStr = "service"
}
return grpc.Errorf(codes.InvalidArgument, "secret '%s' is in use by the following %s: %v", secretName, serviceStr, serviceNameStr)
return status.Errorf(codes.InvalidArgument, "secret '%s' is in use by the following %s: %v", secretName, serviceStr, serviceNameStr)
}
return store.DeleteSecret(tx, request.SecretID)
})
switch err {
case store.ErrNotExist:
return nil, grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID)
return nil, status.Errorf(codes.NotFound, "secret %s not found", request.SecretID)
case nil:
log.G(ctx).WithFields(logrus.Fields{
"secret.ID": request.SecretID,
@@ -243,7 +243,7 @@ func (s *Server) RemoveSecret(ctx context.Context, request *api.RemoveSecretRequ
func validateSecretSpec(spec *api.SecretSpec) error {
if spec == nil {
return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateConfigOrSecretAnnotations(spec.Annotations); err != nil {
return err
@@ -252,12 +252,12 @@ func validateSecretSpec(spec *api.SecretSpec) error {
if spec.Driver != nil {
// Ensure secret driver has a name
if spec.Driver.Name == "" {
return grpc.Errorf(codes.InvalidArgument, "secret driver must have a name")
return status.Errorf(codes.InvalidArgument, "secret driver must have a name")
}
return nil
}
if err := validation.ValidateSecretPayload(spec.Data); err != nil {
return grpc.Errorf(codes.InvalidArgument, "%s", err.Error())
return status.Errorf(codes.InvalidArgument, "%s", err.Error())
}
return nil
}
@@ -19,8 +19,8 @@ import (
"github.com/docker/swarmkit/template"
gogotypes "github.com/gogo/protobuf/types"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
var (
@@ -37,11 +37,11 @@ func validateResources(r *api.Resources) error {
}
if r.NanoCPUs != 0 && r.NanoCPUs < 1e6 {
return grpc.Errorf(codes.InvalidArgument, "invalid cpu value %g: Must be at least %g", float64(r.NanoCPUs)/1e9, 1e6/1e9)
return status.Errorf(codes.InvalidArgument, "invalid cpu value %g: Must be at least %g", float64(r.NanoCPUs)/1e9, 1e6/1e9)
}
if r.MemoryBytes != 0 && r.MemoryBytes < 4*1024*1024 {
return grpc.Errorf(codes.InvalidArgument, "invalid memory value %d: Must be at least 4MiB", r.MemoryBytes)
return status.Errorf(codes.InvalidArgument, "invalid memory value %d: Must be at least 4MiB", r.MemoryBytes)
}
if err := genericresource.ValidateTask(r); err != nil {
return nil
@@ -70,7 +70,7 @@ func validateRestartPolicy(rp *api.RestartPolicy) error {
return err
}
if delay < 0 {
return grpc.Errorf(codes.InvalidArgument, "TaskSpec: restart-delay cannot be negative")
return status.Errorf(codes.InvalidArgument, "TaskSpec: restart-delay cannot be negative")
}
}
@@ -80,7 +80,7 @@ func validateRestartPolicy(rp *api.RestartPolicy) error {
return err
}
if win < 0 {
return grpc.Errorf(codes.InvalidArgument, "TaskSpec: restart-window cannot be negative")
return status.Errorf(codes.InvalidArgument, "TaskSpec: restart-window cannot be negative")
}
}
@@ -101,7 +101,7 @@ func validateUpdate(uc *api.UpdateConfig) error {
}
if uc.Delay < 0 {
return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-delay cannot be negative")
return status.Errorf(codes.InvalidArgument, "TaskSpec: update-delay cannot be negative")
}
if uc.Monitor != nil {
@@ -110,12 +110,12 @@ func validateUpdate(uc *api.UpdateConfig) error {
return err
}
if monitor < 0 {
return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-monitor cannot be negative")
return status.Errorf(codes.InvalidArgument, "TaskSpec: update-monitor cannot be negative")
}
}
if uc.MaxFailureRatio < 0 || uc.MaxFailureRatio > 1 {
return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-maxfailureratio cannot be less than 0 or bigger than 1")
return status.Errorf(codes.InvalidArgument, "TaskSpec: update-maxfailureratio cannot be less than 0 or bigger than 1")
}
return nil
@@ -147,7 +147,7 @@ func validateContainerSpec(taskSpec api.TaskSpec) error {
LogDriver: taskSpec.LogDriver,
})
if err != nil {
return grpc.Errorf(codes.InvalidArgument, err.Error())
return status.Errorf(codes.InvalidArgument, err.Error())
}
if err := validateImage(container.Image); err != nil {
@@ -164,11 +164,11 @@ func validateContainerSpec(taskSpec api.TaskSpec) error {
// validateImage validates image name in containerSpec
func validateImage(image string) error {
if image == "" {
return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: image reference must be provided")
return status.Errorf(codes.InvalidArgument, "ContainerSpec: image reference must be provided")
}
if _, err := reference.ParseNormalizedNamed(image); err != nil {
return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: %q is not a valid repository/tag", image)
return status.Errorf(codes.InvalidArgument, "ContainerSpec: %q is not a valid repository/tag", image)
}
return nil
}
@@ -178,7 +178,7 @@ func validateMounts(mounts []api.Mount) error {
mountMap := make(map[string]bool)
for _, mount := range mounts {
if _, exists := mountMap[mount.Target]; exists {
return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: duplicate mount point: %s", mount.Target)
return status.Errorf(codes.InvalidArgument, "ContainerSpec: duplicate mount point: %s", mount.Target)
}
mountMap[mount.Target] = true
}
@@ -198,7 +198,7 @@ func validateHealthCheck(hc *api.HealthConfig) error {
return err
}
if interval != 0 && interval < time.Duration(minimumDuration) {
return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: Interval in HealthConfig cannot be less than %s", minimumDuration)
return status.Errorf(codes.InvalidArgument, "ContainerSpec: Interval in HealthConfig cannot be less than %s", minimumDuration)
}
}
@@ -208,7 +208,7 @@ func validateHealthCheck(hc *api.HealthConfig) error {
return err
}
if timeout != 0 && timeout < time.Duration(minimumDuration) {
return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: Timeout in HealthConfig cannot be less than %s", minimumDuration)
return status.Errorf(codes.InvalidArgument, "ContainerSpec: Timeout in HealthConfig cannot be less than %s", minimumDuration)
}
}
@@ -218,12 +218,12 @@ func validateHealthCheck(hc *api.HealthConfig) error {
return err
}
if sp != 0 && sp < time.Duration(minimumDuration) {
return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: StartPeriod in HealthConfig cannot be less than %s", minimumDuration)
return status.Errorf(codes.InvalidArgument, "ContainerSpec: StartPeriod in HealthConfig cannot be less than %s", minimumDuration)
}
}
if hc.Retries < 0 {
return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: Retries in HealthConfig cannot be negative")
return status.Errorf(codes.InvalidArgument, "ContainerSpec: Retries in HealthConfig cannot be negative")
}
return nil
@@ -233,28 +233,28 @@ func validateGenericRuntimeSpec(taskSpec api.TaskSpec) error {
generic := taskSpec.GetGeneric()
if len(generic.Kind) < 3 {
return grpc.Errorf(codes.InvalidArgument, "Generic runtime: Invalid name %q", generic.Kind)
return status.Errorf(codes.InvalidArgument, "Generic runtime: Invalid name %q", generic.Kind)
}
reservedNames := []string{"container", "attachment"}
for _, n := range reservedNames {
if strings.ToLower(generic.Kind) == n {
return grpc.Errorf(codes.InvalidArgument, "Generic runtime: %q is a reserved name", generic.Kind)
return status.Errorf(codes.InvalidArgument, "Generic runtime: %q is a reserved name", generic.Kind)
}
}
payload := generic.Payload
if payload == nil {
return grpc.Errorf(codes.InvalidArgument, "Generic runtime is missing payload")
return status.Errorf(codes.InvalidArgument, "Generic runtime is missing payload")
}
if payload.TypeUrl == "" {
return grpc.Errorf(codes.InvalidArgument, "Generic runtime is missing payload type")
return status.Errorf(codes.InvalidArgument, "Generic runtime is missing payload type")
}
if len(payload.Value) == 0 {
return grpc.Errorf(codes.InvalidArgument, "Generic runtime has an empty payload")
return status.Errorf(codes.InvalidArgument, "Generic runtime has an empty payload")
}
return nil
@@ -284,7 +284,7 @@ func validateTaskSpec(taskSpec api.TaskSpec) error {
}
if taskSpec.GetRuntime() == nil {
return grpc.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime")
return status.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime")
}
switch taskSpec.GetRuntime().(type) {
@@ -297,7 +297,7 @@ func validateTaskSpec(taskSpec api.TaskSpec) error {
return err
}
default:
return grpc.Errorf(codes.Unimplemented, "RuntimeSpec: unimplemented runtime in service spec")
return status.Errorf(codes.Unimplemented, "RuntimeSpec: unimplemented runtime in service spec")
}
return nil
@@ -324,7 +324,7 @@ func validateEndpointSpec(epSpec *api.EndpointSpec) error {
// for the backend network and hence we accept that configuration.
if epSpec.Mode == api.ResolutionModeDNSRoundRobin && port.PublishMode == api.PublishModeIngress {
return grpc.Errorf(codes.InvalidArgument, "EndpointSpec: port published with ingress mode can't be used with dnsrr mode")
return status.Errorf(codes.InvalidArgument, "EndpointSpec: port published with ingress mode can't be used with dnsrr mode")
}
// If published port is not specified, it does not conflict
@@ -335,7 +335,7 @@ func validateEndpointSpec(epSpec *api.EndpointSpec) error {
portSpec := portSpec{publishedPort: port.PublishedPort, protocol: port.Protocol}
if _, ok := portSet[portSpec]; ok {
return grpc.Errorf(codes.InvalidArgument, "EndpointSpec: duplicate published ports provided")
return status.Errorf(codes.InvalidArgument, "EndpointSpec: duplicate published ports provided")
}
portSet[portSpec] = struct{}{}
@@ -358,23 +358,23 @@ func validateSecretRefsSpec(spec api.TaskSpec) error {
for _, secretRef := range container.Secrets {
// SecretID and SecretName are mandatory, we have invalid references without them
if secretRef.SecretID == "" || secretRef.SecretName == "" {
return grpc.Errorf(codes.InvalidArgument, "malformed secret reference")
return status.Errorf(codes.InvalidArgument, "malformed secret reference")
}
// Every secret reference requires a Target
if secretRef.GetTarget() == nil {
return grpc.Errorf(codes.InvalidArgument, "malformed secret reference, no target provided")
return status.Errorf(codes.InvalidArgument, "malformed secret reference, no target provided")
}
// If this is a file target, we will ensure filename uniqueness
if secretRef.GetFile() != nil {
fileName := secretRef.GetFile().Name
if fileName == "" {
return grpc.Errorf(codes.InvalidArgument, "malformed file secret reference, invalid target file name provided")
return status.Errorf(codes.InvalidArgument, "malformed file secret reference, invalid target file name provided")
}
// If this target is already in use, we have conflicting targets
if prevSecretName, ok := existingTargets[fileName]; ok {
return grpc.Errorf(codes.InvalidArgument, "secret references '%s' and '%s' have a conflicting target: '%s'", prevSecretName, secretRef.SecretName, fileName)
return status.Errorf(codes.InvalidArgument, "secret references '%s' and '%s' have a conflicting target: '%s'", prevSecretName, secretRef.SecretName, fileName)
}
existingTargets[fileName] = secretRef.SecretName
@@ -398,12 +398,12 @@ func validateConfigRefsSpec(spec api.TaskSpec) error {
for _, configRef := range container.Configs {
// ConfigID and ConfigName are mandatory, we have invalid references without them
if configRef.ConfigID == "" || configRef.ConfigName == "" {
return grpc.Errorf(codes.InvalidArgument, "malformed config reference")
return status.Errorf(codes.InvalidArgument, "malformed config reference")
}
// Every config reference requires a Target
if configRef.GetTarget() == nil {
return grpc.Errorf(codes.InvalidArgument, "malformed config reference, no target provided")
return status.Errorf(codes.InvalidArgument, "malformed config reference, no target provided")
}
// If this is a file target, we will ensure filename uniqueness
@@ -411,12 +411,12 @@ func validateConfigRefsSpec(spec api.TaskSpec) error {
fileName := configRef.GetFile().Name
// Validate the file name
if fileName == "" {
return grpc.Errorf(codes.InvalidArgument, "malformed file config reference, invalid target file name provided")
return status.Errorf(codes.InvalidArgument, "malformed file config reference, invalid target file name provided")
}
// If this target is already in use, we have conflicting targets
if prevConfigName, ok := existingTargets[fileName]; ok {
return grpc.Errorf(codes.InvalidArgument, "config references '%s' and '%s' have a conflicting target: '%s'", prevConfigName, configRef.ConfigName, fileName)
return status.Errorf(codes.InvalidArgument, "config references '%s' and '%s' have a conflicting target: '%s'", prevConfigName, configRef.ConfigName, fileName)
}
existingTargets[fileName] = configRef.ConfigName
@@ -436,7 +436,7 @@ func (s *Server) validateNetworks(networks []*api.NetworkAttachmentConfig) error
continue
}
if allocator.IsIngressNetwork(network) {
return grpc.Errorf(codes.InvalidArgument,
return status.Errorf(codes.InvalidArgument,
"Service cannot be explicitly attached to the ingress network %q", network.Spec.Annotations.Name)
}
}
@@ -448,11 +448,11 @@ func validateMode(s *api.ServiceSpec) error {
switch m.(type) {
case *api.ServiceSpec_Replicated:
if int64(m.(*api.ServiceSpec_Replicated).Replicated.Replicas) < 0 {
return grpc.Errorf(codes.InvalidArgument, "Number of replicas must be non-negative")
return status.Errorf(codes.InvalidArgument, "Number of replicas must be non-negative")
}
case *api.ServiceSpec_Global:
default:
return grpc.Errorf(codes.InvalidArgument, "Unrecognized service mode")
return status.Errorf(codes.InvalidArgument, "Unrecognized service mode")
}
return nil
@@ -460,7 +460,7 @@ func validateMode(s *api.ServiceSpec) error {
func validateServiceSpec(spec *api.ServiceSpec) error {
if spec == nil {
return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateAnnotations(spec.Annotations); err != nil {
return err
@@ -536,7 +536,7 @@ func (s *Server) checkPortConflicts(spec *api.ServiceSpec, serviceID string) err
switch pc.PublishMode {
case api.PublishModeHost:
if _, ok := ingressPorts[pcToStruct(pc)]; ok {
return grpc.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as a host-published port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID)
return status.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as a host-published port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID)
}
// Multiple services with same port in host publish mode can
@@ -546,7 +546,7 @@ func (s *Server) checkPortConflicts(spec *api.ServiceSpec, serviceID string) err
_, ingressConflict := ingressPorts[pcToStruct(pc)]
_, hostModeConflict := hostModePorts[pcToStruct(pc)]
if ingressConflict || hostModeConflict {
return grpc.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as an ingress port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID)
return status.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s) as an ingress port", pc.PublishedPort, service.Spec.Annotations.Name, service.ID)
}
}
@@ -598,7 +598,7 @@ func (s *Server) checkSecretExistence(tx store.Tx, spec *api.ServiceSpec) error
secretStr = "secret"
}
return grpc.Errorf(codes.InvalidArgument, "%s not found: %v", secretStr, strings.Join(failedSecrets, ", "))
return status.Errorf(codes.InvalidArgument, "%s not found: %v", secretStr, strings.Join(failedSecrets, ", "))
}
@@ -627,7 +627,7 @@ func (s *Server) checkConfigExistence(tx store.Tx, spec *api.ServiceSpec) error
configStr = "config"
}
return grpc.Errorf(codes.InvalidArgument, "%s not found: %v", configStr, strings.Join(failedConfigs, ", "))
return status.Errorf(codes.InvalidArgument, "%s not found: %v", configStr, strings.Join(failedConfigs, ", "))
}
@@ -662,7 +662,7 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe
if allocator.IsIngressNetworkNeeded(service) {
if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress {
return nil, grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present")
return nil, status.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present")
}
}
@@ -694,7 +694,7 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe
// - Returns `NotFound` if the Service is not found.
func (s *Server) GetService(ctx context.Context, request *api.GetServiceRequest) (*api.GetServiceResponse, error) {
if request.ServiceID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
var service *api.Service
@@ -702,7 +702,7 @@ func (s *Server) GetService(ctx context.Context, request *api.GetServiceRequest)
service = store.GetService(tx, request.ServiceID)
})
if service == nil {
return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
return nil, status.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
}
if request.InsertDefaults {
@@ -721,7 +721,7 @@ func (s *Server) GetService(ctx context.Context, request *api.GetServiceRequest)
// - Returns an error if the update fails.
func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRequest) (*api.UpdateServiceResponse, error) {
if request.ServiceID == "" || request.ServiceVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateServiceSpec(request.Spec); err != nil {
return nil, err
@@ -732,7 +732,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe
service = store.GetService(tx, request.ServiceID)
})
if service == nil {
return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
return nil, status.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
}
if request.Spec.Endpoint != nil && !reflect.DeepEqual(request.Spec.Endpoint, service.Spec.Endpoint) {
@@ -744,7 +744,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe
err := s.store.Update(func(tx store.Tx) error {
service = store.GetService(tx, request.ServiceID)
if service == nil {
return grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
return status.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
}
// It's not okay to update Service.Spec.Networks on its own.
@@ -754,7 +754,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe
if (len(request.Spec.Networks) != 0 || len(service.Spec.Networks) != 0) &&
!reflect.DeepEqual(request.Spec.Networks, service.Spec.Networks) &&
reflect.DeepEqual(request.Spec.Task.Networks, service.Spec.Task.Networks) {
return grpc.Errorf(codes.Unimplemented, errNetworkUpdateNotSupported.Error())
return status.Errorf(codes.Unimplemented, errNetworkUpdateNotSupported.Error())
}
// Check to see if all the secrets being added exist as objects
@@ -773,18 +773,18 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe
// with service mode change (comparing current config with previous config).
// proper way to change service mode is to delete and re-add.
if reflect.TypeOf(service.Spec.Mode) != reflect.TypeOf(request.Spec.Mode) {
return grpc.Errorf(codes.Unimplemented, errModeChangeNotAllowed.Error())
return status.Errorf(codes.Unimplemented, errModeChangeNotAllowed.Error())
}
if service.Spec.Annotations.Name != request.Spec.Annotations.Name {
return grpc.Errorf(codes.Unimplemented, errRenameNotSupported.Error())
return status.Errorf(codes.Unimplemented, errRenameNotSupported.Error())
}
service.Meta.Version = *request.ServiceVersion
if request.Rollback == api.UpdateServiceRequest_PREVIOUS {
if service.PreviousSpec == nil {
return grpc.Errorf(codes.FailedPrecondition, "service %s does not have a previous spec", request.ServiceID)
return status.Errorf(codes.FailedPrecondition, "service %s does not have a previous spec", request.ServiceID)
}
curSpec := service.Spec.Copy()
@@ -815,7 +815,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe
if allocator.IsIngressNetworkNeeded(service) {
if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress {
return grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present")
return status.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present")
}
}
@@ -836,7 +836,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe
// - Returns an error if the deletion fails.
func (s *Server) RemoveService(ctx context.Context, request *api.RemoveServiceRequest) (*api.RemoveServiceResponse, error) {
if request.ServiceID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
err := s.store.Update(func(tx store.Tx) error {
@@ -844,7 +844,7 @@ func (s *Server) RemoveService(ctx context.Context, request *api.RemoveServiceRe
})
if err != nil {
if err == store.ErrNotExist {
return nil, grpc.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
return nil, status.Errorf(codes.NotFound, "service %s not found", request.ServiceID)
}
return nil, err
}
@@ -6,8 +6,8 @@ import (
"github.com/docker/swarmkit/manager/orchestrator"
"github.com/docker/swarmkit/manager/state/store"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// GetTask returns a Task given a TaskID.
@@ -15,7 +15,7 @@ import (
// - Returns `NotFound` if the Task is not found.
func (s *Server) GetTask(ctx context.Context, request *api.GetTaskRequest) (*api.GetTaskResponse, error) {
if request.TaskID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
var task *api.Task
@@ -23,7 +23,7 @@ func (s *Server) GetTask(ctx context.Context, request *api.GetTaskRequest) (*api
task = store.GetTask(tx, request.TaskID)
})
if task == nil {
return nil, grpc.Errorf(codes.NotFound, "task %s not found", request.TaskID)
return nil, status.Errorf(codes.NotFound, "task %s not found", request.TaskID)
}
return &api.GetTaskResponse{
Task: task,
@@ -36,7 +36,7 @@ func (s *Server) GetTask(ctx context.Context, request *api.GetTaskRequest) (*api
// - Returns an error if the deletion fails.
func (s *Server) RemoveTask(ctx context.Context, request *api.RemoveTaskRequest) (*api.RemoveTaskResponse, error) {
if request.TaskID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
err := s.store.Update(func(tx store.Tx) error {
@@ -44,7 +44,7 @@ func (s *Server) RemoveTask(ctx context.Context, request *api.RemoveTaskRequest)
})
if err != nil {
if err == store.ErrNotExist {
return nil, grpc.Errorf(codes.NotFound, "task %s not found", request.TaskID)
return nil, status.Errorf(codes.NotFound, "task %s not found", request.TaskID)
}
return nil, err
}
@@ -7,11 +7,8 @@ import (
"sync"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/transport"
"github.com/docker/go-events"
"github.com/docker/go-metrics"
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/api/equality"
"github.com/docker/swarmkit/ca"
@@ -25,6 +22,9 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/grpc/transport"
)
const (
@@ -66,8 +66,18 @@ var (
ErrSessionInvalid = errors.New("session invalid")
// ErrNodeNotFound returned when the Node doesn't exist in raft.
ErrNodeNotFound = errors.New("node not found")
// Scheduling delay timer.
schedulingDelayTimer metrics.Timer
)
func init() {
ns := metrics.NewNamespace("swarm", "dispatcher", nil)
schedulingDelayTimer = ns.NewTimer("scheduling_delay",
"Scheduling delay is the time a task takes to go from NEW to RUNNING state.")
metrics.Register(ns)
}
// Config is configuration for Dispatcher. For default you should use
// DefaultConfig.
type Config struct {
@@ -322,7 +332,7 @@ func (d *Dispatcher) isRunningLocked() (context.Context, error) {
d.mu.Lock()
if !d.isRunning() {
d.mu.Unlock()
return nil, grpc.Errorf(codes.Aborted, "dispatcher is stopped")
return nil, status.Errorf(codes.Aborted, "dispatcher is stopped")
}
ctx := d.ctx
d.mu.Unlock()
@@ -556,7 +566,7 @@ func (d *Dispatcher) UpdateTaskStatus(ctx context.Context, r *api.UpdateTaskStat
}
if t.NodeID != nodeID {
err := grpc.Errorf(codes.PermissionDenied, "cannot update a task not assigned this node")
err := status.Errorf(codes.PermissionDenied, "cannot update a task not assigned this node")
log.WithField("task.id", u.TaskID).Error(err)
return nil, err
}
@@ -632,6 +642,17 @@ func (d *Dispatcher) processUpdates(ctx context.Context) {
return nil
}
// Update scheduling delay metric for running tasks.
// We use the status update time on the leader to calculate the scheduling delay.
// Because of this, the recorded scheduling delay will be an overestimate and include
// the network delay between the worker and the leader.
// This is not ideal, but its a known overestimation, rather than using the status update time
// from the worker node, which may cause unknown incorrect results due to possible clock skew.
if status.State == api.TaskStateRunning {
start := time.Unix(status.AppliedAt.GetSeconds(), int64(status.AppliedAt.GetNanos()))
schedulingDelayTimer.UpdateSince(start)
}
task.Status = *status
task.Status.AppliedBy = d.securityConfig.ClientTLSCreds.NodeID()
task.Status.AppliedAt = ptypes.MustTimestampProto(time.Now())
@@ -1189,7 +1210,7 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio
log.WithError(err).Error("failed to remove node")
}
// still return an abort if the transport closure was ineffective.
return grpc.Errorf(codes.Aborted, "node must disconnect")
return status.Errorf(codes.Aborted, "node must disconnect")
}
for {
@@ -4,12 +4,11 @@ import (
"sync"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/identity"
"github.com/docker/swarmkit/manager/dispatcher/heartbeat"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const rateLimitCount = 3
@@ -36,7 +35,7 @@ func (rn *registeredNode) checkSessionID(sessionID string) error {
// changed. If it has, we will the stream and make the node
// re-register.
if sessionID == "" || rn.SessionID != sessionID {
return grpc.Errorf(codes.InvalidArgument, ErrSessionInvalid.Error())
return status.Errorf(codes.InvalidArgument, ErrSessionInvalid.Error())
}
return nil
@@ -97,7 +96,7 @@ func (s *nodeStore) CheckRateLimit(id string) error {
}
existRn.Attempts++
if existRn.Attempts > rateLimitCount {
return grpc.Errorf(codes.Unavailable, "node %s exceeded rate limit count of registrations", id)
return status.Errorf(codes.Unavailable, "node %s exceeded rate limit count of registrations", id)
}
existRn.Registered = time.Now()
}
@@ -136,7 +135,7 @@ func (s *nodeStore) Get(id string) (*registeredNode, error) {
rn, ok := s.nodes[id]
s.mu.RUnlock()
if !ok {
return nil, grpc.Errorf(codes.NotFound, ErrNodeNotRegistered.Error())
return nil, status.Errorf(codes.NotFound, ErrNodeNotRegistered.Error())
}
return rn, nil
}
@@ -146,7 +145,7 @@ func (s *nodeStore) GetWithSession(id, sid string) (*registeredNode, error) {
rn, ok := s.nodes[id]
s.mu.RUnlock()
if !ok {
return nil, grpc.Errorf(codes.NotFound, ErrNodeNotRegistered.Error())
return nil, status.Errorf(codes.NotFound, ErrNodeNotRegistered.Error())
}
return rn, rn.checkSessionID(sid)
}
@@ -12,8 +12,8 @@ import (
"github.com/docker/swarmkit/api"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// Server represents a Health Check server to check
@@ -46,7 +46,7 @@ func (s *Server) Check(ctx context.Context, in *api.HealthCheckRequest) (*api.He
Status: status,
}, nil
}
return nil, grpc.Errorf(codes.NotFound, "unknown service")
return nil, status.Errorf(codes.NotFound, "unknown service")
}
// SetServingStatus is called when need to reset the serving status of a service
@@ -6,9 +6,6 @@ import (
"io"
"sync"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"github.com/docker/go-events"
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/ca"
@@ -18,6 +15,8 @@ import (
"github.com/docker/swarmkit/watch"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
var (
@@ -93,11 +92,11 @@ func (lb *LogBroker) Stop() error {
func validateSelector(selector *api.LogSelector) error {
if selector == nil {
return grpc.Errorf(codes.InvalidArgument, "log selector must be provided")
return status.Errorf(codes.InvalidArgument, "log selector must be provided")
}
if len(selector.ServiceIDs) == 0 && len(selector.TaskIDs) == 0 && len(selector.NodeIDs) == 0 {
return grpc.Errorf(codes.InvalidArgument, "log selector must not be empty")
return status.Errorf(codes.InvalidArgument, "log selector must not be empty")
}
return nil
@@ -401,17 +400,17 @@ func (lb *LogBroker) PublishLogs(stream api.LogBroker_PublishLogsServer) (err er
}
if logMsg.SubscriptionID == "" {
return grpc.Errorf(codes.InvalidArgument, "missing subscription ID")
return status.Errorf(codes.InvalidArgument, "missing subscription ID")
}
if currentSubscription == nil {
currentSubscription = lb.getSubscription(logMsg.SubscriptionID)
if currentSubscription == nil {
return grpc.Errorf(codes.NotFound, "unknown subscription ID")
return status.Errorf(codes.NotFound, "unknown subscription ID")
}
} else {
if logMsg.SubscriptionID != currentSubscription.message.ID {
return grpc.Errorf(codes.InvalidArgument, "different subscription IDs in the same session")
return status.Errorf(codes.InvalidArgument, "different subscription IDs in the same session")
}
}
@@ -427,7 +426,7 @@ func (lb *LogBroker) PublishLogs(stream api.LogBroker_PublishLogsServer) (err er
// Make sure logs are emitted using the right Node ID to avoid impersonation.
for _, msg := range logMsg.Messages {
if msg.Context.NodeID != remote.NodeID {
return grpc.Errorf(codes.PermissionDenied, "invalid NodeID: expected=%s;received=%s", remote.NodeID, msg.Context.NodeID)
return status.Errorf(codes.PermissionDenied, "invalid NodeID: expected=%s;received=%s", remote.NodeID, msg.Context.NodeID)
}
}
+9 -6
View File
@@ -38,6 +38,7 @@ import (
"github.com/docker/swarmkit/manager/resourceapi"
"github.com/docker/swarmkit/manager/scheduler"
"github.com/docker/swarmkit/manager/state/raft"
"github.com/docker/swarmkit/manager/state/raft/transport"
"github.com/docker/swarmkit/manager/state/store"
"github.com/docker/swarmkit/manager/watchapi"
"github.com/docker/swarmkit/remotes"
@@ -54,9 +55,6 @@ import (
const (
// defaultTaskHistoryRetentionLimit is the number of tasks to keep.
defaultTaskHistoryRetentionLimit = 5
// Default value for grpc max message size.
grpcMaxMessageSize = 128 << 20
)
// RemoteAddrs provides a listening address and an optional advertise address
@@ -234,7 +232,7 @@ func New(config *Config) (*Manager, error) {
grpc.Creds(config.SecurityConfig.ServerTLSCreds),
grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
grpc.MaxMsgSize(grpcMaxMessageSize),
grpc.MaxMsgSize(transport.GRPCMaxMsgSize),
}
m := &Manager{
@@ -404,7 +402,7 @@ func (m *Manager) Run(parent context.Context) error {
)
m.raftNode.MemoryStore().View(func(readTx store.ReadTx) {
clusters, err = store.FindClusters(readTx, store.ByName("default"))
clusters, err = store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
})
@@ -954,13 +952,18 @@ func (m *Manager) becomeLeader(ctx context.Context) {
// store. Don't check the error because
// we expect this to fail unless this
// is a brand new cluster.
store.CreateCluster(tx, defaultClusterObject(
err := store.CreateCluster(tx, defaultClusterObject(
clusterID,
initialCAConfig,
raftCfg,
api.EncryptionConfig{AutoLockManagers: m.config.AutoLockManagers},
unlockKeys,
rootCA))
if err != nil && err != store.ErrExist {
log.G(ctx).WithError(err).Errorf("error creating cluster object")
}
// Add Node entry for ourself, if one
// doesn't exist already.
freshCluster := nil == store.CreateNode(tx, managerNode(nodeID, m.config.Availability))
@@ -73,7 +73,7 @@ func (g *Orchestrator) Run(ctx context.Context) error {
var err error
g.store.View(func(readTx store.ReadTx) {
var clusters []*api.Cluster
clusters, err = store.FindClusters(readTx, store.ByName("default"))
clusters, err = store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
if len(clusters) != 1 {
return // just pick up the cluster when it is created.
@@ -147,7 +147,7 @@ func (g *Orchestrator) Run(ctx context.Context) error {
if !orchestrator.IsGlobalService(v.Service) {
continue
}
orchestrator.DeleteServiceTasks(ctx, g.store, v.Service)
orchestrator.SetServiceTasksRemove(ctx, g.store, v.Service)
// delete the service from service map
delete(g.globalServices, v.Service.ID)
g.restarts.ClearServiceHistory(v.Service.ID)
@@ -17,7 +17,7 @@ import (
// responds to changes in individual tasks (or nodes which run them).
func (r *Orchestrator) initCluster(readTx store.ReadTx) error {
clusters, err := store.FindClusters(readTx, store.ByName("default"))
clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
if err != nil {
return err
}
@@ -50,7 +50,7 @@ func (r *Orchestrator) handleServiceEvent(ctx context.Context, event events.Even
if !orchestrator.IsReplicatedService(v.Service) {
return
}
orchestrator.DeleteServiceTasks(ctx, r.store, v.Service)
orchestrator.SetServiceTasksRemove(ctx, r.store, v.Service)
r.restarts.ClearServiceHistory(v.Service.ID)
delete(r.reconcileServices, v.Service.ID)
case api.EventCreateService:
@@ -86,6 +86,12 @@ func (r *Orchestrator) resolveService(ctx context.Context, task *api.Task) *api.
return service
}
// reconcile decides what actions must be taken depending on the number of
// specificed slots and actual running slots. If the actual running slots are
// fewer than what is requested, it creates new tasks. If the actual running
// slots are more than requested, then it decides which slots must be removed
// and sets desired state of those tasks to REMOVE (the actual removal is handled
// by the task reaper, after the agent shuts the tasks down).
func (r *Orchestrator) reconcile(ctx context.Context, service *api.Service) {
runningSlots, deadSlots, err := r.updatableAndDeadSlots(ctx, service)
if err != nil {
@@ -157,7 +163,11 @@ func (r *Orchestrator) reconcile(ctx context.Context, service *api.Service) {
r.updater.Update(ctx, r.cluster, service, sortedSlots[:specifiedSlots])
err = r.store.Batch(func(batch *store.Batch) error {
r.deleteTasksMap(ctx, batch, deadSlots)
r.deleteTasks(ctx, batch, sortedSlots[specifiedSlots:])
// for all slots that we are removing, we set the desired state of those tasks
// to REMOVE. Then, the agent is responsible for shutting them down, and the
// task reaper is responsible for actually removing them from the store after
// shutdown.
r.setTasksDesiredState(ctx, batch, sortedSlots[specifiedSlots:], api.TaskStateRemove)
return nil
})
if err != nil {
@@ -198,10 +208,34 @@ func (r *Orchestrator) addTasks(ctx context.Context, batch *store.Batch, service
}
}
func (r *Orchestrator) deleteTasks(ctx context.Context, batch *store.Batch, slots []orchestrator.Slot) {
// setTasksDesiredState sets the desired state for all tasks for the given slots to the
// requested state
func (r *Orchestrator) setTasksDesiredState(ctx context.Context, batch *store.Batch, slots []orchestrator.Slot, newDesiredState api.TaskState) {
for _, slot := range slots {
for _, t := range slot {
r.deleteTask(ctx, batch, t)
err := batch.Update(func(tx store.Tx) error {
// time travel is not allowed. if the current desired state is
// above the one we're trying to go to we can't go backwards.
// we have nothing to do and we should skip to the next task
if t.DesiredState > newDesiredState {
// log a warning, though. we shouln't be trying to rewrite
// a state to an earlier state
log.G(ctx).Warnf(
"cannot update task %v in desired state %v to an earlier desired state %v",
t.ID, t.DesiredState, newDesiredState,
)
return nil
}
// update desired state
t.DesiredState = newDesiredState
return store.UpdateTask(tx, t)
})
// log an error if we get one
if err != nil {
log.G(ctx).WithError(err).Errorf("failed to update task to %v", newDesiredState.String())
}
}
}
}
@@ -27,8 +27,10 @@ func IsGlobalService(service *api.Service) bool {
return ok
}
// DeleteServiceTasks deletes the tasks associated with a service.
func DeleteServiceTasks(ctx context.Context, s *store.MemoryStore, service *api.Service) {
// SetServiceTasksRemove sets the desired state of tasks associated with a service
// to REMOVE, so that they can be properly shut down by the agent and later removed
// by the task reaper.
func SetServiceTasksRemove(ctx context.Context, s *store.MemoryStore, service *api.Service) {
var (
tasks []*api.Task
err error
@@ -44,8 +46,23 @@ func DeleteServiceTasks(ctx context.Context, s *store.MemoryStore, service *api.
err = s.Batch(func(batch *store.Batch) error {
for _, t := range tasks {
err := batch.Update(func(tx store.Tx) error {
if err := store.DeleteTask(tx, t.ID); err != nil {
log.G(ctx).WithError(err).Errorf("failed to delete task")
// time travel is not allowed. if the current desired state is
// above the one we're trying to go to we can't go backwards.
// we have nothing to do and we should skip to the next task
if t.DesiredState > api.TaskStateRemove {
// log a warning, though. we shouln't be trying to rewrite
// a state to an earlier state
log.G(ctx).Warnf(
"cannot update task %v in desired state %v to an earlier desired state %v",
t.ID, t.DesiredState, api.TaskStateRemove,
)
return nil
}
// update desired state to REMOVE
t.DesiredState = api.TaskStateRemove
if err := store.UpdateTask(tx, t); err != nil {
log.G(ctx).WithError(err).Errorf("failed transaction: update task desired state to REMOVE")
}
return nil
})
@@ -23,12 +23,19 @@ const (
// exist for the same service/instance or service/nodeid combination.
type TaskReaper struct {
store *store.MemoryStore
// taskHistory is the number of tasks to keep
taskHistory int64
dirty map[orchestrator.SlotTuple]struct{}
orphaned []string
stopChan chan struct{}
doneChan chan struct{}
// List of slot tubles to be inspected for task history cleanup.
dirty map[orchestrator.SlotTuple]struct{}
// List of tasks collected for cleanup, which includes two kinds of tasks
// - serviceless orphaned tasks
// - tasks with desired state REMOVE that have already been shut down
cleanup []string
stopChan chan struct{}
doneChan chan struct{}
}
// New creates a new TaskReaper.
@@ -41,7 +48,13 @@ func New(store *store.MemoryStore) *TaskReaper {
}
}
// Run is the TaskReaper's main loop.
// Run is the TaskReaper's watch loop which collects candidates for cleanup.
// Task history is mainly used in task restarts but is also available for administrative purposes.
// Note that the task history is stored per-slot-per-service for replicated services
// and per-node-per-service for global services. History does not apply to serviceless
// since they are not attached to a service. In addition, the TaskReaper watch loop is also
// responsible for cleaning up tasks associated with slots that were removed as part of
// service scale down or service removal.
func (tr *TaskReaper) Run(ctx context.Context) {
watcher, watchCancel := state.Watch(tr.store.WatchQueue(), api.EventCreateTask{}, api.EventUpdateTask{}, api.EventUpdateCluster{})
@@ -50,7 +63,8 @@ func (tr *TaskReaper) Run(ctx context.Context) {
watchCancel()
}()
var tasks []*api.Task
var orphanedTasks []*api.Task
var removeTasks []*api.Task
tr.store.View(func(readTx store.ReadTx) {
var err error
@@ -59,29 +73,54 @@ func (tr *TaskReaper) Run(ctx context.Context) {
tr.taskHistory = clusters[0].Spec.Orchestration.TaskHistoryRetentionLimit
}
tasks, err = store.FindTasks(readTx, store.ByTaskState(api.TaskStateOrphaned))
// On startup, scan the entire store and inspect orphaned tasks from previous life.
orphanedTasks, err = store.FindTasks(readTx, store.ByTaskState(api.TaskStateOrphaned))
if err != nil {
log.G(ctx).WithError(err).Error("failed to find Orphaned tasks in task reaper init")
}
removeTasks, err = store.FindTasks(readTx, store.ByDesiredState(api.TaskStateRemove))
if err != nil {
log.G(ctx).WithError(err).Error("failed to find tasks with desired state REMOVE in task reaper init")
}
})
if len(tasks) > 0 {
for _, t := range tasks {
// Do not reap service tasks immediately
if len(orphanedTasks)+len(removeTasks) > 0 {
for _, t := range orphanedTasks {
// Do not reap service tasks immediately.
// Let them go through the regular history cleanup process
// of checking TaskHistoryRetentionLimit.
if t.ServiceID != "" {
continue
}
tr.orphaned = append(tr.orphaned, t.ID)
// Serviceless tasks can be cleaned up right away since they are not attached to a service.
tr.cleanup = append(tr.cleanup, t.ID)
}
if len(tr.orphaned) > 0 {
// tasks with desired state REMOVE that have progressed beyond SHUTDOWN can be cleaned up
// right away
for _, t := range removeTasks {
if t.Status.State >= api.TaskStateShutdown {
tr.cleanup = append(tr.cleanup, t.ID)
}
}
// Clean up tasks in 'cleanup' right away
if len(tr.cleanup) > 0 {
tr.tick()
}
}
// Clean up when we hit TaskHistoryRetentionLimit or when the timer expires,
// whichever happens first.
timer := time.NewTimer(reaperBatchingInterval)
// Watch for:
// 1. EventCreateTask for cleaning slots, which is the best time to cleanup that node/slot.
// 2. EventUpdateTask for cleaning
// - serviceless orphaned tasks (when orchestrator updates the task status to ORPHANED)
// - tasks which have desired state REMOVE and have been shut down by the agent
// (these are tasks which are associated with slots removed as part of service
// remove or scale down)
// 3. EventUpdateCluster for TaskHistoryRetentionLimit update.
for {
select {
case event := <-watcher:
@@ -95,14 +134,21 @@ func (tr *TaskReaper) Run(ctx context.Context) {
}] = struct{}{}
case api.EventUpdateTask:
t := v.Task
// add serviceless orphaned tasks
if t.Status.State >= api.TaskStateOrphaned && t.ServiceID == "" {
tr.orphaned = append(tr.orphaned, t.ID)
tr.cleanup = append(tr.cleanup, t.ID)
}
// add tasks that have progressed beyond SHUTDOWN and have desired state REMOVE. These
// tasks are associated with slots that were removed as part of a service scale down
// or service removal.
if t.DesiredState == api.TaskStateRemove && t.Status.State >= api.TaskStateShutdown {
tr.cleanup = append(tr.cleanup, t.ID)
}
case api.EventUpdateCluster:
tr.taskHistory = v.Cluster.Spec.Orchestration.TaskHistoryRetentionLimit
}
if len(tr.dirty)+len(tr.orphaned) > maxDirty {
if len(tr.dirty)+len(tr.cleanup) > maxDirty {
timer.Stop()
tr.tick()
} else {
@@ -118,19 +164,22 @@ func (tr *TaskReaper) Run(ctx context.Context) {
}
}
// tick performs task history cleanup.
func (tr *TaskReaper) tick() {
if len(tr.dirty) == 0 && len(tr.orphaned) == 0 {
if len(tr.dirty) == 0 && len(tr.cleanup) == 0 {
return
}
defer func() {
tr.orphaned = nil
tr.cleanup = nil
}()
deleteTasks := make(map[string]struct{})
for _, tID := range tr.orphaned {
for _, tID := range tr.cleanup {
deleteTasks[tID] = struct{}{}
}
// Check history of dirty tasks for cleanup.
tr.store.View(func(tx store.ReadTx) {
for dirty := range tr.dirty {
service := store.GetService(tx, dirty.ServiceID)
@@ -141,8 +190,8 @@ func (tr *TaskReaper) tick() {
taskHistory := tr.taskHistory
// If MaxAttempts is set, keep at least one more than
// that number of tasks. This is necessary reconstruct
// restart history when the orchestrator starts up.
// that number of tasks (this overrides TaskHistoryRetentionLimit).
// This is necessary to reconstruct restart history when the orchestrator starts up.
// TODO(aaronl): Consider hiding tasks beyond the normal
// retention limit in the UI.
// TODO(aaronl): There are some ways to cut down the
@@ -156,6 +205,7 @@ func (tr *TaskReaper) tick() {
taskHistory = int64(service.Spec.Task.Restart.MaxAttempts) + 1
}
// Negative value for TaskHistoryRetentionLimit is an indication to never clean up task history.
if taskHistory < 0 {
continue
}
@@ -164,6 +214,7 @@ func (tr *TaskReaper) tick() {
switch service.Spec.GetMode().(type) {
case *api.ServiceSpec_Replicated:
// Clean out the slot for which we received EventCreateTask.
var err error
historicTasks, err = store.FindTasks(tx, store.BySlot(dirty.ServiceID, dirty.Slot))
if err != nil {
@@ -171,6 +222,7 @@ func (tr *TaskReaper) tick() {
}
case *api.ServiceSpec_Global:
// Clean out the node history in case of global services.
tasksByNode, err := store.FindTasks(tx, store.ByNodeID(dirty.NodeID))
if err != nil {
continue
@@ -215,6 +267,7 @@ func (tr *TaskReaper) tick() {
}
})
// Perform cleanup.
if len(deleteTasks) > 0 {
tr.store.Batch(func(batch *store.Batch) error {
for taskID := range deleteTasks {
@@ -10,8 +10,8 @@ import (
"github.com/docker/swarmkit/manager/state/store"
"github.com/docker/swarmkit/protobuf/ptypes"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
var (
@@ -50,11 +50,11 @@ func (ra *ResourceAllocator) AttachNetwork(ctx context.Context, request *api.Att
}
})
if network == nil {
return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.Config.Target)
return nil, status.Errorf(codes.NotFound, "network %s not found", request.Config.Target)
}
if !network.Spec.Attachable {
return nil, grpc.Errorf(codes.PermissionDenied, "network %s not manually attachable", request.Config.Target)
return nil, status.Errorf(codes.PermissionDenied, "network %s not manually attachable", request.Config.Target)
}
t := &api.Task{
@@ -98,7 +98,7 @@ func (ra *ResourceAllocator) AttachNetwork(ctx context.Context, request *api.Att
// - Returns an error if the deletion fails.
func (ra *ResourceAllocator) DetachNetwork(ctx context.Context, request *api.DetachNetworkRequest) (*api.DetachNetworkResponse, error) {
if request.AttachmentID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
return nil, status.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
nodeInfo, err := ca.RemoteNode(ctx)
@@ -109,10 +109,10 @@ func (ra *ResourceAllocator) DetachNetwork(ctx context.Context, request *api.Det
if err := ra.store.Update(func(tx store.Tx) error {
t := store.GetTask(tx, request.AttachmentID)
if t == nil {
return grpc.Errorf(codes.NotFound, "attachment %s not found", request.AttachmentID)
return status.Errorf(codes.NotFound, "attachment %s not found", request.AttachmentID)
}
if t.NodeID != nodeInfo.NodeID {
return grpc.Errorf(codes.PermissionDenied, "attachment %s doesn't belong to this node", request.AttachmentID)
return status.Errorf(codes.PermissionDenied, "attachment %s doesn't belong to this node", request.AttachmentID)
}
return store.DeleteTask(tx, request.AttachmentID)
@@ -169,7 +169,10 @@ func (f *PluginFilter) Check(n *NodeInfo) bool {
}
}
if f.t.Spec.LogDriver != nil && f.t.Spec.LogDriver.Name != "none" {
// It's possible that the LogDriver object does not carry a name, just some
// configuration options. In that case, the plugin filter shouldn't fail to
// schedule the task
if f.t.Spec.LogDriver != nil && f.t.Spec.LogDriver.Name != "none" && f.t.Spec.LogDriver.Name != "" {
// If there are no log driver types in the list at all, most likely this is
// an older daemon that did not report this information. In this case don't filter
if typeFound, exists := f.pluginExistsOnNode("Log", f.t.Spec.LogDriver.Name, nodePlugins); !exists && typeFound {
@@ -294,6 +297,14 @@ func (f *PlatformFilter) platformEqual(imgPlatform, nodePlatform api.Platform) b
nodePlatform.Architecture = "amd64"
}
// normalize "aarch64" architectures to "arm64"
if imgPlatform.Architecture == "aarch64" {
imgPlatform.Architecture = "arm64"
}
if nodePlatform.Architecture == "aarch64" {
nodePlatform.Architecture = "arm64"
}
if (imgPlatform.Architecture == "" || imgPlatform.Architecture == nodePlatform.Architecture) && (imgPlatform.OS == "" || imgPlatform.OS == nodePlatform.OS) {
return true
}
@@ -2,6 +2,7 @@ package raft
import (
"fmt"
"io"
"math"
"math/rand"
"net"
@@ -14,6 +15,7 @@ import (
"github.com/coreos/etcd/raft/raftpb"
"github.com/docker/docker/pkg/signal"
"github.com/docker/go-events"
"github.com/docker/go-metrics"
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/ca"
"github.com/docker/swarmkit/log"
@@ -34,6 +36,7 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
)
var (
@@ -62,6 +65,9 @@ var (
// work around lint
lostQuorumMessage = "The swarm does not have a leader. It's possible that too few managers are online. Make sure more than half of the managers are online."
errLostQuorum = errors.New(lostQuorumMessage)
// Timer to capture ProposeValue() latency.
proposeLatencyTimer metrics.Timer
)
// LeadershipState indicates whether the node is a leader or follower.
@@ -180,12 +186,9 @@ type NodeOptions struct {
ClockSource clock.Clock
// SendTimeout is the timeout on the sending messages to other raft
// nodes. Leave this as 0 to get the default value.
SendTimeout time.Duration
// LargeSendTimeout is the timeout on the sending snapshots to other raft
// nodes. Leave this as 0 to get the default value.
LargeSendTimeout time.Duration
TLSCredentials credentials.TransportCredentials
KeyRotator EncryptionKeyRotator
SendTimeout time.Duration
TLSCredentials credentials.TransportCredentials
KeyRotator EncryptionKeyRotator
// DisableStackDump prevents Run from dumping goroutine stacks when the
// store becomes stuck.
DisableStackDump bool
@@ -193,6 +196,9 @@ type NodeOptions struct {
func init() {
rand.Seed(time.Now().UnixNano())
ns := metrics.NewNamespace("swarm", "raft", nil)
proposeLatencyTimer = ns.NewTimer("transaction_latency", "Raft transaction latency.")
metrics.Register(ns)
}
// NewNode generates a new Raft node
@@ -207,11 +213,6 @@ func NewNode(opts NodeOptions) *Node {
if opts.SendTimeout == 0 {
opts.SendTimeout = 2 * time.Second
}
if opts.LargeSendTimeout == 0 {
// a "slow" 100Mbps connection can send over 240MB data in 20 seconds
// which is well over the gRPC message limit of 128MB allowed by SwarmKit
opts.LargeSendTimeout = 20 * time.Second
}
raftStore := raft.NewMemoryStorage()
@@ -357,7 +358,6 @@ func (n *Node) initTransport() {
transportConfig := &transport.Config{
HeartbeatInterval: time.Duration(n.Config.ElectionTick) * n.opts.TickInterval,
SendTimeout: n.opts.SendTimeout,
LargeSendTimeout: n.opts.LargeSendTimeout,
Credentials: n.opts.TLSCredentials,
Raft: n,
}
@@ -664,7 +664,7 @@ func (n *Node) Run(ctx context.Context) error {
if n.snapshotInProgress == nil &&
(n.needsSnapshot(ctx) || raftConfig.SnapshotInterval > 0 &&
n.appliedIndex-n.snapshotMeta.Index >= raftConfig.SnapshotInterval) {
n.doSnapshot(ctx, raftConfig)
n.triggerSnapshot(ctx, raftConfig)
}
if wasLeader && atomic.LoadUint32(&n.signalledLeadership) != 1 {
@@ -706,7 +706,7 @@ func (n *Node) Run(ctx context.Context) error {
// there was a key rotation that took place before while the snapshot
// was in progress - we have to take another snapshot and encrypt with the new key
n.rotationQueued = false
n.doSnapshot(ctx, raftConfig)
n.triggerSnapshot(ctx, raftConfig)
}
case <-n.keyRotator.RotationNotify():
// There are 2 separate checks: rotationQueued, and n.needsSnapshot().
@@ -719,7 +719,7 @@ func (n *Node) Run(ctx context.Context) error {
case n.snapshotInProgress != nil:
n.rotationQueued = true
case n.needsSnapshot(ctx):
n.doSnapshot(ctx, n.getCurrentRaftConfig())
n.triggerSnapshot(ctx, n.getCurrentRaftConfig())
}
case <-ctx.Done():
return nil
@@ -929,11 +929,11 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons
defer n.membershipLock.Unlock()
if !n.IsMember() {
return nil, grpc.Errorf(codes.FailedPrecondition, "%s", ErrNoRaftMember.Error())
return nil, status.Errorf(codes.FailedPrecondition, "%s", ErrNoRaftMember.Error())
}
if !n.isLeader() {
return nil, grpc.Errorf(codes.FailedPrecondition, "%s", ErrLostLeadership.Error())
return nil, status.Errorf(codes.FailedPrecondition, "%s", ErrLostLeadership.Error())
}
remoteAddr := req.Addr
@@ -944,7 +944,7 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons
requestHost, requestPort, err := net.SplitHostPort(remoteAddr)
if err != nil {
return nil, grpc.Errorf(codes.InvalidArgument, "invalid address %s in raft join request", remoteAddr)
return nil, status.Errorf(codes.InvalidArgument, "invalid address %s in raft join request", remoteAddr)
}
requestIP := net.ParseIP(requestHost)
@@ -1118,7 +1118,7 @@ func (n *Node) UpdateNode(id uint64, addr string) {
// membership to an active member of the raft
func (n *Node) Leave(ctx context.Context, req *api.LeaveRequest) (*api.LeaveResponse, error) {
if req.Node == nil {
return nil, grpc.Errorf(codes.InvalidArgument, "no node information provided")
return nil, status.Errorf(codes.InvalidArgument, "no node information provided")
}
nodeInfo, err := ca.RemoteNode(ctx)
@@ -1302,6 +1302,82 @@ func (n *Node) reportNewAddress(ctx context.Context, id uint64) error {
return n.transport.UpdatePeerAddr(id, newAddr)
}
// StreamRaftMessage is the server endpoint for streaming Raft messages.
// It accepts a stream of raft messages to be processed on this raft member,
// returning a StreamRaftMessageResponse when processing of the streamed
// messages is complete.
// It is called from the Raft leader, which uses it to stream messages
// to this raft member.
// A single stream corresponds to a single raft message,
// which may be disassembled and streamed by the sender
// as individual messages. Therefore, each of the messages
// received by the stream will have the same raft message type and index.
// Currently, only messages of type raftpb.MsgSnap can be disassembled, sent
// and received on the stream.
func (n *Node) StreamRaftMessage(stream api.Raft_StreamRaftMessageServer) error {
// recvdMsg is the current messasge received from the stream.
// assembledMessage is where the data from recvdMsg is appended to.
var recvdMsg, assembledMessage *api.StreamRaftMessageRequest
var err error
// First message index.
var raftMsgIndex uint64
for {
recvdMsg, err = stream.Recv()
if err == io.EOF {
break
} else if err != nil {
log.G(stream.Context()).WithError(err).Error("error while reading from stream")
return err
}
// Initialized the message to be used for assembling
// the raft message.
if assembledMessage == nil {
// For all message types except raftpb.MsgSnap,
// we don't expect more than a single message
// on the stream so we'll get an EOF on the next Recv()
// and go on to process the received message.
assembledMessage = recvdMsg
raftMsgIndex = recvdMsg.Message.Index
continue
}
// Verify raft message index.
if recvdMsg.Message.Index != raftMsgIndex {
errMsg := fmt.Sprintf("Raft message chunk with index %d is different from the previously received raft message index %d",
recvdMsg.Message.Index, raftMsgIndex)
log.G(stream.Context()).Errorf(errMsg)
return status.Errorf(codes.InvalidArgument, "%s", errMsg)
}
// Verify that multiple message received on a stream
// can only be of type raftpb.MsgSnap.
if recvdMsg.Message.Type != raftpb.MsgSnap {
errMsg := fmt.Sprintf("Raft message chunk is not of type %d",
raftpb.MsgSnap)
log.G(stream.Context()).Errorf(errMsg)
return status.Errorf(codes.InvalidArgument, "%s", errMsg)
}
// Append the received snapshot data.
assembledMessage.Message.Snapshot.Data = append(assembledMessage.Message.Snapshot.Data, recvdMsg.Message.Snapshot.Data...)
}
// We should have the complete snapshot. Verify and process.
if err == io.EOF {
_, err = n.ProcessRaftMessage(stream.Context(), &api.ProcessRaftMessageRequest{Message: assembledMessage.Message})
if err == nil {
// Translate the response of ProcessRaftMessage() from
// ProcessRaftMessageResponse to StreamRaftMessageResponse if needed.
return stream.SendAndClose(&api.StreamRaftMessageResponse{})
}
}
return err
}
// ProcessRaftMessage calls 'Step' which advances the
// raft state machine with the provided message on the
// receiving node
@@ -1315,7 +1391,7 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa
// a node in the remove set
if n.cluster.IsIDRemoved(msg.Message.From) {
n.processRaftMessageLogger(ctx, msg).Debug("received message from removed member")
return nil, grpc.Errorf(codes.NotFound, "%s", membership.ErrMemberRemoved.Error())
return nil, status.Errorf(codes.NotFound, "%s", membership.ErrMemberRemoved.Error())
}
ctx, cancel := n.WithContext(ctx)
@@ -1393,7 +1469,7 @@ func (n *Node) ResolveAddress(ctx context.Context, msg *api.ResolveAddressReques
member := n.cluster.GetMember(msg.RaftID)
if member == nil {
return nil, grpc.Errorf(codes.NotFound, "member %x not found", msg.RaftID)
return nil, status.Errorf(codes.NotFound, "member %x not found", msg.RaftID)
}
return &api.ResolveAddressResponse{Addr: member.Addr}, nil
}
@@ -1497,9 +1573,11 @@ func (n *Node) registerNode(node *api.RaftMember) error {
// ProposeValue calls Propose on the underlying raft library(etcd/raft) and waits
// on the commit log action before returning a result
func (n *Node) ProposeValue(ctx context.Context, storeAction []api.StoreAction, cb func()) error {
defer metrics.StartTimer(proposeLatencyTimer)()
ctx, cancel := n.WithContext(ctx)
defer cancel()
_, err := n.processInternalRaftRequest(ctx, &api.InternalRaftRequest{Action: storeAction}, cb)
if err != nil {
return err
}
@@ -1808,12 +1886,13 @@ func (n *Node) processEntry(ctx context.Context, entry raftpb.Entry) error {
}
if !n.wait.trigger(r.ID, r) {
log.G(ctx).Errorf("wait not found for raft request id %x", r.ID)
// There was no wait on this ID, meaning we don't have a
// transaction in progress that would be committed to the
// memory store by the "trigger" call. Either a different node
// wrote this to raft, or we wrote it before losing the leader
// memory store by the "trigger" call. This could mean that:
// 1. Startup is in progress, and the raft WAL is being parsed,
// processed and applied to the store, or
// 2. Either a different node wrote this to raft,
// or we wrote it before losing the leader
// position and cancelling the transaction. This entry still needs
// to be committed since other nodes have already committed it.
// Create a new transaction to commit this entry.
@@ -1827,7 +1906,6 @@ func (n *Node) processEntry(ctx context.Context, entry raftpb.Entry) error {
err := n.memoryStore.ApplyStoreActions(r.Action)
if err != nil {
log.G(ctx).WithError(err).Error("failed to apply actions from raft")
// TODO(anshul) return err here ?
}
}
return nil
@@ -5,6 +5,7 @@ import (
"github.com/coreos/etcd/raft"
"github.com/coreos/etcd/raft/raftpb"
"github.com/docker/go-metrics"
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/log"
"github.com/docker/swarmkit/manager/encryption"
@@ -15,6 +16,18 @@ import (
"golang.org/x/net/context"
)
var (
// Snapshot create latency timer.
snapshotLatencyTimer metrics.Timer
)
func init() {
ns := metrics.NewNamespace("swarm", "raft", nil)
snapshotLatencyTimer = ns.NewTimer("snapshot_latency",
"Raft snapshot create latency.")
metrics.Register(ns)
}
func (n *Node) readFromDisk(ctx context.Context) (*raftpb.Snapshot, storage.WALData, error) {
keys := n.keyRotator.GetKeys()
@@ -169,7 +182,7 @@ func (n *Node) newRaftLogs(nodeID string) (raft.Peer, error) {
return raft.Peer{ID: n.Config.ID, Context: metadata}, nil
}
func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
func (n *Node) triggerSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
snapshot := api.Snapshot{Version: api.Snapshot_V0}
for _, member := range n.cluster.Members() {
snapshot.Membership.Members = append(snapshot.Membership.Members,
@@ -185,6 +198,9 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
n.asyncTasks.Add(1)
n.snapshotInProgress = make(chan raftpb.SnapshotMetadata, 1) // buffered in case Shutdown is called during the snapshot
go func(appliedIndex uint64, snapshotMeta raftpb.SnapshotMetadata) {
// Deferred latency capture.
defer metrics.StartTimer(snapshotLatencyTimer)()
defer func() {
n.asyncTasks.Done()
n.snapshotInProgress <- snapshotMeta
@@ -18,6 +18,11 @@ import (
"github.com/pkg/errors"
)
const (
// GRPCMaxMsgSize is the max allowed gRPC message size for raft messages.
GRPCMaxMsgSize = 4 << 20
)
type peer struct {
id uint64
@@ -132,17 +137,112 @@ func (p *peer) resolveAddr(ctx context.Context, id uint64) (string, error) {
return resp.Addr, nil
}
func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error {
timeout := p.tr.config.SendTimeout
// if a snapshot is being sent, set timeout to LargeSendTimeout because
// sending snapshots can take more time than other messages sent between peers.
// The same applies to AppendEntries as well, where messages can get large.
if m.Type == raftpb.MsgSnap || m.Type == raftpb.MsgApp {
timeout = p.tr.config.LargeSendTimeout
// Returns the raft message struct size (not including the payload size) for the given raftpb.Message.
// The payload is typically the snapshot or append entries.
func raftMessageStructSize(m *raftpb.Message) int {
return (&api.ProcessRaftMessageRequest{Message: m}).Size() - len(m.Snapshot.Data)
}
// Returns the max allowable payload based on MaxRaftMsgSize and
// the struct size for the given raftpb.Message.
func raftMessagePayloadSize(m *raftpb.Message) int {
return GRPCMaxMsgSize - raftMessageStructSize(m)
}
// Split a large raft message into smaller messages.
// Currently this means splitting the []Snapshot.Data into chunks whose size
// is dictacted by MaxRaftMsgSize.
func splitSnapshotData(ctx context.Context, m *raftpb.Message) []api.StreamRaftMessageRequest {
var messages []api.StreamRaftMessageRequest
if m.Type != raftpb.MsgSnap {
return messages
}
ctx, cancel := context.WithTimeout(ctx, timeout)
// get the size of the data to be split.
size := len(m.Snapshot.Data)
// Get the max payload size.
payloadSize := raftMessagePayloadSize(m)
// split the snapshot into smaller messages.
for snapDataIndex := 0; snapDataIndex < size; {
chunkSize := size - snapDataIndex
if chunkSize > payloadSize {
chunkSize = payloadSize
}
raftMsg := *m
// sub-slice for this snapshot chunk.
raftMsg.Snapshot.Data = m.Snapshot.Data[snapDataIndex : snapDataIndex+chunkSize]
snapDataIndex += chunkSize
// add message to the list of messages to be sent.
msg := api.StreamRaftMessageRequest{Message: &raftMsg}
messages = append(messages, msg)
}
return messages
}
// Function to check if this message needs to be split to be streamed
// (because it is larger than GRPCMaxMsgSize).
// Returns true if the message type is MsgSnap
// and size larger than MaxRaftMsgSize.
func needsSplitting(m *raftpb.Message) bool {
raftMsg := api.ProcessRaftMessageRequest{Message: m}
return m.Type == raftpb.MsgSnap && raftMsg.Size() > GRPCMaxMsgSize
}
func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error {
ctx, cancel := context.WithTimeout(ctx, p.tr.config.SendTimeout)
defer cancel()
_, err := api.NewRaftClient(p.conn()).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
var err error
var stream api.Raft_StreamRaftMessageClient
stream, err = api.NewRaftClient(p.conn()).StreamRaftMessage(ctx)
if err == nil {
// Split the message if needed.
// Currently only supported for MsgSnap.
var msgs []api.StreamRaftMessageRequest
if needsSplitting(&m) {
msgs = splitSnapshotData(ctx, &m)
} else {
raftMsg := api.StreamRaftMessageRequest{Message: &m}
msgs = append(msgs, raftMsg)
}
// Stream
for _, msg := range msgs {
err = stream.Send(&msg)
if err != nil {
log.G(ctx).WithError(err).Error("error streaming message to peer")
stream.CloseAndRecv()
break
}
}
// Finished sending all the messages.
// Close and receive response.
if err == nil {
_, err = stream.CloseAndRecv()
if err != nil {
log.G(ctx).WithError(err).Error("error receiving response")
}
}
} else {
log.G(ctx).WithError(err).Error("error sending message to peer")
}
// Try doing a regular rpc if the receiver doesn't support streaming.
if grpc.Code(err) == codes.Unimplemented {
_, err = api.NewRaftClient(p.conn()).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
}
// Handle errors.
if grpc.Code(err) == codes.NotFound && grpc.ErrorDesc(err) == membership.ErrMemberRemoved.Error() {
p.tr.config.NodeRemoved()
}
@@ -36,7 +36,6 @@ type Raft interface {
type Config struct {
HeartbeatInterval time.Duration
SendTimeout time.Duration
LargeSendTimeout time.Duration
Credentials credentials.TransportCredentials
RaftID string
@@ -11,6 +11,7 @@ import (
"time"
"github.com/docker/go-events"
"github.com/docker/go-metrics"
"github.com/docker/swarmkit/api"
pb "github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/manager/state"
@@ -76,8 +77,38 @@ var (
// WedgeTimeout is the maximum amount of time the store lock may be
// held before declaring a suspected deadlock.
WedgeTimeout = 30 * time.Second
// update()/write tx latency timer.
updateLatencyTimer metrics.Timer
// view()/read tx latency timer.
viewLatencyTimer metrics.Timer
// lookup() latency timer.
lookupLatencyTimer metrics.Timer
// Batch() latency timer.
batchLatencyTimer metrics.Timer
// timer to capture the duration for which the memory store mutex is locked.
storeLockDurationTimer metrics.Timer
)
func init() {
ns := metrics.NewNamespace("swarm", "store", nil)
updateLatencyTimer = ns.NewTimer("write_tx_latency",
"Raft store write tx latency.")
viewLatencyTimer = ns.NewTimer("read_tx_latency",
"Raft store read tx latency.")
lookupLatencyTimer = ns.NewTimer("lookup_latency",
"Raft store read latency.")
batchLatencyTimer = ns.NewTimer("batch_latency",
"Raft store batch latency.")
storeLockDurationTimer = ns.NewTimer("memory_store_lock_duration",
"Duration for which the raft memory store lock was held.")
metrics.Register(ns)
}
func register(os ObjectStoreConfig) {
objectStorers = append(objectStorers, os)
schema.Tables[os.Table.Name] = os.Table
@@ -94,8 +125,13 @@ func (m *timedMutex) Lock() {
m.lockedAt.Store(time.Now())
}
// Unlocks the timedMutex and captures the duration
// for which it was locked in a metric.
func (m *timedMutex) Unlock() {
unlockedTimestamp := m.lockedAt.Load()
m.Mutex.Unlock()
lockedFor := time.Since(unlockedTimestamp.(time.Time))
storeLockDurationTimer.Update(lockedFor)
m.lockedAt.Store(time.Time{})
}
@@ -184,6 +220,7 @@ type readTx struct {
// View executes a read transaction.
func (s *MemoryStore) View(cb func(ReadTx)) {
defer metrics.StartTimer(viewLatencyTimer)()
memDBTx := s.memDB.Txn(false)
readTx := readTx{
@@ -280,6 +317,7 @@ func applyStoreAction(tx Tx, sa api.StoreAction) error {
}
func (s *MemoryStore) update(proposer state.Proposer, cb func(Tx) error) error {
defer metrics.StartTimer(updateLatencyTimer)()
s.updateLock.Lock()
memDBTx := s.memDB.Txn(true)
@@ -329,7 +367,6 @@ func (s *MemoryStore) update(proposer state.Proposer, cb func(Tx) error) error {
}
s.updateLock.Unlock()
return err
}
func (s *MemoryStore) updateLocal(cb func(Tx) error) error {
@@ -458,6 +495,7 @@ func (batch *Batch) commit() error {
// If Batch returns an error, no guarantees are made about how many updates
// were committed successfully.
func (s *MemoryStore) Batch(cb func(*Batch) error) error {
defer metrics.StartTimer(batchLatencyTimer)()
s.updateLock.Lock()
batch := Batch{
@@ -498,6 +536,7 @@ func (tx tx) changelistStoreActions() ([]api.StoreAction, error) {
// lookup is an internal typed wrapper around memdb.
func (tx readTx) lookup(table, index, id string) api.StoreObject {
defer metrics.StartTimer(lookupLatencyTimer)()
j, err := tx.memDBTx.First(table, index, id)
if err != nil {
return nil
@@ -1,12 +1,11 @@
package watchapi
import (
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"github.com/docker/swarmkit/api"
"github.com/docker/swarmkit/manager/state"
"github.com/docker/swarmkit/manager/state/store"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// Watch starts a stream that returns any changes to objects that match
@@ -26,7 +25,7 @@ func (s *Server) Watch(request *api.WatchRequest, stream api.Watch_WatchServer)
watchArgs, err := api.ConvertWatchArgs(request.Entries)
if err != nil {
return grpc.Errorf(codes.InvalidArgument, "%s", err.Error())
return status.Errorf(codes.InvalidArgument, "%s", err.Error())
}
watchArgs = append(watchArgs, state.EventCommit{})
+6 -4
View File
@@ -38,15 +38,18 @@ Add a new bridge and add eth1 into it:
package main
import (
"net"
"fmt"
"github.com/vishvananda/netlink"
)
func main() {
la := netlink.NewLinkAttrs()
la.Name = "foo"
mybridge := &netlink.Bridge{la}}
_ := netlink.LinkAdd(mybridge)
mybridge := &netlink.Bridge{LinkAttrs: la}
err := netlink.LinkAdd(mybridge)
if err != nil {
fmt.Printf("could not add %s: %v\n", la.Name, err)
}
eth1, _ := netlink.LinkByName("eth1")
netlink.LinkSetMaster(eth1, mybridge)
}
@@ -63,7 +66,6 @@ Add a new ip address to loopback:
package main
import (
"net"
"github.com/vishvananda/netlink"
)
+40 -12
View File
@@ -2,7 +2,6 @@ package netlink
import (
"fmt"
"log"
"net"
"strings"
"syscall"
@@ -65,7 +64,7 @@ func (h *Handle) addrHandle(link Link, addr *Addr, req *nl.NetlinkRequest) error
msg := nl.NewIfAddrmsg(family)
msg.Index = uint32(base.Index)
msg.Scope = uint8(addr.Scope)
prefixlen, _ := addr.Mask.Size()
prefixlen, masklen := addr.Mask.Size()
msg.Prefixlen = uint8(prefixlen)
req.AddData(msg)
@@ -103,9 +102,14 @@ func (h *Handle) addrHandle(link Link, addr *Addr, req *nl.NetlinkRequest) error
}
}
if addr.Broadcast != nil {
req.AddData(nl.NewRtAttr(syscall.IFA_BROADCAST, addr.Broadcast))
if addr.Broadcast == nil {
calcBroadcast := make(net.IP, masklen/8)
for i := range localAddrData {
calcBroadcast[i] = localAddrData[i] | ^addr.Mask[i]
}
addr.Broadcast = calcBroadcast
}
req.AddData(nl.NewRtAttr(syscall.IFA_BROADCAST, addr.Broadcast))
if addr.Label != "" {
labelData := nl.NewRtAttr(syscall.IFA_LABEL, nl.ZeroTerminated(addr.Label))
@@ -232,16 +236,34 @@ type AddrUpdate struct {
// AddrSubscribe takes a chan down which notifications will be sent
// when addresses change. Close the 'done' chan to stop subscription.
func AddrSubscribe(ch chan<- AddrUpdate, done <-chan struct{}) error {
return addrSubscribe(netns.None(), netns.None(), ch, done)
return addrSubscribeAt(netns.None(), netns.None(), ch, done, nil)
}
// AddrSubscribeAt works like AddrSubscribe plus it allows the caller
// to choose the network namespace in which to subscribe (ns).
func AddrSubscribeAt(ns netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}) error {
return addrSubscribe(ns, netns.None(), ch, done)
return addrSubscribeAt(ns, netns.None(), ch, done, nil)
}
func addrSubscribe(newNs, curNs netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}) error {
// AddrSubscribeOptions contains a set of options to use with
// AddrSubscribeWithOptions.
type AddrSubscribeOptions struct {
Namespace *netns.NsHandle
ErrorCallback func(error)
}
// AddrSubscribeWithOptions work like AddrSubscribe but enable to
// provide additional options to modify the behavior. Currently, the
// namespace can be provided as well as an error callback.
func AddrSubscribeWithOptions(ch chan<- AddrUpdate, done <-chan struct{}, options AddrSubscribeOptions) error {
if options.Namespace == nil {
none := netns.None()
options.Namespace = &none
}
return addrSubscribeAt(*options.Namespace, netns.None(), ch, done, options.ErrorCallback)
}
func addrSubscribeAt(newNs, curNs netns.NsHandle, ch chan<- AddrUpdate, done <-chan struct{}, cberr func(error)) error {
s, err := nl.SubscribeAt(newNs, curNs, syscall.NETLINK_ROUTE, syscall.RTNLGRP_IPV4_IFADDR, syscall.RTNLGRP_IPV6_IFADDR)
if err != nil {
return err
@@ -257,20 +279,26 @@ func addrSubscribe(newNs, curNs netns.NsHandle, ch chan<- AddrUpdate, done <-cha
for {
msgs, err := s.Receive()
if err != nil {
log.Printf("netlink.AddrSubscribe: Receive() error: %v", err)
if cberr != nil {
cberr(err)
}
return
}
for _, m := range msgs {
msgType := m.Header.Type
if msgType != syscall.RTM_NEWADDR && msgType != syscall.RTM_DELADDR {
log.Printf("netlink.AddrSubscribe: bad message type: %d", msgType)
continue
if cberr != nil {
cberr(fmt.Errorf("bad message type: %d", msgType))
}
return
}
addr, _, ifindex, err := parseAddr(m.Data)
if err != nil {
log.Printf("netlink.AddrSubscribe: could not parse address: %v", err)
continue
if cberr != nil {
cberr(fmt.Errorf("could not parse address: %v", err))
}
return
}
ch <- AddrUpdate{LinkAddress: *addr.IPNet,
+115
View File
@@ -0,0 +1,115 @@
package netlink
import (
"fmt"
"syscall"
"github.com/vishvananda/netlink/nl"
)
// BridgeVlanList gets a map of device id to bridge vlan infos.
// Equivalent to: `bridge vlan show`
func BridgeVlanList() (map[int32][]*nl.BridgeVlanInfo, error) {
return pkgHandle.BridgeVlanList()
}
// BridgeVlanList gets a map of device id to bridge vlan infos.
// Equivalent to: `bridge vlan show`
func (h *Handle) BridgeVlanList() (map[int32][]*nl.BridgeVlanInfo, error) {
req := h.newNetlinkRequest(syscall.RTM_GETLINK, syscall.NLM_F_DUMP)
msg := nl.NewIfInfomsg(syscall.AF_BRIDGE)
req.AddData(msg)
req.AddData(nl.NewRtAttr(nl.IFLA_EXT_MASK, nl.Uint32Attr(uint32(nl.RTEXT_FILTER_BRVLAN))))
msgs, err := req.Execute(syscall.NETLINK_ROUTE, syscall.RTM_NEWLINK)
if err != nil {
return nil, err
}
ret := make(map[int32][]*nl.BridgeVlanInfo)
for _, m := range msgs {
msg := nl.DeserializeIfInfomsg(m)
attrs, err := nl.ParseRouteAttr(m[msg.Len():])
if err != nil {
return nil, err
}
for _, attr := range attrs {
switch attr.Attr.Type {
case nl.IFLA_AF_SPEC:
//nested attr
nestAttrs, err := nl.ParseRouteAttr(attr.Value)
if err != nil {
return nil, fmt.Errorf("failed to parse nested attr %v", err)
}
for _, nestAttr := range nestAttrs {
switch nestAttr.Attr.Type {
case nl.IFLA_BRIDGE_VLAN_INFO:
vlanInfo := nl.DeserializeBridgeVlanInfo(nestAttr.Value)
ret[msg.Index] = append(ret[msg.Index], vlanInfo)
}
}
}
}
}
return ret, nil
}
// BridgeVlanAdd adds a new vlan filter entry
// Equivalent to: `bridge vlan add dev DEV vid VID [ pvid ] [ untagged ] [ self ] [ master ]`
func BridgeVlanAdd(link Link, vid uint16, pvid, untagged, self, master bool) error {
return pkgHandle.BridgeVlanAdd(link, vid, pvid, untagged, self, master)
}
// BridgeVlanAdd adds a new vlan filter entry
// Equivalent to: `bridge vlan add dev DEV vid VID [ pvid ] [ untagged ] [ self ] [ master ]`
func (h *Handle) BridgeVlanAdd(link Link, vid uint16, pvid, untagged, self, master bool) error {
return h.bridgeVlanModify(syscall.RTM_SETLINK, link, vid, pvid, untagged, self, master)
}
// BridgeVlanDel adds a new vlan filter entry
// Equivalent to: `bridge vlan del dev DEV vid VID [ pvid ] [ untagged ] [ self ] [ master ]`
func BridgeVlanDel(link Link, vid uint16, pvid, untagged, self, master bool) error {
return pkgHandle.BridgeVlanDel(link, vid, pvid, untagged, self, master)
}
// BridgeVlanDel adds a new vlan filter entry
// Equivalent to: `bridge vlan del dev DEV vid VID [ pvid ] [ untagged ] [ self ] [ master ]`
func (h *Handle) BridgeVlanDel(link Link, vid uint16, pvid, untagged, self, master bool) error {
return h.bridgeVlanModify(syscall.RTM_DELLINK, link, vid, pvid, untagged, self, master)
}
func (h *Handle) bridgeVlanModify(cmd int, link Link, vid uint16, pvid, untagged, self, master bool) error {
base := link.Attrs()
h.ensureIndex(base)
req := h.newNetlinkRequest(cmd, syscall.NLM_F_ACK)
msg := nl.NewIfInfomsg(syscall.AF_BRIDGE)
msg.Index = int32(base.Index)
req.AddData(msg)
br := nl.NewRtAttr(nl.IFLA_AF_SPEC, nil)
var flags uint16
if self {
flags |= nl.BRIDGE_FLAGS_SELF
}
if master {
flags |= nl.BRIDGE_FLAGS_MASTER
}
if flags > 0 {
nl.NewRtAttrChild(br, nl.IFLA_BRIDGE_FLAGS, nl.Uint16Attr(flags))
}
vlanInfo := &nl.BridgeVlanInfo{Vid: vid}
if pvid {
vlanInfo.Flags |= nl.BRIDGE_VLAN_INFO_PVID
}
if untagged {
vlanInfo.Flags |= nl.BRIDGE_VLAN_INFO_UNTAGGED
}
nl.NewRtAttrChild(br, nl.IFLA_BRIDGE_VLAN_INFO, vlanInfo.Serialize())
req.AddData(br)
_, err := req.Execute(syscall.NETLINK_ROUTE, 0)
if err != nil {
return err
}
return nil
}
+36 -9
View File
@@ -22,7 +22,11 @@ const (
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/netfilter/nfnetlink.h -> #define NFNL_SUBSYS_CTNETLINK_EXP 2
ConntrackExpectTable = 2
)
const (
// For Parsing Mark
TCP_PROTO = 6
UDP_PROTO = 17
)
const (
// backward compatibility with golang 1.6 which does not have io.SeekCurrent
seekCurrent = 1
@@ -56,7 +60,7 @@ func ConntrackTableFlush(table ConntrackTableType) error {
// ConntrackDeleteFilter deletes entries on the specified table on the base of the filter
// conntrack -D [table] parameters Delete conntrack or expectation
func ConntrackDeleteFilter(table ConntrackTableType, family InetFamily, filter *ConntrackFilter) (uint, error) {
func ConntrackDeleteFilter(table ConntrackTableType, family InetFamily, filter CustomConntrackFilter) (uint, error) {
return pkgHandle.ConntrackDeleteFilter(table, family, filter)
}
@@ -88,7 +92,7 @@ func (h *Handle) ConntrackTableFlush(table ConntrackTableType) error {
// ConntrackDeleteFilter deletes entries on the specified table on the base of the filter using the netlink handle passed
// conntrack -D [table] parameters Delete conntrack or expectation
func (h *Handle) ConntrackDeleteFilter(table ConntrackTableType, family InetFamily, filter *ConntrackFilter) (uint, error) {
func (h *Handle) ConntrackDeleteFilter(table ConntrackTableType, family InetFamily, filter CustomConntrackFilter) (uint, error) {
res, err := h.dumpConntrackTable(table, family)
if err != nil {
return 0, err
@@ -142,15 +146,16 @@ type ConntrackFlow struct {
FamilyType uint8
Forward ipTuple
Reverse ipTuple
Mark uint32
}
func (s *ConntrackFlow) String() string {
// conntrack cmd output:
// udp 17 src=127.0.0.1 dst=127.0.0.1 sport=4001 dport=1234 [UNREPLIED] src=127.0.0.1 dst=127.0.0.1 sport=1234 dport=4001
return fmt.Sprintf("%s\t%d src=%s dst=%s sport=%d dport=%d\tsrc=%s dst=%s sport=%d dport=%d",
// udp 17 src=127.0.0.1 dst=127.0.0.1 sport=4001 dport=1234 [UNREPLIED] src=127.0.0.1 dst=127.0.0.1 sport=1234 dport=4001 mark=0
return fmt.Sprintf("%s\t%d src=%s dst=%s sport=%d dport=%d\tsrc=%s dst=%s sport=%d dport=%d mark=%d",
nl.L4ProtoMap[s.Forward.Protocol], s.Forward.Protocol,
s.Forward.SrcIP.String(), s.Forward.DstIP.String(), s.Forward.SrcPort, s.Forward.DstPort,
s.Reverse.SrcIP.String(), s.Reverse.DstIP.String(), s.Reverse.SrcPort, s.Reverse.DstPort)
s.Reverse.SrcIP.String(), s.Reverse.DstIP.String(), s.Reverse.SrcPort, s.Reverse.DstPort, s.Mark)
}
// This method parse the ip tuple structure
@@ -160,7 +165,7 @@ func (s *ConntrackFlow) String() string {
// <len, NLA_F_NESTED|nl.CTA_TUPLE_PROTO, 1 byte for the protocol, 3 bytes of padding>
// <len, CTA_PROTO_SRC_PORT, 2 bytes for the source port, 2 bytes of padding>
// <len, CTA_PROTO_DST_PORT, 2 bytes for the source port, 2 bytes of padding>
func parseIpTuple(reader *bytes.Reader, tpl *ipTuple) {
func parseIpTuple(reader *bytes.Reader, tpl *ipTuple) uint8 {
for i := 0; i < 2; i++ {
_, t, _, v := parseNfAttrTLV(reader)
switch t {
@@ -189,6 +194,7 @@ func parseIpTuple(reader *bytes.Reader, tpl *ipTuple) {
// Skip some padding 2 byte
reader.Seek(2, seekCurrent)
}
return tpl.Protocol
}
func parseNfAttrTLV(r *bytes.Reader) (isNested bool, attrType, len uint16, value []byte) {
@@ -216,6 +222,7 @@ func parseBERaw16(r *bytes.Reader, v *uint16) {
func parseRawData(data []byte) *ConntrackFlow {
s := &ConntrackFlow{}
var proto uint8
// First there is the Nfgenmsg header
// consume only the family field
reader := bytes.NewReader(data)
@@ -234,7 +241,7 @@ func parseRawData(data []byte) *ConntrackFlow {
nested, t, l := parseNfAttrTL(reader)
if nested && t == nl.CTA_TUPLE_ORIG {
if nested, t, _ = parseNfAttrTL(reader); nested && t == nl.CTA_TUPLE_IP {
parseIpTuple(reader, &s.Forward)
proto = parseIpTuple(reader, &s.Forward)
}
} else if nested && t == nl.CTA_TUPLE_REPLY {
if nested, t, _ = parseNfAttrTL(reader); nested && t == nl.CTA_TUPLE_IP {
@@ -248,7 +255,19 @@ func parseRawData(data []byte) *ConntrackFlow {
}
}
}
if proto == TCP_PROTO {
reader.Seek(64, seekCurrent)
_, t, _, v := parseNfAttrTLV(reader)
if t == nl.CTA_MARK {
s.Mark = uint32(v[3])
}
} else if proto == UDP_PROTO {
reader.Seek(16, seekCurrent)
_, t, _, v := parseNfAttrTLV(reader)
if t == nl.CTA_MARK {
s.Mark = uint32(v[3])
}
}
return s
}
@@ -290,6 +309,12 @@ const (
ConntrackNatAnyIP // -any-nat ip Source or destination NAT ip
)
type CustomConntrackFilter interface {
// MatchConntrackFlow applies the filter to the flow and returns true if the flow matches
// the filter or false otherwise
MatchConntrackFlow(flow *ConntrackFlow) bool
}
type ConntrackFilter struct {
ipFilter map[ConntrackFilterType]net.IP
}
@@ -342,3 +367,5 @@ func (f *ConntrackFilter) MatchConntrackFlow(flow *ConntrackFlow) bool {
return match
}
var _ CustomConntrackFilter = (*ConntrackFilter)(nil)
-10
View File
@@ -2,8 +2,6 @@ package netlink
import (
"fmt"
"github.com/vishvananda/netlink/nl"
)
type Filter interface {
@@ -184,14 +182,6 @@ func NewMirredAction(redirIndex int) *MirredAction {
}
}
// Constants used in TcU32Sel.Flags.
const (
TC_U32_TERMINAL = nl.TC_U32_TERMINAL
TC_U32_OFFSET = nl.TC_U32_OFFSET
TC_U32_VAROFFSET = nl.TC_U32_VAROFFSET
TC_U32_EAT = nl.TC_U32_EAT
)
// Sel of the U32 filters that contains multiple TcU32Key. This is the copy
// and the frontend representation of nl.TcU32Sel. It is serialized into canonical
// nl.TcU32Sel with the appropriate endianness.
+38 -28
View File
@@ -11,6 +11,14 @@ import (
"github.com/vishvananda/netlink/nl"
)
// Constants used in TcU32Sel.Flags.
const (
TC_U32_TERMINAL = nl.TC_U32_TERMINAL
TC_U32_OFFSET = nl.TC_U32_OFFSET
TC_U32_VAROFFSET = nl.TC_U32_VAROFFSET
TC_U32_EAT = nl.TC_U32_EAT
)
// Fw filter filters on firewall marks
// NOTE: this is in filter_linux because it refers to nl.TcPolice which
// is defined in nl/tc_linux.go
@@ -128,9 +136,11 @@ func (h *Handle) FilterAdd(filter Filter) error {
req.AddData(nl.NewRtAttr(nl.TCA_KIND, nl.ZeroTerminated(filter.Type())))
options := nl.NewRtAttr(nl.TCA_OPTIONS, nil)
if u32, ok := filter.(*U32); ok {
switch filter := filter.(type) {
case *U32:
// Convert TcU32Sel into nl.TcU32Sel as it is without copy.
sel := (*nl.TcU32Sel)(unsafe.Pointer(u32.Sel))
sel := (*nl.TcU32Sel)(unsafe.Pointer(filter.Sel))
if sel == nil {
// match all
sel = &nl.TcU32Sel{
@@ -158,56 +168,56 @@ func (h *Handle) FilterAdd(filter Filter) error {
}
sel.Nkeys = uint8(len(sel.Keys))
nl.NewRtAttrChild(options, nl.TCA_U32_SEL, sel.Serialize())
if u32.ClassId != 0 {
nl.NewRtAttrChild(options, nl.TCA_U32_CLASSID, nl.Uint32Attr(u32.ClassId))
if filter.ClassId != 0 {
nl.NewRtAttrChild(options, nl.TCA_U32_CLASSID, nl.Uint32Attr(filter.ClassId))
}
actionsAttr := nl.NewRtAttrChild(options, nl.TCA_U32_ACT, nil)
// backwards compatibility
if u32.RedirIndex != 0 {
u32.Actions = append([]Action{NewMirredAction(u32.RedirIndex)}, u32.Actions...)
if filter.RedirIndex != 0 {
filter.Actions = append([]Action{NewMirredAction(filter.RedirIndex)}, filter.Actions...)
}
if err := EncodeActions(actionsAttr, u32.Actions); err != nil {
if err := EncodeActions(actionsAttr, filter.Actions); err != nil {
return err
}
} else if fw, ok := filter.(*Fw); ok {
if fw.Mask != 0 {
case *Fw:
if filter.Mask != 0 {
b := make([]byte, 4)
native.PutUint32(b, fw.Mask)
native.PutUint32(b, filter.Mask)
nl.NewRtAttrChild(options, nl.TCA_FW_MASK, b)
}
if fw.InDev != "" {
nl.NewRtAttrChild(options, nl.TCA_FW_INDEV, nl.ZeroTerminated(fw.InDev))
if filter.InDev != "" {
nl.NewRtAttrChild(options, nl.TCA_FW_INDEV, nl.ZeroTerminated(filter.InDev))
}
if (fw.Police != nl.TcPolice{}) {
if (filter.Police != nl.TcPolice{}) {
police := nl.NewRtAttrChild(options, nl.TCA_FW_POLICE, nil)
nl.NewRtAttrChild(police, nl.TCA_POLICE_TBF, fw.Police.Serialize())
if (fw.Police.Rate != nl.TcRateSpec{}) {
payload := SerializeRtab(fw.Rtab)
nl.NewRtAttrChild(police, nl.TCA_POLICE_TBF, filter.Police.Serialize())
if (filter.Police.Rate != nl.TcRateSpec{}) {
payload := SerializeRtab(filter.Rtab)
nl.NewRtAttrChild(police, nl.TCA_POLICE_RATE, payload)
}
if (fw.Police.PeakRate != nl.TcRateSpec{}) {
payload := SerializeRtab(fw.Ptab)
if (filter.Police.PeakRate != nl.TcRateSpec{}) {
payload := SerializeRtab(filter.Ptab)
nl.NewRtAttrChild(police, nl.TCA_POLICE_PEAKRATE, payload)
}
}
if fw.ClassId != 0 {
if filter.ClassId != 0 {
b := make([]byte, 4)
native.PutUint32(b, fw.ClassId)
native.PutUint32(b, filter.ClassId)
nl.NewRtAttrChild(options, nl.TCA_FW_CLASSID, b)
}
} else if bpf, ok := filter.(*BpfFilter); ok {
case *BpfFilter:
var bpfFlags uint32
if bpf.ClassId != 0 {
nl.NewRtAttrChild(options, nl.TCA_BPF_CLASSID, nl.Uint32Attr(bpf.ClassId))
if filter.ClassId != 0 {
nl.NewRtAttrChild(options, nl.TCA_BPF_CLASSID, nl.Uint32Attr(filter.ClassId))
}
if bpf.Fd >= 0 {
nl.NewRtAttrChild(options, nl.TCA_BPF_FD, nl.Uint32Attr((uint32(bpf.Fd))))
if filter.Fd >= 0 {
nl.NewRtAttrChild(options, nl.TCA_BPF_FD, nl.Uint32Attr((uint32(filter.Fd))))
}
if bpf.Name != "" {
nl.NewRtAttrChild(options, nl.TCA_BPF_NAME, nl.ZeroTerminated(bpf.Name))
if filter.Name != "" {
nl.NewRtAttrChild(options, nl.TCA_BPF_NAME, nl.ZeroTerminated(filter.Name))
}
if bpf.DirectAction {
if filter.DirectAction {
bpfFlags |= nl.TCA_BPF_FLAG_ACT_DIRECT
}
nl.NewRtAttrChild(options, nl.TCA_BPF_FLAGS, nl.Uint32Attr(bpfFlags))
+37 -4
View File
@@ -45,12 +45,27 @@ func (h *Handle) SetSocketTimeout(to time.Duration) error {
}
tv := syscall.NsecToTimeval(to.Nanoseconds())
for _, sh := range h.sockets {
fd := sh.Socket.GetFd()
err := syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, &tv)
if err != nil {
if err := sh.Socket.SetSendTimeout(&tv); err != nil {
return err
}
err = syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, syscall.SO_SNDTIMEO, &tv)
if err := sh.Socket.SetReceiveTimeout(&tv); err != nil {
return err
}
}
return nil
}
// SetSocketReceiveBufferSize sets the receive buffer size for each
// socket in the netlink handle. The maximum value is capped by
// /proc/sys/net/core/rmem_max.
func (h *Handle) SetSocketReceiveBufferSize(size int, force bool) error {
opt := syscall.SO_RCVBUF
if force {
opt = syscall.SO_RCVBUFFORCE
}
for _, sh := range h.sockets {
fd := sh.Socket.GetFd()
err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, opt, size)
if err != nil {
return err
}
@@ -58,6 +73,24 @@ func (h *Handle) SetSocketTimeout(to time.Duration) error {
return nil
}
// GetSocketReceiveBufferSize gets the receiver buffer size for each
// socket in the netlink handle. The retrieved value should be the
// double to the one set for SetSocketReceiveBufferSize.
func (h *Handle) GetSocketReceiveBufferSize() ([]int, error) {
results := make([]int, len(h.sockets))
i := 0
for _, sh := range h.sockets {
fd := sh.Socket.GetFd()
size, err := syscall.GetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF)
if err != nil {
return nil, err
}
results[i] = size
i++
}
return results, nil
}
// NewHandle returns a netlink handle on the network namespace
// specified by ns. If ns=netns.None(), current network namespace
// will be assumed
@@ -145,6 +145,10 @@ func (h *Handle) LinkSetFlood(link Link, mode bool) error {
return ErrNotImplemented
}
func (h *Handle) LinkSetTxQLen(link Link, qlen int) error {
return ErrNotImplemented
}
func (h *Handle) setProtinfoAttr(link Link, mode bool, attr int) error {
return ErrNotImplemented
}
+26
View File
@@ -37,6 +37,7 @@ type LinkAttrs struct {
EncapType string
Protinfo *Protinfo
OperState LinkOperState
NetNsID int
}
// LinkOperState represents the values of the IFLA_OPERSTATE link
@@ -171,6 +172,7 @@ type LinkXdp struct {
Fd int
Attached bool
Flags uint32
ProgId uint32
}
// Device links cannot be created via netlink. These links
@@ -339,6 +341,7 @@ type Vxlan struct {
UDPCSum bool
NoAge bool
GBP bool
FlowBased bool
Age int
Limit int
Port int
@@ -684,6 +687,7 @@ type Gretap struct {
EncapType uint16
EncapFlags uint16
Link uint32
FlowBased bool
}
func (gretap *Gretap) Attrs() *LinkAttrs {
@@ -729,6 +733,28 @@ func (iptun *Vti) Type() string {
return "vti"
}
type Gretun struct {
LinkAttrs
Link uint32
IFlags uint16
OFlags uint16
IKey uint32
OKey uint32
Local net.IP
Remote net.IP
Ttl uint8
Tos uint8
PMtuDisc uint8
}
func (gretun *Gretun) Attrs() *LinkAttrs {
return &gretun.LinkAttrs
}
func (gretun *Gretun) Type() string {
return "gre"
}
type Vrf struct {
LinkAttrs
Table uint32
+252 -34
View File
@@ -379,6 +379,74 @@ func (h *Handle) LinkSetVfTxRate(link Link, vf, rate int) error {
return err
}
// LinkSetVfSpoofchk enables/disables spoof check on a vf for the link.
// Equivalent to: `ip link set $link vf $vf spoofchk $check`
func LinkSetVfSpoofchk(link Link, vf int, check bool) error {
return pkgHandle.LinkSetVfSpoofchk(link, vf, check)
}
// LinkSetVfSpookfchk enables/disables spoof check on a vf for the link.
// Equivalent to: `ip link set $link vf $vf spoofchk $check`
func (h *Handle) LinkSetVfSpoofchk(link Link, vf int, check bool) error {
var setting uint32
base := link.Attrs()
h.ensureIndex(base)
req := h.newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK)
msg := nl.NewIfInfomsg(syscall.AF_UNSPEC)
msg.Index = int32(base.Index)
req.AddData(msg)
data := nl.NewRtAttr(nl.IFLA_VFINFO_LIST, nil)
info := nl.NewRtAttrChild(data, nl.IFLA_VF_INFO, nil)
if check {
setting = 1
}
vfmsg := nl.VfSpoofchk{
Vf: uint32(vf),
Setting: setting,
}
nl.NewRtAttrChild(info, nl.IFLA_VF_SPOOFCHK, vfmsg.Serialize())
req.AddData(data)
_, err := req.Execute(syscall.NETLINK_ROUTE, 0)
return err
}
// LinkSetVfTrust enables/disables trust state on a vf for the link.
// Equivalent to: `ip link set $link vf $vf trust $state`
func LinkSetVfTrust(link Link, vf int, state bool) error {
return pkgHandle.LinkSetVfTrust(link, vf, state)
}
// LinkSetVfTrust enables/disables trust state on a vf for the link.
// Equivalent to: `ip link set $link vf $vf trust $state`
func (h *Handle) LinkSetVfTrust(link Link, vf int, state bool) error {
var setting uint32
base := link.Attrs()
h.ensureIndex(base)
req := h.newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK)
msg := nl.NewIfInfomsg(syscall.AF_UNSPEC)
msg.Index = int32(base.Index)
req.AddData(msg)
data := nl.NewRtAttr(nl.IFLA_VFINFO_LIST, nil)
info := nl.NewRtAttrChild(data, nl.IFLA_VF_INFO, nil)
if state {
setting = 1
}
vfmsg := nl.VfTrust{
Vf: uint32(vf),
Setting: setting,
}
nl.NewRtAttrChild(info, nl.IFLA_VF_TRUST, vfmsg.Serialize())
req.AddData(data)
_, err := req.Execute(syscall.NETLINK_ROUTE, 0)
return err
}
// LinkSetMaster sets the master of the link device.
// Equivalent to: `ip link set $link master $master`
func LinkSetMaster(link Link, master *Bridge) error {
@@ -500,6 +568,12 @@ func (h *Handle) LinkSetNsFd(link Link, fd int) error {
// LinkSetXdpFd adds a bpf function to the driver. The fd must be a bpf
// program loaded with bpf(type=BPF_PROG_TYPE_XDP)
func LinkSetXdpFd(link Link, fd int) error {
return LinkSetXdpFdWithFlags(link, fd, 0)
}
// LinkSetXdpFdWithFlags adds a bpf function to the driver with the given
// options. The fd must be a bpf program loaded with bpf(type=BPF_PROG_TYPE_XDP)
func LinkSetXdpFdWithFlags(link Link, fd, flags int) error {
base := link.Attrs()
ensureIndex(base)
req := nl.NewNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK)
@@ -508,7 +582,7 @@ func LinkSetXdpFd(link Link, fd int) error {
msg.Index = int32(base.Index)
req.AddData(msg)
addXdpAttrs(&LinkXdp{Fd: fd}, req)
addXdpAttrs(&LinkXdp{Fd: fd, Flags: uint32(flags)}, req)
_, err := req.Execute(syscall.NETLINK_ROUTE, 0)
return err
@@ -528,7 +602,13 @@ type vxlanPortRange struct {
func addVxlanAttrs(vxlan *Vxlan, linkInfo *nl.RtAttr) {
data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil)
if vxlan.FlowBased {
vxlan.VxlanId = 0
}
nl.NewRtAttrChild(data, nl.IFLA_VXLAN_ID, nl.Uint32Attr(uint32(vxlan.VxlanId)))
if vxlan.VtepDevIndex != 0 {
nl.NewRtAttrChild(data, nl.IFLA_VXLAN_LINK, nl.Uint32Attr(uint32(vxlan.VtepDevIndex)))
}
@@ -569,6 +649,9 @@ func addVxlanAttrs(vxlan *Vxlan, linkInfo *nl.RtAttr) {
if vxlan.GBP {
nl.NewRtAttrChild(data, nl.IFLA_VXLAN_GBP, []byte{})
}
if vxlan.FlowBased {
nl.NewRtAttrChild(data, nl.IFLA_VXLAN_FLOWBASED, boolAttr(vxlan.FlowBased))
}
if vxlan.NoAge {
nl.NewRtAttrChild(data, nl.IFLA_VXLAN_AGEING, nl.Uint32Attr(0))
} else if vxlan.Age > 0 {
@@ -818,16 +901,17 @@ func (h *Handle) linkModify(link Link, flags int) error {
linkInfo := nl.NewRtAttr(syscall.IFLA_LINKINFO, nil)
nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_KIND, nl.NonZeroTerminated(link.Type()))
if vlan, ok := link.(*Vlan); ok {
switch link := link.(type) {
case *Vlan:
b := make([]byte, 2)
native.PutUint16(b, uint16(vlan.VlanId))
native.PutUint16(b, uint16(link.VlanId))
data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil)
nl.NewRtAttrChild(data, nl.IFLA_VLAN_ID, b)
} else if veth, ok := link.(*Veth); ok {
case *Veth:
data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil)
peer := nl.NewRtAttrChild(data, nl.VETH_INFO_PEER, nil)
nl.NewIfInfomsgChild(peer, syscall.AF_UNSPEC)
nl.NewRtAttrChild(peer, syscall.IFLA_IFNAME, nl.ZeroTerminated(veth.PeerName))
nl.NewRtAttrChild(peer, syscall.IFLA_IFNAME, nl.ZeroTerminated(link.PeerName))
if base.TxQLen >= 0 {
nl.NewRtAttrChild(peer, syscall.IFLA_TXQLEN, nl.Uint32Attr(uint32(base.TxQLen)))
}
@@ -835,35 +919,37 @@ func (h *Handle) linkModify(link Link, flags int) error {
nl.NewRtAttrChild(peer, syscall.IFLA_MTU, nl.Uint32Attr(uint32(base.MTU)))
}
} else if vxlan, ok := link.(*Vxlan); ok {
addVxlanAttrs(vxlan, linkInfo)
} else if bond, ok := link.(*Bond); ok {
addBondAttrs(bond, linkInfo)
} else if ipv, ok := link.(*IPVlan); ok {
case *Vxlan:
addVxlanAttrs(link, linkInfo)
case *Bond:
addBondAttrs(link, linkInfo)
case *IPVlan:
data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil)
nl.NewRtAttrChild(data, nl.IFLA_IPVLAN_MODE, nl.Uint16Attr(uint16(ipv.Mode)))
} else if macv, ok := link.(*Macvlan); ok {
if macv.Mode != MACVLAN_MODE_DEFAULT {
nl.NewRtAttrChild(data, nl.IFLA_IPVLAN_MODE, nl.Uint16Attr(uint16(link.Mode)))
case *Macvlan:
if link.Mode != MACVLAN_MODE_DEFAULT {
data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil)
nl.NewRtAttrChild(data, nl.IFLA_MACVLAN_MODE, nl.Uint32Attr(macvlanModes[macv.Mode]))
nl.NewRtAttrChild(data, nl.IFLA_MACVLAN_MODE, nl.Uint32Attr(macvlanModes[link.Mode]))
}
} else if macv, ok := link.(*Macvtap); ok {
if macv.Mode != MACVLAN_MODE_DEFAULT {
case *Macvtap:
if link.Mode != MACVLAN_MODE_DEFAULT {
data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil)
nl.NewRtAttrChild(data, nl.IFLA_MACVLAN_MODE, nl.Uint32Attr(macvlanModes[macv.Mode]))
nl.NewRtAttrChild(data, nl.IFLA_MACVLAN_MODE, nl.Uint32Attr(macvlanModes[link.Mode]))
}
} else if gretap, ok := link.(*Gretap); ok {
addGretapAttrs(gretap, linkInfo)
} else if iptun, ok := link.(*Iptun); ok {
addIptunAttrs(iptun, linkInfo)
} else if vti, ok := link.(*Vti); ok {
addVtiAttrs(vti, linkInfo)
} else if vrf, ok := link.(*Vrf); ok {
addVrfAttrs(vrf, linkInfo)
} else if bridge, ok := link.(*Bridge); ok {
addBridgeAttrs(bridge, linkInfo)
} else if gtp, ok := link.(*GTP); ok {
addGTPAttrs(gtp, linkInfo)
case *Gretap:
addGretapAttrs(link, linkInfo)
case *Iptun:
addIptunAttrs(link, linkInfo)
case *Gretun:
addGretunAttrs(link, linkInfo)
case *Vti:
addVtiAttrs(link, linkInfo)
case *Vrf:
addVrfAttrs(link, linkInfo)
case *Bridge:
addBridgeAttrs(link, linkInfo)
case *GTP:
addGTPAttrs(link, linkInfo)
}
req.AddData(linkInfo)
@@ -1093,6 +1179,8 @@ func LinkDeserialize(hdr *syscall.NlMsghdr, m []byte) (Link, error) {
link = &Gretap{}
case "ipip":
link = &Iptun{}
case "gre":
link = &Gretun{}
case "vti":
link = &Vti{}
case "vrf":
@@ -1124,6 +1212,8 @@ func LinkDeserialize(hdr *syscall.NlMsghdr, m []byte) (Link, error) {
parseGretapData(link, data)
case "ipip":
parseIptunData(link, data)
case "gre":
parseGretunData(link, data)
case "vti":
parseVtiData(link, data)
case "vrf":
@@ -1178,6 +1268,8 @@ func LinkDeserialize(hdr *syscall.NlMsghdr, m []byte) (Link, error) {
}
case syscall.IFLA_OPERSTATE:
base.OperState = LinkOperState(uint8(attr.Value[0]))
case nl.IFLA_LINK_NETNSID:
base.NetNsID = int(native.Uint32(attr.Value[0:4]))
}
}
@@ -1239,16 +1331,34 @@ type LinkUpdate struct {
// LinkSubscribe takes a chan down which notifications will be sent
// when links change. Close the 'done' chan to stop subscription.
func LinkSubscribe(ch chan<- LinkUpdate, done <-chan struct{}) error {
return linkSubscribe(netns.None(), netns.None(), ch, done)
return linkSubscribeAt(netns.None(), netns.None(), ch, done, nil)
}
// LinkSubscribeAt works like LinkSubscribe plus it allows the caller
// to choose the network namespace in which to subscribe (ns).
func LinkSubscribeAt(ns netns.NsHandle, ch chan<- LinkUpdate, done <-chan struct{}) error {
return linkSubscribe(ns, netns.None(), ch, done)
return linkSubscribeAt(ns, netns.None(), ch, done, nil)
}
func linkSubscribe(newNs, curNs netns.NsHandle, ch chan<- LinkUpdate, done <-chan struct{}) error {
// LinkSubscribeOptions contains a set of options to use with
// LinkSubscribeWithOptions.
type LinkSubscribeOptions struct {
Namespace *netns.NsHandle
ErrorCallback func(error)
}
// LinkSubscribeWithOptions work like LinkSubscribe but enable to
// provide additional options to modify the behavior. Currently, the
// namespace can be provided as well as an error callback.
func LinkSubscribeWithOptions(ch chan<- LinkUpdate, done <-chan struct{}, options LinkSubscribeOptions) error {
if options.Namespace == nil {
none := netns.None()
options.Namespace = &none
}
return linkSubscribeAt(*options.Namespace, netns.None(), ch, done, options.ErrorCallback)
}
func linkSubscribeAt(newNs, curNs netns.NsHandle, ch chan<- LinkUpdate, done <-chan struct{}, cberr func(error)) error {
s, err := nl.SubscribeAt(newNs, curNs, syscall.NETLINK_ROUTE, syscall.RTNLGRP_LINK)
if err != nil {
return err
@@ -1264,12 +1374,18 @@ func linkSubscribe(newNs, curNs netns.NsHandle, ch chan<- LinkUpdate, done <-cha
for {
msgs, err := s.Receive()
if err != nil {
if cberr != nil {
cberr(err)
}
return
}
for _, m := range msgs {
ifmsg := nl.DeserializeIfInfomsg(m.Data)
link, err := LinkDeserialize(&m.Header, m.Data)
if err != nil {
if cberr != nil {
cberr(err)
}
return
}
ch <- LinkUpdate{IfInfomsg: *ifmsg, Header: m.Header, Link: link}
@@ -1363,6 +1479,33 @@ func (h *Handle) setProtinfoAttr(link Link, mode bool, attr int) error {
return nil
}
// LinkSetTxQLen sets the transaction queue length for the link.
// Equivalent to: `ip link set $link txqlen $qlen`
func LinkSetTxQLen(link Link, qlen int) error {
return pkgHandle.LinkSetTxQLen(link, qlen)
}
// LinkSetTxQLen sets the transaction queue length for the link.
// Equivalent to: `ip link set $link txqlen $qlen`
func (h *Handle) LinkSetTxQLen(link Link, qlen int) error {
base := link.Attrs()
h.ensureIndex(base)
req := h.newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK)
msg := nl.NewIfInfomsg(syscall.AF_UNSPEC)
msg.Index = int32(base.Index)
req.AddData(msg)
b := make([]byte, 4)
native.PutUint32(b, uint32(qlen))
data := nl.NewRtAttr(syscall.IFLA_TXQLEN, b)
req.AddData(data)
_, err := req.Execute(syscall.NETLINK_ROUTE, 0)
return err
}
func parseVlanData(link Link, data []syscall.NetlinkRouteAttr) {
vlan := link.(*Vlan)
for _, datum := range data {
@@ -1407,6 +1550,8 @@ func parseVxlanData(link Link, data []syscall.NetlinkRouteAttr) {
vxlan.UDPCSum = int8(datum.Value[0]) != 0
case nl.IFLA_VXLAN_GBP:
vxlan.GBP = true
case nl.IFLA_VXLAN_FLOWBASED:
vxlan.FlowBased = int8(datum.Value[0]) != 0
case nl.IFLA_VXLAN_AGEING:
vxlan.Age = int(native.Uint32(datum.Value[0:4]))
vxlan.NoAge = vxlan.Age == 0
@@ -1547,6 +1692,12 @@ func linkFlags(rawFlags uint32) net.Flags {
func addGretapAttrs(gretap *Gretap, linkInfo *nl.RtAttr) {
data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil)
if gretap.FlowBased {
// In flow based mode, no other attributes need to be configured
nl.NewRtAttrChild(data, nl.IFLA_GRE_COLLECT_METADATA, boolAttr(gretap.FlowBased))
return
}
ip := gretap.Local.To4()
if ip != nil {
nl.NewRtAttrChild(data, nl.IFLA_GRE_LOCAL, []byte(ip))
@@ -1613,6 +1764,69 @@ func parseGretapData(link Link, data []syscall.NetlinkRouteAttr) {
gre.EncapType = native.Uint16(datum.Value[0:2])
case nl.IFLA_GRE_ENCAP_FLAGS:
gre.EncapFlags = native.Uint16(datum.Value[0:2])
case nl.IFLA_GRE_COLLECT_METADATA:
gre.FlowBased = int8(datum.Value[0]) != 0
}
}
}
func addGretunAttrs(gre *Gretun, linkInfo *nl.RtAttr) {
data := nl.NewRtAttrChild(linkInfo, nl.IFLA_INFO_DATA, nil)
ip := gre.Local.To4()
if ip != nil {
nl.NewRtAttrChild(data, nl.IFLA_GRE_LOCAL, []byte(ip))
}
ip = gre.Remote.To4()
if ip != nil {
nl.NewRtAttrChild(data, nl.IFLA_GRE_REMOTE, []byte(ip))
}
if gre.IKey != 0 {
nl.NewRtAttrChild(data, nl.IFLA_GRE_IKEY, htonl(gre.IKey))
gre.IFlags |= uint16(nl.GRE_KEY)
}
if gre.OKey != 0 {
nl.NewRtAttrChild(data, nl.IFLA_GRE_OKEY, htonl(gre.OKey))
gre.OFlags |= uint16(nl.GRE_KEY)
}
nl.NewRtAttrChild(data, nl.IFLA_GRE_IFLAGS, htons(gre.IFlags))
nl.NewRtAttrChild(data, nl.IFLA_GRE_OFLAGS, htons(gre.OFlags))
if gre.Link != 0 {
nl.NewRtAttrChild(data, nl.IFLA_GRE_LINK, nl.Uint32Attr(gre.Link))
}
nl.NewRtAttrChild(data, nl.IFLA_GRE_PMTUDISC, nl.Uint8Attr(gre.PMtuDisc))
nl.NewRtAttrChild(data, nl.IFLA_GRE_TTL, nl.Uint8Attr(gre.Ttl))
nl.NewRtAttrChild(data, nl.IFLA_GRE_TOS, nl.Uint8Attr(gre.Tos))
}
func parseGretunData(link Link, data []syscall.NetlinkRouteAttr) {
gre := link.(*Gretun)
for _, datum := range data {
switch datum.Attr.Type {
case nl.IFLA_GRE_OKEY:
gre.IKey = ntohl(datum.Value[0:4])
case nl.IFLA_GRE_IKEY:
gre.OKey = ntohl(datum.Value[0:4])
case nl.IFLA_GRE_LOCAL:
gre.Local = net.IP(datum.Value[0:4])
case nl.IFLA_GRE_REMOTE:
gre.Remote = net.IP(datum.Value[0:4])
case nl.IFLA_GRE_IFLAGS:
gre.IFlags = ntohs(datum.Value[0:2])
case nl.IFLA_GRE_OFLAGS:
gre.OFlags = ntohs(datum.Value[0:2])
case nl.IFLA_GRE_TTL:
gre.Ttl = uint8(datum.Value[0])
case nl.IFLA_GRE_TOS:
gre.Tos = uint8(datum.Value[0])
case nl.IFLA_GRE_PMTUDISC:
gre.PMtuDisc = uint8(datum.Value[0])
}
}
}
@@ -1630,8 +1844,10 @@ func addXdpAttrs(xdp *LinkXdp, req *nl.NetlinkRequest) {
b := make([]byte, 4)
native.PutUint32(b, uint32(xdp.Fd))
nl.NewRtAttrChild(attrs, nl.IFLA_XDP_FD, b)
native.PutUint32(b, xdp.Flags)
nl.NewRtAttrChild(attrs, nl.IFLA_XDP_FLAGS, b)
if xdp.Flags != 0 {
native.PutUint32(b, xdp.Flags)
nl.NewRtAttrChild(attrs, nl.IFLA_XDP_FLAGS, b)
}
req.AddData(attrs)
}
@@ -1649,6 +1865,8 @@ func parseLinkXdp(data []byte) (*LinkXdp, error) {
xdp.Attached = attr.Value[0] != 0
case nl.IFLA_XDP_FLAGS:
xdp.Flags = native.Uint32(attr.Value[0:4])
case nl.IFLA_XDP_PROG_ID:
xdp.ProgId = native.Uint32(attr.Value[0:4])
}
}
return xdp, nil
+1
View File
@@ -14,6 +14,7 @@ type Neigh struct {
Flags int
IP net.IP
HardwareAddr net.HardwareAddr
LLIPAddr net.IP //Used in the case of NHRP
}
// String returns $ip/$hwaddr $label
+27 -2
View File
@@ -128,6 +128,7 @@ func (h *Handle) NeighDel(neigh *Neigh) error {
func neighHandle(neigh *Neigh, req *nl.NetlinkRequest) error {
var family int
if neigh.Family > 0 {
family = neigh.Family
} else {
@@ -151,7 +152,10 @@ func neighHandle(neigh *Neigh, req *nl.NetlinkRequest) error {
dstData := nl.NewRtAttr(NDA_DST, ipData)
req.AddData(dstData)
if neigh.Flags != NTF_PROXY || neigh.HardwareAddr != nil {
if neigh.LLIPAddr != nil {
llIPData := nl.NewRtAttr(NDA_LLADDR, neigh.LLIPAddr.To4())
req.AddData(llIPData)
} else if neigh.Flags != NTF_PROXY || neigh.HardwareAddr != nil {
hwData := nl.NewRtAttr(NDA_LLADDR, []byte(neigh.HardwareAddr))
req.AddData(hwData)
}
@@ -237,12 +241,33 @@ func NeighDeserialize(m []byte) (*Neigh, error) {
return nil, err
}
// This should be cached for perfomance
// once per table dump
link, err := LinkByIndex(neigh.LinkIndex)
if err != nil {
return nil, err
}
encapType := link.Attrs().EncapType
for _, attr := range attrs {
switch attr.Attr.Type {
case NDA_DST:
neigh.IP = net.IP(attr.Value)
case NDA_LLADDR:
neigh.HardwareAddr = net.HardwareAddr(attr.Value)
// BUG: Is this a bug in the netlink library?
// #define RTA_LENGTH(len) (RTA_ALIGN(sizeof(struct rtattr)) + (len))
// #define RTA_PAYLOAD(rta) ((int)((rta)->rta_len) - RTA_LENGTH(0))
attrLen := attr.Attr.Len - syscall.SizeofRtAttr
if attrLen == 4 && (encapType == "ipip" ||
encapType == "sit" ||
encapType == "gre") {
neigh.LLIPAddr = net.IP(attr.Value)
} else if attrLen == 16 &&
encapType == "tunnel6" {
neigh.IP = net.IP(attr.Value)
} else {
neigh.HardwareAddr = net.HardwareAddr(attr.Value)
}
}
}
@@ -108,6 +108,10 @@ func LinkSetFlood(link Link, mode bool) error {
return ErrNotImplemented
}
func LinkSetTxQLen(link Link, qlen int) error {
return ErrNotImplemented
}
func LinkAdd(link Link) error {
return ErrNotImplemented
}
@@ -0,0 +1,74 @@
package nl
import (
"fmt"
"unsafe"
)
const (
SizeofBridgeVlanInfo = 0x04
)
/* Bridge Flags */
const (
BRIDGE_FLAGS_MASTER = iota /* Bridge command to/from master */
BRIDGE_FLAGS_SELF /* Bridge command to/from lowerdev */
)
/* Bridge management nested attributes
* [IFLA_AF_SPEC] = {
* [IFLA_BRIDGE_FLAGS]
* [IFLA_BRIDGE_MODE]
* [IFLA_BRIDGE_VLAN_INFO]
* }
*/
const (
IFLA_BRIDGE_FLAGS = iota
IFLA_BRIDGE_MODE
IFLA_BRIDGE_VLAN_INFO
)
const (
BRIDGE_VLAN_INFO_MASTER = 1 << iota
BRIDGE_VLAN_INFO_PVID
BRIDGE_VLAN_INFO_UNTAGGED
BRIDGE_VLAN_INFO_RANGE_BEGIN
BRIDGE_VLAN_INFO_RANGE_END
)
// struct bridge_vlan_info {
// __u16 flags;
// __u16 vid;
// };
type BridgeVlanInfo struct {
Flags uint16
Vid uint16
}
func (b *BridgeVlanInfo) Serialize() []byte {
return (*(*[SizeofBridgeVlanInfo]byte)(unsafe.Pointer(b)))[:]
}
func DeserializeBridgeVlanInfo(b []byte) *BridgeVlanInfo {
return (*BridgeVlanInfo)(unsafe.Pointer(&b[0:SizeofBridgeVlanInfo][0]))
}
func (b *BridgeVlanInfo) PortVID() bool {
return b.Flags&BRIDGE_VLAN_INFO_PVID > 0
}
func (b *BridgeVlanInfo) EngressUntag() bool {
return b.Flags&BRIDGE_VLAN_INFO_UNTAGGED > 0
}
func (b *BridgeVlanInfo) String() string {
return fmt.Sprintf("%+v", *b)
}
/* New extended info filters for IFLA_EXT_MASK */
const (
RTEXT_FILTER_VF = 1 << iota
RTEXT_FILTER_BRVLAN
RTEXT_FILTER_BRVLAN_COMPRESSED
)
@@ -79,8 +79,8 @@ const (
CTA_TUPLE_ORIG = 1
CTA_TUPLE_REPLY = 2
CTA_STATUS = 3
CTA_TIMEOUT = 8
CTA_MARK = 9
CTA_TIMEOUT = 7
CTA_MARK = 8
CTA_PROTOINFO = 4
)
+34 -2
View File
@@ -231,7 +231,8 @@ const (
* on/off switch
*/
IFLA_VF_STATS /* network device statistics */
IFLA_VF_MAX = IFLA_VF_STATS
IFLA_VF_TRUST /* Trust state of VF */
IFLA_VF_MAX = IFLA_VF_TRUST
)
const (
@@ -259,6 +260,7 @@ const (
SizeofVfSpoofchk = 0x08
SizeofVfLinkState = 0x08
SizeofVfRssQueryEn = 0x08
SizeofVfTrust = 0x08
)
// struct ifla_vf_mac {
@@ -419,12 +421,42 @@ func (msg *VfRssQueryEn) Serialize() []byte {
return (*(*[SizeofVfRssQueryEn]byte)(unsafe.Pointer(msg)))[:]
}
// struct ifla_vf_trust {
// __u32 vf;
// __u32 setting;
// };
type VfTrust struct {
Vf uint32
Setting uint32
}
func (msg *VfTrust) Len() int {
return SizeofVfTrust
}
func DeserializeVfTrust(b []byte) *VfTrust {
return (*VfTrust)(unsafe.Pointer(&b[0:SizeofVfTrust][0]))
}
func (msg *VfTrust) Serialize() []byte {
return (*(*[SizeofVfTrust]byte)(unsafe.Pointer(msg)))[:]
}
const (
XDP_FLAGS_UPDATE_IF_NOEXIST = 1 << iota
XDP_FLAGS_SKB_MODE
XDP_FLAGS_DRV_MODE
XDP_FLAGS_MASK = XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_SKB_MODE | XDP_FLAGS_DRV_MODE
)
const (
IFLA_XDP_UNSPEC = iota
IFLA_XDP_FD /* fd of xdp program to attach, or -1 to remove */
IFLA_XDP_ATTACHED /* read-only bool indicating if prog is attached */
IFLA_XDP_FLAGS /* xdp prog related flags */
IFLA_XDP_MAX = IFLA_XDP_FLAGS
IFLA_XDP_PROG_ID /* xdp prog id */
IFLA_XDP_MAX = IFLA_XDP_PROG_ID
)
const (
+14
View File
@@ -621,6 +621,20 @@ func (s *NetlinkSocket) Receive() ([]syscall.NetlinkMessage, error) {
return syscall.ParseNetlinkMessage(rb)
}
// SetSendTimeout allows to set a send timeout on the socket
func (s *NetlinkSocket) SetSendTimeout(timeout *syscall.Timeval) error {
// Set a send timeout of SOCKET_SEND_TIMEOUT, this will allow the Send to periodically unblock and avoid that a routine
// remains stuck on a send on a closed fd
return syscall.SetsockoptTimeval(int(s.fd), syscall.SOL_SOCKET, syscall.SO_SNDTIMEO, timeout)
}
// SetReceiveTimeout allows to set a receive timeout on the socket
func (s *NetlinkSocket) SetReceiveTimeout(timeout *syscall.Timeval) error {
// Set a read timeout of SOCKET_READ_TIMEOUT, this will allow the Read to periodically unblock and avoid that a routine
// remains stuck on a recvmsg on a closed fd
return syscall.SetsockoptTimeval(int(s.fd), syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, timeout)
}
func (s *NetlinkSocket) GetPid() (uint32, error) {
fd := int(atomic.LoadInt32(&s.fd))
lsa, err := syscall.Getsockname(fd)
+37 -35
View File
@@ -160,71 +160,73 @@ func qdiscPayload(req *nl.NetlinkRequest, qdisc Qdisc) error {
req.AddData(nl.NewRtAttr(nl.TCA_KIND, nl.ZeroTerminated(qdisc.Type())))
options := nl.NewRtAttr(nl.TCA_OPTIONS, nil)
if prio, ok := qdisc.(*Prio); ok {
switch qdisc := qdisc.(type) {
case *Prio:
tcmap := nl.TcPrioMap{
Bands: int32(prio.Bands),
Priomap: prio.PriorityMap,
Bands: int32(qdisc.Bands),
Priomap: qdisc.PriorityMap,
}
options = nl.NewRtAttr(nl.TCA_OPTIONS, tcmap.Serialize())
} else if tbf, ok := qdisc.(*Tbf); ok {
case *Tbf:
opt := nl.TcTbfQopt{}
opt.Rate.Rate = uint32(tbf.Rate)
opt.Peakrate.Rate = uint32(tbf.Peakrate)
opt.Limit = tbf.Limit
opt.Buffer = tbf.Buffer
opt.Rate.Rate = uint32(qdisc.Rate)
opt.Peakrate.Rate = uint32(qdisc.Peakrate)
opt.Limit = qdisc.Limit
opt.Buffer = qdisc.Buffer
nl.NewRtAttrChild(options, nl.TCA_TBF_PARMS, opt.Serialize())
if tbf.Rate >= uint64(1<<32) {
nl.NewRtAttrChild(options, nl.TCA_TBF_RATE64, nl.Uint64Attr(tbf.Rate))
if qdisc.Rate >= uint64(1<<32) {
nl.NewRtAttrChild(options, nl.TCA_TBF_RATE64, nl.Uint64Attr(qdisc.Rate))
}
if tbf.Peakrate >= uint64(1<<32) {
nl.NewRtAttrChild(options, nl.TCA_TBF_PRATE64, nl.Uint64Attr(tbf.Peakrate))
if qdisc.Peakrate >= uint64(1<<32) {
nl.NewRtAttrChild(options, nl.TCA_TBF_PRATE64, nl.Uint64Attr(qdisc.Peakrate))
}
if tbf.Peakrate > 0 {
nl.NewRtAttrChild(options, nl.TCA_TBF_PBURST, nl.Uint32Attr(tbf.Minburst))
if qdisc.Peakrate > 0 {
nl.NewRtAttrChild(options, nl.TCA_TBF_PBURST, nl.Uint32Attr(qdisc.Minburst))
}
} else if htb, ok := qdisc.(*Htb); ok {
case *Htb:
opt := nl.TcHtbGlob{}
opt.Version = htb.Version
opt.Rate2Quantum = htb.Rate2Quantum
opt.Defcls = htb.Defcls
opt.Version = qdisc.Version
opt.Rate2Quantum = qdisc.Rate2Quantum
opt.Defcls = qdisc.Defcls
// TODO: Handle Debug properly. For now default to 0
opt.Debug = htb.Debug
opt.DirectPkts = htb.DirectPkts
opt.Debug = qdisc.Debug
opt.DirectPkts = qdisc.DirectPkts
nl.NewRtAttrChild(options, nl.TCA_HTB_INIT, opt.Serialize())
// nl.NewRtAttrChild(options, nl.TCA_HTB_DIRECT_QLEN, opt.Serialize())
} else if netem, ok := qdisc.(*Netem); ok {
case *Netem:
opt := nl.TcNetemQopt{}
opt.Latency = netem.Latency
opt.Limit = netem.Limit
opt.Loss = netem.Loss
opt.Gap = netem.Gap
opt.Duplicate = netem.Duplicate
opt.Jitter = netem.Jitter
opt.Latency = qdisc.Latency
opt.Limit = qdisc.Limit
opt.Loss = qdisc.Loss
opt.Gap = qdisc.Gap
opt.Duplicate = qdisc.Duplicate
opt.Jitter = qdisc.Jitter
options = nl.NewRtAttr(nl.TCA_OPTIONS, opt.Serialize())
// Correlation
corr := nl.TcNetemCorr{}
corr.DelayCorr = netem.DelayCorr
corr.LossCorr = netem.LossCorr
corr.DupCorr = netem.DuplicateCorr
corr.DelayCorr = qdisc.DelayCorr
corr.LossCorr = qdisc.LossCorr
corr.DupCorr = qdisc.DuplicateCorr
if corr.DelayCorr > 0 || corr.LossCorr > 0 || corr.DupCorr > 0 {
nl.NewRtAttrChild(options, nl.TCA_NETEM_CORR, corr.Serialize())
}
// Corruption
corruption := nl.TcNetemCorrupt{}
corruption.Probability = netem.CorruptProb
corruption.Correlation = netem.CorruptCorr
corruption.Probability = qdisc.CorruptProb
corruption.Correlation = qdisc.CorruptCorr
if corruption.Probability > 0 {
nl.NewRtAttrChild(options, nl.TCA_NETEM_CORRUPT, corruption.Serialize())
}
// Reorder
reorder := nl.TcNetemReorder{}
reorder.Probability = netem.ReorderProb
reorder.Correlation = netem.ReorderCorr
reorder.Probability = qdisc.ReorderProb
reorder.Correlation = qdisc.ReorderCorr
if reorder.Probability > 0 {
nl.NewRtAttrChild(options, nl.TCA_NETEM_REORDER, reorder.Serialize())
}
} else if _, ok := qdisc.(*Ingress); ok {
case *Ingress:
// ingress filters must use the proper handle
if qdisc.Attrs().Parent != HANDLE_INGRESS {
return fmt.Errorf("Ingress filters must set Parent to HANDLE_INGRESS")
+61 -1
View File
@@ -16,6 +16,7 @@ type Destination interface {
Decode([]byte) error
Encode() ([]byte, error)
String() string
Equal(Destination) bool
}
type Encap interface {
@@ -23,6 +24,7 @@ type Encap interface {
Decode([]byte) error
Encode() ([]byte, error)
String() string
Equal(Encap) bool
}
// Route represents a netlink route.
@@ -72,6 +74,25 @@ func (r Route) String() string {
return fmt.Sprintf("{%s}", strings.Join(elems, " "))
}
func (r Route) Equal(x Route) bool {
return r.LinkIndex == x.LinkIndex &&
r.ILinkIndex == x.ILinkIndex &&
r.Scope == x.Scope &&
ipNetEqual(r.Dst, x.Dst) &&
r.Src.Equal(x.Src) &&
r.Gw.Equal(x.Gw) &&
nexthopInfoSlice(r.MultiPath).Equal(x.MultiPath) &&
r.Protocol == x.Protocol &&
r.Priority == x.Priority &&
r.Table == x.Table &&
r.Type == x.Type &&
r.Tos == x.Tos &&
r.Flags == x.Flags &&
(r.MPLSDst == x.MPLSDst || (r.MPLSDst != nil && x.MPLSDst != nil && *r.MPLSDst == *x.MPLSDst)) &&
(r.NewDst == x.NewDst || (r.NewDst != nil && r.NewDst.Equal(x.NewDst))) &&
(r.Encap == x.Encap || (r.Encap != nil && r.Encap.Equal(x.Encap)))
}
func (r *Route) SetFlag(flag NextHopFlag) {
r.Flags |= int(flag)
}
@@ -110,7 +131,46 @@ func (n *NexthopInfo) String() string {
elems = append(elems, fmt.Sprintf("Encap: %s", n.Encap))
}
elems = append(elems, fmt.Sprintf("Weight: %d", n.Hops+1))
elems = append(elems, fmt.Sprintf("Gw: %d", n.Gw))
elems = append(elems, fmt.Sprintf("Gw: %s", n.Gw))
elems = append(elems, fmt.Sprintf("Flags: %s", n.ListFlags()))
return fmt.Sprintf("{%s}", strings.Join(elems, " "))
}
func (n NexthopInfo) Equal(x NexthopInfo) bool {
return n.LinkIndex == x.LinkIndex &&
n.Hops == x.Hops &&
n.Gw.Equal(x.Gw) &&
n.Flags == x.Flags &&
(n.NewDst == x.NewDst || (n.NewDst != nil && n.NewDst.Equal(x.NewDst))) &&
(n.Encap == x.Encap || (n.Encap != nil && n.Encap.Equal(x.Encap)))
}
type nexthopInfoSlice []*NexthopInfo
func (n nexthopInfoSlice) Equal(x []*NexthopInfo) bool {
if len(n) != len(x) {
return false
}
for i := range n {
if n[i] == nil || x[i] == nil {
return false
}
if !n[i].Equal(*x[i]) {
return false
}
}
return true
}
// ipNetEqual returns true iff both IPNet are equal
func ipNetEqual(ipn1 *net.IPNet, ipn2 *net.IPNet) bool {
if ipn1 == ipn2 {
return true
}
if ipn1 == nil || ipn2 == nil {
return false
}
m1, _ := ipn1.Mask.Size()
m2, _ := ipn2.Mask.Size()
return m1 == m2 && ipn1.IP.Equal(ipn2.IP)
}

Some files were not shown because too many files have changed in this diff Show More