From 0ce7aa426e7067da057606dc8a81adea51d69ec2 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Thu, 20 Oct 2016 11:26:04 -0700 Subject: [PATCH] Vendor swarmkit in master Also, update libnetwork and grpc. Signed-off-by: Aaron Lehmann Upstream-commit: 12a4ed03178982fec44b75479c3e63a316503d90 Component: engine --- components/engine/daemon/cluster/cluster.go | 6 +- components/engine/hack/vendor.sh | 6 +- .../src/github.com/docker/libnetwork/Makefile | 45 +- .../docker/libnetwork/cmd/proxy/tcp_proxy.go | 7 +- .../docker/libnetwork/cmd/proxy/udp_proxy.go | 9 +- .../docker/libnetwork/drivers_solaris.go | 10 +- .../libnetwork/netutils/utils_solaris.go | 80 ++- .../docker/libnetwork/networkdb/cluster.go | 25 +- .../docker/libnetwork/networkdb/delegate.go | 5 +- .../docker/libnetwork/networkdb/networkdb.go | 30 +- .../docker/libnetwork/ns/init_linux.go | 16 +- .../docker/libnetwork/osl/sandbox_solaris.go | 24 + .../libnetwork/osl/sandbox_unsupported.go | 2 +- .../libnetwork/portallocator/portallocator.go | 22 - .../portallocator/portallocator_linux.go | 27 + .../portallocator/portallocator_solaris.go | 5 + .../docker/libnetwork/portmapper/proxy.go | 32 - .../libnetwork/portmapper/proxy_linux.go | 38 + .../libnetwork/portmapper/proxy_solaris.go | 34 + .../github.com/docker/libnetwork/sandbox.go | 3 +- .../docker/swarmkit/agent/errors.go | 2 - .../docker/swarmkit/agent/resource.go | 2 +- .../docker/swarmkit/agent/session.go | 8 +- .../docker/swarmkit/api/control.pb.go | 92 +-- .../docker/swarmkit/api/objects.pb.go | 215 ++++-- .../docker/swarmkit/api/objects.proto | 4 + .../github.com/docker/swarmkit/api/raft.pb.go | 4 +- .../docker/swarmkit/api/snapshot.pb.go | 4 +- .../docker/swarmkit/api/specs.pb.go | 244 ++++--- .../docker/swarmkit/api/specs.proto | 6 + .../docker/swarmkit/api/types.pb.go | 674 ++++++++++++------ .../docker/swarmkit/api/types.proto | 10 + .../src/github.com/docker/swarmkit/ca/auth.go | 22 +- .../docker/swarmkit/ca/certificates.go | 14 +- .../github.com/docker/swarmkit/ca/external.go | 8 +- .../github.com/docker/swarmkit/ca/server.go | 118 ++- .../docker/swarmkit/ca/transport.go | 20 + .../docker/swarmkit/identity/randomid.go | 17 +- .../swarmkit/manager/allocator/allocator.go | 4 +- .../swarmkit/manager/allocator/network.go | 97 +-- .../networkallocator/drivers_darwin.go | 13 + .../networkallocator/drivers_unsupported.go | 2 +- .../networkallocator/networkallocator.go | 2 +- .../swarmkit/manager/controlapi/cluster.go | 1 + .../swarmkit/manager/controlapi/common.go | 7 +- .../swarmkit/manager/controlapi/node.go | 38 + .../swarmkit/manager/controlapi/secret.go | 1 - .../swarmkit/manager/controlapi/service.go | 18 +- .../swarmkit/manager/dispatcher/dispatcher.go | 5 - .../docker/swarmkit/manager/manager.go | 140 ++-- .../orchestrator/constraint_enforcer.go | 157 ++++ .../swarmkit/manager/orchestrator/global.go | 6 +- .../swarmkit/manager/orchestrator/services.go | 3 + .../swarmkit/manager/scheduler/filter.go | 37 + .../swarmkit/manager/scheduler/pipeline.go | 46 +- .../swarmkit/manager/scheduler/scheduler.go | 69 +- .../manager/state/raft/membership/cluster.go | 5 +- .../swarmkit/manager/state/raft/raft.go | 243 ++++--- .../swarmkit/manager/state/raft/storage.go | 12 +- .../docker/swarmkit/{agent => node}/node.go | 32 +- .../src/google.golang.org/grpc/balancer.go | 20 +- .../vendor/src/google.golang.org/grpc/call.go | 11 +- .../src/google.golang.org/grpc/clientconn.go | 127 ++-- .../grpc/credentials/credentials.go | 49 +- .../src/google.golang.org/grpc/interceptor.go | 16 + .../grpc/metadata/metadata.go | 13 +- .../src/google.golang.org/grpc/rpc_util.go | 4 +- .../src/google.golang.org/grpc/server.go | 26 +- .../src/google.golang.org/grpc/stream.go | 33 +- .../grpc/transport/handler_server.go | 4 +- .../grpc/transport/http2_client.go | 100 ++- .../grpc/transport/http2_server.go | 12 +- .../grpc/transport/http_util.go | 11 +- .../grpc/transport/transport.go | 31 +- 74 files changed, 2236 insertions(+), 1049 deletions(-) create mode 100644 components/engine/vendor/src/github.com/docker/libnetwork/osl/sandbox_solaris.go create mode 100644 components/engine/vendor/src/github.com/docker/libnetwork/portallocator/portallocator_linux.go create mode 100644 components/engine/vendor/src/github.com/docker/libnetwork/portallocator/portallocator_solaris.go create mode 100644 components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy_linux.go create mode 100644 components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy_solaris.go create mode 100644 components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/drivers_darwin.go create mode 100644 components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/constraint_enforcer.go rename components/engine/vendor/src/github.com/docker/swarmkit/{agent => node}/node.go (97%) diff --git a/components/engine/daemon/cluster/cluster.go b/components/engine/daemon/cluster/cluster.go index 8262537816..3973817ce4 100644 --- a/components/engine/daemon/cluster/cluster.go +++ b/components/engine/daemon/cluster/cluster.go @@ -26,8 +26,8 @@ import ( "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/signal" "github.com/docker/docker/runconfig" - swarmagent "github.com/docker/swarmkit/agent" swarmapi "github.com/docker/swarmkit/api" + swarmnode "github.com/docker/swarmkit/node" "golang.org/x/net/context" ) @@ -123,7 +123,7 @@ type attacher struct { } type node struct { - *swarmagent.Node + *swarmnode.Node done chan struct{} ready bool conn *grpc.ClientConn @@ -279,7 +279,7 @@ func (c *Cluster) startNewNode(forceNewCluster bool, localAddr, remoteAddr, list c.node = nil c.cancelDelay = nil c.stop = false - n, err := swarmagent.NewNode(&swarmagent.NodeConfig{ + n, err := swarmnode.New(&swarmnode.Config{ Hostname: c.config.Name, ForceNewCluster: forceNewCluster, ListenControlAPI: filepath.Join(c.runtimeRoot, controlSocket), diff --git a/components/engine/hack/vendor.sh b/components/engine/hack/vendor.sh index d590f7d8d6..1e81a6b084 100755 --- a/components/engine/hack/vendor.sh +++ b/components/engine/hack/vendor.sh @@ -70,7 +70,7 @@ clone git github.com/RackSec/srslog 365bf33cd9acc21ae1c355209865f17228ca534e clone git github.com/imdario/mergo 0.2.1 #get libnetwork packages -clone git github.com/docker/libnetwork 04025f2a2eebb0d091883e55980dc6916d36842d +clone git github.com/docker/libnetwork 9fbb4ecbb45af655c4ac3c2f3a849b2294cb447a clone git github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894 clone git github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec @@ -101,7 +101,7 @@ clone git github.com/pborman/uuid v1.0 # get desired notary commit, might also need to be updated in Dockerfile clone git github.com/docker/notary v0.4.2 -clone git google.golang.org/grpc v1.0.1-GA https://github.com/grpc/grpc-go.git +clone git google.golang.org/grpc v1.0.2 https://github.com/grpc/grpc-go.git clone git github.com/miekg/pkcs11 df8ae6ca730422dba20c768ff38ef7d79077a59f clone git github.com/docker/go v1.5.1-1-1-gbaf439e clone git github.com/agl/ed25519 d2b94fd789ea21d12fac1a4443dd3a3f79cda72c @@ -146,7 +146,7 @@ clone git github.com/docker/docker-credential-helpers v0.3.0 clone git github.com/docker/containerd 52ef1ceb4b660c42cf4ea9013180a5663968d4c7 # cluster -clone git github.com/docker/swarmkit 1fed8d2a2ccd2a9b6d6fb864d4ad3461fc6dc3eb +clone git github.com/docker/swarmkit 3b221eb0391d34ae0b9dac65df02b5b64de6dff2 clone git github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9 clone git github.com/gogo/protobuf v0.3 clone git github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/Makefile b/components/engine/vendor/src/github.com/docker/libnetwork/Makefile index a130dc656d..f1d25ff339 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/Makefile +++ b/components/engine/vendor/src/github.com/docker/libnetwork/Makefile @@ -8,6 +8,14 @@ ciargs = -e CIRCLECI -e "COVERALLS_TOKEN=$$COVERALLS_TOKEN" -e "INSIDECONTAINER= cidocker = docker run ${dockerargs} ${ciargs} $$EXTRA_ARGS ${container_env} ${build_image} CROSS_PLATFORMS = linux/amd64 linux/386 linux/arm windows/amd64 export PATH := $(CURDIR)/bin:$(PATH) +hostOS = ${shell go env GOHOSTOS} +ifeq (${hostOS}, solaris) + gnufind=gfind + gnutail=gtail +else + gnufind=find + gnutail=tail +endif all: ${build_image}.created build check integration-tests clean @@ -62,7 +70,40 @@ check-format: run-tests: @echo "Running tests... " @echo "mode: count" > coverage.coverprofile - @for dir in $$(find . -maxdepth 10 -not -path './.git*' -not -path '*/_*' -type d); do \ + @for dir in $$( ${gnufind} . -maxdepth 10 -not -path './.git*' -not -path '*/_*' -type d); do \ + if [ ${hostOS} == solaris ]; then \ + case "$$dir" in \ + "./cmd/dnet" ) \ + ;& \ + "./cmd/ovrouter" ) \ + ;& \ + "./ns" ) \ + ;& \ + "./iptables" ) \ + ;& \ + "./ipvs" ) \ + ;& \ + "./drivers/bridge" ) \ + ;& \ + "./drivers/host" ) \ + ;& \ + "./drivers/ipvlan" ) \ + ;& \ + "./drivers/macvlan" ) \ + ;& \ + "./drivers/overlay" ) \ + ;& \ + "./drivers/remote" ) \ + ;& \ + "./drivers/windows" ) \ + echo "Skipping $$dir on solaris host... "; \ + continue; \ + ;; \ + * )\ + echo "Entering $$dir ... "; \ + ;; \ + esac; \ + fi; \ if ls $$dir/*.go &> /dev/null; then \ pushd . &> /dev/null ; \ cd $$dir ; \ @@ -71,7 +112,7 @@ run-tests: if [ $$ret -ne 0 ]; then exit $$ret; fi ;\ popd &> /dev/null; \ if [ -f $$dir/profile.tmp ]; then \ - cat $$dir/profile.tmp | tail -n +2 >> coverage.coverprofile ; \ + cat $$dir/profile.tmp | ${gnutail} -n +2 >> coverage.coverprofile ; \ rm $$dir/profile.tmp ; \ fi ; \ fi ; \ diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/cmd/proxy/tcp_proxy.go b/components/engine/vendor/src/github.com/docker/libnetwork/cmd/proxy/tcp_proxy.go index 514b778773..1c713105db 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/cmd/proxy/tcp_proxy.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/cmd/proxy/tcp_proxy.go @@ -2,11 +2,10 @@ package main import ( "io" + "log" "net" "sync" "syscall" - - "github.com/Sirupsen/logrus" ) // TCPProxy is a proxy for TCP connections. It implements the Proxy interface to @@ -35,7 +34,7 @@ func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) { func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) { backend, err := net.DialTCP("tcp", nil, proxy.backendAddr) if err != nil { - logrus.Printf("Can't forward traffic to backend tcp/%v: %s\n", proxy.backendAddr, err) + log.Printf("Can't forward traffic to backend tcp/%v: %s\n", proxy.backendAddr, err) client.Close() return } @@ -79,7 +78,7 @@ func (proxy *TCPProxy) Run() { for { client, err := proxy.listener.Accept() if err != nil { - logrus.Printf("Stopping proxy on tcp/%v for tcp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err) + log.Printf("Stopping proxy on tcp/%v for tcp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err) return } go proxy.clientLoop(client.(*net.TCPConn), quit) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/cmd/proxy/udp_proxy.go b/components/engine/vendor/src/github.com/docker/libnetwork/cmd/proxy/udp_proxy.go index 2b895ae9d4..12b96db646 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/cmd/proxy/udp_proxy.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/cmd/proxy/udp_proxy.go @@ -2,13 +2,12 @@ package main import ( "encoding/binary" + "log" "net" "strings" "sync" "syscall" "time" - - "github.com/Sirupsen/logrus" ) const ( @@ -112,7 +111,7 @@ func (proxy *UDPProxy) Run() { // ECONNREFUSED like Read do (see comment in // UDPProxy.replyLoop) if !isClosedError(err) { - logrus.Printf("Stopping proxy on udp/%v for udp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err) + log.Printf("Stopping proxy on udp/%v for udp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err) } break } @@ -123,7 +122,7 @@ func (proxy *UDPProxy) Run() { if !hit { proxyConn, err = net.DialUDP("udp", nil, proxy.backendAddr) if err != nil { - logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err) + log.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err) proxy.connTrackLock.Unlock() continue } @@ -134,7 +133,7 @@ func (proxy *UDPProxy) Run() { for i := 0; i != read; { written, err := proxyConn.Write(readBuf[i:read]) if err != nil { - logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err) + log.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err) break } i += written diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers_solaris.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers_solaris.go index 89ae42c58c..ba5d6a93e0 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers_solaris.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers_solaris.go @@ -1,5 +1,13 @@ package libnetwork +import ( + "github.com/docker/libnetwork/drivers/null" + "github.com/docker/libnetwork/drivers/solaris/bridge" +) + func getInitializers() []initializer { - return []initializer{} + return []initializer{ + {bridge.Init, "bridge"}, + {null.Init, "null"}, + } } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/netutils/utils_solaris.go b/components/engine/vendor/src/github.com/docker/libnetwork/netutils/utils_solaris.go index d0356f6262..12d453f7ce 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/netutils/utils_solaris.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/netutils/utils_solaris.go @@ -1,13 +1,26 @@ +// +build solaris + package netutils -// Solaris: TODO - import ( + "fmt" "net" + "os/exec" + "strings" "github.com/docker/libnetwork/ipamutils" + "github.com/vishvananda/netlink" ) +var ( + networkGetRoutesFct func(netlink.Link, int) ([]netlink.Route, error) +) + +// CheckRouteOverlaps checks whether the passed network overlaps with any existing routes +func CheckRouteOverlaps(toCheck *net.IPNet) error { + return nil +} + // ElectInterfaceAddresses looks for an interface on the OS with the specified name // and returns its IPv4 and IPv6 addresses in CIDR form. If the interface does not exist, // it chooses from a predifined list the first IPv4 address which does not conflict @@ -15,18 +28,75 @@ import ( func ElectInterfaceAddresses(name string) (*net.IPNet, []*net.IPNet, error) { var ( v4Net *net.IPNet - err error ) - v4Net, err = FindAvailableNetwork(ipamutils.PredefinedBroadNetworks) + out, err := exec.Command("/usr/sbin/ipadm", "show-addr", + "-p", "-o", "addrobj,addr").Output() if err != nil { + fmt.Println("failed to list interfaces on system") return nil, nil, err } + alist := strings.Fields(string(out)) + for _, a := range alist { + linkandaddr := strings.SplitN(a, ":", 2) + if len(linkandaddr) != 2 { + fmt.Println("failed to check interfaces on system: ", a) + continue + } + gw := fmt.Sprintf("%s_gw0", name) + link := strings.Split(linkandaddr[0], "/")[0] + addr := linkandaddr[1] + if gw != link { + continue + } + _, ipnet, err := net.ParseCIDR(addr) + if err != nil { + fmt.Println("failed to parse address: ", addr) + continue + } + v4Net = ipnet + break + } + if v4Net == nil { + v4Net, err = FindAvailableNetwork(ipamutils.PredefinedBroadNetworks) + if err != nil { + return nil, nil, err + } + } return v4Net, nil, nil } // FindAvailableNetwork returns a network from the passed list which does not // overlap with existing interfaces in the system func FindAvailableNetwork(list []*net.IPNet) (*net.IPNet, error) { - return list[0], nil + out, err := exec.Command("/usr/sbin/ipadm", "show-addr", + "-p", "-o", "addr").Output() + + if err != nil { + fmt.Println("failed to list interfaces on system") + return nil, err + } + ipaddrs := strings.Fields(string(out)) + inuse := []*net.IPNet{} + for _, ip := range ipaddrs { + _, ipnet, err := net.ParseCIDR(ip) + if err != nil { + fmt.Println("failed to check interfaces on system: ", ip) + continue + } + inuse = append(inuse, ipnet) + } + for _, avail := range list { + is_avail := true + for _, ipnet := range inuse { + if NetworkOverlaps(avail, ipnet) { + is_avail = false + break + } + } + if is_avail { + return avail, nil + } + } + return nil, fmt.Errorf("no available network") } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/networkdb/cluster.go b/components/engine/vendor/src/github.com/docker/libnetwork/networkdb/cluster.go index fc6ac698d8..562f971af3 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/networkdb/cluster.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/networkdb/cluster.go @@ -270,19 +270,27 @@ func (nDB *NetworkDB) reconnectNode() { nDB.bulkSync([]string{node.Name}, true) } +// For timing the entry deletion in the repaer APIs that doesn't use monotonic clock +// source (time.Now, Sub etc.) should be avoided. Hence we use reapTime in every +// entry which is set initially to reapInterval and decremented by reapPeriod every time +// the reaper runs. NOTE nDB.reapTableEntries updates the reapTime with a readlock. This +// is safe as long as no other concurrent path touches the reapTime field. func (nDB *NetworkDB) reapState() { nDB.reapNetworks() nDB.reapTableEntries() } func (nDB *NetworkDB) reapNetworks() { - now := time.Now() nDB.Lock() for name, nn := range nDB.networks { for id, n := range nn { - if n.leaving && now.Sub(n.leaveTime) > reapInterval { - delete(nn, id) - nDB.deleteNetworkNode(id, name) + if n.leaving { + if n.reapTime <= 0 { + delete(nn, id) + nDB.deleteNetworkNode(id, name) + continue + } + n.reapTime -= reapPeriod } } } @@ -292,8 +300,6 @@ func (nDB *NetworkDB) reapNetworks() { func (nDB *NetworkDB) reapTableEntries() { var paths []string - now := time.Now() - nDB.RLock() nDB.indexes[byTable].Walk(func(path string, v interface{}) bool { entry, ok := v.(*entry) @@ -301,10 +307,13 @@ func (nDB *NetworkDB) reapTableEntries() { return false } - if !entry.deleting || now.Sub(entry.deleteTime) <= reapInterval { + if !entry.deleting { + return false + } + if entry.reapTime > 0 { + entry.reapTime -= reapPeriod return false } - paths = append(paths, path) return false }) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/networkdb/delegate.go b/components/engine/vendor/src/github.com/docker/libnetwork/networkdb/delegate.go index e886cfcab2..2f8ca48686 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/networkdb/delegate.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/networkdb/delegate.go @@ -4,7 +4,6 @@ import ( "fmt" "net" "strings" - "time" "github.com/Sirupsen/logrus" "github.com/gogo/protobuf/proto" @@ -121,7 +120,7 @@ func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool { n.ltime = nEvent.LTime n.leaving = nEvent.Type == NetworkEventTypeLeave if n.leaving { - n.leaveTime = time.Now() + n.reapTime = reapInterval } nDB.addNetworkNode(nEvent.NetworkID, nEvent.NodeName) @@ -178,7 +177,7 @@ func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool { } if e.deleting { - e.deleteTime = time.Now() + e.reapTime = reapInterval } nDB.Lock() diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/networkdb/networkdb.go b/components/engine/vendor/src/github.com/docker/libnetwork/networkdb/networkdb.go index 816a07ce71..69fc4fe249 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/networkdb/networkdb.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/networkdb/networkdb.go @@ -107,8 +107,9 @@ type network struct { // Node leave is in progress. leaving bool - // The time this node knew about the node's network leave. - leaveTime time.Time + // Number of seconds still left before a deleted network entry gets + // removed from networkDB + reapTime time.Duration // The broadcast queue for table event gossip. This is only // initialized for this node's network attachment entries. @@ -153,8 +154,9 @@ type entry struct { // the cluster for certain amount of time after deletion. deleting bool - // The wall clock time when this node learned about this deletion. - deleteTime time.Time + // Number of seconds still left before a deleted table entry gets + // removed from networkDB + reapTime time.Duration } // New creates a new instance of NetworkDB using the Config passed by @@ -286,11 +288,11 @@ func (nDB *NetworkDB) DeleteEntry(tname, nid, key string) error { } entry := &entry{ - ltime: nDB.tableClock.Increment(), - node: nDB.config.NodeName, - value: value, - deleting: true, - deleteTime: time.Now(), + ltime: nDB.tableClock.Increment(), + node: nDB.config.NodeName, + value: value, + deleting: true, + reapTime: reapInterval, } if err := nDB.sendTableEvent(TableEventTypeDelete, nid, tname, key, entry); err != nil { @@ -339,11 +341,11 @@ func (nDB *NetworkDB) deleteNodeTableEntries(node string) { key := params[2] entry := &entry{ - ltime: oldEntry.ltime, - node: node, - value: oldEntry.value, - deleting: true, - deleteTime: time.Now(), + ltime: oldEntry.ltime, + node: node, + value: oldEntry.value, + deleting: true, + reapTime: reapInterval, } nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/ns/init_linux.go b/components/engine/vendor/src/github.com/docker/libnetwork/ns/init_linux.go index 78529c7fbe..dd31f3e7e7 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/ns/init_linux.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/ns/init_linux.go @@ -69,8 +69,10 @@ func NlHandle() *netlink.Handle { func getSupportedNlFamilies() []int { fams := []int{syscall.NETLINK_ROUTE} if err := loadXfrmModules(); err != nil { - log.Warnf("Could not load necessary modules for IPSEC rules: %v", err) - return fams + if checkXfrmSocket() != nil { + log.Warnf("Could not load necessary modules for IPSEC rules: %v", err) + return fams + } } return append(fams, syscall.NETLINK_XFRM) } @@ -84,3 +86,13 @@ func loadXfrmModules() error { } return nil } + +// API check on required xfrm modules (xfrm_user, xfrm_algo) +func checkXfrmSocket() error { + fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_XFRM) + if err != nil { + return err + } + syscall.Close(fd) + return nil +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/osl/sandbox_solaris.go b/components/engine/vendor/src/github.com/docker/libnetwork/osl/sandbox_solaris.go new file mode 100644 index 0000000000..9de44e5a20 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/libnetwork/osl/sandbox_solaris.go @@ -0,0 +1,24 @@ +package osl + +// NewSandbox provides a new sandbox instance created in an os specific way +// provided a key which uniquely identifies the sandbox +func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) { + return nil, nil +} + +// GenerateKey generates a sandbox key based on the passed +// container id. +func GenerateKey(containerID string) string { + maxLen := 12 + + if len(containerID) < maxLen { + maxLen = len(containerID) + } + + return containerID[:maxLen] +} + +// InitOSContext initializes OS context while configuring network resources +func InitOSContext() func() { + return func() {} +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/osl/sandbox_unsupported.go b/components/engine/vendor/src/github.com/docker/libnetwork/osl/sandbox_unsupported.go index 51a656c806..49184d608d 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/osl/sandbox_unsupported.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/osl/sandbox_unsupported.go @@ -1,4 +1,4 @@ -// +build !linux,!windows,!freebsd +// +build !linux,!windows,!freebsd,!solaris package osl diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/portallocator/portallocator.go b/components/engine/vendor/src/github.com/docker/libnetwork/portallocator/portallocator.go index 240e94fc82..b7f790bed1 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/portallocator/portallocator.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/portallocator/portallocator.go @@ -1,11 +1,9 @@ package portallocator import ( - "bufio" "errors" "fmt" "net" - "os" "sync" ) @@ -106,26 +104,6 @@ func newInstance() *PortAllocator { } } -func getDynamicPortRange() (start int, end int, err error) { - const portRangeKernelParam = "/proc/sys/net/ipv4/ip_local_port_range" - portRangeFallback := fmt.Sprintf("using fallback port range %d-%d", DefaultPortRangeStart, DefaultPortRangeEnd) - file, err := os.Open(portRangeKernelParam) - if err != nil { - return 0, 0, fmt.Errorf("port allocator - %s due to error: %v", portRangeFallback, err) - } - - defer file.Close() - - n, err := fmt.Fscanf(bufio.NewReader(file), "%d\t%d", &start, &end) - if n != 2 || err != nil { - if err == nil { - err = fmt.Errorf("unexpected count of parsed numbers (%d)", n) - } - return 0, 0, fmt.Errorf("port allocator - failed to parse system ephemeral port range from %s - %s: %v", portRangeKernelParam, portRangeFallback, err) - } - return start, end, nil -} - // RequestPort requests new port from global ports pool for specified ip and proto. // If port is 0 it returns first free port. Otherwise it checks port availability // in proto's pool and returns that port or error if port is already busy. diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/portallocator/portallocator_linux.go b/components/engine/vendor/src/github.com/docker/libnetwork/portallocator/portallocator_linux.go new file mode 100644 index 0000000000..687f6dabb7 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/libnetwork/portallocator/portallocator_linux.go @@ -0,0 +1,27 @@ +package portallocator + +import ( + "bufio" + "fmt" + "os" +) + +func getDynamicPortRange() (start int, end int, err error) { + const portRangeKernelParam = "/proc/sys/net/ipv4/ip_local_port_range" + portRangeFallback := fmt.Sprintf("using fallback port range %d-%d", DefaultPortRangeStart, DefaultPortRangeEnd) + file, err := os.Open(portRangeKernelParam) + if err != nil { + return 0, 0, fmt.Errorf("port allocator - %s due to error: %v", portRangeFallback, err) + } + + defer file.Close() + + n, err := fmt.Fscanf(bufio.NewReader(file), "%d\t%d", &start, &end) + if n != 2 || err != nil { + if err == nil { + err = fmt.Errorf("unexpected count of parsed numbers (%d)", n) + } + return 0, 0, fmt.Errorf("port allocator - failed to parse system ephemeral port range from %s - %s: %v", portRangeKernelParam, portRangeFallback, err) + } + return start, end, nil +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/portallocator/portallocator_solaris.go b/components/engine/vendor/src/github.com/docker/libnetwork/portallocator/portallocator_solaris.go new file mode 100644 index 0000000000..ccc20b1373 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/libnetwork/portallocator/portallocator_solaris.go @@ -0,0 +1,5 @@ +package portallocator + +func getDynamicPortRange() (start int, end int, err error) { + return 32768, 65535, nil +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy.go b/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy.go index 6a4adbb872..a5bdc55348 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy.go @@ -7,8 +7,6 @@ import ( "net" "os" "os/exec" - "strconv" - "syscall" "time" ) @@ -25,36 +23,6 @@ type proxyCommand struct { cmd *exec.Cmd } -func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int, proxyPath string) (userlandProxy, error) { - path := proxyPath - if proxyPath == "" { - cmd, err := exec.LookPath(userlandProxyCommandName) - if err != nil { - return nil, err - } - path = cmd - } - - args := []string{ - path, - "-proto", proto, - "-host-ip", hostIP.String(), - "-host-port", strconv.Itoa(hostPort), - "-container-ip", containerIP.String(), - "-container-port", strconv.Itoa(containerPort), - } - - return &proxyCommand{ - cmd: &exec.Cmd{ - Path: path, - Args: args, - SysProcAttr: &syscall.SysProcAttr{ - Pdeathsig: syscall.SIGTERM, // send a sigterm to the proxy if the daemon process dies - }, - }, - }, nil -} - func (p *proxyCommand) Start() error { r, w, err := os.Pipe() if err != nil { diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy_linux.go b/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy_linux.go new file mode 100644 index 0000000000..947cd0ba4b --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy_linux.go @@ -0,0 +1,38 @@ +package portmapper + +import ( + "net" + "os/exec" + "strconv" + "syscall" +) + +func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int, proxyPath string) (userlandProxy, error) { + path := proxyPath + if proxyPath == "" { + cmd, err := exec.LookPath(userlandProxyCommandName) + if err != nil { + return nil, err + } + path = cmd + } + + args := []string{ + path, + "-proto", proto, + "-host-ip", hostIP.String(), + "-host-port", strconv.Itoa(hostPort), + "-container-ip", containerIP.String(), + "-container-port", strconv.Itoa(containerPort), + } + + return &proxyCommand{ + cmd: &exec.Cmd{ + Path: path, + Args: args, + SysProcAttr: &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGTERM, // send a sigterm to the proxy if the daemon process dies + }, + }, + }, nil +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy_solaris.go b/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy_solaris.go new file mode 100644 index 0000000000..dc70b5edce --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/libnetwork/portmapper/proxy_solaris.go @@ -0,0 +1,34 @@ +package portmapper + +import ( + "net" + "os/exec" + "strconv" +) + +func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int, proxyPath string) (userlandProxy, error) { + path := proxyPath + if proxyPath == "" { + cmd, err := exec.LookPath(userlandProxyCommandName) + if err != nil { + return nil, err + } + path = cmd + } + + args := []string{ + path, + "-proto", proto, + "-host-ip", hostIP.String(), + "-host-port", strconv.Itoa(hostPort), + "-container-ip", containerIP.String(), + "-container-port", strconv.Itoa(containerPort), + } + + return &proxyCommand{ + cmd: &exec.Cmd{ + Path: path, + Args: args, + }, + }, nil +} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/sandbox.go b/components/engine/vendor/src/github.com/docker/libnetwork/sandbox.go index f5d86ef20e..7d1eef5b73 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/sandbox.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/sandbox.go @@ -848,8 +848,9 @@ func (sb *sandbox) clearNetworkResources(origEp *endpoint) error { releaseOSSboxResources(osSbox, ep) } - delete(sb.populatedEndpoints, ep.ID()) sb.Lock() + delete(sb.populatedEndpoints, ep.ID()) + if len(sb.endpoints) == 0 { // sb.endpoints should never be empty and this is unexpected error condition // We log an error message to note this down for debugging purposes. diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/agent/errors.go b/components/engine/vendor/src/github.com/docker/swarmkit/agent/errors.go index 711c96bf35..917869a7d2 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/agent/errors.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/agent/errors.go @@ -10,8 +10,6 @@ var ( ErrClosed = errors.New("agent: closed") errNodeNotRegistered = fmt.Errorf("node not registered") - errNodeStarted = errors.New("node: already started") - errNodeNotStarted = errors.New("node: not started") errAgentStarted = errors.New("agent: already started") errAgentNotStarted = errors.New("agent: not started") diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/agent/resource.go b/components/engine/vendor/src/github.com/docker/swarmkit/agent/resource.go index 7744269712..eca7564aa0 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/agent/resource.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/agent/resource.go @@ -16,7 +16,7 @@ type ResourceAllocator interface { // given a target network and a unique ID representing the // connecting entity and optionally a list of ipv4/ipv6 // addresses to be assigned to the attachment. AttachNetwork - // returns a unique ID for the attachment if successfull or an + // returns a unique ID for the attachment if successful or an // error in case of failure. AttachNetwork(ctx context.Context, id, target string, addresses []string) (string, error) diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/agent/session.go b/components/engine/vendor/src/github.com/docker/swarmkit/agent/session.go index 035766c15b..24ad48073a 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/agent/session.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/agent/session.go @@ -76,7 +76,13 @@ func newSession(ctx context.Context, agent *Agent, delay time.Duration, sessionI } func (s *session) run(ctx context.Context, delay time.Duration, description *api.NodeDescription) { - time.Sleep(delay) // delay before registering. + timer := time.NewTimer(delay) // delay before registering. + defer timer.Stop() + select { + case <-timer.C: + case <-ctx.Done(): + return + } if err := s.start(ctx, description); err != nil { select { diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/control.pb.go b/components/engine/vendor/src/github.com/docker/swarmkit/api/control.pb.go index 8ad69313bf..6945199a7f 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/control.pb.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/control.pb.go @@ -831,16 +831,12 @@ func (m *ListNodesRequest_Filters) Copy() *ListNodesRequest_Filters { if m.Names != nil { o.Names = make([]string, 0, len(m.Names)) - for _, v := range m.Names { - o.Names = append(o.Names, v) - } + o.Names = append(o.Names, m.Names...) } if m.IDPrefixes != nil { o.IDPrefixes = make([]string, 0, len(m.IDPrefixes)) - for _, v := range m.IDPrefixes { - o.IDPrefixes = append(o.IDPrefixes, v) - } + o.IDPrefixes = append(o.IDPrefixes, m.IDPrefixes...) } if m.Labels != nil { @@ -852,23 +848,17 @@ func (m *ListNodesRequest_Filters) Copy() *ListNodesRequest_Filters { if m.Memberships != nil { o.Memberships = make([]NodeSpec_Membership, 0, len(m.Memberships)) - for _, v := range m.Memberships { - o.Memberships = append(o.Memberships, v) - } + o.Memberships = append(o.Memberships, m.Memberships...) } if m.Roles != nil { o.Roles = make([]NodeRole, 0, len(m.Roles)) - for _, v := range m.Roles { - o.Roles = append(o.Roles, v) - } + o.Roles = append(o.Roles, m.Roles...) } if m.NamePrefixes != nil { o.NamePrefixes = make([]string, 0, len(m.NamePrefixes)) - for _, v := range m.NamePrefixes { - o.NamePrefixes = append(o.NamePrefixes, v) - } + o.NamePrefixes = append(o.NamePrefixes, m.NamePrefixes...) } return o @@ -1007,16 +997,12 @@ func (m *ListTasksRequest_Filters) Copy() *ListTasksRequest_Filters { if m.Names != nil { o.Names = make([]string, 0, len(m.Names)) - for _, v := range m.Names { - o.Names = append(o.Names, v) - } + o.Names = append(o.Names, m.Names...) } if m.IDPrefixes != nil { o.IDPrefixes = make([]string, 0, len(m.IDPrefixes)) - for _, v := range m.IDPrefixes { - o.IDPrefixes = append(o.IDPrefixes, v) - } + o.IDPrefixes = append(o.IDPrefixes, m.IDPrefixes...) } if m.Labels != nil { @@ -1028,30 +1014,22 @@ func (m *ListTasksRequest_Filters) Copy() *ListTasksRequest_Filters { if m.ServiceIDs != nil { o.ServiceIDs = make([]string, 0, len(m.ServiceIDs)) - for _, v := range m.ServiceIDs { - o.ServiceIDs = append(o.ServiceIDs, v) - } + o.ServiceIDs = append(o.ServiceIDs, m.ServiceIDs...) } if m.NodeIDs != nil { o.NodeIDs = make([]string, 0, len(m.NodeIDs)) - for _, v := range m.NodeIDs { - o.NodeIDs = append(o.NodeIDs, v) - } + o.NodeIDs = append(o.NodeIDs, m.NodeIDs...) } if m.DesiredStates != nil { o.DesiredStates = make([]TaskState, 0, len(m.DesiredStates)) - for _, v := range m.DesiredStates { - o.DesiredStates = append(o.DesiredStates, v) - } + o.DesiredStates = append(o.DesiredStates, m.DesiredStates...) } if m.NamePrefixes != nil { o.NamePrefixes = make([]string, 0, len(m.NamePrefixes)) - for _, v := range m.NamePrefixes { - o.NamePrefixes = append(o.NamePrefixes, v) - } + o.NamePrefixes = append(o.NamePrefixes, m.NamePrefixes...) } return o @@ -1191,16 +1169,12 @@ func (m *ListServicesRequest_Filters) Copy() *ListServicesRequest_Filters { if m.Names != nil { o.Names = make([]string, 0, len(m.Names)) - for _, v := range m.Names { - o.Names = append(o.Names, v) - } + o.Names = append(o.Names, m.Names...) } if m.IDPrefixes != nil { o.IDPrefixes = make([]string, 0, len(m.IDPrefixes)) - for _, v := range m.IDPrefixes { - o.IDPrefixes = append(o.IDPrefixes, v) - } + o.IDPrefixes = append(o.IDPrefixes, m.IDPrefixes...) } if m.Labels != nil { @@ -1212,9 +1186,7 @@ func (m *ListServicesRequest_Filters) Copy() *ListServicesRequest_Filters { if m.NamePrefixes != nil { o.NamePrefixes = make([]string, 0, len(m.NamePrefixes)) - for _, v := range m.NamePrefixes { - o.NamePrefixes = append(o.NamePrefixes, v) - } + o.NamePrefixes = append(o.NamePrefixes, m.NamePrefixes...) } return o @@ -1330,16 +1302,12 @@ func (m *ListNetworksRequest_Filters) Copy() *ListNetworksRequest_Filters { if m.Names != nil { o.Names = make([]string, 0, len(m.Names)) - for _, v := range m.Names { - o.Names = append(o.Names, v) - } + o.Names = append(o.Names, m.Names...) } if m.IDPrefixes != nil { o.IDPrefixes = make([]string, 0, len(m.IDPrefixes)) - for _, v := range m.IDPrefixes { - o.IDPrefixes = append(o.IDPrefixes, v) - } + o.IDPrefixes = append(o.IDPrefixes, m.IDPrefixes...) } if m.Labels != nil { @@ -1351,9 +1319,7 @@ func (m *ListNetworksRequest_Filters) Copy() *ListNetworksRequest_Filters { if m.NamePrefixes != nil { o.NamePrefixes = make([]string, 0, len(m.NamePrefixes)) - for _, v := range m.NamePrefixes { - o.NamePrefixes = append(o.NamePrefixes, v) - } + o.NamePrefixes = append(o.NamePrefixes, m.NamePrefixes...) } return o @@ -1421,16 +1387,12 @@ func (m *ListClustersRequest_Filters) Copy() *ListClustersRequest_Filters { if m.Names != nil { o.Names = make([]string, 0, len(m.Names)) - for _, v := range m.Names { - o.Names = append(o.Names, v) - } + o.Names = append(o.Names, m.Names...) } if m.IDPrefixes != nil { o.IDPrefixes = make([]string, 0, len(m.IDPrefixes)) - for _, v := range m.IDPrefixes { - o.IDPrefixes = append(o.IDPrefixes, v) - } + o.IDPrefixes = append(o.IDPrefixes, m.IDPrefixes...) } if m.Labels != nil { @@ -1442,9 +1404,7 @@ func (m *ListClustersRequest_Filters) Copy() *ListClustersRequest_Filters { if m.NamePrefixes != nil { o.NamePrefixes = make([]string, 0, len(m.NamePrefixes)) - for _, v := range m.NamePrefixes { - o.NamePrefixes = append(o.NamePrefixes, v) - } + o.NamePrefixes = append(o.NamePrefixes, m.NamePrefixes...) } return o @@ -1552,16 +1512,12 @@ func (m *ListSecretsRequest_Filters) Copy() *ListSecretsRequest_Filters { if m.Names != nil { o.Names = make([]string, 0, len(m.Names)) - for _, v := range m.Names { - o.Names = append(o.Names, v) - } + o.Names = append(o.Names, m.Names...) } if m.IDPrefixes != nil { o.IDPrefixes = make([]string, 0, len(m.IDPrefixes)) - for _, v := range m.IDPrefixes { - o.IDPrefixes = append(o.IDPrefixes, v) - } + o.IDPrefixes = append(o.IDPrefixes, m.IDPrefixes...) } if m.Labels != nil { @@ -1573,9 +1529,7 @@ func (m *ListSecretsRequest_Filters) Copy() *ListSecretsRequest_Filters { if m.NamePrefixes != nil { o.NamePrefixes = make([]string, 0, len(m.NamePrefixes)) - for _, v := range m.NamePrefixes { - o.NamePrefixes = append(o.NamePrefixes, v) - } + o.NamePrefixes = append(o.NamePrefixes, m.NamePrefixes...) } return o diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/objects.pb.go b/components/engine/vendor/src/github.com/docker/swarmkit/api/objects.pb.go index 3cad399730..70d87b43ce 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/objects.pb.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/objects.pb.go @@ -227,6 +227,9 @@ type Cluster struct { // and agents to unambiguously identify the older key to be deleted when // a new key is allocated on key rotation. EncryptionKeyLamportClock uint64 `protobuf:"varint,6,opt,name=encryption_key_lamport_clock,json=encryptionKeyLamportClock,proto3" json:"encryption_key_lamport_clock,omitempty"` + // RemovedNodes is the list of nodes that have been removed from the + // swarm. Their certificates should effectively be blacklisted. + RemovedNodes []*RemovedNode `protobuf:"bytes,7,rep,name=removed_nodes,json=removedNodes" json:"removed_nodes,omitempty"` } func (m *Cluster) Reset() { *m = Cluster{} } @@ -401,16 +404,12 @@ func (m *NetworkAttachment) Copy() *NetworkAttachment { if m.Addresses != nil { o.Addresses = make([]string, 0, len(m.Addresses)) - for _, v := range m.Addresses { - o.Addresses = append(o.Addresses, v) - } + o.Addresses = append(o.Addresses, m.Addresses...) } if m.Aliases != nil { o.Aliases = make([]string, 0, len(m.Aliases)) - for _, v := range m.Aliases { - o.Aliases = append(o.Aliases, v) - } + o.Aliases = append(o.Aliases, m.Aliases...) } return o @@ -452,6 +451,13 @@ func (m *Cluster) Copy() *Cluster { } } + if m.RemovedNodes != nil { + o.RemovedNodes = make([]*RemovedNode, 0, len(m.RemovedNodes)) + for _, v := range m.RemovedNodes { + o.RemovedNodes = append(o.RemovedNodes, v.Copy()) + } + } + return o } @@ -625,7 +631,7 @@ func (this *Cluster) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 10) + s := make([]string, 0, 11) s = append(s, "&api.Cluster{") s = append(s, "ID: "+fmt.Sprintf("%#v", this.ID)+",\n") s = append(s, "Meta: "+strings.Replace(this.Meta.GoString(), `&`, ``, 1)+",\n") @@ -635,6 +641,9 @@ func (this *Cluster) GoString() string { s = append(s, "NetworkBootstrapKeys: "+fmt.Sprintf("%#v", this.NetworkBootstrapKeys)+",\n") } s = append(s, "EncryptionKeyLamportClock: "+fmt.Sprintf("%#v", this.EncryptionKeyLamportClock)+",\n") + if this.RemovedNodes != nil { + s = append(s, "RemovedNodes: "+fmt.Sprintf("%#v", this.RemovedNodes)+",\n") + } s = append(s, "}") return strings.Join(s, "") } @@ -1261,6 +1270,18 @@ func (m *Cluster) MarshalTo(data []byte) (int, error) { i++ i = encodeVarintObjects(data, i, uint64(m.EncryptionKeyLamportClock)) } + if len(m.RemovedNodes) > 0 { + for _, msg := range m.RemovedNodes { + data[i] = 0x3a + i++ + i = encodeVarintObjects(data, i, uint64(msg.Size())) + n, err := msg.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n + } + } return i, nil } @@ -1575,6 +1596,12 @@ func (m *Cluster) Size() (n int) { if m.EncryptionKeyLamportClock != 0 { n += 1 + sovObjects(uint64(m.EncryptionKeyLamportClock)) } + if len(m.RemovedNodes) > 0 { + for _, e := range m.RemovedNodes { + l = e.Size() + n += 1 + l + sovObjects(uint64(l)) + } + } return n } @@ -1741,6 +1768,7 @@ func (this *Cluster) String() string { `RootCA:` + strings.Replace(strings.Replace(this.RootCA.String(), "RootCA", "RootCA", 1), `&`, ``, 1) + `,`, `NetworkBootstrapKeys:` + strings.Replace(fmt.Sprintf("%v", this.NetworkBootstrapKeys), "EncryptionKey", "EncryptionKey", 1) + `,`, `EncryptionKeyLamportClock:` + fmt.Sprintf("%v", this.EncryptionKeyLamportClock) + `,`, + `RemovedNodes:` + strings.Replace(fmt.Sprintf("%v", this.RemovedNodes), "RemovedNode", "RemovedNode", 1) + `,`, `}`, }, "") return s @@ -3669,6 +3697,37 @@ func (m *Cluster) Unmarshal(data []byte) error { break } } + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RemovedNodes", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowObjects + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthObjects + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RemovedNodes = append(m.RemovedNodes, &RemovedNode{}) + if err := m.RemovedNodes[len(m.RemovedNodes)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipObjects(data[iNdEx:]) @@ -4005,74 +4064,76 @@ var ( func init() { proto.RegisterFile("objects.proto", fileDescriptorObjects) } var fileDescriptorObjects = []byte{ - // 1101 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4f, 0x6f, 0x1b, 0xc5, - 0x1b, 0xee, 0xda, 0x5b, 0xdb, 0xfb, 0x3a, 0x8e, 0xf4, 0x9b, 0x5f, 0x15, 0x6d, 0x43, 0xb0, 0x83, - 0x2b, 0x50, 0x0f, 0x95, 0x2b, 0x4a, 0x41, 0xad, 0x68, 0x85, 0xfc, 0x27, 0x02, 0xab, 0x04, 0xa2, - 0x49, 0x49, 0x8f, 0xab, 0xc9, 0xee, 0xd4, 0x0c, 0xb6, 0x77, 0x56, 0x33, 0x63, 0x57, 0xe9, 0x09, - 0xf1, 0x01, 0xf8, 0x08, 0x7c, 0x15, 0xae, 0x39, 0x70, 0xe0, 0x06, 0x27, 0x8b, 0xf8, 0x80, 0xc4, - 0x09, 0x3e, 0x02, 0x9a, 0xd9, 0x59, 0xc7, 0x91, 0xd7, 0x21, 0x95, 0xaa, 0xdc, 0xe6, 0xf5, 0x3c, - 0xcf, 0xf3, 0xfe, 0x99, 0x77, 0xde, 0x59, 0x43, 0x8d, 0x1f, 0x7f, 0x47, 0x43, 0x25, 0x5b, 0x89, - 0xe0, 0x8a, 0x23, 0x14, 0xf1, 0x70, 0x48, 0x45, 0x4b, 0xbe, 0x22, 0x62, 0x3c, 0x64, 0xaa, 0x35, - 0xfd, 0x70, 0xbb, 0xaa, 0x4e, 0x12, 0x6a, 0x01, 0xdb, 0x55, 0x99, 0xd0, 0x30, 0x33, 0x6e, 0x2b, - 0x36, 0xa6, 0x52, 0x91, 0x71, 0x72, 0x7f, 0xb1, 0xb2, 0x5b, 0xb7, 0x06, 0x7c, 0xc0, 0xcd, 0xf2, - 0xbe, 0x5e, 0xa5, 0xbf, 0x36, 0x7f, 0x76, 0xc0, 0xdd, 0xa7, 0x8a, 0xa0, 0x4f, 0xa1, 0x3c, 0xa5, - 0x42, 0x32, 0x1e, 0xfb, 0xce, 0xae, 0x73, 0xb7, 0xfa, 0xe0, 0x9d, 0xd6, 0xaa, 0xe7, 0xd6, 0x51, - 0x0a, 0xe9, 0xb8, 0xa7, 0xb3, 0xc6, 0x0d, 0x9c, 0x31, 0xd0, 0x13, 0x80, 0x50, 0x50, 0xa2, 0x68, - 0x14, 0x10, 0xe5, 0x17, 0x0c, 0xff, 0xdd, 0x3c, 0xfe, 0xf3, 0x2c, 0x28, 0xec, 0x59, 0x42, 0x5b, - 0x69, 0xf6, 0x24, 0x89, 0x32, 0x76, 0xf1, 0x4a, 0x6c, 0x4b, 0x68, 0xab, 0xe6, 0x5f, 0x45, 0x70, - 0xbf, 0xe2, 0x11, 0x45, 0x5b, 0x50, 0x60, 0x91, 0x09, 0xde, 0xeb, 0x94, 0xe6, 0xb3, 0x46, 0xa1, - 0xdf, 0xc3, 0x05, 0x16, 0xa1, 0x07, 0xe0, 0x8e, 0xa9, 0x22, 0x36, 0x2c, 0x3f, 0x4f, 0x58, 0x57, - 0xc0, 0xe6, 0x64, 0xb0, 0xe8, 0x13, 0x70, 0x75, 0x59, 0x6d, 0x30, 0x3b, 0x79, 0x1c, 0xed, 0xf3, - 0x30, 0xa1, 0x61, 0xc6, 0xd3, 0x78, 0xb4, 0x07, 0xd5, 0x88, 0xca, 0x50, 0xb0, 0x44, 0xe9, 0x4a, - 0xba, 0x86, 0x7e, 0x67, 0x1d, 0xbd, 0x77, 0x0e, 0xc5, 0xcb, 0x3c, 0xf4, 0x04, 0x4a, 0x52, 0x11, - 0x35, 0x91, 0xfe, 0x4d, 0xa3, 0x50, 0x5f, 0x1b, 0x80, 0x41, 0xd9, 0x10, 0x2c, 0x07, 0x7d, 0x01, - 0x9b, 0x63, 0x12, 0x93, 0x01, 0x15, 0x81, 0x55, 0x29, 0x19, 0x95, 0xf7, 0x72, 0x53, 0x4f, 0x91, - 0xa9, 0x10, 0xae, 0x8d, 0x97, 0x4d, 0xb4, 0x07, 0x40, 0x94, 0x22, 0xe1, 0xb7, 0x63, 0x1a, 0x2b, - 0xbf, 0x6c, 0x54, 0xde, 0xcf, 0x8d, 0x85, 0xaa, 0x57, 0x5c, 0x0c, 0xdb, 0x0b, 0x30, 0x5e, 0x22, - 0xa2, 0xcf, 0xa1, 0x1a, 0x52, 0xa1, 0xd8, 0x4b, 0x16, 0x12, 0x45, 0xfd, 0x8a, 0xd1, 0x69, 0xe4, - 0xe9, 0x74, 0xcf, 0x61, 0x36, 0xa9, 0x65, 0x66, 0xf3, 0xb7, 0x02, 0x94, 0x0f, 0xa9, 0x98, 0xb2, - 0xf0, 0xed, 0x1e, 0xf7, 0xe3, 0x0b, 0xc7, 0x9d, 0x1b, 0x99, 0x75, 0xbb, 0x72, 0xe2, 0x8f, 0xa0, - 0x42, 0xe3, 0x28, 0xe1, 0x2c, 0x56, 0xf6, 0xb8, 0x73, 0xbb, 0x65, 0xcf, 0x62, 0xf0, 0x02, 0x8d, - 0xf6, 0xa0, 0x96, 0x76, 0x71, 0x70, 0xe1, 0xac, 0x77, 0xf3, 0xe8, 0xdf, 0x18, 0xa0, 0x3d, 0xa4, - 0x8d, 0xc9, 0x92, 0x85, 0x7a, 0x50, 0x4b, 0x04, 0x9d, 0x32, 0x3e, 0x91, 0x81, 0x49, 0xa2, 0x74, - 0xa5, 0x24, 0xf0, 0x46, 0xc6, 0xd2, 0x56, 0xf3, 0xa7, 0x02, 0x54, 0xb2, 0x18, 0xd1, 0x43, 0x5b, - 0x0e, 0x67, 0x7d, 0x40, 0x19, 0xd6, 0x48, 0xa5, 0x95, 0x78, 0x08, 0x37, 0x13, 0x2e, 0x94, 0xf4, - 0x0b, 0xbb, 0xc5, 0x75, 0x3d, 0x7b, 0xc0, 0x85, 0xea, 0xf2, 0xf8, 0x25, 0x1b, 0xe0, 0x14, 0x8c, - 0x5e, 0x40, 0x75, 0xca, 0x84, 0x9a, 0x90, 0x51, 0xc0, 0x12, 0xe9, 0x17, 0x0d, 0xf7, 0x83, 0xcb, - 0x5c, 0xb6, 0x8e, 0x52, 0x7c, 0xff, 0xa0, 0xb3, 0x39, 0x9f, 0x35, 0x60, 0x61, 0x4a, 0x0c, 0x56, - 0xaa, 0x9f, 0xc8, 0xed, 0x7d, 0xf0, 0x16, 0x3b, 0xe8, 0x1e, 0x40, 0x9c, 0xb6, 0x68, 0xb0, 0x68, - 0x9a, 0xda, 0x7c, 0xd6, 0xf0, 0x6c, 0xe3, 0xf6, 0x7b, 0xd8, 0xb3, 0x80, 0x7e, 0x84, 0x10, 0xb8, - 0x24, 0x8a, 0x84, 0x69, 0x21, 0x0f, 0x9b, 0x75, 0xf3, 0x97, 0x9b, 0xe0, 0x3e, 0x27, 0x72, 0x78, - 0xdd, 0x63, 0x46, 0xfb, 0x5c, 0x69, 0xba, 0x7b, 0x00, 0x32, 0x3d, 0x4a, 0x9d, 0x8e, 0x7b, 0x9e, - 0x8e, 0x3d, 0x60, 0x9d, 0x8e, 0x05, 0xa4, 0xe9, 0xc8, 0x11, 0x57, 0xa6, 0xbf, 0x5c, 0x6c, 0xd6, - 0xe8, 0x0e, 0x94, 0x63, 0x1e, 0x19, 0x7a, 0xc9, 0xd0, 0x61, 0x3e, 0x6b, 0x94, 0xf4, 0x48, 0xe9, - 0xf7, 0x70, 0x49, 0x6f, 0xf5, 0x23, 0x7d, 0x6f, 0x49, 0x1c, 0x73, 0x45, 0xf4, 0x50, 0x92, 0xf6, - 0xfe, 0xe7, 0x36, 0x56, 0xfb, 0x1c, 0x96, 0xdd, 0xdb, 0x25, 0x26, 0x3a, 0x82, 0xff, 0x67, 0xf1, - 0x2e, 0x0b, 0x56, 0xde, 0x44, 0x10, 0x59, 0x85, 0xa5, 0x9d, 0xa5, 0x39, 0xe9, 0xad, 0x9f, 0x93, - 0xa6, 0x82, 0x79, 0x73, 0xb2, 0x03, 0xb5, 0x88, 0x4a, 0x26, 0x68, 0x64, 0x6e, 0x20, 0xf5, 0x61, - 0xd7, 0xb9, 0xbb, 0xb9, 0xe6, 0xe9, 0xb1, 0x22, 0x14, 0x6f, 0x58, 0x8e, 0xb1, 0x50, 0x1b, 0x2a, - 0xb6, 0x6f, 0xa4, 0x5f, 0x35, 0xbd, 0x7b, 0xc5, 0xf9, 0xb8, 0xa0, 0x5d, 0x98, 0x20, 0x1b, 0x6f, - 0x34, 0x41, 0x1e, 0x03, 0x8c, 0xf8, 0x20, 0x88, 0x04, 0x9b, 0x52, 0xe1, 0xd7, 0x0c, 0x77, 0x3b, - 0x8f, 0xdb, 0x33, 0x08, 0xec, 0x8d, 0xf8, 0x20, 0x5d, 0x36, 0x7f, 0x70, 0xe0, 0x7f, 0x2b, 0x41, - 0xa1, 0x8f, 0xa1, 0x6c, 0xc3, 0xba, 0xec, 0x23, 0xc0, 0xf2, 0x70, 0x86, 0x45, 0x3b, 0xe0, 0xe9, - 0x3b, 0x42, 0xa5, 0xa4, 0xe9, 0xed, 0xf7, 0xf0, 0xf9, 0x0f, 0xc8, 0x87, 0x32, 0x19, 0x31, 0xa2, - 0xf7, 0x8a, 0x66, 0x2f, 0x33, 0x9b, 0x3f, 0x16, 0xa0, 0x6c, 0xc5, 0xae, 0x7b, 0x9c, 0x5b, 0xb7, - 0x2b, 0x37, 0xeb, 0x29, 0x6c, 0xa4, 0xe5, 0xb4, 0x2d, 0xe1, 0xfe, 0x67, 0x51, 0xab, 0x29, 0x3e, - 0x6d, 0x87, 0xa7, 0xe0, 0xb2, 0x84, 0x8c, 0xed, 0x28, 0xcf, 0xf5, 0xdc, 0x3f, 0x68, 0xef, 0x7f, - 0x9d, 0xa4, 0x9d, 0x5d, 0x99, 0xcf, 0x1a, 0xae, 0xfe, 0x01, 0x1b, 0x5a, 0xf3, 0xef, 0x02, 0x94, - 0xbb, 0xa3, 0x89, 0x54, 0x54, 0x5c, 0x77, 0x41, 0xac, 0xdb, 0x95, 0x82, 0x74, 0xa1, 0x2c, 0x38, - 0x57, 0x41, 0x48, 0x2e, 0xab, 0x05, 0xe6, 0x5c, 0x75, 0xdb, 0x9d, 0x4d, 0x4d, 0xd4, 0x83, 0x24, - 0xb5, 0x71, 0x49, 0x53, 0xbb, 0x04, 0xbd, 0x80, 0xad, 0x6c, 0xfc, 0x1e, 0x73, 0xae, 0xa4, 0x12, - 0x24, 0x09, 0x86, 0xf4, 0x44, 0xbf, 0x79, 0xc5, 0x75, 0x5f, 0x26, 0x7b, 0x71, 0x28, 0x4e, 0x4c, - 0xa1, 0x9e, 0xd1, 0x13, 0x7c, 0xcb, 0x0a, 0x74, 0x32, 0xfe, 0x33, 0x7a, 0x22, 0xd1, 0x67, 0xb0, - 0x43, 0x17, 0x30, 0xad, 0x18, 0x8c, 0xc8, 0x58, 0x3f, 0x2c, 0x41, 0x38, 0xe2, 0xe1, 0xd0, 0xcc, - 0x36, 0x17, 0xdf, 0xa6, 0xcb, 0x52, 0x5f, 0xa6, 0x88, 0xae, 0x06, 0x34, 0xff, 0x74, 0xa0, 0x74, - 0x48, 0x43, 0x41, 0xd5, 0x5b, 0x2d, 0xf8, 0xa3, 0x0b, 0x05, 0xaf, 0xe7, 0xbf, 0xc5, 0xda, 0xeb, - 0x4a, 0xbd, 0xb7, 0xa0, 0x14, 0xb1, 0x01, 0x95, 0xe9, 0xd7, 0x84, 0x87, 0xad, 0x85, 0x9a, 0xe0, - 0x4a, 0xf6, 0x9a, 0x9a, 0xce, 0x2a, 0xa6, 0x0f, 0x9f, 0x55, 0x60, 0xaf, 0x29, 0x36, 0x7b, 0x68, - 0x1b, 0x2a, 0x2c, 0x56, 0x54, 0xc4, 0x64, 0x64, 0x32, 0xaf, 0xe0, 0x85, 0xdd, 0xd9, 0x39, 0x3d, - 0xab, 0xdf, 0xf8, 0xfd, 0xac, 0x7e, 0xe3, 0x9f, 0xb3, 0xba, 0xf3, 0xfd, 0xbc, 0xee, 0x9c, 0xce, - 0xeb, 0xce, 0xaf, 0xf3, 0xba, 0xf3, 0xc7, 0xbc, 0xee, 0x1c, 0x97, 0xcc, 0xbf, 0x81, 0x8f, 0xfe, - 0x0d, 0x00, 0x00, 0xff, 0xff, 0x4e, 0x76, 0x6f, 0x66, 0x7d, 0x0c, 0x00, 0x00, + // 1126 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4d, 0x6f, 0x1b, 0xc5, + 0x1b, 0xef, 0xda, 0x5b, 0xbf, 0x3c, 0xb6, 0x23, 0xfd, 0xe7, 0x5f, 0x45, 0xdb, 0x10, 0xec, 0xe0, + 0x0a, 0xd4, 0x43, 0xe5, 0x8a, 0x52, 0x50, 0x2b, 0x5a, 0x21, 0xbf, 0x44, 0x60, 0x95, 0x40, 0x34, + 0x29, 0xe9, 0x71, 0x35, 0xd9, 0x9d, 0x9a, 0xc5, 0xf6, 0xce, 0x6a, 0x66, 0xec, 0x2a, 0x3d, 0x21, + 0x3e, 0x00, 0x1f, 0x81, 0x6f, 0xc1, 0x99, 0x6b, 0x0e, 0x1c, 0xb8, 0xc1, 0xc9, 0x22, 0x3e, 0x20, + 0x71, 0xe3, 0x23, 0xa0, 0x79, 0x59, 0xdb, 0x91, 0xd7, 0x21, 0x95, 0xaa, 0xdc, 0xe6, 0xf1, 0xfc, + 0x7e, 0xbf, 0x79, 0xde, 0xe6, 0x99, 0x35, 0xd4, 0xd8, 0xc9, 0x77, 0x34, 0x90, 0xa2, 0x95, 0x70, + 0x26, 0x19, 0x42, 0x21, 0x0b, 0x86, 0x94, 0xb7, 0xc4, 0x2b, 0xc2, 0xc7, 0xc3, 0x48, 0xb6, 0xa6, + 0x1f, 0xee, 0x54, 0xe4, 0x69, 0x42, 0x2d, 0x60, 0xa7, 0x22, 0x12, 0x1a, 0xa4, 0xc6, 0x6d, 0x19, + 0x8d, 0xa9, 0x90, 0x64, 0x9c, 0xdc, 0x5f, 0xac, 0xec, 0xd6, 0xad, 0x01, 0x1b, 0x30, 0xbd, 0xbc, + 0xaf, 0x56, 0xe6, 0xd7, 0xe6, 0x2f, 0x0e, 0xb8, 0x07, 0x54, 0x12, 0xf4, 0x29, 0x14, 0xa7, 0x94, + 0x8b, 0x88, 0xc5, 0x9e, 0xb3, 0xe7, 0xdc, 0xad, 0x3c, 0x78, 0xa7, 0xb5, 0x7e, 0x72, 0xeb, 0xd8, + 0x40, 0x3a, 0xee, 0xd9, 0xac, 0x71, 0x03, 0xa7, 0x0c, 0xf4, 0x04, 0x20, 0xe0, 0x94, 0x48, 0x1a, + 0xfa, 0x44, 0x7a, 0x39, 0xcd, 0x7f, 0x37, 0x8b, 0xff, 0x3c, 0x75, 0x0a, 0x97, 0x2d, 0xa1, 0x2d, + 0x15, 0x7b, 0x92, 0x84, 0x29, 0x3b, 0x7f, 0x25, 0xb6, 0x25, 0xb4, 0x65, 0xf3, 0xef, 0x3c, 0xb8, + 0x5f, 0xb1, 0x90, 0xa2, 0x6d, 0xc8, 0x45, 0xa1, 0x76, 0xbe, 0xdc, 0x29, 0xcc, 0x67, 0x8d, 0x5c, + 0xbf, 0x87, 0x73, 0x51, 0x88, 0x1e, 0x80, 0x3b, 0xa6, 0x92, 0x58, 0xb7, 0xbc, 0x2c, 0x61, 0x95, + 0x01, 0x1b, 0x93, 0xc6, 0xa2, 0x4f, 0xc0, 0x55, 0x69, 0xb5, 0xce, 0xec, 0x66, 0x71, 0xd4, 0x99, + 0x47, 0x09, 0x0d, 0x52, 0x9e, 0xc2, 0xa3, 0x7d, 0xa8, 0x84, 0x54, 0x04, 0x3c, 0x4a, 0xa4, 0xca, + 0xa4, 0xab, 0xe9, 0x77, 0x36, 0xd1, 0x7b, 0x4b, 0x28, 0x5e, 0xe5, 0xa1, 0x27, 0x50, 0x10, 0x92, + 0xc8, 0x89, 0xf0, 0x6e, 0x6a, 0x85, 0xfa, 0x46, 0x07, 0x34, 0xca, 0xba, 0x60, 0x39, 0xe8, 0x0b, + 0xd8, 0x1a, 0x93, 0x98, 0x0c, 0x28, 0xf7, 0xad, 0x4a, 0x41, 0xab, 0xbc, 0x97, 0x19, 0xba, 0x41, + 0x1a, 0x21, 0x5c, 0x1b, 0xaf, 0x9a, 0x68, 0x1f, 0x80, 0x48, 0x49, 0x82, 0x6f, 0xc7, 0x34, 0x96, + 0x5e, 0x51, 0xab, 0xbc, 0x9f, 0xe9, 0x0b, 0x95, 0xaf, 0x18, 0x1f, 0xb6, 0x17, 0x60, 0xbc, 0x42, + 0x44, 0x9f, 0x43, 0x25, 0xa0, 0x5c, 0x46, 0x2f, 0xa3, 0x80, 0x48, 0xea, 0x95, 0xb4, 0x4e, 0x23, + 0x4b, 0xa7, 0xbb, 0x84, 0xd9, 0xa0, 0x56, 0x99, 0xcd, 0xdf, 0x73, 0x50, 0x3c, 0xa2, 0x7c, 0x1a, + 0x05, 0x6f, 0xb7, 0xdc, 0x8f, 0x2f, 0x94, 0x3b, 0xd3, 0x33, 0x7b, 0xec, 0x5a, 0xc5, 0x1f, 0x41, + 0x89, 0xc6, 0x61, 0xc2, 0xa2, 0x58, 0xda, 0x72, 0x67, 0x76, 0xcb, 0xbe, 0xc5, 0xe0, 0x05, 0x1a, + 0xed, 0x43, 0xcd, 0x74, 0xb1, 0x7f, 0xa1, 0xd6, 0x7b, 0x59, 0xf4, 0x6f, 0x34, 0xd0, 0x16, 0xa9, + 0x3a, 0x59, 0xb1, 0x50, 0x0f, 0x6a, 0x09, 0xa7, 0xd3, 0x88, 0x4d, 0x84, 0xaf, 0x83, 0x28, 0x5c, + 0x29, 0x08, 0x5c, 0x4d, 0x59, 0xca, 0x6a, 0xfe, 0x94, 0x83, 0x52, 0xea, 0x23, 0x7a, 0x68, 0xd3, + 0xe1, 0x6c, 0x76, 0x28, 0xc5, 0x6a, 0x29, 0x93, 0x89, 0x87, 0x70, 0x33, 0x61, 0x5c, 0x0a, 0x2f, + 0xb7, 0x97, 0xdf, 0xd4, 0xb3, 0x87, 0x8c, 0xcb, 0x2e, 0x8b, 0x5f, 0x46, 0x03, 0x6c, 0xc0, 0xe8, + 0x05, 0x54, 0xa6, 0x11, 0x97, 0x13, 0x32, 0xf2, 0xa3, 0x44, 0x78, 0x79, 0xcd, 0xfd, 0xe0, 0xb2, + 0x23, 0x5b, 0xc7, 0x06, 0xdf, 0x3f, 0xec, 0x6c, 0xcd, 0x67, 0x0d, 0x58, 0x98, 0x02, 0x83, 0x95, + 0xea, 0x27, 0x62, 0xe7, 0x00, 0xca, 0x8b, 0x1d, 0x74, 0x0f, 0x20, 0x36, 0x2d, 0xea, 0x2f, 0x9a, + 0xa6, 0x36, 0x9f, 0x35, 0xca, 0xb6, 0x71, 0xfb, 0x3d, 0x5c, 0xb6, 0x80, 0x7e, 0x88, 0x10, 0xb8, + 0x24, 0x0c, 0xb9, 0x6e, 0xa1, 0x32, 0xd6, 0xeb, 0xe6, 0xaf, 0x37, 0xc1, 0x7d, 0x4e, 0xc4, 0xf0, + 0xba, 0xc7, 0x8c, 0x3a, 0x73, 0xad, 0xe9, 0xee, 0x01, 0x08, 0x53, 0x4a, 0x15, 0x8e, 0xbb, 0x0c, + 0xc7, 0x16, 0x58, 0x85, 0x63, 0x01, 0x26, 0x1c, 0x31, 0x62, 0x52, 0xf7, 0x97, 0x8b, 0xf5, 0x1a, + 0xdd, 0x81, 0x62, 0xcc, 0x42, 0x4d, 0x2f, 0x68, 0x3a, 0xcc, 0x67, 0x8d, 0x82, 0x1a, 0x29, 0xfd, + 0x1e, 0x2e, 0xa8, 0xad, 0x7e, 0xa8, 0xee, 0x2d, 0x89, 0x63, 0x26, 0x89, 0x1a, 0x4a, 0xc2, 0xde, + 0xff, 0xcc, 0xc6, 0x6a, 0x2f, 0x61, 0xe9, 0xbd, 0x5d, 0x61, 0xa2, 0x63, 0xf8, 0x7f, 0xea, 0xef, + 0xaa, 0x60, 0xe9, 0x4d, 0x04, 0x91, 0x55, 0x58, 0xd9, 0x59, 0x99, 0x93, 0xe5, 0xcd, 0x73, 0x52, + 0x67, 0x30, 0x6b, 0x4e, 0x76, 0xa0, 0x16, 0x52, 0x11, 0x71, 0x1a, 0xea, 0x1b, 0x48, 0x3d, 0xd8, + 0x73, 0xee, 0x6e, 0x6d, 0x78, 0x7a, 0xac, 0x08, 0xc5, 0x55, 0xcb, 0xd1, 0x16, 0x6a, 0x43, 0xc9, + 0xf6, 0x8d, 0xf0, 0x2a, 0xba, 0x77, 0xaf, 0x38, 0x1f, 0x17, 0xb4, 0x0b, 0x13, 0xa4, 0xfa, 0x46, + 0x13, 0xe4, 0x31, 0xc0, 0x88, 0x0d, 0xfc, 0x90, 0x47, 0x53, 0xca, 0xbd, 0x9a, 0xe6, 0xee, 0x64, + 0x71, 0x7b, 0x1a, 0x81, 0xcb, 0x23, 0x36, 0x30, 0xcb, 0xe6, 0x0f, 0x0e, 0xfc, 0x6f, 0xcd, 0x29, + 0xf4, 0x31, 0x14, 0xad, 0x5b, 0x97, 0x7d, 0x04, 0x58, 0x1e, 0x4e, 0xb1, 0x68, 0x17, 0xca, 0xea, + 0x8e, 0x50, 0x21, 0xa8, 0xb9, 0xfd, 0x65, 0xbc, 0xfc, 0x01, 0x79, 0x50, 0x24, 0xa3, 0x88, 0xa8, + 0xbd, 0xbc, 0xde, 0x4b, 0xcd, 0xe6, 0x8f, 0x39, 0x28, 0x5a, 0xb1, 0xeb, 0x1e, 0xe7, 0xf6, 0xd8, + 0xb5, 0x9b, 0xf5, 0x14, 0xaa, 0x26, 0x9d, 0xb6, 0x25, 0xdc, 0xff, 0x4c, 0x6a, 0xc5, 0xe0, 0x4d, + 0x3b, 0x3c, 0x05, 0x37, 0x4a, 0xc8, 0xd8, 0x8e, 0xf2, 0xcc, 0x93, 0xfb, 0x87, 0xed, 0x83, 0xaf, + 0x13, 0xd3, 0xd9, 0xa5, 0xf9, 0xac, 0xe1, 0xaa, 0x1f, 0xb0, 0xa6, 0x35, 0x7f, 0xce, 0x43, 0xb1, + 0x3b, 0x9a, 0x08, 0x49, 0xf9, 0x75, 0x27, 0xc4, 0x1e, 0xbb, 0x96, 0x90, 0x2e, 0x14, 0x39, 0x63, + 0xd2, 0x0f, 0xc8, 0x65, 0xb9, 0xc0, 0x8c, 0xc9, 0x6e, 0xbb, 0xb3, 0xa5, 0x88, 0x6a, 0x90, 0x18, + 0x1b, 0x17, 0x14, 0xb5, 0x4b, 0xd0, 0x0b, 0xd8, 0x4e, 0xc7, 0xef, 0x09, 0x63, 0x52, 0x48, 0x4e, + 0x12, 0x7f, 0x48, 0x4f, 0xd5, 0x9b, 0x97, 0xdf, 0xf4, 0x65, 0xb2, 0x1f, 0x07, 0xfc, 0x54, 0x27, + 0xea, 0x19, 0x3d, 0xc5, 0xb7, 0xac, 0x40, 0x27, 0xe5, 0x3f, 0xa3, 0xa7, 0x02, 0x7d, 0x06, 0xbb, + 0x74, 0x01, 0x53, 0x8a, 0xfe, 0x88, 0x8c, 0xd5, 0xc3, 0xe2, 0x07, 0x23, 0x16, 0x0c, 0xf5, 0x6c, + 0x73, 0xf1, 0x6d, 0xba, 0x2a, 0xf5, 0xa5, 0x41, 0x74, 0x15, 0x40, 0xbd, 0x9e, 0x9c, 0x8e, 0xd9, + 0x94, 0x86, 0xbe, 0x1a, 0x7a, 0x6a, 0xc8, 0xe5, 0x37, 0xa5, 0x08, 0x1b, 0xa0, 0x1a, 0x92, 0xb8, + 0xca, 0x97, 0x86, 0x68, 0xfe, 0xe5, 0x40, 0xe1, 0x88, 0x06, 0x9c, 0xca, 0xb7, 0x5a, 0xb6, 0x47, + 0x17, 0xca, 0x56, 0xcf, 0x7e, 0xd1, 0xd5, 0xa9, 0x6b, 0x55, 0xdb, 0x86, 0x42, 0x18, 0x0d, 0xa8, + 0x30, 0xdf, 0x24, 0x65, 0x6c, 0x2d, 0xd4, 0x04, 0x57, 0x44, 0xaf, 0xa9, 0xee, 0xcf, 0xbc, 0x79, + 0x3e, 0xad, 0x42, 0xf4, 0x9a, 0x62, 0xbd, 0x87, 0x76, 0xa0, 0x14, 0xc5, 0x92, 0xf2, 0x98, 0x8c, + 0x74, 0xfe, 0x4a, 0x78, 0x61, 0x77, 0x76, 0xcf, 0xce, 0xeb, 0x37, 0xfe, 0x38, 0xaf, 0xdf, 0xf8, + 0xe7, 0xbc, 0xee, 0x7c, 0x3f, 0xaf, 0x3b, 0x67, 0xf3, 0xba, 0xf3, 0xdb, 0xbc, 0xee, 0xfc, 0x39, + 0xaf, 0x3b, 0x27, 0x05, 0xfd, 0x9f, 0xe2, 0xa3, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xf7, 0x71, + 0xe0, 0xa4, 0xc3, 0x0c, 0x00, 0x00, } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/objects.proto b/components/engine/vendor/src/github.com/docker/swarmkit/api/objects.proto index ab47b1a6fb..894926f3d1 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/objects.proto +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/objects.proto @@ -225,6 +225,10 @@ message Cluster { // and agents to unambiguously identify the older key to be deleted when // a new key is allocated on key rotation. uint64 encryption_key_lamport_clock = 6; + + // RemovedNodes is the list of nodes that have been removed from the + // swarm. Their certificates should effectively be blacklisted. + repeated RemovedNode removed_nodes = 7; } // Secret represents a secret that should be passed to a container or a node, diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/raft.pb.go b/components/engine/vendor/src/github.com/docker/swarmkit/api/raft.pb.go index d1d6102557..e824d66a9c 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/raft.pb.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/raft.pb.go @@ -528,9 +528,7 @@ func (m *JoinResponse) Copy() *JoinResponse { if m.RemovedMembers != nil { o.RemovedMembers = make([]uint64, 0, len(m.RemovedMembers)) - for _, v := range m.RemovedMembers { - o.RemovedMembers = append(o.RemovedMembers, v) - } + o.RemovedMembers = append(o.RemovedMembers, m.RemovedMembers...) } return o diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/snapshot.pb.go b/components/engine/vendor/src/github.com/docker/swarmkit/api/snapshot.pb.go index f57e70ea8b..88b030eeef 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/snapshot.pb.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/snapshot.pb.go @@ -151,9 +151,7 @@ func (m *ClusterSnapshot) Copy() *ClusterSnapshot { if m.Removed != nil { o.Removed = make([]uint64, 0, len(m.Removed)) - for _, v := range m.Removed { - o.Removed = append(o.Removed, v) - } + o.Removed = append(o.Removed, m.Removed...) } return o diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.pb.go b/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.pb.go index 81f9b477a9..7dbaa0c41c 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.pb.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.pb.go @@ -297,6 +297,11 @@ type TaskSpec struct { // configurations (which specify the network and per-network // aliases) that this task spec is bound to. Networks []*NetworkAttachmentConfig `protobuf:"bytes,7,rep,name=networks" json:"networks,omitempty"` + // ForceUpdate is a counter that triggers an update even if no relevant + // parameters have been changed. We do this to allow forced restarts + // using the same reconcilation-based mechanism that performs rolling + // updates. + ForceUpdate uint64 `protobuf:"varint,9,opt,name=force_update,json=forceUpdate,proto3" json:"force_update,omitempty"` } func (m *TaskSpec) Reset() { *m = TaskSpec{} } @@ -667,10 +672,11 @@ func (m *TaskSpec) Copy() *TaskSpec { } o := &TaskSpec{ - Resources: m.Resources.Copy(), - Restart: m.Restart.Copy(), - Placement: m.Placement.Copy(), - LogDriver: m.LogDriver.Copy(), + Resources: m.Resources.Copy(), + Restart: m.Restart.Copy(), + Placement: m.Placement.Copy(), + LogDriver: m.LogDriver.Copy(), + ForceUpdate: m.ForceUpdate, } if m.Networks != nil { @@ -732,30 +738,22 @@ func (m *ContainerSpec) Copy() *ContainerSpec { if m.Command != nil { o.Command = make([]string, 0, len(m.Command)) - for _, v := range m.Command { - o.Command = append(o.Command, v) - } + o.Command = append(o.Command, m.Command...) } if m.Args != nil { o.Args = make([]string, 0, len(m.Args)) - for _, v := range m.Args { - o.Args = append(o.Args, v) - } + o.Args = append(o.Args, m.Args...) } if m.Env != nil { o.Env = make([]string, 0, len(m.Env)) - for _, v := range m.Env { - o.Env = append(o.Env, v) - } + o.Env = append(o.Env, m.Env...) } if m.Groups != nil { o.Groups = make([]string, 0, len(m.Groups)) - for _, v := range m.Groups { - o.Groups = append(o.Groups, v) - } + o.Groups = append(o.Groups, m.Groups...) } if m.Mounts != nil { @@ -929,7 +927,7 @@ func (this *TaskSpec) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 11) + s := make([]string, 0, 12) s = append(s, "&api.TaskSpec{") if this.Runtime != nil { s = append(s, "Runtime: "+fmt.Sprintf("%#v", this.Runtime)+",\n") @@ -949,6 +947,7 @@ func (this *TaskSpec) GoString() string { if this.Networks != nil { s = append(s, "Networks: "+fmt.Sprintf("%#v", this.Networks)+",\n") } + s = append(s, "ForceUpdate: "+fmt.Sprintf("%#v", this.ForceUpdate)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -1371,6 +1370,11 @@ func (m *TaskSpec) MarshalTo(data []byte) (int, error) { i += n } } + if m.ForceUpdate != 0 { + data[i] = 0x48 + i++ + i = encodeVarintSpecs(data, i, uint64(m.ForceUpdate)) + } return i, nil } @@ -1959,6 +1963,9 @@ func (m *TaskSpec) Size() (n int) { n += 1 + l + sovSpecs(uint64(l)) } } + if m.ForceUpdate != 0 { + n += 1 + sovSpecs(uint64(m.ForceUpdate)) + } return n } @@ -2233,6 +2240,7 @@ func (this *TaskSpec) String() string { `Placement:` + strings.Replace(fmt.Sprintf("%v", this.Placement), "Placement", "Placement", 1) + `,`, `LogDriver:` + strings.Replace(fmt.Sprintf("%v", this.LogDriver), "Driver", "Driver", 1) + `,`, `Networks:` + strings.Replace(fmt.Sprintf("%v", this.Networks), "NetworkAttachmentConfig", "NetworkAttachmentConfig", 1) + `,`, + `ForceUpdate:` + fmt.Sprintf("%v", this.ForceUpdate) + `,`, `}`, }, "") return s @@ -3152,6 +3160,25 @@ func (m *TaskSpec) Unmarshal(data []byte) error { } m.Runtime = &TaskSpec_Attachment{v} iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ForceUpdate", wireType) + } + m.ForceUpdate = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpecs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + m.ForceUpdate |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipSpecs(data[iNdEx:]) @@ -4613,96 +4640,97 @@ var ( func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) } var fileDescriptorSpecs = []byte{ - // 1443 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x6e, 0x1b, 0xc9, - 0x11, 0xe6, 0x48, 0x23, 0x8a, 0xac, 0xa1, 0x6c, 0xba, 0xe1, 0x9f, 0x31, 0xed, 0x50, 0x34, 0xed, - 0x38, 0x72, 0x82, 0x48, 0x09, 0x13, 0x38, 0x76, 0x1c, 0x23, 0xe1, 0x5f, 0x64, 0x46, 0x91, 0x4c, - 0xb4, 0x6c, 0x03, 0x39, 0x11, 0xad, 0x99, 0x16, 0x35, 0xd0, 0x70, 0x7a, 0xd2, 0xd3, 0x43, 0x43, - 0xb7, 0x1c, 0x0d, 0x1d, 0xf2, 0x06, 0x3a, 0x2d, 0xb0, 0x6f, 0xb0, 0xef, 0xe0, 0xe3, 0x1e, 0xf7, - 0x24, 0xac, 0x74, 0x5d, 0x2c, 0xb0, 0xc0, 0xbe, 0xc0, 0xa2, 0x7b, 0x9a, 0xe4, 0x70, 0x3d, 0xb2, - 0x0d, 0xac, 0x6e, 0xd5, 0xd5, 0xdf, 0x57, 0x5d, 0xac, 0xfa, 0xa6, 0xba, 0x09, 0x56, 0x14, 0x52, - 0x27, 0x5a, 0x0f, 0x39, 0x13, 0x0c, 0x21, 0x97, 0x39, 0x87, 0x94, 0xaf, 0x47, 0x6f, 0x09, 0x1f, - 0x1d, 0x7a, 0x62, 0x7d, 0xfc, 0xc7, 0x8a, 0x25, 0x8e, 0x42, 0xaa, 0x01, 0x95, 0xeb, 0x43, 0x36, - 0x64, 0xca, 0xdc, 0x90, 0x96, 0xf6, 0xde, 0x72, 0x63, 0x4e, 0x84, 0xc7, 0x82, 0x8d, 0x89, 0x91, - 0x6c, 0xd4, 0xff, 0x6f, 0x42, 0x61, 0x87, 0xb9, 0x74, 0x37, 0xa4, 0x0e, 0xda, 0x04, 0x8b, 0x04, - 0x01, 0x13, 0x0a, 0x10, 0xd9, 0x46, 0xcd, 0x58, 0xb3, 0x1a, 0xab, 0xeb, 0x1f, 0x1e, 0xb9, 0xde, - 0x9c, 0xc1, 0x5a, 0xe6, 0xfb, 0xd3, 0xd5, 0x1c, 0x4e, 0x33, 0xd1, 0x1f, 0xc0, 0xe4, 0xcc, 0xa7, - 0xf6, 0x42, 0xcd, 0x58, 0xbb, 0xd2, 0xb8, 0x9b, 0x15, 0x41, 0x1e, 0x8a, 0x99, 0x4f, 0xb1, 0x42, - 0xa2, 0x4d, 0x80, 0x11, 0x1d, 0xed, 0x51, 0x1e, 0x1d, 0x78, 0xa1, 0xbd, 0xa8, 0x78, 0xbf, 0xb9, - 0x88, 0x27, 0x93, 0x5d, 0xdf, 0x9e, 0xc2, 0x71, 0x8a, 0x8a, 0xb6, 0xa1, 0x44, 0xc6, 0xc4, 0xf3, - 0xc9, 0x9e, 0xe7, 0x7b, 0xe2, 0xc8, 0x36, 0x55, 0xa8, 0x47, 0x1f, 0x0d, 0xd5, 0x4c, 0x11, 0xf0, - 0x1c, 0xbd, 0xee, 0x02, 0xcc, 0x0e, 0x42, 0x0f, 0x61, 0xb9, 0xdf, 0xdd, 0xe9, 0xf4, 0x76, 0x36, - 0xcb, 0xb9, 0xca, 0xed, 0xe3, 0x93, 0xda, 0x0d, 0x19, 0x63, 0x06, 0xe8, 0xd3, 0xc0, 0xf5, 0x82, - 0x21, 0x5a, 0x83, 0x42, 0xb3, 0xdd, 0xee, 0xf6, 0x5f, 0x75, 0x3b, 0x65, 0xa3, 0x52, 0x39, 0x3e, - 0xa9, 0xdd, 0x9c, 0x07, 0x36, 0x1d, 0x87, 0x86, 0x82, 0xba, 0x15, 0xf3, 0xdd, 0x17, 0xd5, 0x5c, - 0xfd, 0x9d, 0x01, 0xa5, 0x74, 0x12, 0xe8, 0x21, 0xe4, 0x9b, 0xed, 0x57, 0xbd, 0x37, 0xdd, 0x72, - 0x6e, 0x46, 0x4f, 0x23, 0x9a, 0x8e, 0xf0, 0xc6, 0x14, 0x3d, 0x80, 0xa5, 0x7e, 0xf3, 0xf5, 0x6e, - 0xb7, 0x6c, 0xcc, 0xd2, 0x49, 0xc3, 0xfa, 0x24, 0x8e, 0x14, 0xaa, 0x83, 0x9b, 0xbd, 0x9d, 0xf2, - 0x42, 0x36, 0xaa, 0xc3, 0x89, 0x17, 0xe8, 0x54, 0xce, 0x16, 0xc1, 0xda, 0xa5, 0x7c, 0xec, 0x39, + // 1457 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x4d, 0x6f, 0x1b, 0xb9, + 0x19, 0xd6, 0xd8, 0x63, 0x59, 0x7a, 0x47, 0x4e, 0x14, 0x22, 0x1f, 0x13, 0x25, 0x95, 0x15, 0x25, + 0x4d, 0x9d, 0x16, 0xb5, 0x5b, 0xb5, 0x48, 0x93, 0xa6, 0x41, 0xab, 0xaf, 0x3a, 0xaa, 0x6b, 0x47, + 0xa0, 0x93, 0x00, 0x3d, 0x09, 0xf4, 0x0c, 0x2d, 0x0f, 0x3c, 0x1a, 0x4e, 0x39, 0x1c, 0x05, 0xbe, + 0xed, 0x31, 0xf0, 0x61, 0xff, 0x81, 0x4f, 0x0b, 0xec, 0x3f, 0xd8, 0xff, 0x90, 0xe3, 0x1e, 0xf7, + 0x64, 0xac, 0x7d, 0x5f, 0x60, 0x81, 0xfd, 0x03, 0x0b, 0x72, 0x28, 0x69, 0xb4, 0x19, 0x27, 0x01, + 0xd6, 0x37, 0xf2, 0xe5, 0xf3, 0xbc, 0xa4, 0xde, 0xf7, 0x99, 0x87, 0x14, 0x58, 0x51, 0x48, 0x9d, + 0x68, 0x3d, 0xe4, 0x4c, 0x30, 0x84, 0x5c, 0xe6, 0x1c, 0x52, 0xbe, 0x1e, 0xbd, 0x25, 0x7c, 0x74, + 0xe8, 0x89, 0xf5, 0xf1, 0x9f, 0x2b, 0x96, 0x38, 0x0a, 0xa9, 0x06, 0x54, 0xae, 0x0f, 0xd9, 0x90, + 0xa9, 0xe1, 0x86, 0x1c, 0xe9, 0xe8, 0x2d, 0x37, 0xe6, 0x44, 0x78, 0x2c, 0xd8, 0x98, 0x0c, 0x92, + 0x85, 0xfa, 0x97, 0x26, 0x14, 0x76, 0x98, 0x4b, 0x77, 0x43, 0xea, 0xa0, 0x4d, 0xb0, 0x48, 0x10, + 0x30, 0xa1, 0x00, 0x91, 0x6d, 0xd4, 0x8c, 0x35, 0xab, 0xb1, 0xba, 0xfe, 0xe1, 0x96, 0xeb, 0xcd, + 0x19, 0xac, 0x65, 0xbe, 0x3f, 0x5d, 0xcd, 0xe1, 0x34, 0x13, 0xfd, 0x09, 0x4c, 0xce, 0x7c, 0x6a, + 0x2f, 0xd4, 0x8c, 0xb5, 0x2b, 0x8d, 0xbb, 0x59, 0x19, 0xe4, 0xa6, 0x98, 0xf9, 0x14, 0x2b, 0x24, + 0xda, 0x04, 0x18, 0xd1, 0xd1, 0x1e, 0xe5, 0xd1, 0x81, 0x17, 0xda, 0x8b, 0x8a, 0xf7, 0xbb, 0x8b, + 0x78, 0xf2, 0xb0, 0xeb, 0xdb, 0x53, 0x38, 0x4e, 0x51, 0xd1, 0x36, 0x94, 0xc8, 0x98, 0x78, 0x3e, + 0xd9, 0xf3, 0x7c, 0x4f, 0x1c, 0xd9, 0xa6, 0x4a, 0xf5, 0xe8, 0xa3, 0xa9, 0x9a, 0x29, 0x02, 0x9e, + 0xa3, 0xd7, 0x5d, 0x80, 0xd9, 0x46, 0xe8, 0x21, 0x2c, 0xf7, 0xbb, 0x3b, 0x9d, 0xde, 0xce, 0x66, + 0x39, 0x57, 0xb9, 0x7d, 0x7c, 0x52, 0xbb, 0x21, 0x73, 0xcc, 0x00, 0x7d, 0x1a, 0xb8, 0x5e, 0x30, + 0x44, 0x6b, 0x50, 0x68, 0xb6, 0xdb, 0xdd, 0xfe, 0xab, 0x6e, 0xa7, 0x6c, 0x54, 0x2a, 0xc7, 0x27, + 0xb5, 0x9b, 0xf3, 0xc0, 0xa6, 0xe3, 0xd0, 0x50, 0x50, 0xb7, 0x62, 0xbe, 0xfb, 0xaa, 0x9a, 0xab, + 0xbf, 0x33, 0xa0, 0x94, 0x3e, 0x04, 0x7a, 0x08, 0xf9, 0x66, 0xfb, 0x55, 0xef, 0x4d, 0xb7, 0x9c, + 0x9b, 0xd1, 0xd3, 0x88, 0xa6, 0x23, 0xbc, 0x31, 0x45, 0x0f, 0x60, 0xa9, 0xdf, 0x7c, 0xbd, 0xdb, + 0x2d, 0x1b, 0xb3, 0xe3, 0xa4, 0x61, 0x7d, 0x12, 0x47, 0x0a, 0xd5, 0xc1, 0xcd, 0xde, 0x4e, 0x79, + 0x21, 0x1b, 0xd5, 0xe1, 0xc4, 0x0b, 0xf4, 0x51, 0xce, 0x16, 0xc1, 0xda, 0xa5, 0x7c, 0xec, 0x39, 0x97, 0xac, 0x89, 0xc7, 0x60, 0x0a, 0x12, 0x1d, 0x2a, 0x4d, 0x58, 0xd9, 0x9a, 0x78, 0x45, 0xa2, - 0x43, 0x79, 0xa8, 0xa6, 0x2b, 0xbc, 0x54, 0x06, 0xa7, 0xa1, 0xef, 0x39, 0x44, 0x50, 0x57, 0x29, - 0xc3, 0x6a, 0xfc, 0x3a, 0x8b, 0x8d, 0xa7, 0x28, 0x9d, 0xff, 0x8b, 0x1c, 0x4e, 0x51, 0xd1, 0x33, - 0xc8, 0x0f, 0x7d, 0xb6, 0x47, 0x7c, 0xa5, 0x09, 0xab, 0x71, 0x2f, 0x2b, 0xc8, 0xa6, 0x42, 0xcc, - 0x02, 0x68, 0x0a, 0x7a, 0x02, 0xf9, 0x38, 0x74, 0x89, 0xa0, 0x76, 0x5e, 0x91, 0x6b, 0x59, 0xe4, - 0xd7, 0x0a, 0xd1, 0x66, 0xc1, 0xbe, 0x37, 0xc4, 0x1a, 0x8f, 0xb6, 0xa0, 0x10, 0x50, 0xf1, 0x96, - 0xf1, 0xc3, 0xc8, 0x5e, 0xae, 0x2d, 0xae, 0x59, 0x8d, 0xdf, 0x65, 0x8a, 0x31, 0xc1, 0x34, 0x85, - 0x20, 0xce, 0xc1, 0x88, 0x06, 0x22, 0x09, 0xd3, 0x5a, 0xb0, 0x0d, 0x3c, 0x0d, 0x80, 0xfe, 0x06, - 0x05, 0x1a, 0xb8, 0x21, 0xf3, 0x02, 0x61, 0x17, 0x2e, 0x4e, 0xa4, 0xab, 0x31, 0xb2, 0x98, 0x78, - 0xca, 0x68, 0xe5, 0xc1, 0x1c, 0x31, 0x97, 0xd6, 0x37, 0xe0, 0xda, 0x07, 0xc5, 0x42, 0x15, 0x28, - 0xe8, 0x62, 0x25, 0x5d, 0x36, 0xf1, 0x74, 0x5d, 0xbf, 0x0a, 0x2b, 0x73, 0x85, 0xa9, 0x7f, 0xb7, - 0x08, 0x85, 0x49, 0xb7, 0x50, 0x13, 0x8a, 0x0e, 0x0b, 0x04, 0xf1, 0x02, 0xca, 0xb5, 0x40, 0x32, - 0x6b, 0xdb, 0x9e, 0x80, 0x24, 0xeb, 0x45, 0x0e, 0xcf, 0x58, 0xe8, 0x9f, 0x50, 0xe4, 0x34, 0x62, - 0x31, 0x77, 0x68, 0xa4, 0x15, 0xb2, 0x96, 0xdd, 0xe3, 0x04, 0x84, 0xe9, 0x7f, 0x63, 0x8f, 0x53, - 0x59, 0xa7, 0x08, 0xcf, 0xa8, 0xe8, 0x19, 0x2c, 0x73, 0x1a, 0x09, 0xc2, 0xc5, 0xc7, 0x9a, 0x8c, - 0x13, 0x48, 0x9f, 0xf9, 0x9e, 0x73, 0x84, 0x27, 0x0c, 0xf4, 0x0c, 0x8a, 0xa1, 0x4f, 0x1c, 0x15, - 0xd5, 0x5e, 0x52, 0xf4, 0x5f, 0x65, 0xd1, 0xfb, 0x13, 0x10, 0x9e, 0xe1, 0xd1, 0x53, 0x00, 0x9f, - 0x0d, 0x07, 0x2e, 0xf7, 0xc6, 0x94, 0x6b, 0x91, 0x54, 0xb2, 0xd8, 0x1d, 0x85, 0xc0, 0x45, 0x9f, - 0x0d, 0x13, 0x13, 0x6d, 0xfe, 0x22, 0x85, 0xa4, 0xd4, 0xb1, 0x05, 0x40, 0xa6, 0xbb, 0x5a, 0x1f, - 0x8f, 0x3e, 0x2b, 0x94, 0xee, 0x48, 0x8a, 0xde, 0x2a, 0xc2, 0x32, 0x8f, 0x03, 0xe1, 0x8d, 0x68, - 0x7d, 0x0b, 0x6e, 0x64, 0x32, 0x50, 0x03, 0x4a, 0xd3, 0x1e, 0x0e, 0x3c, 0x57, 0x35, 0xbf, 0xd8, - 0xba, 0x7a, 0x7e, 0xba, 0x6a, 0x4d, 0x9b, 0xdd, 0xeb, 0x60, 0x6b, 0x0a, 0xea, 0xb9, 0xf5, 0xef, - 0x4d, 0x58, 0x99, 0x53, 0x02, 0xba, 0x0e, 0x4b, 0xde, 0x88, 0x0c, 0x69, 0x42, 0xc7, 0xc9, 0x02, - 0x75, 0x21, 0xef, 0x93, 0x3d, 0xea, 0x4b, 0x3d, 0xc8, 0x9a, 0xfc, 0xfe, 0x93, 0x92, 0x5a, 0xff, - 0xb7, 0xc2, 0x77, 0x03, 0xc1, 0x8f, 0xb0, 0x26, 0x23, 0x1b, 0x96, 0x1d, 0x36, 0x1a, 0x91, 0x40, - 0xce, 0x8e, 0xc5, 0xb5, 0x22, 0x9e, 0x2c, 0x11, 0x02, 0x93, 0xf0, 0x61, 0x64, 0x9b, 0xca, 0xad, - 0x6c, 0x54, 0x86, 0x45, 0x1a, 0x8c, 0xed, 0x25, 0xe5, 0x92, 0xa6, 0xf4, 0xb8, 0x5e, 0xd2, 0xd0, - 0x22, 0x96, 0xa6, 0xe4, 0xc5, 0x11, 0xe5, 0xf6, 0xb2, 0x72, 0x29, 0x1b, 0xfd, 0x05, 0xf2, 0x23, - 0x16, 0x07, 0x22, 0xb2, 0x0b, 0x2a, 0xd9, 0xdb, 0x59, 0xc9, 0x6e, 0x4b, 0x84, 0x9e, 0x6d, 0x1a, - 0x8e, 0x5e, 0xc0, 0xb5, 0x48, 0xb0, 0x70, 0x30, 0xe4, 0xc4, 0xa1, 0x83, 0x90, 0x72, 0x8f, 0xb9, - 0x76, 0xf1, 0xe2, 0x11, 0xd9, 0xd1, 0xd7, 0x37, 0xbe, 0x2a, 0x69, 0x9b, 0x92, 0xd5, 0x57, 0x24, - 0xd4, 0x87, 0x52, 0x18, 0xfb, 0xfe, 0x80, 0x85, 0xc9, 0xa4, 0x06, 0x15, 0xe4, 0x33, 0xaa, 0xd6, - 0x8f, 0x7d, 0xff, 0x65, 0x42, 0xc2, 0x56, 0x38, 0x5b, 0xa0, 0x9b, 0x90, 0x1f, 0x72, 0x16, 0x87, - 0x91, 0x6d, 0xa9, 0x7a, 0xe8, 0x15, 0x7a, 0x0e, 0xcb, 0x11, 0x75, 0x38, 0x15, 0x91, 0x5d, 0x52, - 0xbf, 0xf6, 0x7e, 0xd6, 0x21, 0xbb, 0x0a, 0x82, 0xe9, 0x3e, 0xe5, 0x34, 0x70, 0x28, 0x9e, 0x70, - 0x2a, 0x4f, 0xc1, 0x4a, 0x35, 0x4a, 0x16, 0xf8, 0x90, 0x1e, 0xe9, 0xde, 0x4b, 0x53, 0xea, 0x61, - 0x4c, 0xfc, 0x38, 0x79, 0x3e, 0x14, 0x71, 0xb2, 0xf8, 0xeb, 0xc2, 0x13, 0xa3, 0xd2, 0x00, 0x2b, - 0x95, 0x2d, 0xba, 0x0f, 0x2b, 0x9c, 0x0e, 0xbd, 0x48, 0xf0, 0xa3, 0x01, 0x89, 0xc5, 0x81, 0xfd, - 0x0f, 0x45, 0x28, 0x4d, 0x9c, 0xcd, 0x58, 0x1c, 0xd4, 0x7f, 0x34, 0xa0, 0x94, 0x9e, 0x87, 0xa8, - 0x9d, 0x4c, 0x41, 0x75, 0xe2, 0x95, 0xc6, 0xc6, 0xa7, 0xe6, 0xa7, 0x9a, 0x39, 0x7e, 0x2c, 0x4f, - 0xdc, 0x96, 0x6f, 0x16, 0x45, 0x46, 0x7f, 0x86, 0xa5, 0x90, 0x71, 0x31, 0x11, 0x67, 0x35, 0x73, - 0x4e, 0x30, 0x3e, 0xf9, 0x46, 0x13, 0x70, 0xfd, 0x00, 0xae, 0xcc, 0x47, 0x43, 0x0f, 0x60, 0xf1, - 0x4d, 0xaf, 0x5f, 0xce, 0x55, 0xee, 0x1c, 0x9f, 0xd4, 0x6e, 0xcd, 0x6f, 0xbe, 0xf1, 0xb8, 0x88, - 0x89, 0xdf, 0xeb, 0xa3, 0xdf, 0xc2, 0x52, 0x67, 0x67, 0x17, 0xe3, 0xb2, 0x51, 0x59, 0x3d, 0x3e, - 0xa9, 0xdd, 0x99, 0xc7, 0xc9, 0x2d, 0x16, 0x07, 0x2e, 0x66, 0x7b, 0xd3, 0x6b, 0xfc, 0xab, 0x05, - 0xb0, 0xf4, 0x37, 0x7b, 0xb9, 0xd7, 0xf8, 0xdf, 0x61, 0x25, 0x99, 0x71, 0x03, 0x47, 0xfd, 0x34, - 0x3d, 0xad, 0x3f, 0x36, 0xea, 0x4a, 0x09, 0x21, 0x29, 0x05, 0xba, 0x07, 0x25, 0x2f, 0x1c, 0x3f, - 0x1e, 0xd0, 0x80, 0xec, 0xf9, 0xfa, 0x46, 0x2f, 0x60, 0x4b, 0xfa, 0xba, 0x89, 0x4b, 0x5e, 0x45, - 0x5e, 0x20, 0x28, 0x0f, 0xf4, 0x5d, 0x5d, 0xc0, 0xd3, 0x35, 0x7a, 0x0e, 0xa6, 0x17, 0x92, 0x91, - 0x9e, 0xcf, 0x99, 0xbf, 0xa0, 0xd7, 0x6f, 0x6e, 0x6b, 0x89, 0xb4, 0x0a, 0xe7, 0xa7, 0xab, 0xa6, - 0x74, 0x60, 0x45, 0x43, 0xd5, 0xc9, 0x88, 0x94, 0x27, 0xa9, 0xaf, 0xba, 0x80, 0x53, 0x9e, 0xfa, - 0x97, 0x26, 0x58, 0x6d, 0x3f, 0x8e, 0x84, 0x9e, 0x4d, 0x97, 0x56, 0xb7, 0xff, 0xc0, 0x35, 0xa2, - 0x1e, 0x7d, 0x24, 0x90, 0x1f, 0xba, 0xba, 0x7a, 0x74, 0xed, 0x1e, 0x64, 0x86, 0x9b, 0x82, 0x93, - 0x6b, 0xaa, 0x95, 0x97, 0x31, 0x6d, 0x03, 0x97, 0xc9, 0xcf, 0x76, 0xd0, 0x2e, 0xac, 0x30, 0xee, - 0x1c, 0xd0, 0x48, 0x24, 0xb3, 0x41, 0x3f, 0x92, 0x32, 0x9f, 0xcf, 0x2f, 0xd3, 0x40, 0xfd, 0xc4, - 0x48, 0xb2, 0x9d, 0x8f, 0x81, 0x9e, 0x80, 0xc9, 0xc9, 0xfe, 0xe4, 0x1a, 0xcd, 0xd4, 0x37, 0x26, - 0xfb, 0x62, 0x2e, 0x84, 0x62, 0xa0, 0x7f, 0x01, 0xb8, 0x5e, 0x14, 0x12, 0xe1, 0x1c, 0x50, 0xae, - 0xfb, 0x94, 0xf9, 0x13, 0x3b, 0x53, 0xd4, 0x5c, 0x94, 0x14, 0x1b, 0x6d, 0x41, 0xd1, 0x21, 0x13, - 0xa5, 0xe5, 0x2f, 0x1e, 0x8b, 0xed, 0xa6, 0x0e, 0x51, 0x96, 0x21, 0xce, 0x4f, 0x57, 0x0b, 0x13, - 0x0f, 0x2e, 0x38, 0x44, 0x2b, 0x6f, 0x0b, 0x56, 0xe4, 0x8b, 0x72, 0xe0, 0xd2, 0x7d, 0x12, 0xfb, - 0x22, 0x52, 0x13, 0xfc, 0x82, 0x17, 0x94, 0x7c, 0xdc, 0x74, 0x34, 0x4e, 0xe7, 0x55, 0x12, 0x29, - 0x5f, 0xdd, 0x03, 0x48, 0x26, 0xdc, 0xe5, 0xca, 0x04, 0x81, 0xe9, 0x12, 0x41, 0x94, 0x32, 0x4a, - 0x58, 0xd9, 0xad, 0xbb, 0xef, 0xcf, 0xaa, 0xb9, 0x6f, 0xce, 0xaa, 0xb9, 0x1f, 0xce, 0xaa, 0xc6, - 0xff, 0xce, 0xab, 0xc6, 0xfb, 0xf3, 0xaa, 0xf1, 0xf5, 0x79, 0xd5, 0xf8, 0xf6, 0xbc, 0x6a, 0xec, - 0xe5, 0xd5, 0x1f, 0xb9, 0x3f, 0xfd, 0x14, 0x00, 0x00, 0xff, 0xff, 0xc3, 0xfc, 0x45, 0x4e, 0x27, - 0x0e, 0x00, 0x00, + 0x43, 0xb9, 0xa9, 0xa6, 0x2b, 0xbc, 0x54, 0x06, 0xa7, 0xa1, 0xef, 0x39, 0x44, 0x50, 0x57, 0x29, + 0xc3, 0x6a, 0xfc, 0x36, 0x8b, 0x8d, 0xa7, 0x28, 0x7d, 0xfe, 0x17, 0x39, 0x9c, 0xa2, 0xa2, 0x67, + 0x90, 0x1f, 0xfa, 0x6c, 0x8f, 0xf8, 0x4a, 0x13, 0x56, 0xe3, 0x5e, 0x56, 0x92, 0x4d, 0x85, 0x98, + 0x25, 0xd0, 0x14, 0xf4, 0x04, 0xf2, 0x71, 0xe8, 0x12, 0x41, 0xed, 0xbc, 0x22, 0xd7, 0xb2, 0xc8, + 0xaf, 0x15, 0xa2, 0xcd, 0x82, 0x7d, 0x6f, 0x88, 0x35, 0x1e, 0x6d, 0x41, 0x21, 0xa0, 0xe2, 0x2d, + 0xe3, 0x87, 0x91, 0xbd, 0x5c, 0x5b, 0x5c, 0xb3, 0x1a, 0x7f, 0xc8, 0x14, 0x63, 0x82, 0x69, 0x0a, + 0x41, 0x9c, 0x83, 0x11, 0x0d, 0x44, 0x92, 0xa6, 0xb5, 0x60, 0x1b, 0x78, 0x9a, 0x00, 0xfd, 0x03, + 0x0a, 0x34, 0x70, 0x43, 0xe6, 0x05, 0xc2, 0x2e, 0x5c, 0x7c, 0x90, 0xae, 0xc6, 0xc8, 0x62, 0xe2, + 0x29, 0xa3, 0x95, 0x07, 0x73, 0xc4, 0x5c, 0x5a, 0xdf, 0x80, 0x6b, 0x1f, 0x14, 0x0b, 0x55, 0xa0, + 0xa0, 0x8b, 0x95, 0x74, 0xd9, 0xc4, 0xd3, 0x79, 0xfd, 0x2a, 0xac, 0xcc, 0x15, 0x46, 0xd9, 0xc6, + 0xa4, 0x5b, 0xa8, 0x09, 0x45, 0x87, 0x05, 0x82, 0x78, 0x01, 0xe5, 0x5a, 0x20, 0x99, 0xb5, 0x6d, + 0x4f, 0x40, 0x92, 0xf5, 0x22, 0x87, 0x67, 0x2c, 0xf4, 0x6f, 0x28, 0x72, 0x1a, 0xb1, 0x98, 0x3b, + 0x34, 0xd2, 0x0a, 0x59, 0xcb, 0xee, 0x71, 0x02, 0xc2, 0xf4, 0xff, 0xb1, 0xc7, 0xa9, 0xac, 0x53, + 0x84, 0x67, 0x54, 0xf4, 0x0c, 0x96, 0x39, 0x8d, 0x04, 0xe1, 0xe2, 0x63, 0x4d, 0xc6, 0x09, 0xa4, + 0xcf, 0x7c, 0xcf, 0x39, 0xc2, 0x13, 0x06, 0x7a, 0x06, 0xc5, 0xd0, 0x27, 0x8e, 0xca, 0x6a, 0x2f, + 0x29, 0xfa, 0x6f, 0xb2, 0xe8, 0xfd, 0x09, 0x08, 0xcf, 0xf0, 0xe8, 0x29, 0x80, 0xcf, 0x86, 0x03, + 0x97, 0x7b, 0x63, 0xca, 0xb5, 0x48, 0x2a, 0x59, 0xec, 0x8e, 0x42, 0xe0, 0xa2, 0xcf, 0x86, 0xc9, + 0x10, 0x6d, 0xfe, 0x2a, 0x85, 0xa4, 0xd4, 0xb1, 0x05, 0x40, 0xa6, 0xab, 0x5a, 0x1f, 0x8f, 0x3e, + 0x2b, 0x95, 0xee, 0x48, 0x8a, 0x8e, 0xee, 0x41, 0x69, 0x9f, 0x71, 0x87, 0x0e, 0xb4, 0xee, 0x8b, + 0x4a, 0x13, 0x96, 0x8a, 0x25, 0x42, 0x6f, 0x15, 0x61, 0x99, 0xc7, 0x81, 0xf0, 0x46, 0xb4, 0xbe, + 0x05, 0x37, 0x32, 0x93, 0xa2, 0x06, 0x94, 0xa6, 0x6d, 0x1e, 0x78, 0xae, 0xd2, 0x47, 0xb1, 0x75, + 0xf5, 0xfc, 0x74, 0xd5, 0x9a, 0xea, 0xa1, 0xd7, 0xc1, 0xd6, 0x14, 0xd4, 0x73, 0xeb, 0x3f, 0x98, + 0xb0, 0x32, 0x27, 0x16, 0x74, 0x1d, 0x96, 0xbc, 0x11, 0x19, 0xd2, 0x84, 0x8e, 0x93, 0x09, 0xea, + 0x42, 0xde, 0x27, 0x7b, 0xd4, 0x97, 0x92, 0x91, 0x65, 0xfb, 0xe3, 0x27, 0x55, 0xb7, 0xfe, 0x5f, + 0x85, 0xef, 0x06, 0x82, 0x1f, 0x61, 0x4d, 0x46, 0x36, 0x2c, 0x3b, 0x6c, 0x34, 0x22, 0x81, 0xb4, + 0x97, 0xc5, 0xb5, 0x22, 0x9e, 0x4c, 0x11, 0x02, 0x93, 0xf0, 0x61, 0x64, 0x9b, 0x2a, 0xac, 0xc6, + 0xa8, 0x0c, 0x8b, 0x34, 0x18, 0xdb, 0x4b, 0x2a, 0x24, 0x87, 0x32, 0xe2, 0x7a, 0x49, 0xcf, 0x8b, + 0x58, 0x0e, 0x25, 0x2f, 0x8e, 0x28, 0xb7, 0x97, 0x55, 0x48, 0x8d, 0xd1, 0xdf, 0x20, 0x3f, 0x62, + 0x71, 0x20, 0x22, 0xbb, 0xa0, 0x0e, 0x7b, 0x3b, 0xeb, 0xb0, 0xdb, 0x12, 0xa1, 0xed, 0x4f, 0xc3, + 0xd1, 0x0b, 0xb8, 0x16, 0x09, 0x16, 0x0e, 0x86, 0x9c, 0x38, 0x74, 0x10, 0x52, 0xee, 0x31, 0x57, + 0x75, 0xe3, 0x02, 0x17, 0xed, 0xe8, 0x1b, 0x1e, 0x5f, 0x95, 0xb4, 0x4d, 0xc9, 0xea, 0x2b, 0x12, + 0xea, 0x43, 0x29, 0x8c, 0x7d, 0x7f, 0xc0, 0xc2, 0xc4, 0xcc, 0x41, 0x25, 0xf9, 0x8c, 0xaa, 0xf5, + 0x63, 0xdf, 0x7f, 0x99, 0x90, 0xb0, 0x15, 0xce, 0x26, 0xe8, 0x26, 0xe4, 0x87, 0x9c, 0xc5, 0x61, + 0x64, 0x5b, 0xaa, 0x1e, 0x7a, 0x86, 0x9e, 0xc3, 0x72, 0x44, 0x1d, 0x4e, 0x45, 0x64, 0x97, 0xd4, + 0xaf, 0xbd, 0x9f, 0xb5, 0xc9, 0xae, 0x82, 0x60, 0xba, 0x4f, 0x39, 0x0d, 0x1c, 0x8a, 0x27, 0x9c, + 0xca, 0x53, 0xb0, 0x52, 0x8d, 0x92, 0x05, 0x3e, 0xa4, 0x47, 0xba, 0xf7, 0x72, 0x28, 0xf5, 0x30, + 0x26, 0x7e, 0x9c, 0xbc, 0x30, 0x8a, 0x38, 0x99, 0xfc, 0x7d, 0xe1, 0x89, 0x51, 0x69, 0x80, 0x95, + 0x3a, 0x2d, 0xba, 0x0f, 0x2b, 0x9c, 0x0e, 0xbd, 0x48, 0xf0, 0xa3, 0x01, 0x89, 0xc5, 0x81, 0xfd, + 0x2f, 0x45, 0x28, 0x4d, 0x82, 0xcd, 0x58, 0x1c, 0xd4, 0x7f, 0x32, 0xa0, 0x94, 0xb6, 0x4c, 0xd4, + 0x4e, 0x8c, 0x52, 0xed, 0x78, 0xa5, 0xb1, 0xf1, 0x29, 0x8b, 0x55, 0xb6, 0xe4, 0xc7, 0x72, 0xc7, + 0x6d, 0xf9, 0xac, 0x51, 0x64, 0xf4, 0x57, 0x58, 0x0a, 0x19, 0x17, 0x13, 0x71, 0x56, 0x33, 0xad, + 0x84, 0xf1, 0xc9, 0x67, 0x9c, 0x80, 0xeb, 0x07, 0x70, 0x65, 0x3e, 0x1b, 0x7a, 0x00, 0x8b, 0x6f, + 0x7a, 0xfd, 0x72, 0xae, 0x72, 0xe7, 0xf8, 0xa4, 0x76, 0x6b, 0x7e, 0xf1, 0x8d, 0xc7, 0x45, 0x4c, + 0xfc, 0x5e, 0x1f, 0xfd, 0x1e, 0x96, 0x3a, 0x3b, 0xbb, 0x18, 0x97, 0x8d, 0xca, 0xea, 0xf1, 0x49, + 0xed, 0xce, 0x3c, 0x4e, 0x2e, 0xb1, 0x38, 0x70, 0x31, 0xdb, 0x9b, 0xde, 0xf4, 0xdf, 0x2c, 0x80, + 0xa5, 0xbf, 0xd9, 0xcb, 0xbd, 0xe9, 0xff, 0x09, 0x2b, 0x89, 0x0d, 0x0e, 0x1c, 0xf5, 0xd3, 0xb4, + 0xa1, 0x7f, 0xcc, 0x0d, 0x4b, 0x09, 0x21, 0x29, 0x85, 0xb4, 0x1e, 0x2f, 0x1c, 0x3f, 0x1e, 0xd0, + 0x80, 0xec, 0xf9, 0xfa, 0xd2, 0x2f, 0x60, 0x4b, 0xc6, 0xba, 0x49, 0x48, 0xde, 0x56, 0x5e, 0x20, + 0x28, 0x0f, 0xf4, 0x75, 0x5e, 0xc0, 0xd3, 0x39, 0x7a, 0x0e, 0xa6, 0x17, 0x92, 0x91, 0xb6, 0xf0, + 0xcc, 0x5f, 0xd0, 0xeb, 0x37, 0xb7, 0xb5, 0x44, 0x5a, 0x85, 0xf3, 0xd3, 0x55, 0x53, 0x06, 0xb0, + 0xa2, 0xa1, 0xea, 0xc4, 0x45, 0xe5, 0x4e, 0xea, 0xab, 0x2e, 0xe0, 0x54, 0xa4, 0xfe, 0xb5, 0x09, + 0x56, 0xdb, 0x8f, 0x23, 0xa1, 0xbd, 0xe9, 0xd2, 0xea, 0xf6, 0x3f, 0xb8, 0x46, 0xd4, 0xbb, 0x90, + 0x04, 0xf2, 0x43, 0x57, 0xb7, 0x93, 0xae, 0xdd, 0x83, 0xcc, 0x74, 0x53, 0x70, 0x72, 0x93, 0xb5, + 0xf2, 0x32, 0xa7, 0x6d, 0xe0, 0x32, 0xf9, 0xc5, 0x0a, 0xda, 0x85, 0x15, 0xc6, 0x9d, 0x03, 0x1a, + 0x89, 0xc4, 0x1b, 0xf4, 0x3b, 0x2a, 0xf3, 0x85, 0xfd, 0x32, 0x0d, 0xd4, 0xaf, 0x90, 0xe4, 0xb4, + 0xf3, 0x39, 0xd0, 0x13, 0x30, 0x39, 0xd9, 0x9f, 0xdc, 0xb4, 0x99, 0xfa, 0xc6, 0x64, 0x5f, 0xcc, + 0xa5, 0x50, 0x0c, 0xf4, 0x1f, 0x00, 0xd7, 0x8b, 0x42, 0x22, 0x9c, 0x03, 0xca, 0x75, 0x9f, 0x32, + 0x7f, 0x62, 0x67, 0x8a, 0x9a, 0xcb, 0x92, 0x62, 0xa3, 0x2d, 0x28, 0x3a, 0x64, 0xa2, 0xb4, 0xfc, + 0xc5, 0xb6, 0xd8, 0x6e, 0xea, 0x14, 0x65, 0x99, 0xe2, 0xfc, 0x74, 0xb5, 0x30, 0x89, 0xe0, 0x82, + 0x43, 0xb4, 0xf2, 0xb6, 0x60, 0x45, 0x3e, 0x3a, 0x07, 0x2e, 0xdd, 0x27, 0xb1, 0x2f, 0x22, 0xe5, + 0xe0, 0x17, 0x3c, 0xb2, 0xe4, 0xfb, 0xa7, 0xa3, 0x71, 0xfa, 0x5c, 0x25, 0x91, 0x8a, 0xd5, 0x3d, + 0x80, 0xc4, 0xe1, 0x2e, 0x57, 0x26, 0x08, 0x4c, 0x97, 0x08, 0xa2, 0x94, 0x51, 0xc2, 0x6a, 0xdc, + 0xba, 0xfb, 0xfe, 0xac, 0x9a, 0xfb, 0xee, 0xac, 0x9a, 0xfb, 0xf1, 0xac, 0x6a, 0x7c, 0x71, 0x5e, + 0x35, 0xde, 0x9f, 0x57, 0x8d, 0x6f, 0xcf, 0xab, 0xc6, 0xf7, 0xe7, 0x55, 0x63, 0x2f, 0xaf, 0xfe, + 0xeb, 0xfd, 0xe5, 0xe7, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb6, 0x26, 0x4a, 0x64, 0x4a, 0x0e, 0x00, + 0x00, } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.proto b/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.proto index ddf7eae10e..093b82272c 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.proto +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.proto @@ -112,6 +112,12 @@ message TaskSpec { // configurations (which specify the network and per-network // aliases) that this task spec is bound to. repeated NetworkAttachmentConfig networks = 7; + + // ForceUpdate is a counter that triggers an update even if no relevant + // parameters have been changed. We do this to allow forced restarts + // using the same reconcilation-based mechanism that performs rolling + // updates. + uint64 force_update = 9; } // NetworkAttachmentSpec specifies runtime parameters required to attach diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/types.pb.go b/components/engine/vendor/src/github.com/docker/swarmkit/api/types.pb.go index 2065f07a99..314bc9698a 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/types.pb.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/types.pb.go @@ -57,6 +57,7 @@ EncryptionKey ManagerStatus SecretReference + RemovedNode NodeSpec ServiceSpec ReplicatedService @@ -1396,6 +1397,19 @@ func (m *SecretReference) Reset() { *m = SecretReference{} } func (*SecretReference) ProtoMessage() {} func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{38} } +// RemovedNode is a record for a node that has been removed from the swarm. +type RemovedNode struct { + // ID is the ID of the removed node. + ID string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // Expiry is the latest known expiration time of a certificate that + // was issued to this node. + Expiry *docker_swarmkit_v1.Timestamp `protobuf:"bytes,2,opt,name=expiry" json:"expiry,omitempty"` +} + +func (m *RemovedNode) Reset() { *m = RemovedNode{} } +func (*RemovedNode) ProtoMessage() {} +func (*RemovedNode) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{39} } + func init() { proto.RegisterType((*Version)(nil), "docker.swarmkit.v1.Version") proto.RegisterType((*Annotations)(nil), "docker.swarmkit.v1.Annotations") @@ -1441,6 +1455,7 @@ func init() { proto.RegisterType((*EncryptionKey)(nil), "docker.swarmkit.v1.EncryptionKey") proto.RegisterType((*ManagerStatus)(nil), "docker.swarmkit.v1.ManagerStatus") proto.RegisterType((*SecretReference)(nil), "docker.swarmkit.v1.SecretReference") + proto.RegisterType((*RemovedNode)(nil), "docker.swarmkit.v1.RemovedNode") proto.RegisterEnum("docker.swarmkit.v1.TaskState", TaskState_name, TaskState_value) proto.RegisterEnum("docker.swarmkit.v1.NodeRole", NodeRole_name, NodeRole_value) proto.RegisterEnum("docker.swarmkit.v1.RaftMemberStatus_Reachability", RaftMemberStatus_Reachability_name, RaftMemberStatus_Reachability_value) @@ -1779,16 +1794,12 @@ func (m *NetworkAttachmentConfig) Copy() *NetworkAttachmentConfig { if m.Aliases != nil { o.Aliases = make([]string, 0, len(m.Aliases)) - for _, v := range m.Aliases { - o.Aliases = append(o.Aliases, v) - } + o.Aliases = append(o.Aliases, m.Aliases...) } if m.Addresses != nil { o.Addresses = make([]string, 0, len(m.Addresses)) - for _, v := range m.Addresses { - o.Addresses = append(o.Addresses, v) - } + o.Addresses = append(o.Addresses, m.Addresses...) } return o @@ -2052,9 +2063,7 @@ func (m *Placement) Copy() *Placement { if m.Constraints != nil { o.Constraints = make([]string, 0, len(m.Constraints)) - for _, v := range m.Constraints { - o.Constraints = append(o.Constraints, v) - } + o.Constraints = append(o.Constraints, m.Constraints...) } return o @@ -2149,6 +2158,19 @@ func (m *SecretReference) Copy() *SecretReference { return o } +func (m *RemovedNode) Copy() *RemovedNode { + if m == nil { + return nil + } + + o := &RemovedNode{ + ID: m.ID, + Expiry: m.Expiry.Copy(), + } + + return o +} + func (this *Version) GoString() string { if this == nil { return "nil" @@ -2800,6 +2822,19 @@ func (this *SecretReference) GoString() string { s = append(s, "}") return strings.Join(s, "") } +func (this *RemovedNode) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 6) + s = append(s, "&api.RemovedNode{") + s = append(s, "ID: "+fmt.Sprintf("%#v", this.ID)+",\n") + if this.Expiry != nil { + s = append(s, "Expiry: "+fmt.Sprintf("%#v", this.Expiry)+",\n") + } + s = append(s, "}") + return strings.Join(s, "") +} func valueToGoStringTypes(v interface{}, typ string) string { rv := reflect.ValueOf(v) if rv.IsNil() { @@ -4560,6 +4595,40 @@ func (m *SecretReference) MarshalTo(data []byte) (int, error) { return i, nil } +func (m *RemovedNode) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *RemovedNode) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.ID) > 0 { + data[i] = 0xa + i++ + i = encodeVarintTypes(data, i, uint64(len(m.ID))) + i += copy(data[i:], m.ID) + } + if m.Expiry != nil { + data[i] = 0x12 + i++ + i = encodeVarintTypes(data, i, uint64(m.Expiry.Size())) + n27, err := m.Expiry.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n27 + } + return i, nil +} + func encodeFixed64Types(data []byte, offset int, v uint64) int { data[offset] = uint8(v) data[offset+1] = uint8(v >> 8) @@ -5341,6 +5410,20 @@ func (m *SecretReference) Size() (n int) { return n } +func (m *RemovedNode) Size() (n int) { + var l int + _ = l + l = len(m.ID) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + if m.Expiry != nil { + l = m.Expiry.Size() + n += 1 + l + sovTypes(uint64(l)) + } + return n +} + func sovTypes(x uint64) (n int) { for { n++ @@ -5943,6 +6026,17 @@ func (this *SecretReference) String() string { }, "") return s } +func (this *RemovedNode) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&RemovedNode{`, + `ID:` + fmt.Sprintf("%v", this.ID) + `,`, + `Expiry:` + strings.Replace(fmt.Sprintf("%v", this.Expiry), "Timestamp", "docker_swarmkit_v1.Timestamp", 1) + `,`, + `}`, + }, "") + return s +} func valueToStringTypes(v interface{}) string { rv := reflect.ValueOf(v) if rv.IsNil() { @@ -11920,6 +12014,118 @@ func (m *SecretReference) Unmarshal(data []byte) error { } return nil } +func (m *RemovedNode) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RemovedNode: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RemovedNode: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ID = string(data[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Expiry", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Expiry == nil { + m.Expiry = &docker_swarmkit_v1.Timestamp{} + } + if err := m.Expiry.Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipTypes(data []byte) (n int, err error) { l := len(data) iNdEx := 0 @@ -12028,231 +12234,233 @@ var ( func init() { proto.RegisterFile("types.proto", fileDescriptorTypes) } var fileDescriptorTypes = []byte{ - // 3603 bytes of a gzipped FileDescriptorProto + // 3635 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x59, 0xcd, 0x6f, 0x23, 0x47, 0x76, 0x17, 0x3f, 0x45, 0x3e, 0x52, 0x52, 0x4f, 0xcd, 0xec, 0x58, 0x43, 0x8f, 0x25, 0xba, 0xc7, - 0xb3, 0x1e, 0x7b, 0x1d, 0xda, 0x96, 0x37, 0xc6, 0xac, 0x67, 0xb3, 0x76, 0x8b, 0xa4, 0x66, 0xb8, - 0x23, 0x51, 0x44, 0x91, 0xd4, 0xc0, 0x08, 0x10, 0xa2, 0xd4, 0x5d, 0x22, 0xdb, 0x6a, 0x76, 0x33, - 0xdd, 0x45, 0x69, 0x98, 0x20, 0xc0, 0x24, 0x87, 0x24, 0xd0, 0x29, 0xf7, 0x40, 0x58, 0x04, 0x09, - 0x72, 0xcb, 0x39, 0x40, 0x4e, 0x3e, 0xfa, 0xb8, 0x41, 0x80, 0x60, 0xb1, 0x41, 0x84, 0x58, 0xf9, - 0x07, 0x16, 0x08, 0x82, 0x3d, 0x24, 0x87, 0xa0, 0x3e, 0xba, 0xd9, 0xe4, 0x50, 0xf2, 0x4c, 0x76, - 0x4f, 0x64, 0xbd, 0xfa, 0xbd, 0x57, 0xaf, 0xaa, 0x5e, 0xbd, 0xfa, 0xbd, 0x6a, 0x28, 0xb0, 0xc9, - 0x88, 0x06, 0x95, 0x91, 0xef, 0x31, 0x0f, 0x21, 0xcb, 0x33, 0x8f, 0xa9, 0x5f, 0x09, 0x4e, 0x89, - 0x3f, 0x3c, 0xb6, 0x59, 0xe5, 0xe4, 0xe3, 0xd2, 0x1d, 0x66, 0x0f, 0x69, 0xc0, 0xc8, 0x70, 0xf4, - 0x61, 0xf4, 0x4f, 0xc2, 0x4b, 0x6f, 0x58, 0x63, 0x9f, 0x30, 0xdb, 0x73, 0x3f, 0x0c, 0xff, 0xa8, - 0x8e, 0x5b, 0x7d, 0xaf, 0xef, 0x89, 0xbf, 0x1f, 0xf2, 0x7f, 0x52, 0xaa, 0x6f, 0xc2, 0xf2, 0x01, - 0xf5, 0x03, 0xdb, 0x73, 0xd1, 0x2d, 0xc8, 0xd8, 0xae, 0x45, 0x9f, 0xaf, 0x27, 0xca, 0x89, 0x07, - 0x69, 0x2c, 0x1b, 0xfa, 0xdf, 0x24, 0xa0, 0x60, 0xb8, 0xae, 0xc7, 0x84, 0xad, 0x00, 0x21, 0x48, - 0xbb, 0x64, 0x48, 0x05, 0x28, 0x8f, 0xc5, 0x7f, 0x54, 0x85, 0xac, 0x43, 0x0e, 0xa9, 0x13, 0xac, - 0x27, 0xcb, 0xa9, 0x07, 0x85, 0xad, 0x1f, 0x54, 0x5e, 0xf6, 0xb9, 0x12, 0x33, 0x52, 0xd9, 0x15, - 0xe8, 0xba, 0xcb, 0xfc, 0x09, 0x56, 0xaa, 0xa5, 0x1f, 0x41, 0x21, 0x26, 0x46, 0x1a, 0xa4, 0x8e, - 0xe9, 0x44, 0x0d, 0xc3, 0xff, 0x72, 0xff, 0x4e, 0x88, 0x33, 0xa6, 0xeb, 0x49, 0x21, 0x93, 0x8d, - 0xcf, 0x92, 0x0f, 0x13, 0xfa, 0x97, 0x90, 0xc7, 0x34, 0xf0, 0xc6, 0xbe, 0x49, 0x03, 0xf4, 0x1e, - 0xe4, 0x5d, 0xe2, 0x7a, 0x3d, 0x73, 0x34, 0x0e, 0x84, 0x7a, 0x6a, 0xbb, 0x78, 0x79, 0xb1, 0x99, - 0x6b, 0x12, 0xd7, 0xab, 0xb6, 0xba, 0x01, 0xce, 0xf1, 0xee, 0xea, 0x68, 0x1c, 0xa0, 0xb7, 0xa1, - 0x38, 0xa4, 0x43, 0xcf, 0x9f, 0xf4, 0x0e, 0x27, 0x8c, 0x06, 0xc2, 0x70, 0x0a, 0x17, 0xa4, 0x6c, - 0x9b, 0x8b, 0xf4, 0xbf, 0x4a, 0xc0, 0xad, 0xd0, 0x36, 0xa6, 0x7f, 0x38, 0xb6, 0x7d, 0x3a, 0xa4, - 0x2e, 0x0b, 0xd0, 0xef, 0x42, 0xd6, 0xb1, 0x87, 0x36, 0x93, 0x63, 0x14, 0xb6, 0xde, 0x5a, 0x34, - 0xe7, 0xc8, 0x2b, 0xac, 0xc0, 0xc8, 0x80, 0xa2, 0x4f, 0x03, 0xea, 0x9f, 0xc8, 0x95, 0x10, 0x43, - 0x7e, 0xa7, 0xf2, 0x8c, 0x8a, 0xbe, 0x03, 0xb9, 0x96, 0x43, 0xd8, 0x91, 0xe7, 0x0f, 0x91, 0x0e, - 0x45, 0xe2, 0x9b, 0x03, 0x9b, 0x51, 0x93, 0x8d, 0xfd, 0x70, 0x57, 0x66, 0x64, 0xe8, 0x36, 0x24, - 0x3d, 0x39, 0x50, 0x7e, 0x3b, 0x7b, 0x79, 0xb1, 0x99, 0xdc, 0x6f, 0xe3, 0xa4, 0x17, 0xe8, 0x8f, - 0xe0, 0x46, 0xcb, 0x19, 0xf7, 0x6d, 0xb7, 0x46, 0x03, 0xd3, 0xb7, 0x47, 0xdc, 0x3a, 0xdf, 0x5e, - 0x1e, 0x7c, 0xe1, 0xf6, 0xf2, 0xff, 0xd1, 0x96, 0x27, 0xa7, 0x5b, 0xae, 0xff, 0x45, 0x12, 0x6e, - 0xd4, 0xdd, 0xbe, 0xed, 0xd2, 0xb8, 0xf6, 0x7d, 0x58, 0xa5, 0x42, 0xd8, 0x3b, 0x91, 0x41, 0xa5, - 0xec, 0xac, 0x48, 0x69, 0x18, 0x69, 0x8d, 0xb9, 0x78, 0xf9, 0x78, 0xd1, 0xf4, 0x5f, 0xb2, 0xbe, - 0x28, 0x6a, 0x50, 0x1d, 0x96, 0x47, 0x62, 0x12, 0xc1, 0x7a, 0x4a, 0xd8, 0xba, 0xbf, 0xc8, 0xd6, - 0x4b, 0xf3, 0xdc, 0x4e, 0x7f, 0x73, 0xb1, 0xb9, 0x84, 0x43, 0xdd, 0xdf, 0x24, 0xf8, 0xfe, 0x33, - 0x01, 0x6b, 0x4d, 0xcf, 0x9a, 0x59, 0x87, 0x12, 0xe4, 0x06, 0x5e, 0xc0, 0x62, 0x07, 0x25, 0x6a, - 0xa3, 0x87, 0x90, 0x1b, 0xa9, 0xed, 0x53, 0xbb, 0x7f, 0x77, 0xb1, 0xcb, 0x12, 0x83, 0x23, 0x34, - 0x7a, 0x04, 0x79, 0x3f, 0x8c, 0x89, 0xf5, 0xd4, 0xab, 0x04, 0xce, 0x14, 0x8f, 0x7e, 0x0f, 0xb2, - 0x72, 0x13, 0xd6, 0xd3, 0x42, 0xf3, 0xfe, 0x2b, 0xad, 0x39, 0x56, 0x4a, 0xfa, 0x2f, 0x12, 0xa0, - 0x61, 0x72, 0xc4, 0xf6, 0xe8, 0xf0, 0x90, 0xfa, 0x6d, 0x46, 0xd8, 0x38, 0x40, 0xb7, 0x21, 0xeb, - 0x50, 0x62, 0x51, 0x5f, 0x4c, 0x32, 0x87, 0x55, 0x0b, 0x75, 0x79, 0x90, 0x13, 0x73, 0x40, 0x0e, - 0x6d, 0xc7, 0x66, 0x13, 0x31, 0xcd, 0xd5, 0xc5, 0xbb, 0x3c, 0x6f, 0xb3, 0x82, 0x63, 0x8a, 0x78, - 0xc6, 0x0c, 0x5a, 0x87, 0xe5, 0x21, 0x0d, 0x02, 0xd2, 0xa7, 0x62, 0xf6, 0x79, 0x1c, 0x36, 0xf5, - 0x47, 0x50, 0x8c, 0xeb, 0xa1, 0x02, 0x2c, 0x77, 0x9b, 0x4f, 0x9b, 0xfb, 0xcf, 0x9a, 0xda, 0x12, - 0x5a, 0x83, 0x42, 0xb7, 0x89, 0xeb, 0x46, 0xf5, 0x89, 0xb1, 0xbd, 0x5b, 0xd7, 0x12, 0x68, 0x05, - 0xf2, 0xd3, 0x66, 0x52, 0xff, 0x59, 0x02, 0x80, 0x6f, 0xa0, 0x9a, 0xd4, 0x67, 0x90, 0x09, 0x18, - 0x61, 0x72, 0xe3, 0x56, 0xb7, 0xde, 0x59, 0xe4, 0xf5, 0x14, 0x5e, 0xe1, 0x3f, 0x14, 0x4b, 0x95, - 0xb8, 0x87, 0xc9, 0x79, 0x0f, 0x33, 0x02, 0x39, 0xeb, 0x5a, 0x0e, 0xd2, 0x35, 0xfe, 0x2f, 0x81, - 0xf2, 0x90, 0xc1, 0x75, 0xa3, 0xf6, 0xa5, 0x96, 0x44, 0x1a, 0x14, 0x6b, 0x8d, 0x76, 0x75, 0xbf, - 0xd9, 0xac, 0x57, 0x3b, 0xf5, 0x9a, 0x96, 0xd2, 0xef, 0x43, 0xa6, 0x31, 0x24, 0x7d, 0x8a, 0xee, - 0xf2, 0x08, 0x38, 0xa2, 0x3e, 0x75, 0xcd, 0x30, 0xb0, 0xa6, 0x02, 0xfd, 0xe7, 0x79, 0xc8, 0xec, - 0x79, 0x63, 0x97, 0xa1, 0xad, 0xd8, 0x29, 0x5e, 0xdd, 0xda, 0x58, 0x34, 0x05, 0x01, 0xac, 0x74, - 0x26, 0x23, 0xaa, 0x4e, 0xf9, 0x6d, 0xc8, 0xca, 0x58, 0x51, 0xae, 0xab, 0x16, 0x97, 0x33, 0xe2, - 0xf7, 0x29, 0x53, 0x8b, 0xae, 0x5a, 0xe8, 0x01, 0xe4, 0x7c, 0x4a, 0x2c, 0xcf, 0x75, 0x26, 0x22, - 0xa4, 0x72, 0x32, 0xcd, 0x62, 0x4a, 0xac, 0x7d, 0xd7, 0x99, 0xe0, 0xa8, 0x17, 0x3d, 0x81, 0xe2, - 0xa1, 0xed, 0x5a, 0x3d, 0x6f, 0x24, 0x73, 0x5e, 0xe6, 0xea, 0x00, 0x94, 0x5e, 0x6d, 0xdb, 0xae, - 0xb5, 0x2f, 0xc1, 0xb8, 0x70, 0x38, 0x6d, 0xa0, 0x26, 0xac, 0x9e, 0x78, 0xce, 0x78, 0x48, 0x23, - 0x5b, 0x59, 0x61, 0xeb, 0xdd, 0xab, 0x6d, 0x1d, 0x08, 0x7c, 0x68, 0x6d, 0xe5, 0x24, 0xde, 0x44, - 0x4f, 0x61, 0x85, 0x0d, 0x47, 0x47, 0x41, 0x64, 0x6e, 0x59, 0x98, 0xfb, 0xfe, 0x35, 0x0b, 0xc6, - 0xe1, 0xa1, 0xb5, 0x22, 0x8b, 0xb5, 0x4a, 0x7f, 0x96, 0x82, 0x42, 0xcc, 0x73, 0xd4, 0x86, 0xc2, - 0xc8, 0xf7, 0x46, 0xa4, 0x2f, 0xf2, 0xb6, 0xda, 0x8b, 0x8f, 0x5f, 0x69, 0xd6, 0x95, 0xd6, 0x54, - 0x11, 0xc7, 0xad, 0xe8, 0xe7, 0x49, 0x28, 0xc4, 0x3a, 0xd1, 0xfb, 0x90, 0xc3, 0x2d, 0xdc, 0x38, - 0x30, 0x3a, 0x75, 0x6d, 0xa9, 0x74, 0xf7, 0xec, 0xbc, 0xbc, 0x2e, 0xac, 0xc5, 0x0d, 0xb4, 0x7c, - 0xfb, 0x84, 0x87, 0xde, 0x03, 0x58, 0x0e, 0xa1, 0x89, 0xd2, 0x9b, 0x67, 0xe7, 0xe5, 0x37, 0xe6, - 0xa1, 0x31, 0x24, 0x6e, 0x3f, 0x31, 0x70, 0xbd, 0xa6, 0x25, 0x17, 0x23, 0x71, 0x7b, 0x40, 0x7c, - 0x6a, 0xa1, 0xef, 0x43, 0x56, 0x01, 0x53, 0xa5, 0xd2, 0xd9, 0x79, 0xf9, 0xf6, 0x3c, 0x70, 0x8a, - 0xc3, 0xed, 0x5d, 0xe3, 0xa0, 0xae, 0xa5, 0x17, 0xe3, 0x70, 0xdb, 0x21, 0x27, 0x14, 0xbd, 0x03, - 0x19, 0x09, 0xcb, 0x94, 0xee, 0x9c, 0x9d, 0x97, 0xbf, 0xf7, 0x92, 0x39, 0x8e, 0x2a, 0xad, 0xff, - 0xe5, 0xdf, 0x6e, 0x2c, 0xfd, 0xd3, 0xdf, 0x6d, 0x68, 0xf3, 0xdd, 0xa5, 0xff, 0x4d, 0xc0, 0xca, - 0xcc, 0x96, 0x23, 0x1d, 0xb2, 0xae, 0x67, 0x7a, 0x23, 0x99, 0xce, 0x73, 0xdb, 0x70, 0x79, 0xb1, - 0x99, 0x6d, 0x7a, 0x55, 0x6f, 0x34, 0xc1, 0xaa, 0x07, 0x3d, 0x9d, 0xbb, 0x90, 0x3e, 0x79, 0xc5, - 0x78, 0x5a, 0x78, 0x25, 0x7d, 0x0e, 0x2b, 0x96, 0x6f, 0x9f, 0x50, 0xbf, 0x67, 0x7a, 0xee, 0x91, - 0xdd, 0x57, 0xa9, 0xba, 0xb4, 0xc8, 0x66, 0x4d, 0x00, 0x71, 0x51, 0x2a, 0x54, 0x05, 0xfe, 0x37, - 0xb8, 0x8c, 0x4a, 0x07, 0x50, 0x8c, 0x47, 0x28, 0x7a, 0x0b, 0x20, 0xb0, 0xff, 0x88, 0x2a, 0x7e, - 0x23, 0xd8, 0x10, 0xce, 0x73, 0x89, 0x60, 0x37, 0xe8, 0x5d, 0x48, 0x0f, 0x3d, 0x4b, 0xda, 0xc9, - 0x6c, 0xdf, 0xe4, 0x77, 0xe2, 0x2f, 0x2f, 0x36, 0x0b, 0x5e, 0x50, 0xd9, 0xb1, 0x1d, 0xba, 0xe7, - 0x59, 0x14, 0x0b, 0x80, 0x7e, 0x02, 0x69, 0x9e, 0x2a, 0xd0, 0x9b, 0x90, 0xde, 0x6e, 0x34, 0x6b, - 0xda, 0x52, 0xe9, 0xc6, 0xd9, 0x79, 0x79, 0x45, 0x2c, 0x09, 0xef, 0xe0, 0xb1, 0x8b, 0x36, 0x21, - 0x7b, 0xb0, 0xbf, 0xdb, 0xdd, 0xe3, 0xe1, 0x75, 0xf3, 0xec, 0xbc, 0xbc, 0x16, 0x75, 0xcb, 0x45, - 0x43, 0x6f, 0x41, 0xa6, 0xb3, 0xd7, 0xda, 0x69, 0x6b, 0xc9, 0x12, 0x3a, 0x3b, 0x2f, 0xaf, 0x46, - 0xfd, 0xc2, 0xe7, 0xd2, 0x0d, 0xb5, 0xab, 0xf9, 0x48, 0xae, 0xff, 0x4f, 0x12, 0x56, 0x30, 0xe7, - 0xb7, 0x3e, 0x6b, 0x79, 0x8e, 0x6d, 0x4e, 0x50, 0x0b, 0xf2, 0xa6, 0xe7, 0x5a, 0x76, 0xec, 0x4c, - 0x6d, 0x5d, 0x71, 0x09, 0x4e, 0xb5, 0xc2, 0x56, 0x35, 0xd4, 0xc4, 0x53, 0x23, 0x68, 0x0b, 0x32, - 0x16, 0x75, 0xc8, 0xe4, 0xba, 0xdb, 0xb8, 0xa6, 0xb8, 0x34, 0x96, 0x50, 0xc1, 0x1c, 0xc9, 0xf3, - 0x1e, 0x61, 0x8c, 0x0e, 0x47, 0x4c, 0xde, 0xc6, 0x69, 0x5c, 0x18, 0x92, 0xe7, 0x86, 0x12, 0xa1, - 0x1f, 0x42, 0xf6, 0xd4, 0x76, 0x2d, 0xef, 0x54, 0x5d, 0xb8, 0xd7, 0xdb, 0x55, 0x58, 0xfd, 0x8c, - 0xdf, 0xb3, 0x73, 0xce, 0xf2, 0x55, 0x6f, 0xee, 0x37, 0xeb, 0xe1, 0xaa, 0xab, 0xfe, 0x7d, 0xb7, - 0xe9, 0xb9, 0xfc, 0xc4, 0xc0, 0x7e, 0xb3, 0xb7, 0x63, 0x34, 0x76, 0xbb, 0x98, 0xaf, 0xfc, 0xad, - 0xb3, 0xf3, 0xb2, 0x16, 0x41, 0x76, 0x88, 0xed, 0x70, 0x12, 0x78, 0x07, 0x52, 0x46, 0xf3, 0x4b, - 0x2d, 0x59, 0xd2, 0xce, 0xce, 0xcb, 0xc5, 0xa8, 0xdb, 0x70, 0x27, 0xd3, 0xc3, 0x34, 0x3f, 0xae, - 0xfe, 0xef, 0x49, 0x28, 0x76, 0x47, 0x16, 0x61, 0x54, 0x46, 0x26, 0x2a, 0x43, 0x61, 0x44, 0x7c, - 0xe2, 0x38, 0xd4, 0xb1, 0x83, 0xa1, 0x2a, 0x14, 0xe2, 0x22, 0xf4, 0xf0, 0x35, 0x16, 0x53, 0x91, - 0x30, 0xb5, 0xa4, 0x5d, 0x58, 0x3d, 0x92, 0xce, 0xf6, 0x88, 0x29, 0x76, 0x37, 0x25, 0x76, 0xb7, - 0xb2, 0xc8, 0x44, 0xdc, 0xab, 0x8a, 0x9a, 0xa3, 0x21, 0xb4, 0xf0, 0xca, 0x51, 0xbc, 0x89, 0x3e, - 0x85, 0xe5, 0xa1, 0xe7, 0xda, 0xcc, 0xf3, 0x5f, 0x69, 0x1f, 0x42, 0x30, 0x7a, 0x1f, 0x6e, 0xf0, - 0x1d, 0x0e, 0x5d, 0x12, 0xdd, 0xe2, 0xe6, 0x4a, 0xe2, 0xb5, 0x21, 0x79, 0xae, 0xc6, 0xc4, 0x5c, - 0xac, 0x7f, 0x0a, 0x2b, 0x33, 0x3e, 0xf0, 0xdb, 0xbc, 0x65, 0x74, 0xdb, 0x75, 0x6d, 0x09, 0x15, - 0x21, 0x57, 0xdd, 0x6f, 0x76, 0x1a, 0xcd, 0x2e, 0xa7, 0x1e, 0x45, 0xc8, 0xe1, 0xfd, 0xdd, 0xdd, - 0x6d, 0xa3, 0xfa, 0x54, 0x4b, 0xea, 0xff, 0x1d, 0xad, 0xaf, 0xe2, 0x1e, 0xdb, 0xb3, 0xdc, 0xe3, - 0x83, 0xab, 0xa7, 0xae, 0xd8, 0xc7, 0xb4, 0x11, 0x71, 0x90, 0x1f, 0x03, 0x88, 0x6d, 0xa4, 0x56, - 0x8f, 0xb0, 0xeb, 0xea, 0x8b, 0x4e, 0x58, 0x39, 0xe2, 0xbc, 0x52, 0x30, 0x18, 0xfa, 0x02, 0x8a, - 0xa6, 0x37, 0x1c, 0x39, 0x54, 0xe9, 0xa7, 0x5e, 0x45, 0xbf, 0x10, 0xa9, 0x18, 0x2c, 0xce, 0x81, - 0xd2, 0xb3, 0x1c, 0xe8, 0xcf, 0x13, 0x50, 0x88, 0x39, 0x3c, 0x4b, 0x85, 0x8a, 0x90, 0xeb, 0xb6, - 0x6a, 0x46, 0xa7, 0xd1, 0x7c, 0xac, 0x25, 0x10, 0x40, 0x56, 0x2c, 0x60, 0x4d, 0x4b, 0x72, 0xba, - 0x56, 0xdd, 0xdf, 0x6b, 0xed, 0xd6, 0x05, 0x19, 0x42, 0xb7, 0x40, 0x0b, 0x97, 0xb0, 0xd7, 0xee, - 0x18, 0x98, 0x4b, 0xd3, 0xe8, 0x26, 0xac, 0x45, 0x52, 0xa5, 0x99, 0x41, 0xb7, 0x01, 0x45, 0xc2, - 0xa9, 0x89, 0xac, 0xfe, 0x27, 0xb0, 0x56, 0xf5, 0x5c, 0x46, 0x6c, 0x37, 0xa2, 0xb2, 0x5b, 0x7c, - 0xde, 0x4a, 0xd4, 0xb3, 0x2d, 0x99, 0x6d, 0xb7, 0xd7, 0x2e, 0x2f, 0x36, 0x0b, 0x11, 0xb4, 0x51, - 0xe3, 0x33, 0x0d, 0x1b, 0x16, 0x3f, 0x53, 0x23, 0xdb, 0x52, 0xc9, 0x73, 0xf9, 0xf2, 0x62, 0x33, - 0xd5, 0x6a, 0xd4, 0x30, 0x97, 0xa1, 0x37, 0x21, 0x4f, 0x9f, 0xdb, 0xac, 0x67, 0xf2, 0xec, 0xca, - 0xd7, 0x30, 0x83, 0x73, 0x5c, 0x50, 0xe5, 0xc9, 0xf4, 0x4f, 0x93, 0x00, 0x1d, 0x12, 0x1c, 0xab, - 0xa1, 0x1f, 0x41, 0x3e, 0x2a, 0xe2, 0xaf, 0x2b, 0x26, 0x63, 0xfb, 0x15, 0xe1, 0xd1, 0x27, 0x61, - 0xc4, 0x48, 0x8e, 0xbd, 0x58, 0x51, 0x8d, 0xb5, 0x88, 0xa6, 0xce, 0x12, 0x69, 0x7e, 0xd7, 0x50, - 0xdf, 0x57, 0x1b, 0xc7, 0xff, 0xa2, 0xaa, 0xc8, 0xb7, 0x72, 0xce, 0x8a, 0xb9, 0xdd, 0x5b, 0x34, - 0xc8, 0xdc, 0x82, 0x3e, 0x59, 0xc2, 0x53, 0xbd, 0x6d, 0x0d, 0x56, 0xfd, 0xb1, 0xcb, 0xbd, 0xee, - 0x05, 0xa2, 0x5b, 0xb7, 0xe1, 0x8d, 0x26, 0x65, 0xa7, 0x9e, 0x7f, 0x6c, 0x30, 0x46, 0xcc, 0x01, - 0x2f, 0xaa, 0x55, 0x92, 0x99, 0x12, 0xce, 0xc4, 0x0c, 0xe1, 0x5c, 0x87, 0x65, 0xe2, 0xd8, 0x24, - 0xa0, 0xf2, 0x96, 0xce, 0xe3, 0xb0, 0xc9, 0x69, 0x31, 0xb1, 0x2c, 0x9f, 0x06, 0x01, 0x95, 0x65, - 0x60, 0x1e, 0x4f, 0x05, 0xfa, 0xbf, 0x24, 0x01, 0x1a, 0x2d, 0x63, 0x4f, 0x99, 0xaf, 0x41, 0xf6, - 0x88, 0x0c, 0x6d, 0x67, 0x72, 0xdd, 0x21, 0x9b, 0xe2, 0x2b, 0x86, 0x34, 0xb4, 0x23, 0x74, 0xb0, - 0xd2, 0x15, 0x6c, 0x79, 0x7c, 0xe8, 0x52, 0x16, 0xb1, 0x65, 0xd1, 0xe2, 0x57, 0xb3, 0x4f, 0xdc, - 0x68, 0x61, 0x65, 0x83, 0xbb, 0xde, 0x27, 0x8c, 0x9e, 0x92, 0x49, 0x78, 0x26, 0x54, 0x13, 0x3d, - 0xe1, 0x2c, 0x9a, 0x17, 0xf7, 0xd4, 0x5a, 0xcf, 0x08, 0xee, 0xf1, 0x5d, 0xfe, 0x60, 0x05, 0x97, - 0xa4, 0x23, 0xd2, 0x2e, 0x3d, 0x12, 0x37, 0xe5, 0xb4, 0xeb, 0xb5, 0x8a, 0xd8, 0x8f, 0x60, 0x65, - 0x66, 0x9e, 0x2f, 0x95, 0x29, 0x8d, 0xd6, 0xc1, 0x0f, 0xb5, 0xb4, 0xfa, 0xf7, 0xa9, 0x96, 0xd5, - 0xff, 0x2b, 0x01, 0xd0, 0xf2, 0xfc, 0x70, 0xd3, 0x16, 0x3f, 0x0b, 0xe5, 0xc4, 0x23, 0x93, 0xe9, - 0x39, 0x2a, 0x3c, 0x17, 0xf2, 0xf4, 0xa9, 0x15, 0x4e, 0x7b, 0x05, 0x1c, 0x47, 0x8a, 0x68, 0x13, - 0x0a, 0x72, 0xff, 0x7b, 0x23, 0xcf, 0x97, 0xf9, 0x68, 0x05, 0x83, 0x14, 0x71, 0x4d, 0x74, 0x1f, - 0x56, 0x47, 0xe3, 0x43, 0xc7, 0x0e, 0x06, 0xd4, 0x92, 0x98, 0xb4, 0xc0, 0xac, 0x44, 0x52, 0x0e, - 0xd3, 0x6b, 0x90, 0x0b, 0xad, 0xa3, 0x75, 0x48, 0x75, 0xaa, 0x2d, 0x6d, 0xa9, 0xb4, 0x76, 0x76, - 0x5e, 0x2e, 0x84, 0xe2, 0x4e, 0xb5, 0xc5, 0x7b, 0xba, 0xb5, 0x96, 0x96, 0x98, 0xed, 0xe9, 0xd6, - 0x5a, 0xa5, 0x34, 0xbf, 0x25, 0xf5, 0xbf, 0x4e, 0x40, 0x56, 0x72, 0xb6, 0x85, 0x33, 0x36, 0x60, - 0x39, 0xac, 0x24, 0x24, 0x91, 0x7c, 0xf7, 0x6a, 0xd2, 0x57, 0x51, 0x1c, 0x4d, 0xee, 0x63, 0xa8, - 0x57, 0xfa, 0x0c, 0x8a, 0xf1, 0x8e, 0xd7, 0xda, 0xc5, 0x3f, 0x86, 0x02, 0x0f, 0x94, 0x90, 0xfc, - 0x6d, 0x41, 0x56, 0xf2, 0x4a, 0x95, 0x55, 0xae, 0x63, 0xa0, 0x0a, 0x89, 0x1e, 0xc2, 0xb2, 0x64, - 0xad, 0xe1, 0x7b, 0xca, 0xc6, 0xf5, 0xe1, 0x88, 0x43, 0xb8, 0xfe, 0x39, 0xa4, 0x5b, 0x94, 0xfa, - 0xe8, 0x1e, 0x2c, 0xbb, 0x9e, 0x45, 0xa7, 0x49, 0x54, 0x11, 0x6e, 0x8b, 0x36, 0x6a, 0x9c, 0x70, - 0x5b, 0xb4, 0x61, 0xf1, 0xc5, 0xe3, 0x07, 0x34, 0x7c, 0x52, 0xe2, 0xff, 0xf5, 0x0e, 0x14, 0x9f, - 0x51, 0xbb, 0x3f, 0x60, 0xd4, 0x12, 0x86, 0x3e, 0x80, 0xf4, 0x88, 0x46, 0xce, 0xaf, 0x2f, 0x0c, - 0x1d, 0x4a, 0x7d, 0x2c, 0x50, 0xfc, 0x40, 0x9e, 0x0a, 0x6d, 0xf5, 0x8a, 0xa7, 0x5a, 0xfa, 0x3f, - 0x24, 0x61, 0xb5, 0x11, 0x04, 0x63, 0xe2, 0x9a, 0xe1, 0x2d, 0xfb, 0x93, 0xd9, 0x5b, 0xf6, 0xc1, - 0xc2, 0x19, 0xce, 0xa8, 0xcc, 0x56, 0xf9, 0x2a, 0x49, 0x26, 0xa3, 0x24, 0xa9, 0x7f, 0x93, 0x08, - 0xcb, 0xfb, 0xfb, 0xb1, 0x73, 0x53, 0x5a, 0x3f, 0x3b, 0x2f, 0xdf, 0x8a, 0x5b, 0xa2, 0x5d, 0xf7, - 0xd8, 0xf5, 0x4e, 0x5d, 0xf4, 0x36, 0x2f, 0xf7, 0x9b, 0xf5, 0x67, 0x5a, 0xa2, 0x74, 0xfb, 0xec, - 0xbc, 0x8c, 0x66, 0x40, 0x98, 0xba, 0xf4, 0x94, 0x5b, 0x6a, 0xd5, 0x9b, 0x35, 0x7e, 0x1f, 0x26, - 0x17, 0x58, 0x6a, 0x51, 0xd7, 0xb2, 0xdd, 0x3e, 0xba, 0x07, 0xd9, 0x46, 0xbb, 0xdd, 0x15, 0x05, - 0xd8, 0x1b, 0x67, 0xe7, 0xe5, 0x9b, 0x33, 0x28, 0xde, 0xa0, 0x16, 0x07, 0x71, 0x82, 0xc8, 0x6f, - 0xca, 0x05, 0x20, 0xce, 0x5d, 0xa8, 0xa5, 0x22, 0xfc, 0xdf, 0x92, 0xa0, 0x19, 0xa6, 0x49, 0x47, - 0x8c, 0xf7, 0x2b, 0xd2, 0xdd, 0x81, 0xdc, 0x88, 0xff, 0xb3, 0x45, 0x11, 0xc1, 0xc3, 0xe2, 0xe1, - 0xc2, 0x27, 0xde, 0x39, 0xbd, 0x0a, 0xf6, 0x1c, 0x6a, 0x58, 0x43, 0x3b, 0x08, 0x78, 0x71, 0x29, - 0x64, 0x38, 0xb2, 0x54, 0xfa, 0x55, 0x02, 0x6e, 0x2e, 0x40, 0xa0, 0x8f, 0x20, 0xed, 0x7b, 0x4e, - 0xb8, 0x3d, 0x77, 0xaf, 0x7a, 0x80, 0xe1, 0xaa, 0x58, 0x20, 0xd1, 0x06, 0x00, 0x19, 0x33, 0x8f, - 0x88, 0xf1, 0xc5, 0xc6, 0xe4, 0x70, 0x4c, 0x82, 0x9e, 0x41, 0x36, 0xa0, 0xa6, 0x4f, 0x43, 0x3e, - 0xf3, 0xf9, 0xff, 0xd7, 0xfb, 0x4a, 0x5b, 0x98, 0xc1, 0xca, 0x5c, 0xa9, 0x02, 0x59, 0x29, 0xe1, - 0x11, 0x6d, 0x11, 0x46, 0x84, 0xd3, 0x45, 0x2c, 0xfe, 0xf3, 0x40, 0x21, 0x4e, 0x3f, 0x0c, 0x14, - 0xe2, 0xf4, 0xf5, 0x9f, 0x25, 0x01, 0xea, 0xcf, 0x19, 0xf5, 0x5d, 0xe2, 0x54, 0x0d, 0x54, 0x8f, - 0x65, 0x48, 0x39, 0xdb, 0xf7, 0x16, 0x3e, 0xcb, 0x45, 0x1a, 0x95, 0xaa, 0xb1, 0x20, 0x47, 0xde, - 0x81, 0xd4, 0xd8, 0x77, 0xd4, 0x13, 0xaf, 0x20, 0x22, 0x5d, 0xbc, 0x8b, 0xb9, 0x0c, 0xd5, 0xa7, - 0x19, 0x29, 0x75, 0xf5, 0xdb, 0x7c, 0x6c, 0x80, 0xdf, 0x7e, 0x56, 0xfa, 0x00, 0x60, 0xea, 0x35, - 0xda, 0x80, 0x4c, 0x75, 0xa7, 0xdd, 0xde, 0xd5, 0x96, 0x64, 0x8d, 0x38, 0xed, 0x12, 0x62, 0xfd, - 0xef, 0x13, 0x90, 0xab, 0x1a, 0xea, 0x56, 0xd9, 0x01, 0x4d, 0xe4, 0x12, 0x93, 0xfa, 0xac, 0x47, - 0x9f, 0x8f, 0x6c, 0x7f, 0xa2, 0xd2, 0xc1, 0xf5, 0x2c, 0x7e, 0x95, 0x6b, 0x55, 0xa9, 0xcf, 0xea, - 0x42, 0x07, 0x61, 0x28, 0x52, 0x35, 0xc5, 0x9e, 0x49, 0xc2, 0xe4, 0xbc, 0x71, 0xfd, 0x52, 0x48, - 0xf6, 0x37, 0x6d, 0x07, 0xb8, 0x10, 0x1a, 0xa9, 0x92, 0x40, 0x3f, 0x80, 0x9b, 0xfb, 0xbe, 0x39, - 0xa0, 0x01, 0x93, 0x83, 0x2a, 0x97, 0x3f, 0x87, 0xbb, 0x8c, 0x04, 0xc7, 0xbd, 0x81, 0x1d, 0x30, - 0xcf, 0x9f, 0xf4, 0x7c, 0xca, 0xa8, 0xcb, 0xfb, 0x7b, 0xe2, 0x0b, 0x80, 0xaa, 0xc1, 0xef, 0x70, - 0xcc, 0x13, 0x09, 0xc1, 0x21, 0x62, 0x97, 0x03, 0xf4, 0x06, 0x14, 0x39, 0x61, 0xab, 0xd1, 0x23, - 0x32, 0x76, 0x58, 0x80, 0x7e, 0x04, 0xe0, 0x78, 0xfd, 0xde, 0x2b, 0x67, 0xf2, 0xbc, 0xe3, 0xf5, - 0xe5, 0x5f, 0xfd, 0xf7, 0x41, 0xab, 0xd9, 0xc1, 0x88, 0x30, 0x73, 0x10, 0x3e, 0x2e, 0xa0, 0xc7, - 0xa0, 0x0d, 0x28, 0xf1, 0xd9, 0x21, 0x25, 0xac, 0x37, 0xa2, 0xbe, 0xed, 0x59, 0xaf, 0xb4, 0xa4, - 0x6b, 0x91, 0x56, 0x4b, 0x28, 0xe9, 0xbf, 0x4e, 0x00, 0x60, 0x72, 0x14, 0x12, 0x80, 0x1f, 0xc0, - 0x8d, 0xc0, 0x25, 0xa3, 0x60, 0xe0, 0xb1, 0x9e, 0xed, 0x32, 0xea, 0x9f, 0x10, 0x47, 0x15, 0x88, - 0x5a, 0xd8, 0xd1, 0x50, 0x72, 0xf4, 0x01, 0xa0, 0x63, 0x4a, 0x47, 0x3d, 0xcf, 0xb1, 0x7a, 0x61, - 0xa7, 0xfc, 0x44, 0x91, 0xc6, 0x1a, 0xef, 0xd9, 0x77, 0xac, 0x76, 0x28, 0x47, 0xdb, 0xb0, 0xc1, - 0x57, 0x80, 0xba, 0xcc, 0xb7, 0x69, 0xd0, 0x3b, 0xf2, 0xfc, 0x5e, 0xe0, 0x78, 0xa7, 0xbd, 0x23, - 0xcf, 0x71, 0xbc, 0x53, 0xea, 0x87, 0xe5, 0x77, 0xc9, 0xf1, 0xfa, 0x75, 0x09, 0xda, 0xf1, 0xfc, - 0xb6, 0xe3, 0x9d, 0xee, 0x84, 0x08, 0xce, 0x12, 0xa6, 0xd3, 0x66, 0xb6, 0x79, 0x1c, 0xb2, 0x84, - 0x48, 0xda, 0xb1, 0xcd, 0x63, 0x74, 0x0f, 0x56, 0xa8, 0x43, 0x45, 0x11, 0x27, 0x51, 0x19, 0x81, - 0x2a, 0x86, 0x42, 0x0e, 0xd2, 0x7f, 0x07, 0xf2, 0x2d, 0x87, 0x98, 0xe2, 0x43, 0x10, 0x2f, 0x89, - 0x4d, 0xcf, 0xe5, 0x41, 0x60, 0xbb, 0x4c, 0x66, 0xc7, 0x3c, 0x8e, 0x8b, 0xf4, 0x9f, 0x00, 0xfc, - 0xd4, 0xb3, 0xdd, 0x8e, 0x77, 0x4c, 0x5d, 0xf1, 0x66, 0xce, 0x59, 0xaf, 0xda, 0xca, 0x3c, 0x56, - 0x2d, 0xc1, 0xc9, 0x89, 0x4b, 0xfa, 0xd4, 0x8f, 0x9e, 0x8e, 0x65, 0x93, 0x5f, 0x2e, 0x59, 0xec, - 0x79, 0xac, 0x6a, 0xa0, 0x32, 0x64, 0x4d, 0xd2, 0x0b, 0x4f, 0x5e, 0x71, 0x3b, 0x7f, 0x79, 0xb1, - 0x99, 0xa9, 0x1a, 0x4f, 0xe9, 0x04, 0x67, 0x4c, 0xf2, 0x94, 0x4e, 0xf8, 0xed, 0x6b, 0x12, 0x71, - 0x5e, 0x84, 0x99, 0xa2, 0xbc, 0x7d, 0xab, 0x06, 0x3f, 0x0c, 0x38, 0x6b, 0x12, 0xfe, 0x8b, 0x3e, - 0x82, 0xa2, 0x02, 0xf5, 0x06, 0x24, 0x18, 0x48, 0xae, 0xba, 0xbd, 0x7a, 0x79, 0xb1, 0x09, 0x12, - 0xf9, 0x84, 0x04, 0x03, 0x0c, 0x12, 0xcd, 0xff, 0xa3, 0x3a, 0x14, 0xbe, 0xf2, 0x6c, 0xb7, 0xc7, - 0xc4, 0x24, 0x54, 0x25, 0xbd, 0xf0, 0xfc, 0x4c, 0xa7, 0xaa, 0xca, 0x7b, 0xf8, 0x2a, 0x92, 0xe8, - 0xff, 0x9a, 0x80, 0x02, 0xb7, 0x69, 0x1f, 0xd9, 0x26, 0xbf, 0x2d, 0x5f, 0x3f, 0xd3, 0xdf, 0x81, - 0x94, 0x19, 0xf8, 0x6a, 0x6e, 0x22, 0xd5, 0x55, 0xdb, 0x18, 0x73, 0x19, 0xfa, 0x02, 0xb2, 0xb2, - 0xb8, 0x50, 0x49, 0x5e, 0xff, 0xee, 0x7b, 0x5d, 0xb9, 0xa8, 0xf4, 0xc4, 0x5e, 0x4e, 0xbd, 0x13, - 0xb3, 0x2c, 0xe2, 0xb8, 0x08, 0xdd, 0x86, 0xa4, 0xe9, 0x8a, 0xa0, 0x50, 0xdf, 0xd2, 0xaa, 0x4d, - 0x9c, 0x34, 0x5d, 0xfd, 0x9f, 0x13, 0xb0, 0x52, 0x77, 0x4d, 0x7f, 0x22, 0x92, 0x24, 0xdf, 0x88, - 0xbb, 0x90, 0x0f, 0xc6, 0x87, 0xc1, 0x24, 0x60, 0x74, 0x18, 0x3e, 0xd5, 0x47, 0x02, 0xd4, 0x80, - 0x3c, 0x71, 0xfa, 0x9e, 0x6f, 0xb3, 0xc1, 0x50, 0x71, 0xe3, 0xc5, 0x89, 0x39, 0x6e, 0xb3, 0x62, - 0x84, 0x2a, 0x78, 0xaa, 0x1d, 0xa6, 0xe2, 0x94, 0x70, 0x56, 0xa4, 0xe2, 0xb7, 0xa1, 0xe8, 0x90, - 0x21, 0xa7, 0xc2, 0x3d, 0x5e, 0x72, 0x89, 0x79, 0xa4, 0x71, 0x41, 0xc9, 0x78, 0x19, 0xa9, 0xeb, - 0x90, 0x8f, 0x8c, 0xa1, 0x35, 0x28, 0x18, 0xf5, 0x76, 0xef, 0xe3, 0xad, 0x87, 0xbd, 0xc7, 0xd5, - 0x3d, 0x6d, 0x49, 0x31, 0x81, 0x7f, 0x4c, 0xc0, 0xca, 0x9e, 0x8c, 0x41, 0x45, 0x9c, 0xee, 0xc1, - 0xb2, 0x4f, 0x8e, 0x58, 0x48, 0xed, 0xd2, 0x32, 0xb8, 0x78, 0x12, 0xe0, 0xd4, 0x8e, 0x77, 0x2d, - 0xa6, 0x76, 0xb1, 0x0f, 0x45, 0xa9, 0x6b, 0x3f, 0x14, 0xa5, 0x7f, 0x2b, 0x1f, 0x8a, 0xf4, 0x5f, - 0x26, 0x60, 0x4d, 0x5d, 0xd4, 0xe1, 0xc7, 0x11, 0xf4, 0x1e, 0xe4, 0xe5, 0x9d, 0x3d, 0x25, 0xa6, - 0xe2, 0x7b, 0x85, 0xc4, 0x35, 0x6a, 0x38, 0x27, 0xbb, 0x1b, 0x16, 0xfa, 0x71, 0xec, 0x55, 0xf4, - 0x0a, 0x7a, 0x38, 0x67, 0xbd, 0x32, 0x7d, 0x2a, 0xbd, 0xf2, 0x7b, 0xc9, 0x26, 0x14, 0x94, 0x03, - 0xa2, 0x6c, 0x90, 0x75, 0x20, 0x48, 0x51, 0x93, 0x0c, 0xa9, 0x7e, 0x1f, 0xd2, 0xdc, 0x0c, 0x02, - 0xc8, 0xb6, 0xbf, 0x6c, 0x77, 0xea, 0x7b, 0xb2, 0xf2, 0xda, 0x69, 0x88, 0x8f, 0x56, 0xcb, 0x90, - 0xaa, 0x37, 0x0f, 0xb4, 0xe4, 0xfb, 0xbf, 0x4e, 0x41, 0x3e, 0xaa, 0xe8, 0xf9, 0x79, 0xe0, 0x34, - 0x72, 0x49, 0xbe, 0xeb, 0x45, 0xf2, 0xa6, 0x20, 0x90, 0x79, 0x63, 0x77, 0x77, 0xbf, 0x6a, 0x74, - 0xea, 0x35, 0xed, 0x0b, 0xc9, 0x33, 0x23, 0x80, 0xe1, 0x38, 0x1e, 0x8f, 0x68, 0x0b, 0xe9, 0x53, - 0x9e, 0xf9, 0x42, 0xbd, 0x1e, 0x46, 0xa8, 0x90, 0x64, 0xbe, 0x03, 0x39, 0xa3, 0xdd, 0x6e, 0x3c, - 0x6e, 0xd6, 0x6b, 0xda, 0xd7, 0x89, 0xd2, 0xf7, 0xce, 0xce, 0xcb, 0x37, 0xa6, 0xa6, 0x82, 0xc0, - 0xee, 0xbb, 0xd4, 0x12, 0xa8, 0x6a, 0xb5, 0xde, 0xe2, 0xe3, 0xbd, 0x48, 0xce, 0xa3, 0x04, 0xbb, - 0x12, 0x5f, 0x02, 0xf2, 0x2d, 0x5c, 0x6f, 0x19, 0x98, 0x8f, 0xf8, 0x75, 0x72, 0xce, 0xaf, 0x96, - 0x4f, 0x47, 0xc4, 0xe7, 0x63, 0x6e, 0x84, 0x5f, 0xc4, 0x5e, 0xa4, 0xe4, 0x6b, 0xf1, 0xf4, 0x19, - 0x83, 0x12, 0x6b, 0xc2, 0x47, 0x13, 0xcf, 0x3f, 0xc2, 0x4c, 0x6a, 0x6e, 0xb4, 0x36, 0x23, 0x3e, - 0xe3, 0x56, 0x74, 0x58, 0xc6, 0xdd, 0x66, 0x53, 0xcc, 0x2e, 0x3d, 0x37, 0x3b, 0x3c, 0x76, 0x5d, - 0x8e, 0xb9, 0x0f, 0xb9, 0xf0, 0x75, 0x48, 0xfb, 0x3a, 0x3d, 0xe7, 0x50, 0x35, 0x7c, 0xda, 0x12, - 0x03, 0x3e, 0xe9, 0x76, 0xc4, 0x07, 0xbb, 0x17, 0x99, 0xf9, 0x01, 0x07, 0x63, 0x66, 0x71, 0x66, - 0x5f, 0x8e, 0xa8, 0xf6, 0xd7, 0x19, 0xc9, 0x70, 0x22, 0x8c, 0xe4, 0xd9, 0xdc, 0x0e, 0xae, 0xff, - 0x54, 0x7e, 0xdb, 0x7b, 0x91, 0x9d, 0xb3, 0x83, 0xe9, 0x57, 0xd4, 0x64, 0xd4, 0x9a, 0x3e, 0x86, - 0x47, 0x5d, 0xef, 0xff, 0x01, 0xe4, 0xc2, 0x6c, 0x88, 0x36, 0x20, 0xfb, 0x6c, 0x1f, 0x3f, 0xad, - 0x63, 0x6d, 0x49, 0xae, 0x4e, 0xd8, 0xf3, 0x4c, 0x5e, 0x27, 0x65, 0x58, 0xde, 0x33, 0x9a, 0xc6, - 0xe3, 0x3a, 0x0e, 0x1f, 0xe3, 0x43, 0x80, 0x3a, 0xd2, 0x25, 0x4d, 0x0d, 0x10, 0xd9, 0xdc, 0xbe, - 0xfb, 0xcd, 0xb7, 0x1b, 0x4b, 0xbf, 0xf8, 0x76, 0x63, 0xe9, 0x57, 0xdf, 0x6e, 0x24, 0x5e, 0x5c, - 0x6e, 0x24, 0xbe, 0xb9, 0xdc, 0x48, 0xfc, 0xfc, 0x72, 0x23, 0xf1, 0x1f, 0x97, 0x1b, 0x89, 0xc3, - 0xac, 0xa0, 0x9b, 0x9f, 0xfc, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x79, 0xf6, 0x7c, 0x35, 0x9d, + 0xb3, 0x1e, 0x7b, 0x1d, 0xda, 0x96, 0x77, 0x8d, 0x59, 0xcf, 0x66, 0xed, 0x16, 0x49, 0xcd, 0x70, + 0x47, 0xa2, 0x88, 0x22, 0x39, 0x03, 0x23, 0x41, 0x88, 0x52, 0x77, 0x89, 0x6a, 0xab, 0xd9, 0xc5, + 0x74, 0x17, 0xa5, 0x61, 0x82, 0x00, 0x93, 0x1c, 0x92, 0x40, 0xa7, 0xdc, 0x03, 0x61, 0x11, 0x24, + 0xc8, 0x2d, 0xe7, 0x00, 0x39, 0xf9, 0xe8, 0xe3, 0x06, 0x01, 0x82, 0xc5, 0x06, 0x19, 0xc4, 0xca, + 0x3f, 0xb0, 0x40, 0x10, 0xec, 0x21, 0x39, 0x04, 0xf5, 0xd1, 0xcd, 0x8f, 0xe1, 0xc8, 0x72, 0xd6, + 0x27, 0xb2, 0x5e, 0xfd, 0xde, 0xab, 0x57, 0x55, 0xaf, 0x5e, 0xfd, 0x5e, 0x35, 0x14, 0xf8, 0x78, + 0x48, 0xc3, 0xca, 0x30, 0x60, 0x9c, 0x21, 0xe4, 0x30, 0xfb, 0x98, 0x06, 0x95, 0xf0, 0x94, 0x04, + 0x83, 0x63, 0x97, 0x57, 0x4e, 0x3e, 0x2c, 0xdd, 0xe2, 0xee, 0x80, 0x86, 0x9c, 0x0c, 0x86, 0xef, + 0xc7, 0xff, 0x14, 0xbc, 0xf4, 0x9a, 0x33, 0x0a, 0x08, 0x77, 0x99, 0xff, 0x7e, 0xf4, 0x47, 0x77, + 0xdc, 0xe8, 0xb3, 0x3e, 0x93, 0x7f, 0xdf, 0x17, 0xff, 0x94, 0xd4, 0xdc, 0x84, 0xe5, 0x27, 0x34, + 0x08, 0x5d, 0xe6, 0xa3, 0x1b, 0x90, 0x71, 0x7d, 0x87, 0x3e, 0x5b, 0x4f, 0x94, 0x13, 0xf7, 0xd2, + 0x58, 0x35, 0xcc, 0xbf, 0x49, 0x40, 0xc1, 0xf2, 0x7d, 0xc6, 0xa5, 0xad, 0x10, 0x21, 0x48, 0xfb, + 0x64, 0x40, 0x25, 0x28, 0x8f, 0xe5, 0x7f, 0x54, 0x85, 0xac, 0x47, 0x0e, 0xa8, 0x17, 0xae, 0x27, + 0xcb, 0xa9, 0x7b, 0x85, 0xad, 0x1f, 0x54, 0x5e, 0xf6, 0xb9, 0x32, 0x65, 0xa4, 0xb2, 0x2b, 0xd1, + 0x75, 0x9f, 0x07, 0x63, 0xac, 0x55, 0x4b, 0x3f, 0x86, 0xc2, 0x94, 0x18, 0x19, 0x90, 0x3a, 0xa6, + 0x63, 0x3d, 0x8c, 0xf8, 0x2b, 0xfc, 0x3b, 0x21, 0xde, 0x88, 0xae, 0x27, 0xa5, 0x4c, 0x35, 0x3e, + 0x49, 0xde, 0x4f, 0x98, 0x9f, 0x43, 0x1e, 0xd3, 0x90, 0x8d, 0x02, 0x9b, 0x86, 0xe8, 0x1d, 0xc8, + 0xfb, 0xc4, 0x67, 0x3d, 0x7b, 0x38, 0x0a, 0xa5, 0x7a, 0x6a, 0xbb, 0x78, 0xf1, 0x62, 0x33, 0xd7, + 0x24, 0x3e, 0xab, 0xb6, 0xba, 0x21, 0xce, 0x89, 0xee, 0xea, 0x70, 0x14, 0xa2, 0x37, 0xa1, 0x38, + 0xa0, 0x03, 0x16, 0x8c, 0x7b, 0x07, 0x63, 0x4e, 0x43, 0x69, 0x38, 0x85, 0x0b, 0x4a, 0xb6, 0x2d, + 0x44, 0xe6, 0x5f, 0x25, 0xe0, 0x46, 0x64, 0x1b, 0xd3, 0x3f, 0x1c, 0xb9, 0x01, 0x1d, 0x50, 0x9f, + 0x87, 0xe8, 0x47, 0x90, 0xf5, 0xdc, 0x81, 0xcb, 0xd5, 0x18, 0x85, 0xad, 0x37, 0x16, 0xcd, 0x39, + 0xf6, 0x0a, 0x6b, 0x30, 0xb2, 0xa0, 0x18, 0xd0, 0x90, 0x06, 0x27, 0x6a, 0x25, 0xe4, 0x90, 0xdf, + 0xa8, 0x3c, 0xa3, 0x62, 0xee, 0x40, 0xae, 0xe5, 0x11, 0x7e, 0xc8, 0x82, 0x01, 0x32, 0xa1, 0x48, + 0x02, 0xfb, 0xc8, 0xe5, 0xd4, 0xe6, 0xa3, 0x20, 0xda, 0x95, 0x19, 0x19, 0xba, 0x09, 0x49, 0xa6, + 0x06, 0xca, 0x6f, 0x67, 0x2f, 0x5e, 0x6c, 0x26, 0xf7, 0xdb, 0x38, 0xc9, 0x42, 0xf3, 0x01, 0x5c, + 0x6b, 0x79, 0xa3, 0xbe, 0xeb, 0xd7, 0x68, 0x68, 0x07, 0xee, 0x50, 0x58, 0x17, 0xdb, 0x2b, 0x82, + 0x2f, 0xda, 0x5e, 0xf1, 0x3f, 0xde, 0xf2, 0xe4, 0x64, 0xcb, 0xcd, 0xbf, 0x48, 0xc2, 0xb5, 0xba, + 0xdf, 0x77, 0x7d, 0x3a, 0xad, 0x7d, 0x17, 0x56, 0xa9, 0x14, 0xf6, 0x4e, 0x54, 0x50, 0x69, 0x3b, + 0x2b, 0x4a, 0x1a, 0x45, 0x5a, 0x63, 0x2e, 0x5e, 0x3e, 0x5c, 0x34, 0xfd, 0x97, 0xac, 0x2f, 0x8a, + 0x1a, 0x54, 0x87, 0xe5, 0xa1, 0x9c, 0x44, 0xb8, 0x9e, 0x92, 0xb6, 0xee, 0x2e, 0xb2, 0xf5, 0xd2, + 0x3c, 0xb7, 0xd3, 0x5f, 0xbd, 0xd8, 0x5c, 0xc2, 0x91, 0xee, 0x6f, 0x13, 0x7c, 0xff, 0x99, 0x80, + 0xb5, 0x26, 0x73, 0x66, 0xd6, 0xa1, 0x04, 0xb9, 0x23, 0x16, 0xf2, 0xa9, 0x83, 0x12, 0xb7, 0xd1, + 0x7d, 0xc8, 0x0d, 0xf5, 0xf6, 0xe9, 0xdd, 0xbf, 0xbd, 0xd8, 0x65, 0x85, 0xc1, 0x31, 0x1a, 0x3d, + 0x80, 0x7c, 0x10, 0xc5, 0xc4, 0x7a, 0xea, 0x2a, 0x81, 0x33, 0xc1, 0xa3, 0xdf, 0x85, 0xac, 0xda, + 0x84, 0xf5, 0xb4, 0xd4, 0xbc, 0x7b, 0xa5, 0x35, 0xc7, 0x5a, 0xc9, 0xfc, 0x65, 0x02, 0x0c, 0x4c, + 0x0e, 0xf9, 0x1e, 0x1d, 0x1c, 0xd0, 0xa0, 0xcd, 0x09, 0x1f, 0x85, 0xe8, 0x26, 0x64, 0x3d, 0x4a, + 0x1c, 0x1a, 0xc8, 0x49, 0xe6, 0xb0, 0x6e, 0xa1, 0xae, 0x08, 0x72, 0x62, 0x1f, 0x91, 0x03, 0xd7, + 0x73, 0xf9, 0x58, 0x4e, 0x73, 0x75, 0xf1, 0x2e, 0xcf, 0xdb, 0xac, 0xe0, 0x29, 0x45, 0x3c, 0x63, + 0x06, 0xad, 0xc3, 0xf2, 0x80, 0x86, 0x21, 0xe9, 0x53, 0x39, 0xfb, 0x3c, 0x8e, 0x9a, 0xe6, 0x03, + 0x28, 0x4e, 0xeb, 0xa1, 0x02, 0x2c, 0x77, 0x9b, 0x8f, 0x9b, 0xfb, 0x4f, 0x9b, 0xc6, 0x12, 0x5a, + 0x83, 0x42, 0xb7, 0x89, 0xeb, 0x56, 0xf5, 0x91, 0xb5, 0xbd, 0x5b, 0x37, 0x12, 0x68, 0x05, 0xf2, + 0x93, 0x66, 0xd2, 0xfc, 0x79, 0x02, 0x40, 0x6c, 0xa0, 0x9e, 0xd4, 0x27, 0x90, 0x09, 0x39, 0xe1, + 0x6a, 0xe3, 0x56, 0xb7, 0xde, 0x5a, 0xe4, 0xf5, 0x04, 0x5e, 0x11, 0x3f, 0x14, 0x2b, 0x95, 0x69, + 0x0f, 0x93, 0xf3, 0x1e, 0x66, 0x24, 0x72, 0xd6, 0xb5, 0x1c, 0xa4, 0x6b, 0xe2, 0x5f, 0x02, 0xe5, + 0x21, 0x83, 0xeb, 0x56, 0xed, 0x73, 0x23, 0x89, 0x0c, 0x28, 0xd6, 0x1a, 0xed, 0xea, 0x7e, 0xb3, + 0x59, 0xaf, 0x76, 0xea, 0x35, 0x23, 0x65, 0xde, 0x85, 0x4c, 0x63, 0x40, 0xfa, 0x14, 0xdd, 0x16, + 0x11, 0x70, 0x48, 0x03, 0xea, 0xdb, 0x51, 0x60, 0x4d, 0x04, 0xe6, 0x2f, 0xf2, 0x90, 0xd9, 0x63, + 0x23, 0x9f, 0xa3, 0xad, 0xa9, 0x53, 0xbc, 0xba, 0xb5, 0xb1, 0x68, 0x0a, 0x12, 0x58, 0xe9, 0x8c, + 0x87, 0x54, 0x9f, 0xf2, 0x9b, 0x90, 0x55, 0xb1, 0xa2, 0x5d, 0xd7, 0x2d, 0x21, 0xe7, 0x24, 0xe8, + 0x53, 0xae, 0x17, 0x5d, 0xb7, 0xd0, 0x3d, 0xc8, 0x05, 0x94, 0x38, 0xcc, 0xf7, 0xc6, 0x32, 0xa4, + 0x72, 0x2a, 0xcd, 0x62, 0x4a, 0x9c, 0x7d, 0xdf, 0x1b, 0xe3, 0xb8, 0x17, 0x3d, 0x82, 0xe2, 0x81, + 0xeb, 0x3b, 0x3d, 0x36, 0x54, 0x39, 0x2f, 0xf3, 0xea, 0x00, 0x54, 0x5e, 0x6d, 0xbb, 0xbe, 0xb3, + 0xaf, 0xc0, 0xb8, 0x70, 0x30, 0x69, 0xa0, 0x26, 0xac, 0x9e, 0x30, 0x6f, 0x34, 0xa0, 0xb1, 0xad, + 0xac, 0xb4, 0xf5, 0xf6, 0xab, 0x6d, 0x3d, 0x91, 0xf8, 0xc8, 0xda, 0xca, 0xc9, 0x74, 0x13, 0x3d, + 0x86, 0x15, 0x3e, 0x18, 0x1e, 0x86, 0xb1, 0xb9, 0x65, 0x69, 0xee, 0xfb, 0x97, 0x2c, 0x98, 0x80, + 0x47, 0xd6, 0x8a, 0x7c, 0xaa, 0x55, 0xfa, 0xb3, 0x14, 0x14, 0xa6, 0x3c, 0x47, 0x6d, 0x28, 0x0c, + 0x03, 0x36, 0x24, 0x7d, 0x99, 0xb7, 0xf5, 0x5e, 0x7c, 0x78, 0xa5, 0x59, 0x57, 0x5a, 0x13, 0x45, + 0x3c, 0x6d, 0xc5, 0x3c, 0x4f, 0x42, 0x61, 0xaa, 0x13, 0xbd, 0x0b, 0x39, 0xdc, 0xc2, 0x8d, 0x27, + 0x56, 0xa7, 0x6e, 0x2c, 0x95, 0x6e, 0x9f, 0x9d, 0x97, 0xd7, 0xa5, 0xb5, 0x69, 0x03, 0xad, 0xc0, + 0x3d, 0x11, 0xa1, 0x77, 0x0f, 0x96, 0x23, 0x68, 0xa2, 0xf4, 0xfa, 0xd9, 0x79, 0xf9, 0xb5, 0x79, + 0xe8, 0x14, 0x12, 0xb7, 0x1f, 0x59, 0xb8, 0x5e, 0x33, 0x92, 0x8b, 0x91, 0xb8, 0x7d, 0x44, 0x02, + 0xea, 0xa0, 0xef, 0x43, 0x56, 0x03, 0x53, 0xa5, 0xd2, 0xd9, 0x79, 0xf9, 0xe6, 0x3c, 0x70, 0x82, + 0xc3, 0xed, 0x5d, 0xeb, 0x49, 0xdd, 0x48, 0x2f, 0xc6, 0xe1, 0xb6, 0x47, 0x4e, 0x28, 0x7a, 0x0b, + 0x32, 0x0a, 0x96, 0x29, 0xdd, 0x3a, 0x3b, 0x2f, 0x7f, 0xef, 0x25, 0x73, 0x02, 0x55, 0x5a, 0xff, + 0xcb, 0xbf, 0xdd, 0x58, 0xfa, 0xa7, 0xbf, 0xdb, 0x30, 0xe6, 0xbb, 0x4b, 0xff, 0x9b, 0x80, 0x95, + 0x99, 0x2d, 0x47, 0x26, 0x64, 0x7d, 0x66, 0xb3, 0xa1, 0x4a, 0xe7, 0xb9, 0x6d, 0xb8, 0x78, 0xb1, + 0x99, 0x6d, 0xb2, 0x2a, 0x1b, 0x8e, 0xb1, 0xee, 0x41, 0x8f, 0xe7, 0x2e, 0xa4, 0x8f, 0xae, 0x18, + 0x4f, 0x0b, 0xaf, 0xa4, 0x4f, 0x61, 0xc5, 0x09, 0xdc, 0x13, 0x1a, 0xf4, 0x6c, 0xe6, 0x1f, 0xba, + 0x7d, 0x9d, 0xaa, 0x4b, 0x8b, 0x6c, 0xd6, 0x24, 0x10, 0x17, 0x95, 0x42, 0x55, 0xe2, 0x7f, 0x8b, + 0xcb, 0xa8, 0xf4, 0x04, 0x8a, 0xd3, 0x11, 0x8a, 0xde, 0x00, 0x08, 0xdd, 0x3f, 0xa2, 0x9a, 0xdf, + 0x48, 0x36, 0x84, 0xf3, 0x42, 0x22, 0xd9, 0x0d, 0x7a, 0x1b, 0xd2, 0x03, 0xe6, 0x28, 0x3b, 0x99, + 0xed, 0xeb, 0xe2, 0x4e, 0xfc, 0xd5, 0x8b, 0xcd, 0x02, 0x0b, 0x2b, 0x3b, 0xae, 0x47, 0xf7, 0x98, + 0x43, 0xb1, 0x04, 0x98, 0x27, 0x90, 0x16, 0xa9, 0x02, 0xbd, 0x0e, 0xe9, 0xed, 0x46, 0xb3, 0x66, + 0x2c, 0x95, 0xae, 0x9d, 0x9d, 0x97, 0x57, 0xe4, 0x92, 0x88, 0x0e, 0x11, 0xbb, 0x68, 0x13, 0xb2, + 0x4f, 0xf6, 0x77, 0xbb, 0x7b, 0x22, 0xbc, 0xae, 0x9f, 0x9d, 0x97, 0xd7, 0xe2, 0x6e, 0xb5, 0x68, + 0xe8, 0x0d, 0xc8, 0x74, 0xf6, 0x5a, 0x3b, 0x6d, 0x23, 0x59, 0x42, 0x67, 0xe7, 0xe5, 0xd5, 0xb8, + 0x5f, 0xfa, 0x5c, 0xba, 0xa6, 0x77, 0x35, 0x1f, 0xcb, 0xcd, 0xff, 0x49, 0xc2, 0x0a, 0x16, 0xfc, + 0x36, 0xe0, 0x2d, 0xe6, 0xb9, 0xf6, 0x18, 0xb5, 0x20, 0x6f, 0x33, 0xdf, 0x71, 0xa7, 0xce, 0xd4, + 0xd6, 0x2b, 0x2e, 0xc1, 0x89, 0x56, 0xd4, 0xaa, 0x46, 0x9a, 0x78, 0x62, 0x04, 0x6d, 0x41, 0xc6, + 0xa1, 0x1e, 0x19, 0x5f, 0x76, 0x1b, 0xd7, 0x34, 0x97, 0xc6, 0x0a, 0x2a, 0x99, 0x23, 0x79, 0xd6, + 0x23, 0x9c, 0xd3, 0xc1, 0x90, 0xab, 0xdb, 0x38, 0x8d, 0x0b, 0x03, 0xf2, 0xcc, 0xd2, 0x22, 0xf4, + 0x43, 0xc8, 0x9e, 0xba, 0xbe, 0xc3, 0x4e, 0xf5, 0x85, 0x7b, 0xb9, 0x5d, 0x8d, 0x35, 0xcf, 0xc4, + 0x3d, 0x3b, 0xe7, 0xac, 0x58, 0xf5, 0xe6, 0x7e, 0xb3, 0x1e, 0xad, 0xba, 0xee, 0xdf, 0xf7, 0x9b, + 0xcc, 0x17, 0x27, 0x06, 0xf6, 0x9b, 0xbd, 0x1d, 0xab, 0xb1, 0xdb, 0xc5, 0x62, 0xe5, 0x6f, 0x9c, + 0x9d, 0x97, 0x8d, 0x18, 0xb2, 0x43, 0x5c, 0x4f, 0x90, 0xc0, 0x5b, 0x90, 0xb2, 0x9a, 0x9f, 0x1b, + 0xc9, 0x92, 0x71, 0x76, 0x5e, 0x2e, 0xc6, 0xdd, 0x96, 0x3f, 0x9e, 0x1c, 0xa6, 0xf9, 0x71, 0xcd, + 0x7f, 0x4f, 0x42, 0xb1, 0x3b, 0x74, 0x08, 0xa7, 0x2a, 0x32, 0x51, 0x19, 0x0a, 0x43, 0x12, 0x10, + 0xcf, 0xa3, 0x9e, 0x1b, 0x0e, 0x74, 0xa1, 0x30, 0x2d, 0x42, 0xf7, 0xbf, 0xc5, 0x62, 0x6a, 0x12, + 0xa6, 0x97, 0xb4, 0x0b, 0xab, 0x87, 0xca, 0xd9, 0x1e, 0xb1, 0xe5, 0xee, 0xa6, 0xe4, 0xee, 0x56, + 0x16, 0x99, 0x98, 0xf6, 0xaa, 0xa2, 0xe7, 0x68, 0x49, 0x2d, 0xbc, 0x72, 0x38, 0xdd, 0x44, 0x1f, + 0xc3, 0xf2, 0x80, 0xf9, 0x2e, 0x67, 0xc1, 0x95, 0xf6, 0x21, 0x02, 0xa3, 0x77, 0xe1, 0x9a, 0xd8, + 0xe1, 0xc8, 0x25, 0xd9, 0x2d, 0x6f, 0xae, 0x24, 0x5e, 0x1b, 0x90, 0x67, 0x7a, 0x4c, 0x2c, 0xc4, + 0xe6, 0xc7, 0xb0, 0x32, 0xe3, 0x83, 0xb8, 0xcd, 0x5b, 0x56, 0xb7, 0x5d, 0x37, 0x96, 0x50, 0x11, + 0x72, 0xd5, 0xfd, 0x66, 0xa7, 0xd1, 0xec, 0x0a, 0xea, 0x51, 0x84, 0x1c, 0xde, 0xdf, 0xdd, 0xdd, + 0xb6, 0xaa, 0x8f, 0x8d, 0xa4, 0xf9, 0xdf, 0xf1, 0xfa, 0x6a, 0xee, 0xb1, 0x3d, 0xcb, 0x3d, 0xde, + 0x7b, 0xf5, 0xd4, 0x35, 0xfb, 0x98, 0x34, 0x62, 0x0e, 0xf2, 0x13, 0x00, 0xb9, 0x8d, 0xd4, 0xe9, + 0x11, 0x7e, 0x59, 0x7d, 0xd1, 0x89, 0x2a, 0x47, 0x9c, 0xd7, 0x0a, 0x16, 0x47, 0x9f, 0x41, 0xd1, + 0x66, 0x83, 0xa1, 0x47, 0xb5, 0x7e, 0xea, 0x2a, 0xfa, 0x85, 0x58, 0xc5, 0xe2, 0xd3, 0x1c, 0x28, + 0x3d, 0xcb, 0x81, 0xfe, 0x3c, 0x01, 0x85, 0x29, 0x87, 0x67, 0xa9, 0x50, 0x11, 0x72, 0xdd, 0x56, + 0xcd, 0xea, 0x34, 0x9a, 0x0f, 0x8d, 0x04, 0x02, 0xc8, 0xca, 0x05, 0xac, 0x19, 0x49, 0x41, 0xd7, + 0xaa, 0xfb, 0x7b, 0xad, 0xdd, 0xba, 0x24, 0x43, 0xe8, 0x06, 0x18, 0xd1, 0x12, 0xf6, 0xda, 0x1d, + 0x0b, 0x0b, 0x69, 0x1a, 0x5d, 0x87, 0xb5, 0x58, 0xaa, 0x35, 0x33, 0xe8, 0x26, 0xa0, 0x58, 0x38, + 0x31, 0x91, 0x35, 0xff, 0x04, 0xd6, 0xaa, 0xcc, 0xe7, 0xc4, 0xf5, 0x63, 0x2a, 0xbb, 0x25, 0xe6, + 0xad, 0x45, 0x3d, 0xd7, 0x51, 0xd9, 0x76, 0x7b, 0xed, 0xe2, 0xc5, 0x66, 0x21, 0x86, 0x36, 0x6a, + 0x62, 0xa6, 0x51, 0xc3, 0x11, 0x67, 0x6a, 0xe8, 0x3a, 0x3a, 0x79, 0x2e, 0x5f, 0xbc, 0xd8, 0x4c, + 0xb5, 0x1a, 0x35, 0x2c, 0x64, 0xe8, 0x75, 0xc8, 0xd3, 0x67, 0x2e, 0xef, 0xd9, 0x22, 0xbb, 0x8a, + 0x35, 0xcc, 0xe0, 0x9c, 0x10, 0x54, 0x45, 0x32, 0xfd, 0xd3, 0x24, 0x40, 0x87, 0x84, 0xc7, 0x7a, + 0xe8, 0x07, 0x90, 0x8f, 0x8b, 0xf8, 0xcb, 0x8a, 0xc9, 0xa9, 0xfd, 0x8a, 0xf1, 0xe8, 0xa3, 0x28, + 0x62, 0x14, 0xc7, 0x5e, 0xac, 0xa8, 0xc7, 0x5a, 0x44, 0x53, 0x67, 0x89, 0xb4, 0xb8, 0x6b, 0x68, + 0x10, 0xe8, 0x8d, 0x13, 0x7f, 0x51, 0x55, 0xe6, 0x5b, 0x35, 0x67, 0xcd, 0xdc, 0xee, 0x2c, 0x1a, + 0x64, 0x6e, 0x41, 0x1f, 0x2d, 0xe1, 0x89, 0xde, 0xb6, 0x01, 0xab, 0xc1, 0xc8, 0x17, 0x5e, 0xf7, + 0x42, 0xd9, 0x6d, 0xba, 0xf0, 0x5a, 0x93, 0xf2, 0x53, 0x16, 0x1c, 0x5b, 0x9c, 0x13, 0xfb, 0x48, + 0x14, 0xd5, 0x3a, 0xc9, 0x4c, 0x08, 0x67, 0x62, 0x86, 0x70, 0xae, 0xc3, 0x32, 0xf1, 0x5c, 0x12, + 0x52, 0x75, 0x4b, 0xe7, 0x71, 0xd4, 0x14, 0xb4, 0x98, 0x38, 0x4e, 0x40, 0xc3, 0x90, 0xaa, 0x32, + 0x30, 0x8f, 0x27, 0x02, 0xf3, 0x5f, 0x92, 0x00, 0x8d, 0x96, 0xb5, 0xa7, 0xcd, 0xd7, 0x20, 0x7b, + 0x48, 0x06, 0xae, 0x37, 0xbe, 0xec, 0x90, 0x4d, 0xf0, 0x15, 0x4b, 0x19, 0xda, 0x91, 0x3a, 0x58, + 0xeb, 0x4a, 0xb6, 0x3c, 0x3a, 0xf0, 0x29, 0x8f, 0xd9, 0xb2, 0x6c, 0x89, 0xab, 0x39, 0x20, 0x7e, + 0xbc, 0xb0, 0xaa, 0x21, 0x5c, 0xef, 0x13, 0x4e, 0x4f, 0xc9, 0x38, 0x3a, 0x13, 0xba, 0x89, 0x1e, + 0x09, 0x16, 0x2d, 0x8a, 0x7b, 0xea, 0xac, 0x67, 0x24, 0xf7, 0xf8, 0x26, 0x7f, 0xb0, 0x86, 0x2b, + 0xd2, 0x11, 0x6b, 0x97, 0x1e, 0xc8, 0x9b, 0x72, 0xd2, 0xf5, 0xad, 0x8a, 0xd8, 0x0f, 0x60, 0x65, + 0x66, 0x9e, 0x2f, 0x95, 0x29, 0x8d, 0xd6, 0x93, 0x1f, 0x1a, 0x69, 0xfd, 0xef, 0x63, 0x23, 0x6b, + 0xfe, 0x57, 0x02, 0xa0, 0xc5, 0x82, 0x68, 0xd3, 0x16, 0x3f, 0x0b, 0xe5, 0xe4, 0x23, 0x93, 0xcd, + 0x3c, 0x1d, 0x9e, 0x0b, 0x79, 0xfa, 0xc4, 0x8a, 0xa0, 0xbd, 0x12, 0x8e, 0x63, 0x45, 0xb4, 0x09, + 0x05, 0xb5, 0xff, 0xbd, 0x21, 0x0b, 0x54, 0x3e, 0x5a, 0xc1, 0xa0, 0x44, 0x42, 0x13, 0xdd, 0x85, + 0xd5, 0xe1, 0xe8, 0xc0, 0x73, 0xc3, 0x23, 0xea, 0x28, 0x4c, 0x5a, 0x62, 0x56, 0x62, 0xa9, 0x80, + 0x99, 0x35, 0xc8, 0x45, 0xd6, 0xd1, 0x3a, 0xa4, 0x3a, 0xd5, 0x96, 0xb1, 0x54, 0x5a, 0x3b, 0x3b, + 0x2f, 0x17, 0x22, 0x71, 0xa7, 0xda, 0x12, 0x3d, 0xdd, 0x5a, 0xcb, 0x48, 0xcc, 0xf6, 0x74, 0x6b, + 0xad, 0x52, 0x5a, 0xdc, 0x92, 0xe6, 0x5f, 0x27, 0x20, 0xab, 0x38, 0xdb, 0xc2, 0x19, 0x5b, 0xb0, + 0x1c, 0x55, 0x12, 0x8a, 0x48, 0xbe, 0xfd, 0x6a, 0xd2, 0x57, 0xd1, 0x1c, 0x4d, 0xed, 0x63, 0xa4, + 0x57, 0xfa, 0x04, 0x8a, 0xd3, 0x1d, 0xdf, 0x6a, 0x17, 0xff, 0x18, 0x0a, 0x22, 0x50, 0x22, 0xf2, + 0xb7, 0x05, 0x59, 0xc5, 0x2b, 0x75, 0x56, 0xb9, 0x8c, 0x81, 0x6a, 0x24, 0xba, 0x0f, 0xcb, 0x8a, + 0xb5, 0x46, 0xef, 0x29, 0x1b, 0x97, 0x87, 0x23, 0x8e, 0xe0, 0xe6, 0xa7, 0x90, 0x6e, 0x51, 0x1a, + 0xa0, 0x3b, 0xb0, 0xec, 0x33, 0x87, 0x4e, 0x92, 0xa8, 0x26, 0xdc, 0x0e, 0x6d, 0xd4, 0x04, 0xe1, + 0x76, 0x68, 0xc3, 0x11, 0x8b, 0x27, 0x0e, 0x68, 0xf4, 0xa4, 0x24, 0xfe, 0x9b, 0x1d, 0x28, 0x3e, + 0xa5, 0x6e, 0xff, 0x88, 0x53, 0x47, 0x1a, 0x7a, 0x0f, 0xd2, 0x43, 0x1a, 0x3b, 0xbf, 0xbe, 0x30, + 0x74, 0x28, 0x0d, 0xb0, 0x44, 0x89, 0x03, 0x79, 0x2a, 0xb5, 0xf5, 0x2b, 0x9e, 0x6e, 0x99, 0xff, + 0x90, 0x84, 0xd5, 0x46, 0x18, 0x8e, 0x88, 0x6f, 0x47, 0xb7, 0xec, 0x4f, 0x67, 0x6f, 0xd9, 0x7b, + 0x0b, 0x67, 0x38, 0xa3, 0x32, 0x5b, 0xe5, 0xeb, 0x24, 0x99, 0x8c, 0x93, 0xa4, 0xf9, 0x55, 0x22, + 0x2a, 0xef, 0xef, 0x4e, 0x9d, 0x9b, 0xd2, 0xfa, 0xd9, 0x79, 0xf9, 0xc6, 0xb4, 0x25, 0xda, 0xf5, + 0x8f, 0x7d, 0x76, 0xea, 0xa3, 0x37, 0x45, 0xb9, 0xdf, 0xac, 0x3f, 0x35, 0x12, 0xa5, 0x9b, 0x67, + 0xe7, 0x65, 0x34, 0x03, 0xc2, 0xd4, 0xa7, 0xa7, 0xc2, 0x52, 0xab, 0xde, 0xac, 0x89, 0xfb, 0x30, + 0xb9, 0xc0, 0x52, 0x8b, 0xfa, 0x8e, 0xeb, 0xf7, 0xd1, 0x1d, 0xc8, 0x36, 0xda, 0xed, 0xae, 0x2c, + 0xc0, 0x5e, 0x3b, 0x3b, 0x2f, 0x5f, 0x9f, 0x41, 0x89, 0x06, 0x75, 0x04, 0x48, 0x10, 0x44, 0x71, + 0x53, 0x2e, 0x00, 0x09, 0xee, 0x42, 0x1d, 0x1d, 0xe1, 0xff, 0x96, 0x04, 0xc3, 0xb2, 0x6d, 0x3a, + 0xe4, 0xa2, 0x5f, 0x93, 0xee, 0x0e, 0xe4, 0x86, 0xe2, 0x9f, 0x2b, 0x8b, 0x08, 0x11, 0x16, 0xf7, + 0x17, 0x3e, 0xf1, 0xce, 0xe9, 0x55, 0x30, 0xf3, 0xa8, 0xe5, 0x0c, 0xdc, 0x30, 0x14, 0xc5, 0xa5, + 0x94, 0xe1, 0xd8, 0x52, 0xe9, 0xd7, 0x09, 0xb8, 0xbe, 0x00, 0x81, 0x3e, 0x80, 0x74, 0xc0, 0xbc, + 0x68, 0x7b, 0x6e, 0xbf, 0xea, 0x01, 0x46, 0xa8, 0x62, 0x89, 0x44, 0x1b, 0x00, 0x64, 0xc4, 0x19, + 0x91, 0xe3, 0xcb, 0x8d, 0xc9, 0xe1, 0x29, 0x09, 0x7a, 0x0a, 0xd9, 0x90, 0xda, 0x01, 0x8d, 0xf8, + 0xcc, 0xa7, 0xff, 0x5f, 0xef, 0x2b, 0x6d, 0x69, 0x06, 0x6b, 0x73, 0xa5, 0x0a, 0x64, 0x95, 0x44, + 0x44, 0xb4, 0x43, 0x38, 0x91, 0x4e, 0x17, 0xb1, 0xfc, 0x2f, 0x02, 0x85, 0x78, 0xfd, 0x28, 0x50, + 0x88, 0xd7, 0x37, 0x7f, 0x9e, 0x04, 0xa8, 0x3f, 0xe3, 0x34, 0xf0, 0x89, 0x57, 0xb5, 0x50, 0x7d, + 0x2a, 0x43, 0xaa, 0xd9, 0xbe, 0xb3, 0xf0, 0x59, 0x2e, 0xd6, 0xa8, 0x54, 0xad, 0x05, 0x39, 0xf2, + 0x16, 0xa4, 0x46, 0x81, 0xa7, 0x9f, 0x78, 0x25, 0x11, 0xe9, 0xe2, 0x5d, 0x2c, 0x64, 0xa8, 0x3e, + 0xc9, 0x48, 0xa9, 0x57, 0xbf, 0xcd, 0x4f, 0x0d, 0xf0, 0xdd, 0x67, 0xa5, 0xf7, 0x00, 0x26, 0x5e, + 0xa3, 0x0d, 0xc8, 0x54, 0x77, 0xda, 0xed, 0x5d, 0x63, 0x49, 0xd5, 0x88, 0x93, 0x2e, 0x29, 0x36, + 0xff, 0x3e, 0x01, 0xb9, 0xaa, 0xa5, 0x6f, 0x95, 0x1d, 0x30, 0x64, 0x2e, 0xb1, 0x69, 0xc0, 0x7b, + 0xf4, 0xd9, 0xd0, 0x0d, 0xc6, 0x3a, 0x1d, 0x5c, 0xce, 0xe2, 0x57, 0x85, 0x56, 0x95, 0x06, 0xbc, + 0x2e, 0x75, 0x10, 0x86, 0x22, 0xd5, 0x53, 0xec, 0xd9, 0x24, 0x4a, 0xce, 0x1b, 0x97, 0x2f, 0x85, + 0x62, 0x7f, 0x93, 0x76, 0x88, 0x0b, 0x91, 0x91, 0x2a, 0x09, 0xcd, 0x27, 0x70, 0x7d, 0x3f, 0xb0, + 0x8f, 0x68, 0xc8, 0xd5, 0xa0, 0xda, 0xe5, 0x4f, 0xe1, 0x36, 0x27, 0xe1, 0x71, 0xef, 0xc8, 0x0d, + 0x39, 0x0b, 0xc6, 0xbd, 0x80, 0x72, 0xea, 0x8b, 0xfe, 0x9e, 0xfc, 0x02, 0xa0, 0x6b, 0xf0, 0x5b, + 0x02, 0xf3, 0x48, 0x41, 0x70, 0x84, 0xd8, 0x15, 0x00, 0xb3, 0x01, 0x45, 0x41, 0xd8, 0x6a, 0xf4, + 0x90, 0x8c, 0x3c, 0x1e, 0xa2, 0x1f, 0x03, 0x78, 0xac, 0xdf, 0xbb, 0x72, 0x26, 0xcf, 0x7b, 0xac, + 0xaf, 0xfe, 0x9a, 0xbf, 0x07, 0x46, 0xcd, 0x0d, 0x87, 0x84, 0xdb, 0x47, 0xd1, 0xe3, 0x02, 0x7a, + 0x08, 0xc6, 0x11, 0x25, 0x01, 0x3f, 0xa0, 0x84, 0xf7, 0x86, 0x34, 0x70, 0x99, 0x73, 0xa5, 0x25, + 0x5d, 0x8b, 0xb5, 0x5a, 0x52, 0xc9, 0xfc, 0x4d, 0x02, 0x00, 0x93, 0xc3, 0x88, 0x00, 0xfc, 0x00, + 0xae, 0x85, 0x3e, 0x19, 0x86, 0x47, 0x8c, 0xf7, 0x5c, 0x9f, 0xd3, 0xe0, 0x84, 0x78, 0xba, 0x40, + 0x34, 0xa2, 0x8e, 0x86, 0x96, 0xa3, 0xf7, 0x00, 0x1d, 0x53, 0x3a, 0xec, 0x31, 0xcf, 0xe9, 0x45, + 0x9d, 0xea, 0x13, 0x45, 0x1a, 0x1b, 0xa2, 0x67, 0xdf, 0x73, 0xda, 0x91, 0x1c, 0x6d, 0xc3, 0x86, + 0x58, 0x01, 0xea, 0xf3, 0xc0, 0xa5, 0x61, 0xef, 0x90, 0x05, 0xbd, 0xd0, 0x63, 0xa7, 0xbd, 0x43, + 0xe6, 0x79, 0xec, 0x94, 0x06, 0x51, 0xf9, 0x5d, 0xf2, 0x58, 0xbf, 0xae, 0x40, 0x3b, 0x2c, 0x68, + 0x7b, 0xec, 0x74, 0x27, 0x42, 0x08, 0x96, 0x30, 0x99, 0x36, 0x77, 0xed, 0xe3, 0x88, 0x25, 0xc4, + 0xd2, 0x8e, 0x6b, 0x1f, 0xa3, 0x3b, 0xb0, 0x42, 0x3d, 0x2a, 0x8b, 0x38, 0x85, 0xca, 0x48, 0x54, + 0x31, 0x12, 0x0a, 0x90, 0xf9, 0x3b, 0x90, 0x6f, 0x79, 0xc4, 0x96, 0x1f, 0x82, 0x44, 0x49, 0x6c, + 0x33, 0x5f, 0x04, 0x81, 0xeb, 0x73, 0x95, 0x1d, 0xf3, 0x78, 0x5a, 0x64, 0xfe, 0x14, 0xe0, 0x67, + 0xcc, 0xf5, 0x3b, 0xec, 0x98, 0xfa, 0xf2, 0xcd, 0x5c, 0xb0, 0x5e, 0xbd, 0x95, 0x79, 0xac, 0x5b, + 0x92, 0x93, 0x13, 0x9f, 0xf4, 0x69, 0x10, 0x3f, 0x1d, 0xab, 0xa6, 0xb8, 0x5c, 0xb2, 0x98, 0x31, + 0x5e, 0xb5, 0x50, 0x19, 0xb2, 0x36, 0xe9, 0x45, 0x27, 0xaf, 0xb8, 0x9d, 0xbf, 0x78, 0xb1, 0x99, + 0xa9, 0x5a, 0x8f, 0xe9, 0x18, 0x67, 0x6c, 0xf2, 0x98, 0x8e, 0xc5, 0xed, 0x6b, 0x13, 0x79, 0x5e, + 0xa4, 0x99, 0xa2, 0xba, 0x7d, 0xab, 0x96, 0x38, 0x0c, 0x38, 0x6b, 0x13, 0xf1, 0x8b, 0x3e, 0x80, + 0xa2, 0x06, 0xf5, 0x8e, 0x48, 0x78, 0xa4, 0xb8, 0xea, 0xf6, 0xea, 0xc5, 0x8b, 0x4d, 0x50, 0xc8, + 0x47, 0x24, 0x3c, 0xc2, 0xa0, 0xd0, 0xe2, 0x3f, 0xaa, 0x43, 0xe1, 0x0b, 0xe6, 0xfa, 0x3d, 0x2e, + 0x27, 0xa1, 0x2b, 0xe9, 0x85, 0xe7, 0x67, 0x32, 0x55, 0x5d, 0xde, 0xc3, 0x17, 0xb1, 0xc4, 0xfc, + 0xd7, 0x04, 0x14, 0x84, 0x4d, 0xf7, 0xd0, 0xb5, 0xc5, 0x6d, 0xf9, 0xed, 0x33, 0xfd, 0x2d, 0x48, + 0xd9, 0x61, 0xa0, 0xe7, 0x26, 0x53, 0x5d, 0xb5, 0x8d, 0xb1, 0x90, 0xa1, 0xcf, 0x20, 0xab, 0x8a, + 0x0b, 0x9d, 0xe4, 0xcd, 0x6f, 0xbe, 0xd7, 0xb5, 0x8b, 0x5a, 0x4f, 0xee, 0xe5, 0xc4, 0x3b, 0x39, + 0xcb, 0x22, 0x9e, 0x16, 0xa1, 0x9b, 0x90, 0xb4, 0x7d, 0x19, 0x14, 0xfa, 0x5b, 0x5a, 0xb5, 0x89, + 0x93, 0xb6, 0x6f, 0xfe, 0x73, 0x02, 0x56, 0xea, 0xbe, 0x1d, 0x8c, 0x65, 0x92, 0x14, 0x1b, 0x71, + 0x1b, 0xf2, 0xe1, 0xe8, 0x20, 0x1c, 0x87, 0x9c, 0x0e, 0xa2, 0xa7, 0xfa, 0x58, 0x80, 0x1a, 0x90, + 0x27, 0x5e, 0x9f, 0x05, 0x2e, 0x3f, 0x1a, 0x68, 0x6e, 0xbc, 0x38, 0x31, 0x4f, 0xdb, 0xac, 0x58, + 0x91, 0x0a, 0x9e, 0x68, 0x47, 0xa9, 0x38, 0x25, 0x9d, 0x95, 0xa9, 0xf8, 0x4d, 0x28, 0x7a, 0x64, + 0x20, 0xa8, 0x70, 0x4f, 0x94, 0x5c, 0x72, 0x1e, 0x69, 0x5c, 0xd0, 0x32, 0x51, 0x46, 0x9a, 0x26, + 0xe4, 0x63, 0x63, 0x68, 0x0d, 0x0a, 0x56, 0xbd, 0xdd, 0xfb, 0x70, 0xeb, 0x7e, 0xef, 0x61, 0x75, + 0xcf, 0x58, 0xd2, 0x4c, 0xe0, 0x1f, 0x13, 0xb0, 0xb2, 0xa7, 0x62, 0x50, 0x13, 0xa7, 0x3b, 0xb0, + 0x1c, 0x90, 0x43, 0x1e, 0x51, 0xbb, 0xb4, 0x0a, 0x2e, 0x91, 0x04, 0x04, 0xb5, 0x13, 0x5d, 0x8b, + 0xa9, 0xdd, 0xd4, 0x87, 0xa2, 0xd4, 0xa5, 0x1f, 0x8a, 0xd2, 0xdf, 0xc9, 0x87, 0x22, 0xf3, 0x57, + 0x09, 0x58, 0xd3, 0x17, 0x75, 0xf4, 0x71, 0x04, 0xbd, 0x03, 0x79, 0x75, 0x67, 0x4f, 0x88, 0xa9, + 0xfc, 0x5e, 0xa1, 0x70, 0x8d, 0x1a, 0xce, 0xa9, 0xee, 0x86, 0x83, 0x7e, 0x32, 0xf5, 0x2a, 0xfa, + 0x0a, 0x7a, 0x38, 0x67, 0xbd, 0x32, 0x79, 0x2a, 0x7d, 0xe5, 0xf7, 0x92, 0x4d, 0x28, 0x68, 0x07, + 0x64, 0xd9, 0xa0, 0xea, 0x40, 0x50, 0xa2, 0x26, 0x19, 0x50, 0xf3, 0x2e, 0xa4, 0x85, 0x19, 0x04, + 0x90, 0x6d, 0x7f, 0xde, 0xee, 0xd4, 0xf7, 0x54, 0xe5, 0xb5, 0xd3, 0x90, 0x1f, 0xad, 0x96, 0x21, + 0x55, 0x6f, 0x3e, 0x31, 0x92, 0xe6, 0xef, 0x43, 0x01, 0xd3, 0x01, 0x3b, 0xa1, 0x4e, 0x53, 0x0d, + 0x97, 0x8c, 0x27, 0x24, 0x23, 0xb2, 0x51, 0xc3, 0x49, 0xd7, 0x41, 0x3f, 0x82, 0xac, 0xbe, 0x30, + 0xaf, 0xf4, 0x04, 0xa4, 0xc1, 0xef, 0xfe, 0x26, 0x05, 0xf9, 0xf8, 0xbd, 0x40, 0x9c, 0x36, 0x41, + 0x52, 0x97, 0xd4, 0xab, 0x61, 0x2c, 0x6f, 0x4a, 0x7a, 0x9a, 0xb7, 0x76, 0x77, 0xf7, 0xab, 0x56, + 0xa7, 0x5e, 0x33, 0x3e, 0x53, 0x2c, 0x36, 0x06, 0x58, 0x9e, 0xc7, 0xc4, 0x79, 0x71, 0x90, 0x39, + 0x61, 0xb1, 0xcf, 0xf5, 0xdb, 0x64, 0x8c, 0x8a, 0x28, 0xec, 0x5b, 0x90, 0xb3, 0xda, 0xed, 0xc6, + 0xc3, 0x66, 0xbd, 0x66, 0x7c, 0x99, 0x28, 0x7d, 0xef, 0xec, 0xbc, 0x7c, 0x6d, 0x62, 0x2a, 0x0c, + 0xdd, 0xbe, 0x4f, 0x1d, 0x89, 0xaa, 0x56, 0xeb, 0x2d, 0x31, 0xde, 0xf3, 0xe4, 0x3c, 0x4a, 0x72, + 0x37, 0xf9, 0x9d, 0x21, 0xdf, 0xc2, 0xf5, 0x96, 0x85, 0xc5, 0x88, 0x5f, 0x26, 0xe7, 0xfc, 0x6a, + 0x05, 0x74, 0x48, 0x02, 0x31, 0xe6, 0x46, 0xf4, 0xbd, 0xed, 0x79, 0x4a, 0xbd, 0x45, 0x4f, 0x1e, + 0x49, 0x28, 0x71, 0xc6, 0x62, 0x34, 0xf9, 0xb8, 0x24, 0xcd, 0xa4, 0xe6, 0x46, 0x6b, 0x73, 0x12, + 0x70, 0x61, 0xc5, 0x84, 0x65, 0xdc, 0x6d, 0x36, 0xe5, 0xec, 0xd2, 0x73, 0xb3, 0xc3, 0x23, 0xdf, + 0x17, 0x98, 0xbb, 0x90, 0x8b, 0xde, 0x9e, 0x8c, 0x2f, 0xd3, 0x73, 0x0e, 0x55, 0xa3, 0x87, 0x33, + 0x39, 0xe0, 0xa3, 0x6e, 0x47, 0x7e, 0x0e, 0x7c, 0x9e, 0x99, 0x1f, 0xf0, 0x68, 0xc4, 0x1d, 0x51, + 0x37, 0x94, 0x63, 0x22, 0xff, 0x65, 0x46, 0xf1, 0xa7, 0x18, 0xa3, 0x58, 0xbc, 0xb0, 0x83, 0xeb, + 0x3f, 0x53, 0x5f, 0x0e, 0x9f, 0x67, 0xe7, 0xec, 0x60, 0xfa, 0x05, 0xb5, 0x39, 0x75, 0x26, 0x4f, + 0xed, 0x71, 0xd7, 0xbb, 0x7f, 0x00, 0xb9, 0x28, 0xd7, 0xa2, 0x0d, 0xc8, 0x3e, 0xdd, 0xc7, 0x8f, + 0xeb, 0xd8, 0x58, 0x52, 0xab, 0x13, 0xf5, 0x3c, 0x55, 0x97, 0x55, 0x19, 0x96, 0xf7, 0xac, 0xa6, + 0xf5, 0xb0, 0x8e, 0xa3, 0xa7, 0xfe, 0x08, 0xa0, 0x13, 0x46, 0xc9, 0xd0, 0x03, 0xc4, 0x36, 0xb7, + 0x6f, 0x7f, 0xf5, 0xf5, 0xc6, 0xd2, 0x2f, 0xbf, 0xde, 0x58, 0xfa, 0xf5, 0xd7, 0x1b, 0x89, 0xe7, + 0x17, 0x1b, 0x89, 0xaf, 0x2e, 0x36, 0x12, 0xbf, 0xb8, 0xd8, 0x48, 0xfc, 0xc7, 0xc5, 0x46, 0xe2, + 0x20, 0x2b, 0xc9, 0xec, 0x47, 0xff, 0x17, 0x00, 0x00, 0xff, 0xff, 0x1f, 0x9b, 0xee, 0x67, 0xfb, 0x22, 0x00, 0x00, } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/types.proto b/components/engine/vendor/src/github.com/docker/swarmkit/api/types.proto index 61bf84e83b..a72b65fb45 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/types.proto +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/types.proto @@ -776,3 +776,13 @@ message SecretReference { // lookup/display purposes. The secret in the reference will be identified by its ID. string secret_name = 4; } + +// RemovedNode is a record for a node that has been removed from the swarm. +message RemovedNode { + // ID is the ID of the removed node. + string id = 1 [(gogoproto.customname) = "ID"]; + + // Expiry is the latest known expiration time of a certificate that + // was issued to this node. + Timestamp expiry = 2; +} diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/ca/auth.go b/components/engine/vendor/src/github.com/docker/swarmkit/ca/auth.go index 0ed9e65c79..724416c77c 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/ca/auth.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/ca/auth.go @@ -7,6 +7,7 @@ import ( "github.com/Sirupsen/logrus" + "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" "golang.org/x/net/context" "google.golang.org/grpc" @@ -79,7 +80,7 @@ func certSubjectFromContext(ctx context.Context) (pkix.Name, error) { // AuthorizeOrgAndRole takes in a context and a list of roles, and returns // the Node ID of the node. -func AuthorizeOrgAndRole(ctx context.Context, org string, ou ...string) (string, error) { +func AuthorizeOrgAndRole(ctx context.Context, org string, removedNodes []*api.RemovedNode, ou ...string) (string, error) { certSubj, err := certSubjectFromContext(ctx) if err != nil { return "", err @@ -87,18 +88,19 @@ func AuthorizeOrgAndRole(ctx context.Context, org string, ou ...string) (string, // Check if the current certificate has an OU that authorizes // access to this method if intersectArrays(certSubj.OrganizationalUnit, ou) { - return authorizeOrg(ctx, org) + return authorizeOrg(certSubj, org, removedNodes) } return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: remote certificate not part of OUs: %v", ou) } -// authorizeOrg takes in a context and an organization, and returns +// authorizeOrg takes in a certificate subject and an organization, and returns // the Node ID of the node. -func authorizeOrg(ctx context.Context, org string) (string, error) { - certSubj, err := certSubjectFromContext(ctx) - if err != nil { - return "", err +func authorizeOrg(certSubj pkix.Name, org string, removedNodes []*api.RemovedNode) (string, error) { + for _, removedNode := range removedNodes { + if removedNode.ID == certSubj.CommonName { + return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: node %s was removed from swarm", certSubj.CommonName) + } } if len(certSubj.Organization) > 0 && certSubj.Organization[0] == org { @@ -112,9 +114,9 @@ func authorizeOrg(ctx context.Context, org string) (string, error) { // been proxied by a manager, in which case the manager is authenticated and // so is the certificate information that it forwarded. It returns the node ID // of the original client. -func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarderRoles []string, org string) (string, error) { +func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarderRoles []string, org string, removedNodes []*api.RemovedNode) (string, error) { if isForwardedRequest(ctx) { - _, err := AuthorizeOrgAndRole(ctx, org, forwarderRoles...) + _, err := AuthorizeOrgAndRole(ctx, org, removedNodes, forwarderRoles...) if err != nil { return "", grpc.Errorf(codes.PermissionDenied, "Permission denied: unauthorized forwarder role: %v", err) } @@ -140,7 +142,7 @@ func AuthorizeForwardedRoleAndOrg(ctx context.Context, authorizedRoles, forwarde } // There wasn't any node being forwarded, check if this is a direct call by the expected role - nodeID, err := AuthorizeOrgAndRole(ctx, org, authorizedRoles...) + nodeID, err := AuthorizeOrgAndRole(ctx, org, removedNodes, authorizedRoles...) if err == nil { return nodeID, nil } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/ca/certificates.go b/components/engine/vendor/src/github.com/docker/swarmkit/ca/certificates.go index a29dbda245..d12b896095 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/ca/certificates.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/ca/certificates.go @@ -69,13 +69,23 @@ const ( MinNodeCertExpiration = 1 * time.Hour ) +// A recoverableErr is an non-fatal error encountered signing a certificate, +// which means that the certificate issuance may be retried at a later time. +type recoverableErr struct { + err error +} + +func (r recoverableErr) Error() string { + return r.err.Error() +} + // ErrNoLocalRootCA is an error type used to indicate that the local root CA // certificate file does not exist. var ErrNoLocalRootCA = errors.New("local root CA certificate does not exist") // ErrNoValidSigner is an error type used to indicate that our RootCA doesn't have the ability to // sign certificates. -var ErrNoValidSigner = errors.New("no valid signer found") +var ErrNoValidSigner = recoverableErr{err: errors.New("no valid signer found")} func init() { cflog.Level = 5 @@ -679,7 +689,7 @@ func readCertValidity(paths CertPaths) (time.Time, time.Time, error) { } // Create an x509 certificate out of the contents on disk - certBlock, _ := pem.Decode([]byte(cert)) + certBlock, _ := pem.Decode(cert) if certBlock == nil { return zeroTime, zeroTime, errors.New("failed to decode certificate block") } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/ca/external.go b/components/engine/vendor/src/github.com/docker/swarmkit/ca/external.go index 48dbf81725..e728961c03 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/ca/external.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/ca/external.go @@ -99,23 +99,23 @@ func (eca *ExternalCA) Sign(req signer.SignRequest) (cert []byte, err error) { func makeExternalSignRequest(client *http.Client, url string, csrJSON []byte) (cert []byte, err error) { resp, err := client.Post(url, "application/json", bytes.NewReader(csrJSON)) if err != nil { - return nil, errors.Wrap(err, "unable to perform certificate signing request") + return nil, recoverableErr{err: errors.Wrap(err, "unable to perform certificate signing request")} } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, errors.Wrap(err, "unable to read CSR response body") + return nil, recoverableErr{err: errors.Wrap(err, "unable to read CSR response body")} } if resp.StatusCode != http.StatusOK { - return nil, errors.Errorf("unexpected status code in CSR response: %d - %s", resp.StatusCode, string(body)) + return nil, recoverableErr{err: errors.Errorf("unexpected status code in CSR response: %d - %s", resp.StatusCode, string(body))} } var apiResponse api.Response if err := json.Unmarshal(body, &apiResponse); err != nil { log.Debugf("unable to JSON-parse CFSSL API response body: %s", string(body)) - return nil, errors.Wrap(err, "unable to parse JSON response") + return nil, recoverableErr{err: errors.Wrap(err, "unable to parse JSON response")} } if !apiResponse.Success || apiResponse.Result == nil { diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/ca/server.go b/components/engine/vendor/src/github.com/docker/swarmkit/ca/server.go index ebd3047ba2..c35c404086 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/ca/server.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/ca/server.go @@ -3,6 +3,7 @@ package ca import ( "crypto/subtle" "sync" + "time" "github.com/Sirupsen/logrus" "github.com/docker/swarmkit/api" @@ -17,18 +18,27 @@ import ( "google.golang.org/grpc/codes" ) +const ( + defaultReconciliationRetryInterval = 10 * time.Second +) + // Server is the CA and NodeCA API gRPC server. -// TODO(diogo): At some point we may want to have separate implementations of +// TODO(aaronl): At some point we may want to have separate implementations of // CA, NodeCA, and other hypothetical future CA services. At the moment, // breaking it apart doesn't seem worth it. type Server struct { - mu sync.Mutex - wg sync.WaitGroup - ctx context.Context - cancel func() - store *store.MemoryStore - securityConfig *SecurityConfig - joinTokens *api.JoinTokens + mu sync.Mutex + wg sync.WaitGroup + ctx context.Context + cancel func() + store *store.MemoryStore + securityConfig *SecurityConfig + joinTokens *api.JoinTokens + reconciliationRetryInterval time.Duration + + // pending is a map of nodes with pending certificates issuance or + // renewal. They are indexed by node ID. + pending map[string]*api.Node // Started is a channel which gets closed once the server is running // and able to service RPCs. @@ -45,12 +55,20 @@ func DefaultCAConfig() api.CAConfig { // NewServer creates a CA API server. func NewServer(store *store.MemoryStore, securityConfig *SecurityConfig) *Server { return &Server{ - store: store, - securityConfig: securityConfig, - started: make(chan struct{}), + store: store, + securityConfig: securityConfig, + pending: make(map[string]*api.Node), + started: make(chan struct{}), + reconciliationRetryInterval: defaultReconciliationRetryInterval, } } +// SetReconciliationRetryInterval changes the time interval between +// reconciliation attempts. This function must be called before Run. +func (s *Server) SetReconciliationRetryInterval(reconciliationRetryInterval time.Duration) { + s.reconciliationRetryInterval = reconciliationRetryInterval +} + // NodeCertificateStatus returns the current issuance status of an issuance request identified by the nodeID func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCertificateStatusRequest) (*api.NodeCertificateStatusResponse, error) { if request.NodeID == "" { @@ -149,16 +167,33 @@ func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNod } defer s.doneTask() + var ( + removedNodes []*api.RemovedNode + clusters []*api.Cluster + err error + ) + + s.store.View(func(readTx store.ReadTx) { + clusters, err = store.FindClusters(readTx, store.ByName("default")) + + }) + + // Not having a cluster object yet means we can't check + // the blacklist. + if err == nil && len(clusters) == 1 { + removedNodes = clusters[0].RemovedNodes + } + // If the remote node is a worker (either forwarded by a manager, or calling directly), // issue a renew worker certificate entry with the correct ID - nodeID, err := AuthorizeForwardedRoleAndOrg(ctx, []string{WorkerRole}, []string{ManagerRole}, s.securityConfig.ClientTLSCreds.Organization()) + nodeID, err := AuthorizeForwardedRoleAndOrg(ctx, []string{WorkerRole}, []string{ManagerRole}, s.securityConfig.ClientTLSCreds.Organization(), removedNodes) if err == nil { return s.issueRenewCertificate(ctx, nodeID, request.CSR) } // If the remote node is a manager (either forwarded by another manager, or calling directly), // issue a renew certificate entry with the correct ID - nodeID, err = AuthorizeForwardedRoleAndOrg(ctx, []string{ManagerRole}, []string{ManagerRole}, s.securityConfig.ClientTLSCreds.Organization()) + nodeID, err = AuthorizeForwardedRoleAndOrg(ctx, []string{ManagerRole}, []string{ManagerRole}, s.securityConfig.ClientTLSCreds.Organization(), removedNodes) if err == nil { return s.issueRenewCertificate(ctx, nodeID, request.CSR) } @@ -240,7 +275,6 @@ func (s *Server) issueRenewCertificate(ctx context.Context, nodeID string, csr [ node *api.Node ) err := s.store.Update(func(tx store.Tx) error { - // Attempt to retrieve the node with nodeID node = store.GetNode(tx, nodeID) if node == nil { @@ -356,6 +390,9 @@ func (s *Server) Run(ctx context.Context) error { }).WithError(err).Errorf("error attempting to reconcile certificates") } + ticker := time.NewTicker(s.reconciliationRetryInterval) + defer ticker.Stop() + // Watch for new nodes being created, new nodes being updated, and changes // to the cluster for { @@ -373,7 +410,16 @@ func (s *Server) Run(ctx context.Context) error { case state.EventUpdateCluster: s.updateCluster(ctx, v.Cluster) } - + case <-ticker.C: + for _, node := range s.pending { + if err := s.evaluateAndSignNodeCert(ctx, node); err != nil { + // If this sign operation did not succeed, the rest are + // unlikely to. Yield so that we don't hammer an external CA. + // Since the map iteration order is randomized, there is no + // risk of getting stuck on a problematic CSR. + break + } + } case <-ctx.Done(): return ctx.Err() case <-s.ctx.Done(): @@ -493,28 +539,29 @@ func (s *Server) updateCluster(ctx context.Context, cluster *api.Cluster) { } // evaluateAndSignNodeCert implements the logic of which certificates to sign -func (s *Server) evaluateAndSignNodeCert(ctx context.Context, node *api.Node) { +func (s *Server) evaluateAndSignNodeCert(ctx context.Context, node *api.Node) error { // If the desired membership and actual state are in sync, there's // nothing to do. if node.Spec.Membership == api.NodeMembershipAccepted && node.Certificate.Status.State == api.IssuanceStateIssued { - return + return nil } // If the certificate state is renew, then it is a server-sided accepted cert (cert renewals) if node.Certificate.Status.State == api.IssuanceStateRenew { - s.signNodeCert(ctx, node) - return + return s.signNodeCert(ctx, node) } // Sign this certificate if a user explicitly changed it to Accepted, and // the certificate is in pending state if node.Spec.Membership == api.NodeMembershipAccepted && node.Certificate.Status.State == api.IssuanceStatePending { - s.signNodeCert(ctx, node) + return s.signNodeCert(ctx, node) } + + return nil } // signNodeCert does the bulk of the work for signing a certificate -func (s *Server) signNodeCert(ctx context.Context, node *api.Node) { +func (s *Server) signNodeCert(ctx context.Context, node *api.Node) error { rootCA := s.securityConfig.RootCA() externalCA := s.securityConfig.externalCA @@ -527,9 +574,11 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) { "node.id": node.ID, "method": "(*Server).signNodeCert", }).WithError(err).Errorf("failed to parse role") - return + return errors.New("failed to parse role") } + s.pending[node.ID] = node + // Attempt to sign the CSR var ( rawCSR = node.Certificate.CSR @@ -550,16 +599,19 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) { "node.id": node.ID, "method": "(*Server).signNodeCert", }).WithError(err).Errorf("failed to sign CSR") - // If this error is due the lack of signer, maybe some other - // manager in the future will pick it up. Return without - // changing the state of the certificate. - if err == ErrNoValidSigner { - return - } + // If the current state is already Failed, no need to change it if node.Certificate.Status.State == api.IssuanceStateFailed { - return + delete(s.pending, node.ID) + return errors.New("failed to sign CSR") } + + if _, ok := err.(recoverableErr); ok { + // Return without changing the state of the certificate. We may + // retry signing it in the future. + return errors.New("failed to sign CSR") + } + // We failed to sign this CSR, change the state to FAILED err = s.store.Update(func(tx store.Tx) error { node := store.GetNode(tx, nodeID) @@ -580,7 +632,9 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) { "method": "(*Server).signNodeCert", }).WithError(err).Errorf("transaction failed when setting state to FAILED") } - return + + delete(s.pending, node.ID) + return errors.New("failed to sign CSR") } // We were able to successfully sign the new CSR. Let's try to update the nodeStore @@ -606,6 +660,7 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) { "node.role": node.Certificate.Role, "method": "(*Server).signNodeCert", }).Debugf("certificate issued") + delete(s.pending, node.ID) break } if err == store.ErrSequenceConflict { @@ -616,8 +671,9 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) { "node.id": nodeID, "method": "(*Server).signNodeCert", }).WithError(err).Errorf("transaction failed") - return + return errors.New("transaction failed") } + return nil } // reconcileNodeCertificates is a helper method that calles evaluateAndSignNodeCert on all the diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/ca/transport.go b/components/engine/vendor/src/github.com/docker/swarmkit/ca/transport.go index 8c634f9d73..834f2c1170 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/ca/transport.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/ca/transport.go @@ -44,6 +44,26 @@ func (c *MutableTLSCreds) Info() credentials.ProtocolInfo { } } +// Clone returns new MutableTLSCreds created from underlying *tls.Config. +// It panics if validation of underlying config fails. +func (c *MutableTLSCreds) Clone() credentials.TransportCredentials { + c.Lock() + newCfg, err := NewMutableTLS(c.config) + if err != nil { + panic("validation error on Clone") + } + c.Unlock() + return newCfg +} + +// OverrideServerName overrides *tls.Config.ServerName. +func (c *MutableTLSCreds) OverrideServerName(name string) error { + c.Lock() + c.config.ServerName = name + c.Unlock() + return nil +} + // GetRequestMetadata implements the credentials.TransportCredentials interface func (c *MutableTLSCreds) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { return nil, nil diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/identity/randomid.go b/components/engine/vendor/src/github.com/docker/swarmkit/identity/randomid.go index f59179b164..e9078bc9fe 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/identity/randomid.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/identity/randomid.go @@ -16,16 +16,20 @@ var ( // parameters for random identifier generation. We can tweak this when there is // time for further analysis. const ( - randomIDEntropyBytes = 16 + randomIDEntropyBytes = 17 randomIDBase = 36 // To ensure that all identifiers are fixed length, we make sure they - // get padded out to 25 characters, which is the maximum for the base36 - // representation of 128-bit identifiers. + // get padded out or truncated to 25 characters. // // For academics, f5lxx1zz5pnorynqglhzmsp33 == 2^128 - 1. This value // was calculated from floor(log(2^128-1, 36)) + 1. // + // While 128 bits is the largest whole-byte size that fits into 25 + // base-36 characters, we generate an extra byte of entropy to fill + // in the high bits, which would otherwise be 0. This gives us a more + // even distribution of the first character. + // // See http://mathworld.wolfram.com/NumberLength.html for more information. maxRandomIDLength = 25 ) @@ -34,7 +38,7 @@ const ( // collision probability are required. // // With the parameters in this package, the generated identifier will provide -// 128 bits of entropy encoded with base36. Leading padding is added if the +// ~129 bits of entropy encoded with base36. Leading padding is added if the // string is less 25 bytes. We do not intend to maintain this interface, so // identifiers should be treated opaquely. func NewID() string { @@ -44,7 +48,6 @@ func NewID() string { panic(fmt.Errorf("failed to read random bytes: %v", err)) } - var nn big.Int - nn.SetBytes(p[:]) - return fmt.Sprintf("%0[1]*s", maxRandomIDLength, nn.Text(randomIDBase)) + p[0] |= 0x80 // set high bit to avoid the need for padding + return (&big.Int{}).SetBytes(p[:]).Text(randomIDBase)[1 : maxRandomIDLength+1] } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/allocator.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/allocator.go index b1c0058065..08a0ca71c9 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/allocator.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/allocator.go @@ -125,6 +125,8 @@ func (a *Allocator) Run(ctx context.Context) error { aaCopy := aa actor := func() error { wg.Add(1) + defer wg.Done() + // init might return an allocator specific context // which is a child of the passed in context to hold // allocator specific state @@ -133,10 +135,10 @@ func (a *Allocator) Run(ctx context.Context) error { // if we are failing in the init of // this allocator. aa.cancel() - wg.Done() return err } + wg.Add(1) go func() { defer wg.Done() a.run(ctx, aaCopy) diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/network.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/network.go index 658af795d5..3e6cdefdc8 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/network.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/network.go @@ -68,7 +68,7 @@ type networkContext struct { unallocatedNetworks map[string]*api.Network } -func (a *Allocator) doNetworkInit(ctx context.Context) error { +func (a *Allocator) doNetworkInit(ctx context.Context) (err error) { na, err := networkallocator.New() if err != nil { return err @@ -81,6 +81,13 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error { unallocatedNetworks: make(map[string]*api.Network), ingressNetwork: newIngressNetwork(), } + a.netCtx = nc + defer func() { + // Clear a.netCtx if initialization was unsuccessful. + if err != nil { + a.netCtx = nil + } + }() // Check if we have the ingress network. If not found create // it before reading all network objects for allocation. @@ -125,7 +132,7 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error { // that the we can get the preferred subnet for ingress // network. if !na.IsAllocated(nc.ingressNetwork) { - if err := a.allocateNetwork(ctx, nc, nc.ingressNetwork); err != nil { + if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil { log.G(ctx).WithError(err).Error("failed allocating ingress network during init") } @@ -155,7 +162,7 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error { continue } - if err := a.allocateNetwork(ctx, nc, n); err != nil { + if err := a.allocateNetwork(ctx, n); err != nil { log.G(ctx).WithError(err).Errorf("failed allocating network %s during init", n.ID) } } @@ -179,7 +186,7 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error { } node.Attachment.Network = nc.ingressNetwork.Copy() - if err := a.allocateNode(ctx, nc, node); err != nil { + if err := a.allocateNode(ctx, node); err != nil { log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s during init", node.ID) } } @@ -198,7 +205,7 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error { continue } - if err := a.allocateService(ctx, nc, s); err != nil { + if err := a.allocateService(ctx, s); err != nil { log.G(ctx).WithError(err).Errorf("failed allocating service %s during init", s.ID) } } @@ -260,7 +267,7 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error { } err := batch.Update(func(tx store.Tx) error { - _, err := a.allocateTask(ctx, nc, tx, t) + _, err := a.allocateTask(ctx, tx, t) return err }) if err != nil { @@ -274,7 +281,6 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error { return err } - a.netCtx = nc return nil } @@ -288,7 +294,7 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) { break } - if err := a.allocateNetwork(ctx, nc, n); err != nil { + if err := a.allocateNetwork(ctx, n); err != nil { log.G(ctx).WithError(err).Errorf("Failed allocation for network %s", n.ID) break } @@ -309,7 +315,7 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) { break } - if err := a.allocateService(ctx, nc, s); err != nil { + if err := a.allocateService(ctx, s); err != nil { log.G(ctx).WithError(err).Errorf("Failed allocation for service %s", s.ID) break } @@ -320,7 +326,7 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) { break } - if err := a.allocateService(ctx, nc, s); err != nil { + if err := a.allocateService(ctx, s); err != nil { log.G(ctx).WithError(err).Errorf("Failed allocation during update of service %s", s.ID) break } @@ -335,18 +341,18 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) { // it's still there. delete(nc.unallocatedServices, s.ID) case state.EventCreateNode, state.EventUpdateNode, state.EventDeleteNode: - a.doNodeAlloc(ctx, nc, ev) + a.doNodeAlloc(ctx, ev) case state.EventCreateTask, state.EventUpdateTask, state.EventDeleteTask: - a.doTaskAlloc(ctx, nc, ev) + a.doTaskAlloc(ctx, ev) case state.EventCommit: - a.procUnallocatedNetworks(ctx, nc) - a.procUnallocatedServices(ctx, nc) - a.procUnallocatedTasksNetwork(ctx, nc) + a.procUnallocatedNetworks(ctx) + a.procUnallocatedServices(ctx) + a.procUnallocatedTasksNetwork(ctx) return } } -func (a *Allocator) doNodeAlloc(ctx context.Context, nc *networkContext, ev events.Event) { +func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) { var ( isDelete bool node *api.Node @@ -362,6 +368,8 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, nc *networkContext, ev even node = v.Node.Copy() } + nc := a.netCtx + if isDelete { if nc.nwkAllocator.IsNodeAllocated(node) { if err := nc.nwkAllocator.DeallocateNode(node); err != nil { @@ -377,7 +385,7 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, nc *networkContext, ev even } node.Attachment.Network = nc.ingressNetwork.Copy() - if err := a.allocateNode(ctx, nc, node); err != nil { + if err := a.allocateNode(ctx, node); err != nil { log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s", node.ID) } } @@ -445,26 +453,22 @@ func (a *Allocator) taskCreateNetworkAttachments(t *api.Task, s *api.Service) { for _, na := range specNetworks { n := store.GetNetwork(tx, na.Target) - if n != nil { - var aliases []string - var addresses []string - - for _, a := range na.Aliases { - aliases = append(aliases, a) - } - for _, a := range na.Addresses { - addresses = append(addresses, a) - } - - networks = append(networks, &api.NetworkAttachment{Network: n, Aliases: aliases, Addresses: addresses}) + if n == nil { + continue } + + attachment := api.NetworkAttachment{Network: n} + attachment.Aliases = append(attachment.Aliases, na.Aliases...) + attachment.Addresses = append(attachment.Addresses, na.Addresses...) + + networks = append(networks, &attachment) } }) taskUpdateNetworks(t, networks) } -func (a *Allocator) doTaskAlloc(ctx context.Context, nc *networkContext, ev events.Event) { +func (a *Allocator) doTaskAlloc(ctx context.Context, ev events.Event) { var ( isDelete bool t *api.Task @@ -480,6 +484,8 @@ func (a *Allocator) doTaskAlloc(ctx context.Context, nc *networkContext, ev even t = v.Task.Copy() } + nc := a.netCtx + // If the task has stopped running or it's being deleted then // we should free the network resources associated with the // task right away. @@ -530,7 +536,9 @@ func (a *Allocator) doTaskAlloc(ctx context.Context, nc *networkContext, ev even nc.unallocatedTasks[t.ID] = t } -func (a *Allocator) allocateNode(ctx context.Context, nc *networkContext, node *api.Node) error { +func (a *Allocator) allocateNode(ctx context.Context, node *api.Node) error { + nc := a.netCtx + if err := nc.nwkAllocator.AllocateNode(node); err != nil { return err } @@ -563,7 +571,9 @@ func (a *Allocator) allocateNode(ctx context.Context, nc *networkContext, node * return nil } -func (a *Allocator) allocateService(ctx context.Context, nc *networkContext, s *api.Service) error { +func (a *Allocator) allocateService(ctx context.Context, s *api.Service) error { + nc := a.netCtx + if s.Spec.Endpoint != nil { // service has user-defined endpoint if s.Endpoint == nil { @@ -648,7 +658,9 @@ func (a *Allocator) allocateService(ctx context.Context, nc *networkContext, s * return nil } -func (a *Allocator) allocateNetwork(ctx context.Context, nc *networkContext, n *api.Network) error { +func (a *Allocator) allocateNetwork(ctx context.Context, n *api.Network) error { + nc := a.netCtx + if err := nc.nwkAllocator.Allocate(n); err != nil { nc.unallocatedNetworks[n.ID] = n return errors.Wrapf(err, "failed during network allocation for network %s", n.ID) @@ -670,7 +682,7 @@ func (a *Allocator) allocateNetwork(ctx context.Context, nc *networkContext, n * return nil } -func (a *Allocator) allocateTask(ctx context.Context, nc *networkContext, tx store.Tx, t *api.Task) (*api.Task, error) { +func (a *Allocator) allocateTask(ctx context.Context, tx store.Tx, t *api.Task) (*api.Task, error) { taskUpdated := false // Get the latest task state from the store before updating. @@ -679,6 +691,8 @@ func (a *Allocator) allocateTask(ctx context.Context, nc *networkContext, tx sto return nil, fmt.Errorf("could not find task %s while trying to update network allocation", t.ID) } + nc := a.netCtx + // We might be here even if a task allocation has already // happened but wasn't successfully committed to store. In such // cases skip allocation and go straight ahead to updating the @@ -738,10 +752,11 @@ func (a *Allocator) allocateTask(ctx context.Context, nc *networkContext, tx sto return storeT, nil } -func (a *Allocator) procUnallocatedNetworks(ctx context.Context, nc *networkContext) { +func (a *Allocator) procUnallocatedNetworks(ctx context.Context) { + nc := a.netCtx for _, n := range nc.unallocatedNetworks { if !nc.nwkAllocator.IsAllocated(n) { - if err := a.allocateNetwork(ctx, nc, n); err != nil { + if err := a.allocateNetwork(ctx, n); err != nil { log.G(ctx).Debugf("Failed allocation of unallocated network %s: %v", n.ID, err) continue } @@ -751,10 +766,11 @@ func (a *Allocator) procUnallocatedNetworks(ctx context.Context, nc *networkCont } } -func (a *Allocator) procUnallocatedServices(ctx context.Context, nc *networkContext) { +func (a *Allocator) procUnallocatedServices(ctx context.Context) { + nc := a.netCtx for _, s := range nc.unallocatedServices { if !nc.nwkAllocator.IsServiceAllocated(s) { - if err := a.allocateService(ctx, nc, s); err != nil { + if err := a.allocateService(ctx, s); err != nil { log.G(ctx).Debugf("Failed allocation of unallocated service %s: %v", s.ID, err) continue } @@ -764,7 +780,8 @@ func (a *Allocator) procUnallocatedServices(ctx context.Context, nc *networkCont } } -func (a *Allocator) procUnallocatedTasksNetwork(ctx context.Context, nc *networkContext) { +func (a *Allocator) procUnallocatedTasksNetwork(ctx context.Context) { + nc := a.netCtx tasks := make([]*api.Task, 0, len(nc.unallocatedTasks)) committed, err := a.store.Batch(func(batch *store.Batch) error { @@ -772,7 +789,7 @@ func (a *Allocator) procUnallocatedTasksNetwork(ctx context.Context, nc *network var allocatedT *api.Task err := batch.Update(func(tx store.Tx) error { var err error - allocatedT, err = a.allocateTask(ctx, nc, tx, t) + allocatedT, err = a.allocateTask(ctx, tx, t) return err }) diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/drivers_darwin.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/drivers_darwin.go new file mode 100644 index 0000000000..b8ad76b092 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/drivers_darwin.go @@ -0,0 +1,13 @@ +package networkallocator + +import ( + "github.com/docker/libnetwork/drivers/overlay/ovmanager" + "github.com/docker/libnetwork/drivers/remote" +) + +func getInitializers() []initializer { + return []initializer{ + {remote.Init, "remote"}, + {ovmanager.Init, "overlay"}, + } +} diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/drivers_unsupported.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/drivers_unsupported.go index 4bca7583c5..e4bef5848f 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/drivers_unsupported.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/drivers_unsupported.go @@ -1,4 +1,4 @@ -// +build !linux +// +build !linux,!darwin package networkallocator diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go index c989070975..f741a21fe2 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go @@ -518,7 +518,7 @@ func (na *NetworkAllocator) allocateNetworkIPs(nAttach *api.NetworkAttachment) e } addresses := nAttach.Addresses - if addresses == nil { + if len(addresses) == 0 { addresses = []string{""} } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/cluster.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/cluster.go index be5f46fe75..d3654ca72a 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/cluster.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/cluster.go @@ -202,6 +202,7 @@ func redactClusters(clusters []*api.Cluster) []*api.Cluster { CACertHash: cluster.RootCA.CACertHash, JoinTokens: cluster.RootCA.JoinTokens, }, + RemovedNodes: cluster.RemovedNodes, } redactedClusters = append(redactedClusters, newCluster) diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/common.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/common.go index 4551878e6a..f180cbd6a6 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/common.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/common.go @@ -64,10 +64,15 @@ func filterMatchLabels(match map[string]string, candidates map[string]string) bo func validateAnnotations(m api.Annotations) error { if m.Name == "" { return grpc.Errorf(codes.InvalidArgument, "meta: name must be provided") - } else if !isValidName.MatchString(m.Name) { + } + if !isValidName.MatchString(m.Name) { // if the name doesn't match the regex return grpc.Errorf(codes.InvalidArgument, "name must be valid as a DNS name component") } + if len(m.Name) > 63 { + // DNS labels are limited to 63 characters + return grpc.Errorf(codes.InvalidArgument, "name must be 63 characters or fewer") + } return nil } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go index 2a7df6f44f..44e62b3222 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/node.go @@ -1,9 +1,13 @@ package controlapi import ( + "crypto/x509" + "encoding/pem" + "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/state/raft/membership" "github.com/docker/swarmkit/manager/state/store" + "github.com/docker/swarmkit/protobuf/ptypes" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -286,6 +290,40 @@ func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest) if !request.Force && node.Status.State == api.NodeStatus_READY { return grpc.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID) } + + // lookup the cluster + clusters, err := store.FindClusters(tx, store.ByName("default")) + if err != nil { + return err + } + if len(clusters) != 1 { + return grpc.Errorf(codes.Internal, "could not fetch cluster object") + } + cluster := clusters[0] + + removedNode := &api.RemovedNode{ID: node.ID} + + // Set an expiry time for this RemovedNode if a certificate + // exists and can be parsed. + if len(node.Certificate.Certificate) != 0 { + certBlock, _ := pem.Decode(node.Certificate.Certificate) + if certBlock != nil { + X509Cert, err := x509.ParseCertificate(certBlock.Bytes) + if err == nil && !X509Cert.NotAfter.IsZero() { + expiry, err := ptypes.TimestampProto(X509Cert.NotAfter) + if err == nil { + removedNode.Expiry = expiry + } + } + } + } + + cluster.RemovedNodes = append(cluster.RemovedNodes, removedNode) + + if err := store.UpdateCluster(tx, cluster); err != nil { + return err + } + return store.DeleteNode(tx, request.NodeID) }) if err != nil { diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/secret.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/secret.go index 4a2a04981e..a1164619ae 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/secret.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/secret.go @@ -48,7 +48,6 @@ func (s *Server) GetSecret(ctx context.Context, request *api.GetSecretRequest) ( return nil, grpc.Errorf(codes.NotFound, "secret %s not found", request.SecretID) } - secret.Spec.Data = nil // clean the actual secret data so it's never returned return &api.GetSecretResponse{Secret: secret}, nil } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go index 6cb69ff063..d484ced37e 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go @@ -158,13 +158,25 @@ func validateEndpointSpec(epSpec *api.EndpointSpec) error { return grpc.Errorf(codes.InvalidArgument, "EndpointSpec: ports can't be used with dnsrr mode") } - portSet := make(map[uint32]struct{}) + type portSpec struct { + publishedPort uint32 + protocol api.PortConfig_Protocol + } + + portSet := make(map[portSpec]struct{}) for _, port := range epSpec.Ports { - if _, ok := portSet[port.PublishedPort]; ok { + // If published port is not specified, it does not conflict + // with any others. + if port.PublishedPort == 0 { + continue + } + + portSpec := portSpec{publishedPort: port.PublishedPort, protocol: port.Protocol} + if _, ok := portSet[portSpec]; ok { return grpc.Errorf(codes.InvalidArgument, "EndpointSpec: duplicate published ports provided") } - portSet[port.PublishedPort] = struct{}{} + portSet[portSpec] = struct{}{} } return nil diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go index 7e463a24ca..38f2fb2d32 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go @@ -1283,8 +1283,3 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio } } } - -// NodeCount returns number of nodes which connected to this dispatcher. -func (d *Dispatcher) NodeCount() int { - return d.nodes.Len() -} diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/manager.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/manager.go index 0727c0270f..f329a30831 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/manager.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/manager.go @@ -9,6 +9,7 @@ import ( "path/filepath" "sync" "syscall" + "time" "github.com/Sirupsen/logrus" "github.com/docker/go-events" @@ -80,16 +81,17 @@ type Manager struct { listeners map[string]net.Listener caserver *ca.Server - Dispatcher *dispatcher.Dispatcher + dispatcher *dispatcher.Dispatcher replicatedOrchestrator *orchestrator.ReplicatedOrchestrator globalOrchestrator *orchestrator.GlobalOrchestrator taskReaper *orchestrator.TaskReaper + constraintEnforcer *orchestrator.ConstraintEnforcer scheduler *scheduler.Scheduler allocator *allocator.Allocator keyManager *keymanager.KeyManager server *grpc.Server localserver *grpc.Server - RaftNode *raft.Node + raftNode *raft.Node mu sync.Mutex @@ -210,7 +212,7 @@ func New(config *Config) (*Manager, error) { ForceNewCluster: config.ForceNewCluster, TLSCredentials: config.SecurityConfig.ClientTLSCreds, } - RaftNode := raft.NewNode(context.TODO(), newNodeOpts) + raftNode := raft.NewNode(newNodeOpts) opts := []grpc.ServerOption{ grpc.Creds(config.SecurityConfig.ServerTLSCreds)} @@ -218,11 +220,11 @@ func New(config *Config) (*Manager, error) { m := &Manager{ config: config, listeners: listeners, - caserver: ca.NewServer(RaftNode.MemoryStore(), config.SecurityConfig), - Dispatcher: dispatcher.New(RaftNode, dispatcherConfig), + caserver: ca.NewServer(raftNode.MemoryStore(), config.SecurityConfig), + dispatcher: dispatcher.New(raftNode, dispatcherConfig), server: grpc.NewServer(opts...), localserver: grpc.NewServer(opts...), - RaftNode: RaftNode, + raftNode: raftNode, started: make(chan struct{}), stopped: make(chan struct{}), } @@ -254,36 +256,53 @@ func (m *Manager) Run(parent context.Context) error { } }() - leadershipCh, cancel := m.RaftNode.SubscribeLeadership() + leadershipCh, cancel := m.raftNode.SubscribeLeadership() defer cancel() go m.handleLeadershipEvents(ctx, leadershipCh) authorize := func(ctx context.Context, roles []string) error { + var ( + removedNodes []*api.RemovedNode + clusters []*api.Cluster + err error + ) + + m.raftNode.MemoryStore().View(func(readTx store.ReadTx) { + clusters, err = store.FindClusters(readTx, store.ByName("default")) + + }) + + // Not having a cluster object yet means we can't check + // the blacklist. + if err == nil && len(clusters) == 1 { + removedNodes = clusters[0].RemovedNodes + } + // Authorize the remote roles, ensure they can only be forwarded by managers - _, err := ca.AuthorizeForwardedRoleAndOrg(ctx, roles, []string{ca.ManagerRole}, m.config.SecurityConfig.ClientTLSCreds.Organization()) + _, err = ca.AuthorizeForwardedRoleAndOrg(ctx, roles, []string{ca.ManagerRole}, m.config.SecurityConfig.ClientTLSCreds.Organization(), removedNodes) return err } - baseControlAPI := controlapi.NewServer(m.RaftNode.MemoryStore(), m.RaftNode, m.config.SecurityConfig.RootCA()) - baseResourceAPI := resourceapi.New(m.RaftNode.MemoryStore()) + baseControlAPI := controlapi.NewServer(m.raftNode.MemoryStore(), m.raftNode, m.config.SecurityConfig.RootCA()) + baseResourceAPI := resourceapi.New(m.raftNode.MemoryStore()) healthServer := health.NewHealthServer() localHealthServer := health.NewHealthServer() authenticatedControlAPI := api.NewAuthenticatedWrapperControlServer(baseControlAPI, authorize) authenticatedResourceAPI := api.NewAuthenticatedWrapperResourceAllocatorServer(baseResourceAPI, authorize) - authenticatedDispatcherAPI := api.NewAuthenticatedWrapperDispatcherServer(m.Dispatcher, authorize) + authenticatedDispatcherAPI := api.NewAuthenticatedWrapperDispatcherServer(m.dispatcher, authorize) authenticatedCAAPI := api.NewAuthenticatedWrapperCAServer(m.caserver, authorize) authenticatedNodeCAAPI := api.NewAuthenticatedWrapperNodeCAServer(m.caserver, authorize) - authenticatedRaftAPI := api.NewAuthenticatedWrapperRaftServer(m.RaftNode, authorize) + authenticatedRaftAPI := api.NewAuthenticatedWrapperRaftServer(m.raftNode, authorize) authenticatedHealthAPI := api.NewAuthenticatedWrapperHealthServer(healthServer, authorize) - authenticatedRaftMembershipAPI := api.NewAuthenticatedWrapperRaftMembershipServer(m.RaftNode, authorize) + authenticatedRaftMembershipAPI := api.NewAuthenticatedWrapperRaftMembershipServer(m.raftNode, authorize) - proxyDispatcherAPI := api.NewRaftProxyDispatcherServer(authenticatedDispatcherAPI, m.RaftNode, ca.WithMetadataForwardTLSInfo) - proxyCAAPI := api.NewRaftProxyCAServer(authenticatedCAAPI, m.RaftNode, ca.WithMetadataForwardTLSInfo) - proxyNodeCAAPI := api.NewRaftProxyNodeCAServer(authenticatedNodeCAAPI, m.RaftNode, ca.WithMetadataForwardTLSInfo) - proxyRaftMembershipAPI := api.NewRaftProxyRaftMembershipServer(authenticatedRaftMembershipAPI, m.RaftNode, ca.WithMetadataForwardTLSInfo) - proxyResourceAPI := api.NewRaftProxyResourceAllocatorServer(authenticatedResourceAPI, m.RaftNode, ca.WithMetadataForwardTLSInfo) + proxyDispatcherAPI := api.NewRaftProxyDispatcherServer(authenticatedDispatcherAPI, m.raftNode, ca.WithMetadataForwardTLSInfo) + proxyCAAPI := api.NewRaftProxyCAServer(authenticatedCAAPI, m.raftNode, ca.WithMetadataForwardTLSInfo) + proxyNodeCAAPI := api.NewRaftProxyNodeCAServer(authenticatedNodeCAAPI, m.raftNode, ca.WithMetadataForwardTLSInfo) + proxyRaftMembershipAPI := api.NewRaftProxyRaftMembershipServer(authenticatedRaftMembershipAPI, m.raftNode, ca.WithMetadataForwardTLSInfo) + proxyResourceAPI := api.NewRaftProxyResourceAllocatorServer(authenticatedResourceAPI, m.raftNode, ca.WithMetadataForwardTLSInfo) // localProxyControlAPI is a special kind of proxy. It is only wired up // to receive requests from a trusted local socket, and these requests @@ -292,7 +311,7 @@ func (m *Manager) Run(parent context.Context) error { // this manager rather than forwarded requests (it has no TLS // information to put in the metadata map). forwardAsOwnRequest := func(ctx context.Context) (context.Context, error) { return ctx, nil } - localProxyControlAPI := api.NewRaftProxyControlServer(baseControlAPI, m.RaftNode, forwardAsOwnRequest) + localProxyControlAPI := api.NewRaftProxyControlServer(baseControlAPI, m.raftNode, forwardAsOwnRequest) // Everything registered on m.server should be an authenticated // wrapper, or a proxy wrapping an authenticated wrapper! @@ -324,7 +343,7 @@ func (m *Manager) Run(parent context.Context) error { // Set the raft server as serving for the health server healthServer.SetServingStatus("Raft", api.HealthCheckResponse_SERVING) - if err := m.RaftNode.JoinAndStart(); err != nil { + if err := m.raftNode.JoinAndStart(ctx); err != nil { return errors.Wrap(err, "can't initialize raft node") } @@ -333,28 +352,28 @@ func (m *Manager) Run(parent context.Context) error { close(m.started) go func() { - err := m.RaftNode.Run(ctx) + err := m.raftNode.Run(ctx) if err != nil { log.G(ctx).Error(err) m.Stop(ctx) } }() - if err := raft.WaitForLeader(ctx, m.RaftNode); err != nil { + if err := raft.WaitForLeader(ctx, m.raftNode); err != nil { return err } - c, err := raft.WaitForCluster(ctx, m.RaftNode) + c, err := raft.WaitForCluster(ctx, m.raftNode) if err != nil { return err } raftConfig := c.Spec.Raft - if int(raftConfig.ElectionTick) != m.RaftNode.Config.ElectionTick { - log.G(ctx).Warningf("election tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.RaftNode.Config.ElectionTick, raftConfig.ElectionTick) + if int(raftConfig.ElectionTick) != m.raftNode.Config.ElectionTick { + log.G(ctx).Warningf("election tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.raftNode.Config.ElectionTick, raftConfig.ElectionTick) } - if int(raftConfig.HeartbeatTick) != m.RaftNode.Config.HeartbeatTick { - log.G(ctx).Warningf("heartbeat tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.RaftNode.Config.HeartbeatTick, raftConfig.HeartbeatTick) + if int(raftConfig.HeartbeatTick) != m.raftNode.Config.HeartbeatTick { + log.G(ctx).Warningf("heartbeat tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.raftNode.Config.HeartbeatTick, raftConfig.HeartbeatTick) } // wait for an error in serving. @@ -379,11 +398,12 @@ func (m *Manager) Run(parent context.Context) error { } } +const stopTimeout = 8 * time.Second + // Stop stops the manager. It immediately closes all open connections and // active RPCs as well as stopping the scheduler. func (m *Manager) Stop(ctx context.Context) { log.G(ctx).Info("Stopping manager") - // It's not safe to start shutting down while the manager is still // starting up. <-m.started @@ -400,12 +420,17 @@ func (m *Manager) Stop(ctx context.Context) { // do nothing, we're stopping for the first time } - // once we start stopping, send a signal that we're doing so. this tells - // Run that we've started stopping, when it gets the error from errServe - // it also prevents the loop from processing any more stuff. - close(m.stopped) + srvDone, localSrvDone := make(chan struct{}), make(chan struct{}) + go func() { + m.server.GracefulStop() + close(srvDone) + }() + go func() { + m.localserver.GracefulStop() + close(localSrvDone) + }() - m.Dispatcher.Stop() + m.dispatcher.Stop() m.caserver.Stop() if m.allocator != nil { @@ -420,6 +445,9 @@ func (m *Manager) Stop(ctx context.Context) { if m.taskReaper != nil { m.taskReaper.Stop() } + if m.constraintEnforcer != nil { + m.constraintEnforcer.Stop() + } if m.scheduler != nil { m.scheduler.Stop() } @@ -427,10 +455,26 @@ func (m *Manager) Stop(ctx context.Context) { m.keyManager.Stop() } - m.RaftNode.Shutdown() - // some time after this point, Run will receive an error from one of these - m.server.Stop() - m.localserver.Stop() + // once we start stopping, send a signal that we're doing so. this tells + // Run that we've started stopping, when it gets the error from errServe + // it also prevents the loop from processing any more stuff. + close(m.stopped) + + <-m.raftNode.Done() + + timer := time.AfterFunc(stopTimeout, func() { + m.server.Stop() + m.localserver.Stop() + }) + defer timer.Stop() + // TODO: we're not waiting on ctx because it very well could be passed from Run, + // which is already cancelled here. We need to refactor that. + select { + case <-srvDone: + <-localSrvDone + case <-localSrvDone: + <-srvDone + } log.G(ctx).Info("Manager shut down") // mutex is released and Run can return now @@ -452,7 +496,7 @@ func (m *Manager) rotateRootCAKEK(ctx context.Context, clusterID string) error { passphrase := []byte(strPassphrase) passphrasePrev := []byte(strPassphrasePrev) - s := m.RaftNode.MemoryStore() + s := m.raftNode.MemoryStore() var ( cluster *api.Cluster err error @@ -468,7 +512,7 @@ func (m *Manager) rotateRootCAKEK(ctx context.Context, clusterID string) error { // Try to get the private key from the cluster privKeyPEM := cluster.RootCA.CAKey - if privKeyPEM == nil || len(privKeyPEM) == 0 { + if len(privKeyPEM) == 0 { // We have no PEM root private key in this cluster. log.G(ctx).Warnf("cluster %s does not have private key material", clusterID) return nil @@ -575,14 +619,14 @@ func (m *Manager) serveListener(ctx context.Context, errServe chan error, proto // becomeLeader starts the subsystems that are run on the leader. func (m *Manager) becomeLeader(ctx context.Context) { - s := m.RaftNode.MemoryStore() + s := m.raftNode.MemoryStore() rootCA := m.config.SecurityConfig.RootCA() nodeID := m.config.SecurityConfig.ClientTLSCreds.NodeID() raftCfg := raft.DefaultRaftConfig() - raftCfg.ElectionTick = uint32(m.RaftNode.Config.ElectionTick) - raftCfg.HeartbeatTick = uint32(m.RaftNode.Config.HeartbeatTick) + raftCfg.ElectionTick = uint32(m.raftNode.Config.ElectionTick) + raftCfg.HeartbeatTick = uint32(m.raftNode.Config.HeartbeatTick) clusterID := m.config.SecurityConfig.ClientTLSCreds.Organization() @@ -608,6 +652,7 @@ func (m *Manager) becomeLeader(ctx context.Context) { } m.replicatedOrchestrator = orchestrator.NewReplicatedOrchestrator(s) + m.constraintEnforcer = orchestrator.NewConstraintEnforcer(s) m.globalOrchestrator = orchestrator.NewGlobalOrchestrator(s) m.taskReaper = orchestrator.NewTaskReaper(s) m.scheduler = scheduler.New(s) @@ -636,7 +681,7 @@ func (m *Manager) becomeLeader(ctx context.Context) { if err := d.Run(ctx); err != nil { log.G(ctx).WithError(err).Error("Dispatcher exited with an error") } - }(m.Dispatcher) + }(m.dispatcher) go func(server *ca.Server) { if err := server.Run(ctx); err != nil { @@ -661,6 +706,10 @@ func (m *Manager) becomeLeader(ctx context.Context) { } }(m.scheduler) + go func(constraintEnforcer *orchestrator.ConstraintEnforcer) { + constraintEnforcer.Run() + }(m.constraintEnforcer) + go func(taskReaper *orchestrator.TaskReaper) { taskReaper.Run() }(m.taskReaper) @@ -681,7 +730,7 @@ func (m *Manager) becomeLeader(ctx context.Context) { // becomeFollower shuts down the subsystems that are only run by the leader. func (m *Manager) becomeFollower() { - m.Dispatcher.Stop() + m.dispatcher.Stop() m.caserver.Stop() if m.allocator != nil { @@ -689,6 +738,9 @@ func (m *Manager) becomeFollower() { m.allocator = nil } + m.constraintEnforcer.Stop() + m.constraintEnforcer = nil + m.replicatedOrchestrator.Stop() m.replicatedOrchestrator = nil diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/constraint_enforcer.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/constraint_enforcer.go new file mode 100644 index 0000000000..77d522f9cc --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/constraint_enforcer.go @@ -0,0 +1,157 @@ +package orchestrator + +import ( + "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/log" + "github.com/docker/swarmkit/manager/constraint" + "github.com/docker/swarmkit/manager/state" + "github.com/docker/swarmkit/manager/state/store" +) + +// ConstraintEnforcer watches for updates to nodes and shuts down tasks that no +// longer satisfy scheduling constraints or resource limits. +type ConstraintEnforcer struct { + store *store.MemoryStore + stopChan chan struct{} + doneChan chan struct{} +} + +// NewConstraintEnforcer creates a new ConstraintEnforcer. +func NewConstraintEnforcer(store *store.MemoryStore) *ConstraintEnforcer { + return &ConstraintEnforcer{ + store: store, + stopChan: make(chan struct{}), + doneChan: make(chan struct{}), + } +} + +// Run is the ConstraintEnforcer's main loop. +func (ce *ConstraintEnforcer) Run() { + defer close(ce.doneChan) + + watcher, cancelWatch := state.Watch(ce.store.WatchQueue(), state.EventUpdateNode{}) + defer cancelWatch() + + var ( + nodes []*api.Node + err error + ) + ce.store.View(func(readTx store.ReadTx) { + nodes, err = store.FindNodes(readTx, store.All) + }) + if err != nil { + log.L.WithError(err).Error("failed to check nodes for noncompliant tasks") + } else { + for _, node := range nodes { + ce.shutdownNoncompliantTasks(node) + } + } + + for { + select { + case event := <-watcher: + node := event.(state.EventUpdateNode).Node + ce.shutdownNoncompliantTasks(node) + case <-ce.stopChan: + return + } + } +} + +func (ce *ConstraintEnforcer) shutdownNoncompliantTasks(node *api.Node) { + // If the availability is "drain", the orchestrator will + // shut down all tasks. + // If the availability is "pause", we shouldn't touch + // the tasks on this node. + if node.Spec.Availability != api.NodeAvailabilityActive { + return + } + + var ( + tasks []*api.Task + err error + ) + + ce.store.View(func(tx store.ReadTx) { + tasks, err = store.FindTasks(tx, store.ByNodeID(node.ID)) + }) + + if err != nil { + log.L.WithError(err).Errorf("failed to list tasks for node ID %s", node.ID) + } + + var availableMemoryBytes, availableNanoCPUs int64 + if node.Description != nil && node.Description.Resources != nil { + availableMemoryBytes = node.Description.Resources.MemoryBytes + availableNanoCPUs = node.Description.Resources.NanoCPUs + } + + removeTasks := make(map[string]*api.Task) + + // TODO(aaronl): The set of tasks removed will be + // nondeterministic because it depends on the order of + // the slice returned from FindTasks. We could do + // a separate pass over the tasks for each type of + // resource, and sort by the size of the reservation + // to remove the most resource-intensive tasks. + for _, t := range tasks { + if t.DesiredState < api.TaskStateAssigned || t.DesiredState > api.TaskStateRunning { + continue + } + + // Ensure that the task still meets scheduling + // constraints. + if t.Spec.Placement != nil && len(t.Spec.Placement.Constraints) != 0 { + constraints, _ := constraint.Parse(t.Spec.Placement.Constraints) + if !constraint.NodeMatches(constraints, node) { + removeTasks[t.ID] = t + continue + } + } + + // Ensure that the task assigned to the node + // still satisfies the resource limits. + if t.Spec.Resources != nil && t.Spec.Resources.Reservations != nil { + if t.Spec.Resources.Reservations.MemoryBytes > availableMemoryBytes { + removeTasks[t.ID] = t + continue + } + if t.Spec.Resources.Reservations.NanoCPUs > availableNanoCPUs { + removeTasks[t.ID] = t + continue + } + availableMemoryBytes -= t.Spec.Resources.Reservations.MemoryBytes + availableNanoCPUs -= t.Spec.Resources.Reservations.NanoCPUs + } + } + + if len(removeTasks) != 0 { + _, err := ce.store.Batch(func(batch *store.Batch) error { + for _, t := range removeTasks { + err := batch.Update(func(tx store.Tx) error { + t = store.GetTask(tx, t.ID) + if t == nil || t.DesiredState > api.TaskStateRunning { + return nil + } + + t.DesiredState = api.TaskStateShutdown + return store.UpdateTask(tx, t) + }) + if err != nil { + log.L.WithError(err).Errorf("failed to shut down task %s", t.ID) + } + } + return nil + }) + + if err != nil { + log.L.WithError(err).Errorf("failed to shut down tasks") + } + } +} + +// Stop stops the ConstraintEnforcer and waits for the main loop to exit. +func (ce *ConstraintEnforcer) Stop() { + close(ce.stopChan) + <-ce.doneChan +} diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/global.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/global.go index dc160e6d62..d45bc86ac6 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/global.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/global.go @@ -374,11 +374,13 @@ func (g *GlobalOrchestrator) reconcileServicesOneNode(ctx context.Context, servi continue } - meetsConstraints := constraint.NodeMatches(service.constraints, node) + if !constraint.NodeMatches(service.constraints, node) { + continue + } // if restart policy considers this node has finished its task // it should remove all running tasks - if completed[serviceID] || !meetsConstraints { + if completed[serviceID] { g.removeTasks(ctx, batch, tasks[serviceID]) continue } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go index 60a611c6e4..47d03f1cc4 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go @@ -168,6 +168,9 @@ func (r *ReplicatedOrchestrator) reconcile(ctx context.Context, service *api.Ser r.deleteTasksMap(ctx, batch, deadSlots) return nil }) + if err != nil { + log.G(ctx).WithError(err).Errorf("reconcile batch failed") + } // Simple update, no scaling - update all tasks. r.updater.Update(ctx, r.cluster, service, slotsSlice) } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/filter.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/filter.go index 77a43ce512..1ecbb7d08a 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/filter.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/filter.go @@ -1,6 +1,8 @@ package scheduler import ( + "fmt" + "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/manager/constraint" ) @@ -18,6 +20,9 @@ type Filter interface { // into the given node. This function should not be called if SetTask // returned false. Check(*NodeInfo) bool + + // Explain what a failure of this filter means + Explain(nodes int) string } // ReadyFilter checks that the node is ready to schedule tasks. @@ -35,6 +40,14 @@ func (f *ReadyFilter) Check(n *NodeInfo) bool { n.Spec.Availability == api.NodeAvailabilityActive } +// Explain returns an explanation of a failure. +func (f *ReadyFilter) Explain(nodes int) string { + if nodes == 1 { + return "1 node not available for new tasks" + } + return fmt.Sprintf("%d nodes not available for new tasks", nodes) +} + // ResourceFilter checks that the node has enough resources available to run // the task. type ResourceFilter struct { @@ -67,6 +80,14 @@ func (f *ResourceFilter) Check(n *NodeInfo) bool { return true } +// Explain returns an explanation of a failure. +func (f *ResourceFilter) Explain(nodes int) string { + if nodes == 1 { + return "insufficient resources on 1 node" + } + return fmt.Sprintf("insufficient resources on %d nodes", nodes) +} + // PluginFilter checks that the node has a specific volume plugin installed type PluginFilter struct { t *api.Task @@ -133,6 +154,14 @@ func (f *PluginFilter) pluginExistsOnNode(pluginType string, pluginName string, return false } +// Explain returns an explanation of a failure. +func (f *PluginFilter) Explain(nodes int) string { + if nodes == 1 { + return "missing plugin on 1 node" + } + return fmt.Sprintf("missing plugin on %d nodes", nodes) +} + // ConstraintFilter selects only nodes that match certain labels. type ConstraintFilter struct { constraints []constraint.Constraint @@ -159,3 +188,11 @@ func (f *ConstraintFilter) SetTask(t *api.Task) bool { func (f *ConstraintFilter) Check(n *NodeInfo) bool { return constraint.NodeMatches(f.constraints, n.Node) } + +// Explain returns an explanation of a failure. +func (f *ConstraintFilter) Explain(nodes int) string { + if nodes == 1 { + return "scheduling constraints not satisfied on 1 node" + } + return fmt.Sprintf("scheduling constraints not satisfied on %d nodes", nodes) +} diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/pipeline.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/pipeline.go index b1517db961..d9981aa125 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/pipeline.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/pipeline.go @@ -1,6 +1,10 @@ package scheduler -import "github.com/docker/swarmkit/api" +import ( + "sort" + + "github.com/docker/swarmkit/api" +) var ( defaultFilters = []Filter{ @@ -20,10 +24,21 @@ var ( type checklistEntry struct { f Filter enabled bool + + // failureCount counts the number of nodes that this filter failed + // against. + failureCount int } +type checklistByFailures []checklistEntry + +func (c checklistByFailures) Len() int { return len(c) } +func (c checklistByFailures) Swap(i, j int) { c[i], c[j] = c[j], c[i] } +func (c checklistByFailures) Less(i, j int) bool { return c[i].failureCount < c[j].failureCount } + // Pipeline runs a set of filters against nodes. type Pipeline struct { + // checklist is a slice of filters to run checklist []checklistEntry } @@ -41,12 +56,16 @@ func NewPipeline() *Pipeline { // Process a node through the filter pipeline. // Returns true if all filters pass, false otherwise. func (p *Pipeline) Process(n *NodeInfo) bool { - for _, entry := range p.checklist { + for i, entry := range p.checklist { if entry.enabled && !entry.f.Check(n) { // Immediately stop on first failure. + p.checklist[i].failureCount++ return false } } + for i := range p.checklist { + p.checklist[i].failureCount = 0 + } return true } @@ -55,5 +74,28 @@ func (p *Pipeline) Process(n *NodeInfo) bool { func (p *Pipeline) SetTask(t *api.Task) { for i := range p.checklist { p.checklist[i].enabled = p.checklist[i].f.SetTask(t) + p.checklist[i].failureCount = 0 } } + +// Explain returns a string explaining why a task could not be scheduled. +func (p *Pipeline) Explain() string { + var explanation string + + // Sort from most failures to least + + sortedByFailures := make([]checklistEntry, len(p.checklist)) + copy(sortedByFailures, p.checklist) + sort.Sort(sort.Reverse(checklistByFailures(sortedByFailures))) + + for _, entry := range sortedByFailures { + if entry.failureCount > 0 { + if len(explanation) > 0 { + explanation += "; " + } + explanation += entry.f.Explain(entry.failureCount) + } + } + + return explanation +} diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/scheduler.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/scheduler.go index 6d3b0d93e6..c18a278d95 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/scheduler.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/scheduler.go @@ -298,7 +298,9 @@ func (s *Scheduler) processPreassignedTasks(ctx context.Context) { successful, failed := s.applySchedulingDecisions(ctx, schedulingDecisions) for _, decision := range successful { - delete(s.preassignedTasks, decision.old.ID) + if decision.new.Status.State == api.TaskStateAssigned { + delete(s.preassignedTasks, decision.old.ID) + } } for _, decision := range failed { s.allTasks[decision.old.ID] = decision.old @@ -381,11 +383,35 @@ func (s *Scheduler) applySchedulingDecisions(ctx context.Context, schedulingDeci t := store.GetTask(tx, taskID) if t == nil { - // Task no longer exists. Do nothing. - failed = append(failed, decision) + // Task no longer exists + nodeInfo, err := s.nodeSet.nodeInfo(decision.new.NodeID) + if err == nil && nodeInfo.removeTask(decision.new) { + s.nodeSet.updateNode(nodeInfo) + } + delete(s.allTasks, decision.old.ID) + continue } + if t.Status.State == decision.new.Status.State && t.Status.Message == decision.new.Status.Message { + // No changes, ignore + continue + } + + if t.Status.State >= api.TaskStateAssigned { + nodeInfo, err := s.nodeSet.nodeInfo(decision.new.NodeID) + if err != nil { + failed = append(failed, decision) + continue + } + node := store.GetNode(tx, decision.new.NodeID) + if node == nil || node.Meta.Version != nodeInfo.Meta.Version { + // node is out of date + failed = append(failed, decision) + continue + } + } + if err := store.UpdateTask(tx, decision.new); err != nil { log.G(ctx).Debugf("scheduler failed to update task %s; will retry", taskID) failed = append(failed, decision) @@ -418,12 +444,16 @@ func (s *Scheduler) taskFitNode(ctx context.Context, t *api.Task, nodeID string) // node does not exist in set (it may have been deleted) return nil } + newT := *t s.pipeline.SetTask(t) if !s.pipeline.Process(&nodeInfo) { // this node cannot accommodate this task - return nil + newT.Status.Timestamp = ptypes.MustTimestampProto(time.Now()) + newT.Status.Message = s.pipeline.Explain() + s.allTasks[t.ID] = &newT + + return &newT } - newT := *t newT.Status = api.TaskStatus{ State: api.TaskStateAssigned, Timestamp: ptypes.MustTimestampProto(time.Now()), @@ -468,10 +498,7 @@ func (s *Scheduler) scheduleTaskGroup(ctx context.Context, taskGroup map[string] nodes := s.nodeSet.findBestNodes(len(taskGroup), s.pipeline.Process, nodeLess) if len(nodes) == 0 { - for _, t := range taskGroup { - log.G(ctx).WithField("task.id", t.ID).Debug("no suitable node available for task") - s.enqueue(t) - } + s.noSuitableNode(ctx, taskGroup, schedulingDecisions) return } @@ -518,16 +545,32 @@ func (s *Scheduler) scheduleTaskGroup(ctx context.Context, taskGroup map[string] nodeIter++ if nodeIter-origNodeIter == len(nodes) { // None of the nodes meet the constraints anymore. - for _, t := range taskGroup { - log.G(ctx).WithField("task.id", t.ID).Debug("no suitable node available for task") - s.enqueue(t) - } + s.noSuitableNode(ctx, taskGroup, schedulingDecisions) return } } } } +func (s *Scheduler) noSuitableNode(ctx context.Context, taskGroup map[string]*api.Task, schedulingDecisions map[string]schedulingDecision) { + explanation := s.pipeline.Explain() + for _, t := range taskGroup { + log.G(ctx).WithField("task.id", t.ID).Debug("no suitable node available for task") + + newT := *t + newT.Status.Timestamp = ptypes.MustTimestampProto(time.Now()) + if explanation != "" { + newT.Status.Message = "no suitable node (" + explanation + ")" + } else { + newT.Status.Message = "no suitable node" + } + s.allTasks[t.ID] = &newT + schedulingDecisions[t.ID] = schedulingDecision{old: t, new: &newT} + + s.enqueue(&newT) + } +} + func (s *Scheduler) buildNodeSet(tx store.ReadTx, tasksByNode map[string]map[string]*api.Task) error { nodes, err := store.FindNodes(tx, store.All) if err != nil { diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go index 65e753c71f..0a9ef98a3e 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go @@ -53,7 +53,6 @@ type Cluster struct { type Member struct { *api.RaftMember - api.RaftClient Conn *grpc.ClientConn tick int active bool @@ -211,8 +210,7 @@ func (c *Cluster) clearMember(id uint64) error { return nil } -// ReplaceMemberConnection replaces the member's GRPC connection and GRPC -// client. +// ReplaceMemberConnection replaces the member's GRPC connection. func (c *Cluster) ReplaceMemberConnection(id uint64, oldConn *Member, newConn *Member, newAddr string, force bool) error { c.mu.Lock() defer c.mu.Unlock() @@ -236,7 +234,6 @@ func (c *Cluster) ReplaceMemberConnection(id uint64, oldConn *Member, newConn *M newMember.RaftMember = oldMember.RaftMember.Copy() newMember.RaftMember.Addr = newAddr newMember.Conn = newConn.Conn - newMember.RaftClient = newConn.RaftClient c.members[id] = &newMember return nil diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go index 3185d4b4c1..7112f25e68 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go @@ -81,10 +81,6 @@ type Node struct { raftNode raft.Node cluster *membership.Cluster - Server *grpc.Server - Ctx context.Context - cancel func() - raftStore *raft.MemoryStorage memoryStore *store.MemoryStore Config *raft.Config @@ -106,7 +102,6 @@ type Node struct { snapshotIndex uint64 ticker clock.Ticker - stopCh chan struct{} doneCh chan struct{} // removeRaftCh notifies about node deletion from raft cluster removeRaftCh chan struct{} @@ -121,6 +116,12 @@ type Node struct { snapshotInProgress chan uint64 asyncTasks sync.WaitGroup + + // stopped chan is used for notifying grpc handlers that raft node going + // to stop. + stopped chan struct{} + + lastSendToMember map[uint64]chan struct{} } // NodeOptions provides node-level options. @@ -155,8 +156,8 @@ func init() { rand.Seed(time.Now().UnixNano()) } -// NewNode generates a new Raft node. -func NewNode(ctx context.Context, opts NodeOptions) *Node { +// NewNode generates a new Raft node +func NewNode(opts NodeOptions) *Node { cfg := opts.Config if cfg == nil { cfg = DefaultNodeConfig() @@ -170,11 +171,7 @@ func NewNode(ctx context.Context, opts NodeOptions) *Node { raftStore := raft.NewMemoryStorage() - ctx, cancel := context.WithCancel(ctx) - n := &Node{ - Ctx: ctx, - cancel: cancel, cluster: membership.NewCluster(2 * cfg.ElectionTick), raftStore: raftStore, opts: opts, @@ -186,10 +183,11 @@ func NewNode(ctx context.Context, opts NodeOptions) *Node { MaxInflightMsgs: cfg.MaxInflightMsgs, Logger: cfg.Logger, }, - stopCh: make(chan struct{}), doneCh: make(chan struct{}), removeRaftCh: make(chan struct{}), + stopped: make(chan struct{}), leadershipBroadcast: watch.NewQueue(), + lastSendToMember: make(map[uint64]chan struct{}), } n.memoryStore = store.NewMemoryStore(n) @@ -214,15 +212,32 @@ func NewNode(ctx context.Context, opts NodeOptions) *Node { return n } +// WithContext returns context which is cancelled when parent context cancelled +// or node is stopped. +func (n *Node) WithContext(ctx context.Context) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(ctx) + + go func() { + select { + case <-ctx.Done(): + case <-n.stopped: + cancel() + } + }() + return ctx, cancel +} + // JoinAndStart joins and starts the raft server -func (n *Node) JoinAndStart() (err error) { +func (n *Node) JoinAndStart(ctx context.Context) (err error) { + ctx, cancel := n.WithContext(ctx) defer func() { + cancel() if err != nil { n.done() } }() - loadAndStartErr := n.loadAndStart(n.Ctx, n.opts.ForceNewCluster) + loadAndStartErr := n.loadAndStart(ctx, n.opts.ForceNewCluster) if loadAndStartErr != nil && loadAndStartErr != errNoWAL { return loadAndStartErr } @@ -248,9 +263,9 @@ func (n *Node) JoinAndStart() (err error) { _ = c.Conn.Close() }() - ctx, cancel := context.WithTimeout(n.Ctx, 10*time.Second) - defer cancel() - resp, err := client.Join(ctx, &api.JoinRequest{ + joinCtx, joinCancel := context.WithTimeout(ctx, 10*time.Second) + defer joinCancel() + resp, err := client.Join(joinCtx, &api.JoinRequest{ Addr: n.opts.Addr, }) if err != nil { @@ -267,7 +282,7 @@ func (n *Node) JoinAndStart() (err error) { if err := n.registerNodes(resp.Members); err != nil { if walErr := n.wal.Close(); err != nil { - n.Config.Logger.Errorf("raft: error closing WAL: %v", walErr) + log.G(ctx).WithError(walErr).Error("raft: error closing WAL") } return err } @@ -286,7 +301,7 @@ func (n *Node) JoinAndStart() (err error) { } if n.opts.JoinAddr != "" { - n.Config.Logger.Warning("ignoring request to join cluster, because raft state already exists") + log.G(ctx).Warning("ignoring request to join cluster, because raft state already exists") } n.campaignWhenAble = true n.raftNode = raft.RestartNode(n.Config) @@ -340,7 +355,24 @@ func (n *Node) done() { // Before running the main loop, it first starts the raft node based on saved // cluster state. If no saved state exists, it starts a single-node cluster. func (n *Node) Run(ctx context.Context) error { - defer n.done() + ctx = log.WithLogger(ctx, logrus.WithField("raft_id", fmt.Sprintf("%x", n.Config.ID))) + ctx, cancel := context.WithCancel(ctx) + + // nodeRemoved indicates that node was stopped due its removal. + nodeRemoved := false + + defer func() { + cancel() + n.stop(ctx) + if nodeRemoved { + // Move WAL and snapshot out of the way, since + // they are no longer usable. + if err := n.moveWALAndSnap(); err != nil { + log.G(ctx).WithError(err).Error("failed to move wal after node removal") + } + } + n.done() + }() wasLeader := false @@ -360,13 +392,13 @@ func (n *Node) Run(ctx context.Context) error { // Save entries to storage if err := n.saveToStorage(&raftConfig, rd.HardState, rd.Entries, rd.Snapshot); err != nil { - n.Config.Logger.Error(err) + log.G(ctx).WithError(err).Error("failed to save entries to storage") } if len(rd.Messages) != 0 { // Send raft messages to peers - if err := n.send(rd.Messages); err != nil { - n.Config.Logger.Error(err) + if err := n.send(ctx, rd.Messages); err != nil { + log.G(ctx).WithError(err).Error("failed to send message to members") } } @@ -376,7 +408,7 @@ func (n *Node) Run(ctx context.Context) error { if !raft.IsEmptySnap(rd.Snapshot) { // Load the snapshot data into the store if err := n.restoreFromSnapshot(rd.Snapshot.Data, false); err != nil { - n.Config.Logger.Error(err) + log.G(ctx).WithError(err).Error("failed to restore from snapshot") } n.appliedIndex = rd.Snapshot.Metadata.Index n.snapshotIndex = rd.Snapshot.Metadata.Index @@ -420,8 +452,8 @@ func (n *Node) Run(ctx context.Context) error { // Process committed entries for _, entry := range rd.CommittedEntries { - if err := n.processCommitted(entry); err != nil { - n.Config.Logger.Error(err) + if err := n.processCommitted(ctx, entry); err != nil { + log.G(ctx).WithError(err).Error("failed to process committed entries") } } @@ -429,7 +461,7 @@ func (n *Node) Run(ctx context.Context) error { if n.snapshotInProgress == nil && raftConfig.SnapshotInterval > 0 && n.appliedIndex-n.snapshotIndex >= raftConfig.SnapshotInterval { - n.doSnapshot(&raftConfig) + n.doSnapshot(ctx, raftConfig) } if wasLeader && atomic.LoadUint32(&n.signalledLeadership) != 1 { @@ -453,7 +485,7 @@ func (n *Node) Run(ctx context.Context) error { n.campaignWhenAble = false } if len(members) == 1 && members[n.Config.ID] != nil { - if err := n.raftNode.Campaign(n.Ctx); err != nil { + if err := n.raftNode.Campaign(ctx); err != nil { panic("raft: cannot campaign to be the leader on node restore") } } @@ -465,49 +497,35 @@ func (n *Node) Run(ctx context.Context) error { } n.snapshotInProgress = nil case <-n.removeRaftCh: + nodeRemoved = true // If the node was removed from other members, // send back an error to the caller to start // the shutdown process. - n.stop() - - // Move WAL and snapshot out of the way, since - // they are no longer usable. - if err := n.moveWALAndSnap(); err != nil { - n.Config.Logger.Error(err) - } - return ErrMemberRemoved - case <-n.stopCh: - n.stop() + case <-ctx.Done(): return nil } } } -// Shutdown stops the raft node processing loop. -// Calling Shutdown on an already stopped node -// will result in a panic. -func (n *Node) Shutdown() { - select { - case <-n.doneCh: - default: - close(n.stopCh) - <-n.doneCh - } +// Done returns channel which is closed when raft node is fully stopped. +func (n *Node) Done() <-chan struct{} { + return n.doneCh } -func (n *Node) stop() { +func (n *Node) stop(ctx context.Context) { n.stopMu.Lock() defer n.stopMu.Unlock() - n.cancel() + close(n.stopped) + n.waitProp.Wait() n.asyncTasks.Wait() n.raftNode.Stop() n.ticker.Stop() if err := n.wal.Close(); err != nil { - n.Config.Logger.Errorf("raft: error closing WAL: %v", err) + log.G(ctx).WithError(err).Error("raft: failed to close WAL") } atomic.StoreUint32(&n.isMember, 0) // TODO(stevvooe): Handle ctx.Done() @@ -581,11 +599,13 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons fields := logrus.Fields{ "node.id": nodeInfo.NodeID, "method": "(*Node).Join", + "raft_id": fmt.Sprintf("%x", n.Config.ID), } if nodeInfo.ForwardedBy != nil { fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID } log := log.G(ctx).WithFields(fields) + log.Debug("") // can't stop the raft node while an async RPC is in progress n.stopMu.RLock() @@ -639,7 +659,7 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons err = n.addMember(ctx, remoteAddr, raftID, nodeInfo.NodeID) if err != nil { - log.WithError(err).Errorf("failed to add member") + log.WithError(err).Errorf("failed to add member %x", raftID) return nil, err } @@ -738,11 +758,12 @@ func (n *Node) Leave(ctx context.Context, req *api.LeaveRequest) (*api.LeaveResp fields := logrus.Fields{ "node.id": nodeInfo.NodeID, "method": "(*Node).Leave", + "raft_id": fmt.Sprintf("%x", n.Config.ID), } if nodeInfo.ForwardedBy != nil { fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID } - log.G(ctx).WithFields(fields).Debugf("") + log.G(ctx).WithFields(fields).Debug("") // can't stop the raft node while an async RPC is in progress n.stopMu.RLock() @@ -783,8 +804,9 @@ func (n *Node) RemoveMember(ctx context.Context, id uint64) error { NodeID: id, Context: []byte(""), } - + ctx, cancel := n.WithContext(ctx) err := n.configure(ctx, cc) + cancel() return err } @@ -813,11 +835,14 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa n.cluster.ReportActive(msg.Message.From, sourceHost) + ctx, cancel := n.WithContext(ctx) + defer cancel() + // Reject vote requests from unreachable peers if msg.Message.Type == raftpb.MsgVote { member := n.cluster.GetMember(msg.Message.From) if member == nil || member.Conn == nil { - n.Config.Logger.Errorf("received vote request from unknown member %x", msg.Message.From) + log.G(ctx).Errorf("received vote request from unknown member %x", msg.Message.From) return nil, ErrMemberUnknown } @@ -825,7 +850,7 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa defer cancel() if err := member.HealthCheck(healthCtx); err != nil { - n.Config.Logger.Warningf("member %x which sent vote request failed health check: %v", msg.Message.From, err) + log.G(ctx).WithError(err).Warningf("member %x which sent vote request failed health check", msg.Message.From) return nil, errors.Wrap(err, "member unreachable") } } @@ -846,7 +871,7 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa return nil, ErrNoRaftMember } - if err := n.raftNode.Step(n.Ctx, *msg.Message); err != nil { + if err := n.raftNode.Step(ctx, *msg.Message); err != nil { return nil, err } @@ -867,11 +892,12 @@ func (n *Node) ResolveAddress(ctx context.Context, msg *api.ResolveAddressReques fields := logrus.Fields{ "node.id": nodeInfo.NodeID, "method": "(*Node).ResolveAddress", + "raft_id": fmt.Sprintf("%x", n.Config.ID), } if nodeInfo.ForwardedBy != nil { fields["forwarder.id"] = nodeInfo.ForwardedBy.NodeID } - log.G(ctx).WithFields(fields).Debugf("") + log.G(ctx).WithFields(fields).Debug("") member := n.cluster.GetMember(msg.RaftID) if member == nil { @@ -948,7 +974,6 @@ func (n *Node) registerNode(node *api.RaftMember) error { // address. if existingMember.Addr != node.Addr { member.RaftMember = node - member.RaftClient = existingMember.RaftClient member.Conn = existingMember.Conn n.cluster.AddMember(member) } @@ -993,6 +1018,8 @@ func (n *Node) registerNodes(nodes []*api.RaftMember) error { // ProposeValue calls Propose on the raft and waits // on the commit log action before returning a result func (n *Node) ProposeValue(ctx context.Context, storeAction []*api.StoreAction, cb func()) error { + ctx, cancel := n.WithContext(ctx) + defer cancel() _, err := n.processInternalRaftRequest(ctx, &api.InternalRaftRequest{Action: storeAction}, cb) if err != nil { return err @@ -1083,7 +1110,7 @@ func (n *Node) IsMember() bool { // could be submitted and processed. func (n *Node) canSubmitProposal() bool { select { - case <-n.Ctx.Done(): + case <-n.stopped: return false default: return true @@ -1115,7 +1142,7 @@ func (n *Node) saveToStorage(raftConfig *api.RaftConfig, hardState raftpb.HardSt } // Sends a series of messages to members in the raft -func (n *Node) send(messages []raftpb.Message) error { +func (n *Node) send(ctx context.Context, messages []raftpb.Message) error { members := n.cluster.Members() n.stopMu.RLock() @@ -1124,7 +1151,7 @@ func (n *Node) send(messages []raftpb.Message) error { for _, m := range messages { // Process locally if m.To == n.Config.ID { - if err := n.raftNode.Step(n.Ctx, m); err != nil { + if err := n.raftNode.Step(ctx, m); err != nil { return err } continue @@ -1138,24 +1165,37 @@ func (n *Node) send(messages []raftpb.Message) error { continue } + ch := make(chan struct{}) + n.asyncTasks.Add(1) - go n.sendToMember(members, m) + go n.sendToMember(ctx, members, m, n.lastSendToMember[m.To], ch) + + n.lastSendToMember[m.To] = ch } return nil } -func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Message) { +func (n *Node) sendToMember(ctx context.Context, members map[uint64]*membership.Member, m raftpb.Message, lastSend <-chan struct{}, thisSend chan<- struct{}) { defer n.asyncTasks.Done() + defer close(thisSend) + + ctx, cancel := context.WithTimeout(ctx, n.opts.SendTimeout) + defer cancel() + + if lastSend != nil { + select { + case <-lastSend: + case <-ctx.Done(): + return + } + } if n.cluster.IsIDRemoved(m.To) { // Should not send to removed members return } - ctx, cancel := context.WithTimeout(n.Ctx, n.opts.SendTimeout) - defer cancel() - var ( conn *membership.Member ) @@ -1165,7 +1205,7 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess // If we are being asked to send to a member that's not in // our member list, that could indicate that the current leader // was added while we were offline. Try to resolve its address. - n.Config.Logger.Warningf("sending message to an unrecognized member ID %x", m.To) + log.G(ctx).Warningf("sending message to an unrecognized member ID %x", m.To) // Choose a random member var ( @@ -1179,18 +1219,18 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess } if queryMember == nil || queryMember.RaftID == n.Config.ID { - n.Config.Logger.Error("could not find cluster member to query for leader address") + log.G(ctx).Error("could not find cluster member to query for leader address") return } - resp, err := queryMember.ResolveAddress(ctx, &api.ResolveAddressRequest{RaftID: m.To}) + resp, err := api.NewRaftClient(queryMember.Conn).ResolveAddress(ctx, &api.ResolveAddressRequest{RaftID: m.To}) if err != nil { - n.Config.Logger.Errorf("could not resolve address of member ID %x: %v", m.To, err) + log.G(ctx).WithError(err).Errorf("could not resolve address of member ID %x", m.To) return } conn, err = n.ConnectToMember(resp.Addr, n.opts.SendTimeout) if err != nil { - n.Config.Logger.Errorf("could connect to member ID %x at %s: %v", m.To, resp.Addr, err) + log.G(ctx).WithError(err).Errorf("could connect to member ID %x at %s", m.To, resp.Addr) return } // The temporary connection is only used for this message. @@ -1199,7 +1239,7 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess defer conn.Conn.Close() } - _, err := conn.ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m}) + _, err := api.NewRaftClient(conn.Conn).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m}) if err != nil { if grpc.ErrorDesc(err) == ErrMemberRemoved.Error() { n.removeRaftFunc() @@ -1219,9 +1259,9 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess officialHost, officialPort, _ := net.SplitHostPort(conn.Addr) if officialHost != lastSeenHost { reconnectAddr := net.JoinHostPort(lastSeenHost, officialPort) - n.Config.Logger.Warningf("detected address change for %x (%s -> %s)", m.To, conn.Addr, reconnectAddr) - if err := n.handleAddressChange(conn, reconnectAddr); err != nil { - n.Config.Logger.Error(err) + log.G(ctx).Warningf("detected address change for %x (%s -> %s)", m.To, conn.Addr, reconnectAddr) + if err := n.handleAddressChange(ctx, conn, reconnectAddr); err != nil { + log.G(ctx).WithError(err).Error("failed to hande address change") } return } @@ -1230,12 +1270,12 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess // Bounce the connection newConn, err := n.ConnectToMember(conn.Addr, 0) if err != nil { - n.Config.Logger.Errorf("could connect to member ID %x at %s: %v", m.To, conn.Addr, err) + log.G(ctx).WithError(err).Errorf("could connect to member ID %x at %s", m.To, conn.Addr) return } err = n.cluster.ReplaceMemberConnection(m.To, conn, newConn, conn.Addr, false) if err != nil { - n.Config.Logger.Errorf("failed to replace connection to raft member: %v", err) + log.G(ctx).WithError(err).Error("failed to replace connection to raft member") newConn.Conn.Close() } } else if m.Type == raftpb.MsgSnap { @@ -1243,13 +1283,13 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess } } -func (n *Node) handleAddressChange(member *membership.Member, reconnectAddr string) error { +func (n *Node) handleAddressChange(ctx context.Context, member *membership.Member, reconnectAddr string) error { newConn, err := n.ConnectToMember(reconnectAddr, 0) if err != nil { return errors.Wrapf(err, "could connect to member ID %x at observed address %s", member.RaftID, reconnectAddr) } - healthCtx, cancelHealth := context.WithTimeout(n.Ctx, time.Duration(n.Config.ElectionTick)*n.opts.TickInterval) + healthCtx, cancelHealth := context.WithTimeout(ctx, time.Duration(n.Config.ElectionTick)*n.opts.TickInterval) defer cancelHealth() if err := newConn.HealthCheck(healthCtx); err != nil { @@ -1262,7 +1302,7 @@ func (n *Node) handleAddressChange(member *membership.Member, reconnectAddr stri } // If we're the leader, write the address change to raft - updateCtx, cancelUpdate := context.WithTimeout(n.Ctx, time.Duration(n.Config.ElectionTick)*n.opts.TickInterval) + updateCtx, cancelUpdate := context.WithTimeout(ctx, time.Duration(n.Config.ElectionTick)*n.opts.TickInterval) defer cancelUpdate() if err := n.updateMember(updateCtx, reconnectAddr, member.RaftID, member.NodeID); err != nil { return errors.Wrap(err, "failed to update member address in raft") @@ -1295,7 +1335,7 @@ func (n *Node) processInternalRaftRequest(ctx context.Context, r *api.InternalRa // This must be derived from the context which is cancelled by stop() // to avoid a deadlock on shutdown. - waitCtx, cancel := context.WithCancel(n.Ctx) + waitCtx, cancel := context.WithCancel(ctx) ch := n.wait.register(r.ID, cb, cancel) @@ -1361,29 +1401,27 @@ func (n *Node) configure(ctx context.Context, cc raftpb.ConfChange) error { case <-ctx.Done(): n.wait.cancel(cc.ID) return ctx.Err() - case <-n.Ctx.Done(): - return ErrStopped } } -func (n *Node) processCommitted(entry raftpb.Entry) error { +func (n *Node) processCommitted(ctx context.Context, entry raftpb.Entry) error { // Process a normal entry if entry.Type == raftpb.EntryNormal && entry.Data != nil { - if err := n.processEntry(entry); err != nil { + if err := n.processEntry(ctx, entry); err != nil { return err } } // Process a configuration change (add/remove node) if entry.Type == raftpb.EntryConfChange { - n.processConfChange(entry) + n.processConfChange(ctx, entry) } n.appliedIndex = entry.Index return nil } -func (n *Node) processEntry(entry raftpb.Entry) error { +func (n *Node) processEntry(ctx context.Context, entry raftpb.Entry) error { r := &api.InternalRaftRequest{} err := proto.Unmarshal(entry.Data, r) if err != nil { @@ -1409,13 +1447,13 @@ func (n *Node) processEntry(entry raftpb.Entry) error { err := n.memoryStore.ApplyStoreActions(r.Action) if err != nil { - log.G(context.Background()).Errorf("error applying actions from raft: %v", err) + log.G(ctx).WithError(err).Error("failed to apply actions from raft") } } return nil } -func (n *Node) processConfChange(entry raftpb.Entry) { +func (n *Node) processConfChange(ctx context.Context, entry raftpb.Entry) { var ( err error cc raftpb.ConfChange @@ -1433,9 +1471,9 @@ func (n *Node) processConfChange(entry raftpb.Entry) { case raftpb.ConfChangeAddNode: err = n.applyAddNode(cc) case raftpb.ConfChangeUpdateNode: - err = n.applyUpdateNode(cc) + err = n.applyUpdateNode(ctx, cc) case raftpb.ConfChangeRemoveNode: - err = n.applyRemoveNode(cc) + err = n.applyRemoveNode(ctx, cc) } if err != nil { @@ -1469,7 +1507,7 @@ func (n *Node) applyAddNode(cc raftpb.ConfChange) error { // applyUpdateNode is called when we receive a ConfChange from a member in the // raft cluster which update the address of an existing node. -func (n *Node) applyUpdateNode(cc raftpb.ConfChange) error { +func (n *Node) applyUpdateNode(ctx context.Context, cc raftpb.ConfChange) error { newMember := &api.RaftMember{} err := proto.Unmarshal(cc.Context, newMember) if err != nil { @@ -1483,7 +1521,7 @@ func (n *Node) applyUpdateNode(cc raftpb.ConfChange) error { } if oldMember.NodeID != newMember.NodeID { // Should never happen; this is a sanity check - n.Config.Logger.Errorf("node ID mismatch on node update (old: %x, new: %x)", oldMember.NodeID, newMember.NodeID) + log.G(ctx).Errorf("node ID mismatch on node update (old: %x, new: %x)", oldMember.NodeID, newMember.NodeID) return errors.New("node ID mismatch match on node update") } @@ -1507,13 +1545,13 @@ func (n *Node) applyUpdateNode(cc raftpb.ConfChange) error { // applyRemoveNode is called when we receive a ConfChange // from a member in the raft cluster, this removes a node // from the existing raft cluster -func (n *Node) applyRemoveNode(cc raftpb.ConfChange) (err error) { +func (n *Node) applyRemoveNode(ctx context.Context, cc raftpb.ConfChange) (err error) { // If the node from where the remove is issued is // a follower and the leader steps down, Campaign // to be the leader. if cc.NodeID == n.leader() && !n.isLeader() { - if err = n.raftNode.Campaign(n.Ctx); err != nil { + if err = n.raftNode.Campaign(ctx); err != nil { return err } } @@ -1547,8 +1585,7 @@ func (n *Node) ConnectToMember(addr string, timeout time.Duration) (*membership. } return &membership.Member{ - RaftClient: api.NewRaftClient(conn), - Conn: conn, + Conn: conn, }, nil } @@ -1579,7 +1616,7 @@ func createConfigChangeEnts(ids []uint64, self uint64, term, index uint64) []raf } data, err := cc.Marshal() if err != nil { - log.G(context.Background()).Panicf("marshal configuration change should never fail: %v", err) + log.L.WithError(err).Panic("marshal configuration change should never fail") } e := raftpb.Entry{ Type: raftpb.EntryConfChange, @@ -1594,7 +1631,7 @@ func createConfigChangeEnts(ids []uint64, self uint64, term, index uint64) []raf node := &api.RaftMember{RaftID: self} meta, err := node.Marshal() if err != nil { - log.G(context.Background()).Panicf("marshal member should never fail: %v", err) + log.L.WithError(err).Panic("marshal member should never fail") } cc := &raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, @@ -1603,7 +1640,7 @@ func createConfigChangeEnts(ids []uint64, self uint64, term, index uint64) []raf } data, err := cc.Marshal() if err != nil { - log.G(context.Background()).Panicf("marshal configuration change should never fail: %v", err) + log.L.WithError(err).Panic("marshal configuration change should never fail") } e := raftpb.Entry{ Type: raftpb.EntryConfChange, @@ -1637,7 +1674,7 @@ func getIDs(snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 { } var cc raftpb.ConfChange if err := cc.Unmarshal(e.Data); err != nil { - log.G(context.Background()).Panicf("unmarshal configuration change should never fail: %v", err) + log.L.WithError(err).Panic("unmarshal configuration change should never fail") } switch cc.Type { case raftpb.ConfChangeAddNode: @@ -1647,7 +1684,7 @@ func getIDs(snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 { case raftpb.ConfChangeUpdateNode: // do nothing default: - log.G(context.Background()).Panic("ConfChange Type should be either ConfChangeAddNode or ConfChangeRemoveNode!") + log.L.Panic("ConfChange Type should be either ConfChangeAddNode or ConfChangeRemoveNode!") } } var sids []uint64 diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go index 3209cb40f2..c47da6ce68 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go @@ -485,7 +485,7 @@ func (n *Node) saveSnapshot(snapshot raftpb.Snapshot, keepOldSnapshots uint64) e return nil } -func (n *Node) doSnapshot(raftConfig *api.RaftConfig) { +func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { snapshot := api.Snapshot{Version: api.Snapshot_V0} for _, member := range n.cluster.Members() { snapshot.Membership.Members = append(snapshot.Membership.Members, @@ -515,19 +515,19 @@ func (n *Node) doSnapshot(raftConfig *api.RaftConfig) { snapshot.Store = *storeSnapshot }) if err != nil { - n.Config.Logger.Error(err) + log.G(ctx).WithError(err).Error("failed to read snapshot from store") return } d, err := snapshot.Marshal() if err != nil { - n.Config.Logger.Error(err) + log.G(ctx).WithError(err).Error("failed to marshal snapshot") return } snap, err := n.raftStore.CreateSnapshot(appliedIndex, &n.confState, d) if err == nil { if err := n.saveSnapshot(snap, raftConfig.KeepOldSnapshots); err != nil { - n.Config.Logger.Error(err) + log.G(ctx).WithError(err).Error("failed to save snapshot") return } snapshotIndex = appliedIndex @@ -535,11 +535,11 @@ func (n *Node) doSnapshot(raftConfig *api.RaftConfig) { if appliedIndex > raftConfig.LogEntriesForSlowFollowers { err := n.raftStore.Compact(appliedIndex - raftConfig.LogEntriesForSlowFollowers) if err != nil && err != raft.ErrCompacted { - n.Config.Logger.Error(err) + log.G(ctx).WithError(err).Error("failed to compact snapshot") } } } else if err != raft.ErrSnapOutOfDate { - n.Config.Logger.Error(err) + log.G(ctx).WithError(err).Error("failed to create snapshot") } }(n.appliedIndex, n.snapshotIndex) diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/agent/node.go b/components/engine/vendor/src/github.com/docker/swarmkit/node/node.go similarity index 97% rename from components/engine/vendor/src/github.com/docker/swarmkit/agent/node.go rename to components/engine/vendor/src/github.com/docker/swarmkit/node/node.go index 35c720dc18..0367edabad 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/agent/node.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/node/node.go @@ -1,4 +1,4 @@ -package agent +package node import ( "crypto/tls" @@ -14,6 +14,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/boltdb/bolt" + "github.com/docker/swarmkit/agent" "github.com/docker/swarmkit/agent/exec" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" @@ -29,8 +30,13 @@ import ( const stateFilename = "state.json" -// NodeConfig provides values for a Node. -type NodeConfig struct { +var ( + errNodeStarted = errors.New("node: already started") + errNodeNotStarted = errors.New("node: not started") +) + +// Config provides values for a Node. +type Config struct { // Hostname is the name of host for agent instance. Hostname string @@ -80,7 +86,7 @@ type NodeConfig struct { // cluster. Node handles workloads and may also run as a manager. type Node struct { sync.RWMutex - config *NodeConfig + config *Config remotes *persistentRemotes role string roleCond *sync.Cond @@ -96,7 +102,7 @@ type Node struct { certificateRequested chan struct{} // closed when certificate issue request has been sent by node closed chan struct{} err error - agent *Agent + agent *agent.Agent manager *manager.Manager roleChangeReq chan api.NodeRole // used to send role updates from the dispatcher api on promotion/demotion } @@ -116,8 +122,8 @@ func (n *Node) RemoteAPIAddr() (string, error) { return addr.String(), nil } -// NewNode returns new Node instance. -func NewNode(c *NodeConfig) (*Node, error) { +// New returns new Node instance. +func New(c *Config) (*Node, error) { if err := os.MkdirAll(c.StateDir, 0700); err != nil { return nil, err } @@ -369,7 +375,7 @@ func (n *Node) runAgent(ctx context.Context, db *bolt.DB, creds credentials.Tran return ctx.Err() } - agent, err := New(&Config{ + a, err := agent.New(&agent.Config{ Hostname: n.config.Hostname, Managers: n.remotes, Executor: n.config.Executor, @@ -380,12 +386,12 @@ func (n *Node) runAgent(ctx context.Context, db *bolt.DB, creds credentials.Tran if err != nil { return err } - if err := agent.Start(ctx); err != nil { + if err := a.Start(ctx); err != nil { return err } n.Lock() - n.agent = agent + n.agent = a n.Unlock() defer func() { @@ -395,13 +401,13 @@ func (n *Node) runAgent(ctx context.Context, db *bolt.DB, creds credentials.Tran }() go func() { - <-agent.Ready() + <-a.Ready() close(ready) }() // todo: manually call stop on context cancellation? - return agent.Err(context.Background()) + return a.Err(context.Background()) } // Ready returns a channel that is closed after node's initialization has @@ -483,7 +489,7 @@ func (n *Node) Manager() *manager.Manager { } // Agent returns agent instance started by node. May be nil. -func (n *Node) Agent() *Agent { +func (n *Node) Agent() *agent.Agent { n.RLock() defer n.RUnlock() return n.agent diff --git a/components/engine/vendor/src/google.golang.org/grpc/balancer.go b/components/engine/vendor/src/google.golang.org/grpc/balancer.go index 419e214611..e217a2077c 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/balancer.go +++ b/components/engine/vendor/src/google.golang.org/grpc/balancer.go @@ -38,6 +38,7 @@ import ( "sync" "golang.org/x/net/context" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/naming" ) @@ -52,6 +53,14 @@ type Address struct { Metadata interface{} } +// BalancerConfig specifies the configurations for Balancer. +type BalancerConfig struct { + // DialCreds is the transport credential the Balancer implementation can + // use to dial to a remote load balancer server. The Balancer implementations + // can ignore this if it does not need to talk to another party securely. + DialCreds credentials.TransportCredentials +} + // BalancerGetOptions configures a Get call. // This is the EXPERIMENTAL API and may be changed or extended in the future. type BalancerGetOptions struct { @@ -66,11 +75,11 @@ type Balancer interface { // Start does the initialization work to bootstrap a Balancer. For example, // this function may start the name resolution and watch the updates. It will // be called when dialing. - Start(target string) error + Start(target string, config BalancerConfig) error // Up informs the Balancer that gRPC has a connection to the server at // addr. It returns down which is called once the connection to addr gets // lost or closed. - // TODO: It is not clear how to construct and take advantage the meaningful error + // TODO: It is not clear how to construct and take advantage of the meaningful error // parameter for down. Need realistic demands to guide. Up(addr Address) (down func(error)) // Get gets the address of a server for the RPC corresponding to ctx. @@ -205,7 +214,12 @@ func (rr *roundRobin) watchAddrUpdates() error { return nil } -func (rr *roundRobin) Start(target string) error { +func (rr *roundRobin) Start(target string, config BalancerConfig) error { + rr.mu.Lock() + defer rr.mu.Unlock() + if rr.done { + return ErrClientConnClosing + } if rr.r == nil { // If there is no name resolver installed, it is not needed to // do name resolution. In this case, target is added into rr.addrs diff --git a/components/engine/vendor/src/google.golang.org/grpc/call.go b/components/engine/vendor/src/google.golang.org/grpc/call.go index fea07998d7..788b3d9281 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/call.go +++ b/components/engine/vendor/src/google.golang.org/grpc/call.go @@ -96,7 +96,7 @@ func sendRequest(ctx context.Context, codec Codec, compressor Compressor, callHd } outBuf, err := encode(codec, args, compressor, cbuf) if err != nil { - return nil, transport.StreamErrorf(codes.Internal, "grpc: %v", err) + return nil, Errorf(codes.Internal, "grpc: %v", err) } err = t.Write(stream, outBuf, opts) // t.NewStream(...) could lead to an early rejection of the RPC (e.g., the service/method @@ -112,7 +112,14 @@ func sendRequest(ctx context.Context, codec Codec, compressor Compressor, callHd // Invoke sends the RPC request on the wire and returns after response is received. // Invoke is called by generated code. Also users can call Invoke directly when it // is really needed in their use cases. -func Invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) (err error) { +func Invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) error { + if cc.dopts.unaryInt != nil { + return cc.dopts.unaryInt(ctx, method, args, reply, cc, invoke, opts...) + } + return invoke(ctx, method, args, reply, cc, opts...) +} + +func invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) (err error) { c := defaultCallInfo for _, o := range opts { if err := o.before(&c); err != nil { diff --git a/components/engine/vendor/src/google.golang.org/grpc/clientconn.go b/components/engine/vendor/src/google.golang.org/grpc/clientconn.go index 27e74e6f22..11dce44fd3 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/clientconn.go +++ b/components/engine/vendor/src/google.golang.org/grpc/clientconn.go @@ -83,15 +83,17 @@ var ( // dialOptions configure a Dial call. dialOptions are set by the DialOption // values passed to Dial. type dialOptions struct { - codec Codec - cp Compressor - dc Decompressor - bs backoffStrategy - balancer Balancer - block bool - insecure bool - timeout time.Duration - copts transport.ConnectOptions + unaryInt UnaryClientInterceptor + streamInt StreamClientInterceptor + codec Codec + cp Compressor + dc Decompressor + bs backoffStrategy + balancer Balancer + block bool + insecure bool + timeout time.Duration + copts transport.ConnectOptions } // DialOption configures how we set up the connection. @@ -215,19 +217,48 @@ func WithUserAgent(s string) DialOption { } } +// WithUnaryInterceptor returns a DialOption that specifies the interceptor for unary RPCs. +func WithUnaryInterceptor(f UnaryClientInterceptor) DialOption { + return func(o *dialOptions) { + o.unaryInt = f + } +} + +// WithStreamInterceptor returns a DialOption that specifies the interceptor for streaming RPCs. +func WithStreamInterceptor(f StreamClientInterceptor) DialOption { + return func(o *dialOptions) { + o.streamInt = f + } +} + // Dial creates a client connection to the given target. func Dial(target string, opts ...DialOption) (*ClientConn, error) { return DialContext(context.Background(), target, opts...) } -// DialContext creates a client connection to the given target -// using the supplied context. -func DialContext(ctx context.Context, target string, opts ...DialOption) (*ClientConn, error) { +// DialContext creates a client connection to the given target. ctx can be used to +// cancel or expire the pending connecting. Once this function returns, the +// cancellation and expiration of ctx will be noop. Users should call ClientConn.Close +// to terminate all the pending operations after this function returns. +// This is the EXPERIMENTAL API. +func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *ClientConn, err error) { cc := &ClientConn{ target: target, conns: make(map[Address]*addrConn), } - cc.ctx, cc.cancel = context.WithCancel(ctx) + cc.ctx, cc.cancel = context.WithCancel(context.Background()) + defer func() { + select { + case <-ctx.Done(): + conn, err = nil, ctx.Err() + default: + } + + if err != nil { + cc.Close() + } + }() + for _, opt := range opts { opt(&cc.dopts) } @@ -239,31 +270,47 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (*Clien if cc.dopts.bs == nil { cc.dopts.bs = DefaultBackoffConfig } - - var ( - ok bool - addrs []Address - ) - if cc.dopts.balancer == nil { - // Connect to target directly if balancer is nil. - addrs = append(addrs, Address{Addr: target}) + creds := cc.dopts.copts.TransportCredentials + if creds != nil && creds.Info().ServerName != "" { + cc.authority = creds.Info().ServerName } else { - if err := cc.dopts.balancer.Start(target); err != nil { - return nil, err - } - ch := cc.dopts.balancer.Notify() - if ch == nil { - // There is no name resolver installed. - addrs = append(addrs, Address{Addr: target}) - } else { - addrs, ok = <-ch - if !ok || len(addrs) == 0 { - return nil, errNoAddr - } + colonPos := strings.LastIndex(target, ":") + if colonPos == -1 { + colonPos = len(target) } + cc.authority = target[:colonPos] } + var ok bool waitC := make(chan error, 1) go func() { + var addrs []Address + if cc.dopts.balancer == nil { + // Connect to target directly if balancer is nil. + addrs = append(addrs, Address{Addr: target}) + } else { + var credsClone credentials.TransportCredentials + if creds != nil { + credsClone = creds.Clone() + } + config := BalancerConfig{ + DialCreds: credsClone, + } + if err := cc.dopts.balancer.Start(target, config); err != nil { + waitC <- err + return + } + ch := cc.dopts.balancer.Notify() + if ch == nil { + // There is no name resolver installed. + addrs = append(addrs, Address{Addr: target}) + } else { + addrs, ok = <-ch + if !ok || len(addrs) == 0 { + waitC <- errNoAddr + return + } + } + } for _, a := range addrs { if err := cc.resetAddrConn(a, false, nil); err != nil { waitC <- err @@ -277,16 +324,13 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (*Clien timeoutCh = time.After(cc.dopts.timeout) } select { + case <-ctx.Done(): + return nil, ctx.Err() case err := <-waitC: if err != nil { - cc.Close() return nil, err } - case <-cc.ctx.Done(): - cc.Close() - return nil, cc.ctx.Err() case <-timeoutCh: - cc.Close() return nil, ErrClientConnTimeout } // If balancer is nil or balancer.Notify() is nil, ok will be false here. @@ -294,11 +338,6 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (*Clien if ok { go cc.lbWatcher() } - colonPos := strings.LastIndex(target, ":") - if colonPos == -1 { - colonPos = len(target) - } - cc.authority = target[:colonPos] return cc, nil } @@ -652,7 +691,7 @@ func (ac *addrConn) resetTransport(closeTransport bool) error { if e, ok := err.(transport.ConnectionError); ok && !e.Temporary() { return err } - grpclog.Printf("grpc: addrConn.resetTransport failed to create client transport: %v; Reconnecting to %q", err, ac.addr) + grpclog.Printf("grpc: addrConn.resetTransport failed to create client transport: %v; Reconnecting to %v", err, ac.addr) ac.mu.Lock() if ac.state == Shutdown { // ac.tearDown(...) has been invoked. diff --git a/components/engine/vendor/src/google.golang.org/grpc/credentials/credentials.go b/components/engine/vendor/src/google.golang.org/grpc/credentials/credentials.go index 3f17b70628..5555ef024f 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/credentials/credentials.go +++ b/components/engine/vendor/src/google.golang.org/grpc/credentials/credentials.go @@ -40,6 +40,7 @@ package credentials // import "google.golang.org/grpc/credentials" import ( "crypto/tls" "crypto/x509" + "errors" "fmt" "io/ioutil" "net" @@ -71,7 +72,7 @@ type PerRPCCredentials interface { } // ProtocolInfo provides information regarding the gRPC wire protocol version, -// security protocol, security protocol version in use, etc. +// security protocol, security protocol version in use, server name, etc. type ProtocolInfo struct { // ProtocolVersion is the gRPC wire protocol version. ProtocolVersion string @@ -79,6 +80,8 @@ type ProtocolInfo struct { SecurityProtocol string // SecurityVersion is the security protocol version. SecurityVersion string + // ServerName is the user-configured server name. + ServerName string } // AuthInfo defines the common interface for the auth information the users are interested in. @@ -86,6 +89,12 @@ type AuthInfo interface { AuthType() string } +var ( + // ErrConnDispatched indicates that rawConn has been dispatched out of gRPC + // and the caller should not close rawConn. + ErrConnDispatched = errors.New("credentials: rawConn is dispatched out of gRPC") +) + // TransportCredentials defines the common interface for all the live gRPC wire // protocols and supported transport security protocols (e.g., TLS, SSL). type TransportCredentials interface { @@ -100,6 +109,12 @@ type TransportCredentials interface { ServerHandshake(net.Conn) (net.Conn, AuthInfo, error) // Info provides the ProtocolInfo of this TransportCredentials. Info() ProtocolInfo + // Clone makes a copy of this TransportCredentials. + Clone() TransportCredentials + // OverrideServerName overrides the server name used to verify the hostname on the returned certificates from the server. + // gRPC internals also use it to override the virtual hosting name if it is set. + // It must be called before dialing. Currently, this is only used by grpclb. + OverrideServerName(string) error } // TLSInfo contains the auth information for a TLS authenticated connection. @@ -123,19 +138,10 @@ func (c tlsCreds) Info() ProtocolInfo { return ProtocolInfo{ SecurityProtocol: "tls", SecurityVersion: "1.2", + ServerName: c.config.ServerName, } } -// GetRequestMetadata returns nil, nil since TLS credentials does not have -// metadata. -func (c *tlsCreds) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { - return nil, nil -} - -func (c *tlsCreds) RequireTransportSecurity() bool { - return true -} - func (c *tlsCreds) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (_ net.Conn, _ AuthInfo, err error) { // use local cfg to avoid clobbering ServerName if using multiple endpoints cfg := cloneTLSConfig(c.config) @@ -172,6 +178,15 @@ func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) return conn, TLSInfo{conn.ConnectionState()}, nil } +func (c *tlsCreds) Clone() TransportCredentials { + return NewTLS(c.config) +} + +func (c *tlsCreds) OverrideServerName(serverNameOverride string) error { + c.config.ServerName = serverNameOverride + return nil +} + // NewTLS uses c to construct a TransportCredentials based on TLS. func NewTLS(c *tls.Config) TransportCredentials { tc := &tlsCreds{cloneTLSConfig(c)} @@ -180,12 +195,16 @@ func NewTLS(c *tls.Config) TransportCredentials { } // NewClientTLSFromCert constructs a TLS from the input certificate for client. -func NewClientTLSFromCert(cp *x509.CertPool, serverName string) TransportCredentials { - return NewTLS(&tls.Config{ServerName: serverName, RootCAs: cp}) +// serverNameOverride is for testing only. If set to a non empty string, +// it will override the virtual host name of authority (e.g. :authority header field) in requests. +func NewClientTLSFromCert(cp *x509.CertPool, serverNameOverride string) TransportCredentials { + return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}) } // NewClientTLSFromFile constructs a TLS from the input certificate file for client. -func NewClientTLSFromFile(certFile, serverName string) (TransportCredentials, error) { +// serverNameOverride is for testing only. If set to a non empty string, +// it will override the virtual host name of authority (e.g. :authority header field) in requests. +func NewClientTLSFromFile(certFile, serverNameOverride string) (TransportCredentials, error) { b, err := ioutil.ReadFile(certFile) if err != nil { return nil, err @@ -194,7 +213,7 @@ func NewClientTLSFromFile(certFile, serverName string) (TransportCredentials, er if !cp.AppendCertsFromPEM(b) { return nil, fmt.Errorf("credentials: failed to append certificates") } - return NewTLS(&tls.Config{ServerName: serverName, RootCAs: cp}), nil + return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}), nil } // NewServerTLSFromCert constructs a TLS from the input certificate for server. diff --git a/components/engine/vendor/src/google.golang.org/grpc/interceptor.go b/components/engine/vendor/src/google.golang.org/grpc/interceptor.go index 588f59e5ab..8d932efed7 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/interceptor.go +++ b/components/engine/vendor/src/google.golang.org/grpc/interceptor.go @@ -37,6 +37,22 @@ import ( "golang.org/x/net/context" ) +// UnaryInvoker is called by UnaryClientInterceptor to complete RPCs. +type UnaryInvoker func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error + +// UnaryClientInterceptor intercepts the execution of a unary RPC on the client. inovker is the handler to complete the RPC +// and it is the responsibility of the interceptor to call it. +// This is the EXPERIMENTAL API. +type UnaryClientInterceptor func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error + +// Streamer is called by StreamClientInterceptor to create a ClientStream. +type Streamer func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) + +// StreamClientInterceptor intercepts the creation of ClientStream. It may return a custom ClientStream to intercept all I/O +// operations. streamer is the handlder to create a ClientStream and it is the responsibility of the interceptor to call it. +// This is the EXPERIMENTAL API. +type StreamClientInterceptor func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error) + // UnaryServerInfo consists of various information about a unary RPC on // server side. All per-rpc information may be mutated by the interceptor. type UnaryServerInfo struct { diff --git a/components/engine/vendor/src/google.golang.org/grpc/metadata/metadata.go b/components/engine/vendor/src/google.golang.org/grpc/metadata/metadata.go index 954c0f77c6..3c0ca7a36c 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/metadata/metadata.go +++ b/components/engine/vendor/src/google.golang.org/grpc/metadata/metadata.go @@ -117,10 +117,17 @@ func (md MD) Len() int { // Copy returns a copy of md. func (md MD) Copy() MD { + return Join(md) +} + +// Join joins any number of MDs into a single MD. +// The order of values for each key is determined by the order in which +// the MDs containing those values are presented to Join. +func Join(mds ...MD) MD { out := MD{} - for k, v := range md { - for _, i := range v { - out[k] = append(out[k], i) + for _, md := range mds { + for k, v := range md { + out[k] = append(out[k], v...) } } return out diff --git a/components/engine/vendor/src/google.golang.org/grpc/rpc_util.go b/components/engine/vendor/src/google.golang.org/grpc/rpc_util.go index 35ac9cc7b0..6b60095d56 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/rpc_util.go +++ b/components/engine/vendor/src/google.golang.org/grpc/rpc_util.go @@ -303,10 +303,10 @@ func checkRecvPayload(pf payloadFormat, recvCompress string, dc Decompressor) er case compressionNone: case compressionMade: if dc == nil || recvCompress != dc.Type() { - return transport.StreamErrorf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", recvCompress) + return Errorf(codes.Unimplemented, "grpc: Decompressor is not installed for grpc-encoding %q", recvCompress) } default: - return transport.StreamErrorf(codes.Internal, "grpc: received unexpected payload format %d", pf) + return Errorf(codes.Internal, "grpc: received unexpected payload format %d", pf) } return nil } diff --git a/components/engine/vendor/src/google.golang.org/grpc/server.go b/components/engine/vendor/src/google.golang.org/grpc/server.go index 1ed8aac9eb..debbd79aed 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/server.go +++ b/components/engine/vendor/src/google.golang.org/grpc/server.go @@ -324,7 +324,7 @@ func (s *Server) useTransportAuthenticator(rawConn net.Conn) (net.Conn, credenti // Serve accepts incoming connections on the listener lis, creating a new // ServerTransport and service goroutine for each. The service goroutines // read gRPC requests and then call the registered handlers to reply to them. -// Service returns when lis.Accept fails. lis will be closed when +// Serve returns when lis.Accept fails. lis will be closed when // this method returns. func (s *Server) Serve(lis net.Listener) error { s.mu.Lock() @@ -367,7 +367,10 @@ func (s *Server) handleRawConn(rawConn net.Conn) { s.errorf("ServerHandshake(%q) failed: %v", rawConn.RemoteAddr(), err) s.mu.Unlock() grpclog.Printf("grpc: Server.Serve failed to complete security handshake from %q: %v", rawConn.RemoteAddr(), err) - rawConn.Close() + // If serverHandShake returns ErrConnDispatched, keep rawConn open. + if err != credentials.ErrConnDispatched { + rawConn.Close() + } return } @@ -544,7 +547,7 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. return err } if err == io.ErrUnexpectedEOF { - err = transport.StreamError{Code: codes.Internal, Desc: "io.ErrUnexpectedEOF"} + err = Errorf(codes.Internal, io.ErrUnexpectedEOF.Error()) } if err != nil { switch err := err.(type) { @@ -566,8 +569,8 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. if err := checkRecvPayload(pf, stream.RecvCompress(), s.opts.dc); err != nil { switch err := err.(type) { - case transport.StreamError: - if err := t.WriteStatus(stream, err.Code, err.Desc); err != nil { + case *rpcError: + if err := t.WriteStatus(stream, err.code, err.desc); err != nil { grpclog.Printf("grpc: Server.processUnaryRPC failed to write status %v", err) } default: @@ -870,25 +873,28 @@ func SendHeader(ctx context.Context, md metadata.MD) error { } stream, ok := transport.StreamFromContext(ctx) if !ok { - return fmt.Errorf("grpc: failed to fetch the stream from the context %v", ctx) + return Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) } t := stream.ServerTransport() if t == nil { grpclog.Fatalf("grpc: SendHeader: %v has no ServerTransport to send header metadata.", stream) } - return t.WriteHeader(stream, md) + if err := t.WriteHeader(stream, md); err != nil { + return toRPCErr(err) + } + return nil } // SetTrailer sets the trailer metadata that will be sent when an RPC returns. -// It may be called at most once from a unary RPC handler. The ctx is the RPC -// handler's Context or one derived from it. +// When called more than once, all the provided metadata will be merged. +// The ctx is the RPC handler's Context or one derived from it. func SetTrailer(ctx context.Context, md metadata.MD) error { if md.Len() == 0 { return nil } stream, ok := transport.StreamFromContext(ctx) if !ok { - return fmt.Errorf("grpc: failed to fetch the stream from the context %v", ctx) + return Errorf(codes.Internal, "grpc: failed to fetch the stream from the context %v", ctx) } return stream.SetTrailer(md) } diff --git a/components/engine/vendor/src/google.golang.org/grpc/stream.go b/components/engine/vendor/src/google.golang.org/grpc/stream.go index 51df3f01da..68d777b509 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/stream.go +++ b/components/engine/vendor/src/google.golang.org/grpc/stream.go @@ -97,7 +97,14 @@ type ClientStream interface { // NewClientStream creates a new Stream for the client side. This is called // by generated code. -func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (_ ClientStream, err error) { +func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) { + if cc.dopts.streamInt != nil { + return cc.dopts.streamInt(ctx, desc, cc, method, newClientStream, opts...) + } + return newClientStream(ctx, desc, cc, method, opts...) +} + +func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (_ ClientStream, err error) { var ( t transport.ClientTransport s *transport.Stream @@ -296,7 +303,7 @@ func (cs *clientStream) SendMsg(m interface{}) (err error) { } }() if err != nil { - return transport.StreamErrorf(codes.Internal, "grpc: %v", err) + return Errorf(codes.Internal, "grpc: %v", err) } return cs.t.Write(cs.s, out, &transport.Options{Last: false}) } @@ -407,8 +414,8 @@ type ServerStream interface { // after SendProto. It fails if called multiple times or if // called after SendProto. SendHeader(metadata.MD) error - // SetTrailer sets the trailer metadata which will be sent with the - // RPC status. + // SetTrailer sets the trailer metadata which will be sent with the RPC status. + // When called more than once, all the provided metadata will be merged. SetTrailer(metadata.MD) Stream } @@ -468,10 +475,13 @@ func (ss *serverStream) SendMsg(m interface{}) (err error) { } }() if err != nil { - err = transport.StreamErrorf(codes.Internal, "grpc: %v", err) + err = Errorf(codes.Internal, "grpc: %v", err) return err } - return ss.t.Write(ss.s, out, &transport.Options{Last: false}) + if err := ss.t.Write(ss.s, out, &transport.Options{Last: false}); err != nil { + return toRPCErr(err) + } + return nil } func (ss *serverStream) RecvMsg(m interface{}) (err error) { @@ -489,5 +499,14 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) { ss.mu.Unlock() } }() - return recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxMsgSize) + if err := recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxMsgSize); err != nil { + if err == io.EOF { + return err + } + if err == io.ErrUnexpectedEOF { + err = Errorf(codes.Internal, io.ErrUnexpectedEOF.Error()) + } + return toRPCErr(err) + } + return nil } diff --git a/components/engine/vendor/src/google.golang.org/grpc/transport/handler_server.go b/components/engine/vendor/src/google.golang.org/grpc/transport/handler_server.go index 30e21ac0f4..114e34906a 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/transport/handler_server.go +++ b/components/engine/vendor/src/google.golang.org/grpc/transport/handler_server.go @@ -85,7 +85,7 @@ func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request) (ServerTr if v := r.Header.Get("grpc-timeout"); v != "" { to, err := decodeTimeout(v) if err != nil { - return nil, StreamErrorf(codes.Internal, "malformed time-out: %v", err) + return nil, streamErrorf(codes.Internal, "malformed time-out: %v", err) } st.timeoutSet = true st.timeout = to @@ -393,5 +393,5 @@ func mapRecvMsgError(err error) error { } } } - return ConnectionError{Desc: err.Error()} + return connectionErrorf(true, err, err.Error()) } diff --git a/components/engine/vendor/src/google.golang.org/grpc/transport/http2_client.go b/components/engine/vendor/src/google.golang.org/grpc/transport/http2_client.go index 5819cb8a43..3c185541a5 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/transport/http2_client.go +++ b/components/engine/vendor/src/google.golang.org/grpc/transport/http2_client.go @@ -114,14 +114,42 @@ func dial(fn func(context.Context, string) (net.Conn, error), ctx context.Contex return dialContext(ctx, "tcp", addr) } +func isTemporary(err error) bool { + switch err { + case io.EOF: + // Connection closures may be resolved upon retry, and are thus + // treated as temporary. + return true + case context.DeadlineExceeded: + // In Go 1.7, context.DeadlineExceeded implements Timeout(), and this + // special case is not needed. Until then, we need to keep this + // clause. + return true + } + + switch err := err.(type) { + case interface { + Temporary() bool + }: + return err.Temporary() + case interface { + Timeout() bool + }: + // Timeouts may be resolved upon retry, and are thus treated as + // temporary. + return err.Timeout() + } + return false +} + // newHTTP2Client constructs a connected ClientTransport to addr based on HTTP2 // and starts to receive messages on it. Non-nil error returns if construction // fails. func newHTTP2Client(ctx context.Context, addr string, opts ConnectOptions) (_ ClientTransport, err error) { scheme := "http" - conn, connErr := dial(opts.Dialer, ctx, addr) - if connErr != nil { - return nil, ConnectionErrorf(true, connErr, "transport: %v", connErr) + conn, err := dial(opts.Dialer, ctx, addr) + if err != nil { + return nil, connectionErrorf(true, err, "transport: %v", err) } // Any further errors will close the underlying connection defer func(conn net.Conn) { @@ -132,12 +160,13 @@ func newHTTP2Client(ctx context.Context, addr string, opts ConnectOptions) (_ Cl var authInfo credentials.AuthInfo if creds := opts.TransportCredentials; creds != nil { scheme = "https" - conn, authInfo, connErr = creds.ClientHandshake(ctx, addr, conn) - } - if connErr != nil { - // Credentials handshake error is not a temporary error (unless the error - // was the connection closing). - return nil, ConnectionErrorf(connErr == io.EOF, connErr, "transport: %v", connErr) + conn, authInfo, err = creds.ClientHandshake(ctx, addr, conn) + if err != nil { + // Credentials handshake errors are typically considered permanent + // to avoid retrying on e.g. bad certificates. + temp := isTemporary(err) + return nil, connectionErrorf(temp, err, "transport: %v", err) + } } ua := primaryUA if opts.UserAgent != "" { @@ -176,11 +205,11 @@ func newHTTP2Client(ctx context.Context, addr string, opts ConnectOptions) (_ Cl n, err := t.conn.Write(clientPreface) if err != nil { t.Close() - return nil, ConnectionErrorf(true, err, "transport: %v", err) + return nil, connectionErrorf(true, err, "transport: %v", err) } if n != len(clientPreface) { t.Close() - return nil, ConnectionErrorf(true, err, "transport: preface mismatch, wrote %d bytes; want %d", n, len(clientPreface)) + return nil, connectionErrorf(true, err, "transport: preface mismatch, wrote %d bytes; want %d", n, len(clientPreface)) } if initialWindowSize != defaultWindowSize { err = t.framer.writeSettings(true, http2.Setting{ @@ -192,13 +221,13 @@ func newHTTP2Client(ctx context.Context, addr string, opts ConnectOptions) (_ Cl } if err != nil { t.Close() - return nil, ConnectionErrorf(true, err, "transport: %v", err) + return nil, connectionErrorf(true, err, "transport: %v", err) } // Adjust the connection flow control window if needed. if delta := uint32(initialConnWindowSize - defaultWindowSize); delta > 0 { if err := t.framer.writeWindowUpdate(true, 0, delta); err != nil { t.Close() - return nil, ConnectionErrorf(true, err, "transport: %v", err) + return nil, connectionErrorf(true, err, "transport: %v", err) } } go t.controller() @@ -223,8 +252,10 @@ func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream { s.windowHandler = func(n int) { t.updateWindow(s, uint32(n)) } - // Make a stream be able to cancel the pending operations by itself. - s.ctx, s.cancel = context.WithCancel(ctx) + // The client side stream context should have exactly the same life cycle with the user provided context. + // That means, s.ctx should be read-only. And s.ctx is done iff ctx is done. + // So we use the original context here instead of creating a copy. + s.ctx = ctx s.dec = &recvBufferReader{ ctx: s.ctx, goAway: s.goAway, @@ -236,16 +267,6 @@ func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream { // NewStream creates a stream and register it into the transport as "active" // streams. func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Stream, err error) { - // Record the timeout value on the context. - var timeout time.Duration - if dl, ok := ctx.Deadline(); ok { - timeout = dl.Sub(time.Now()) - } - select { - case <-ctx.Done(): - return nil, ContextErr(ctx.Err()) - default: - } pr := &peer.Peer{ Addr: t.conn.RemoteAddr(), } @@ -266,12 +287,12 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea } pos := strings.LastIndex(callHdr.Method, "/") if pos == -1 { - return nil, StreamErrorf(codes.InvalidArgument, "transport: malformed method name: %q", callHdr.Method) + return nil, streamErrorf(codes.InvalidArgument, "transport: malformed method name: %q", callHdr.Method) } audience := "https://" + callHdr.Host + port + callHdr.Method[:pos] data, err := c.GetRequestMetadata(ctx, audience) if err != nil { - return nil, StreamErrorf(codes.InvalidArgument, "transport: %v", err) + return nil, streamErrorf(codes.InvalidArgument, "transport: %v", err) } for k, v := range data { authData[k] = v @@ -352,9 +373,12 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea if callHdr.SendCompress != "" { t.hEnc.WriteField(hpack.HeaderField{Name: "grpc-encoding", Value: callHdr.SendCompress}) } - if timeout > 0 { + if dl, ok := ctx.Deadline(); ok { + // Send out timeout regardless its value. The server can detect timeout context by itself. + timeout := dl.Sub(time.Now()) t.hEnc.WriteField(hpack.HeaderField{Name: "grpc-timeout", Value: encodeTimeout(timeout)}) } + for k, v := range authData { // Capital header names are illegal in HTTP/2. k = strings.ToLower(k) @@ -408,7 +432,7 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea } if err != nil { t.notifyError(err) - return nil, ConnectionErrorf(true, err, "transport: %v", err) + return nil, connectionErrorf(true, err, "transport: %v", err) } } t.writableChan <- 0 @@ -454,7 +478,7 @@ func (t *http2Client) CloseStream(s *Stream, err error) { } s.state = streamDone s.mu.Unlock() - if _, ok := err.(StreamError); ok { + if se, ok := err.(StreamError); ok && se.Code != codes.DeadlineExceeded { t.controlBuf.put(&resetStream{s.id, http2.ErrCodeCancel}) } } @@ -622,7 +646,7 @@ func (t *http2Client) Write(s *Stream, data []byte, opts *Options) error { // invoked. if err := t.framer.writeData(forceFlush, s.id, endStream, p); err != nil { t.notifyError(err) - return ConnectionErrorf(true, err, "transport: %v", err) + return connectionErrorf(true, err, "transport: %v", err) } if t.framer.adjustNumWriters(-1) == 0 { t.framer.flushWrite() @@ -670,7 +694,7 @@ func (t *http2Client) updateWindow(s *Stream, n uint32) { func (t *http2Client) handleData(f *http2.DataFrame) { size := len(f.Data()) if err := t.fc.onData(uint32(size)); err != nil { - t.notifyError(ConnectionErrorf(true, err, "%v", err)) + t.notifyError(connectionErrorf(true, err, "%v", err)) return } // Select the right stream to dispatch. @@ -776,7 +800,7 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { if t.state == reachable || t.state == draining { if f.LastStreamID > 0 && f.LastStreamID%2 != 1 { t.mu.Unlock() - t.notifyError(ConnectionErrorf(true, nil, "received illegal http2 GOAWAY frame: stream ID %d is even", f.LastStreamID)) + t.notifyError(connectionErrorf(true, nil, "received illegal http2 GOAWAY frame: stream ID %d is even", f.LastStreamID)) return } select { @@ -785,7 +809,7 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { // t.goAway has been closed (i.e.,multiple GoAways). if id < f.LastStreamID { t.mu.Unlock() - t.notifyError(ConnectionErrorf(true, nil, "received illegal http2 GOAWAY frame: previously recv GOAWAY frame with LastStramID %d, currently recv %d", id, f.LastStreamID)) + t.notifyError(connectionErrorf(true, nil, "received illegal http2 GOAWAY frame: previously recv GOAWAY frame with LastStramID %d, currently recv %d", id, f.LastStreamID)) return } t.prevGoAwayID = id @@ -823,6 +847,12 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) { state.processHeaderField(hf) } if state.err != nil { + s.mu.Lock() + if !s.headerDone { + close(s.headerChan) + s.headerDone = true + } + s.mu.Unlock() s.write(recvMsg{err: state.err}) // Something wrong. Stops reading even when there is remaining. return @@ -900,7 +930,7 @@ func (t *http2Client) reader() { t.mu.Unlock() if s != nil { // use error detail to provide better err message - handleMalformedHTTP2(s, StreamErrorf(http2ErrConvTab[se.Code], "%v", t.framer.errorDetail())) + handleMalformedHTTP2(s, streamErrorf(http2ErrConvTab[se.Code], "%v", t.framer.errorDetail())) } continue } else { diff --git a/components/engine/vendor/src/google.golang.org/grpc/transport/http2_server.go b/components/engine/vendor/src/google.golang.org/grpc/transport/http2_server.go index 16010d55fb..f753c4f1ea 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/transport/http2_server.go +++ b/components/engine/vendor/src/google.golang.org/grpc/transport/http2_server.go @@ -111,12 +111,12 @@ func newHTTP2Server(conn net.Conn, maxStreams uint32, authInfo credentials.AuthI Val: uint32(initialWindowSize)}) } if err := framer.writeSettings(true, settings...); err != nil { - return nil, ConnectionErrorf(true, err, "transport: %v", err) + return nil, connectionErrorf(true, err, "transport: %v", err) } // Adjust the connection flow control window if needed. if delta := uint32(initialConnWindowSize - defaultWindowSize); delta > 0 { if err := framer.writeWindowUpdate(true, 0, delta); err != nil { - return nil, ConnectionErrorf(true, err, "transport: %v", err) + return nil, connectionErrorf(true, err, "transport: %v", err) } } var buf bytes.Buffer @@ -448,7 +448,7 @@ func (t *http2Server) writeHeaders(s *Stream, b *bytes.Buffer, endStream bool) e } if err != nil { t.Close() - return ConnectionErrorf(true, err, "transport: %v", err) + return connectionErrorf(true, err, "transport: %v", err) } } return nil @@ -544,7 +544,7 @@ func (t *http2Server) Write(s *Stream, data []byte, opts *Options) error { s.mu.Lock() if s.state == streamDone { s.mu.Unlock() - return StreamErrorf(codes.Unknown, "the stream has been done") + return streamErrorf(codes.Unknown, "the stream has been done") } if !s.headerOk { writeHeaderFrame = true @@ -568,7 +568,7 @@ func (t *http2Server) Write(s *Stream, data []byte, opts *Options) error { } if err := t.framer.writeHeaders(false, p); err != nil { t.Close() - return ConnectionErrorf(true, err, "transport: %v", err) + return connectionErrorf(true, err, "transport: %v", err) } t.writableChan <- 0 } @@ -642,7 +642,7 @@ func (t *http2Server) Write(s *Stream, data []byte, opts *Options) error { } if err := t.framer.writeData(forceFlush, s.id, false, p); err != nil { t.Close() - return ConnectionErrorf(true, err, "transport: %v", err) + return connectionErrorf(true, err, "transport: %v", err) } if t.framer.adjustNumWriters(-1) == 0 { t.framer.flushWrite() diff --git a/components/engine/vendor/src/google.golang.org/grpc/transport/http_util.go b/components/engine/vendor/src/google.golang.org/grpc/transport/http_util.go index 79da512640..a3c68d4cac 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/transport/http_util.go +++ b/components/engine/vendor/src/google.golang.org/grpc/transport/http_util.go @@ -162,7 +162,7 @@ func (d *decodeState) processHeaderField(f hpack.HeaderField) { switch f.Name { case "content-type": if !validContentType(f.Value) { - d.setErr(StreamErrorf(codes.FailedPrecondition, "transport: received the unexpected content-type %q", f.Value)) + d.setErr(streamErrorf(codes.FailedPrecondition, "transport: received the unexpected content-type %q", f.Value)) return } case "grpc-encoding": @@ -170,7 +170,7 @@ func (d *decodeState) processHeaderField(f hpack.HeaderField) { case "grpc-status": code, err := strconv.Atoi(f.Value) if err != nil { - d.setErr(StreamErrorf(codes.Internal, "transport: malformed grpc-status: %v", err)) + d.setErr(streamErrorf(codes.Internal, "transport: malformed grpc-status: %v", err)) return } d.statusCode = codes.Code(code) @@ -181,7 +181,7 @@ func (d *decodeState) processHeaderField(f hpack.HeaderField) { var err error d.timeout, err = decodeTimeout(f.Value) if err != nil { - d.setErr(StreamErrorf(codes.Internal, "transport: malformed time-out: %v", err)) + d.setErr(streamErrorf(codes.Internal, "transport: malformed time-out: %v", err)) return } case ":path": @@ -253,6 +253,9 @@ func div(d, r time.Duration) int64 { // TODO(zhaoq): It is the simplistic and not bandwidth efficient. Improve it. func encodeTimeout(t time.Duration) string { + if t <= 0 { + return "0n" + } if d := div(t, time.Nanosecond); d <= maxTimeoutValue { return strconv.FormatInt(d, 10) + "n" } @@ -349,7 +352,7 @@ func decodeGrpcMessageUnchecked(msg string) string { for i := 0; i < lenMsg; i++ { c := msg[i] if c == percentByte && i+2 < lenMsg { - parsed, err := strconv.ParseInt(msg[i+1:i+3], 16, 8) + parsed, err := strconv.ParseUint(msg[i+1:i+3], 16, 8) if err != nil { buf.WriteByte(c) } else { diff --git a/components/engine/vendor/src/google.golang.org/grpc/transport/transport.go b/components/engine/vendor/src/google.golang.org/grpc/transport/transport.go index d59e511372..3d6b6a6d51 100644 --- a/components/engine/vendor/src/google.golang.org/grpc/transport/transport.go +++ b/components/engine/vendor/src/google.golang.org/grpc/transport/transport.go @@ -39,7 +39,6 @@ package transport // import "google.golang.org/grpc/transport" import ( "bytes" - "errors" "fmt" "io" "net" @@ -169,7 +168,8 @@ type Stream struct { // nil for client side Stream. st ServerTransport // ctx is the associated context of the stream. - ctx context.Context + ctx context.Context + // cancel is always nil for client side Stream. cancel context.CancelFunc // done is closed when the final status arrives. done chan struct{} @@ -286,19 +286,12 @@ func (s *Stream) StatusDesc() string { return s.statusDesc } -// ErrIllegalTrailerSet indicates that the trailer has already been set or it -// is too late to do so. -var ErrIllegalTrailerSet = errors.New("transport: trailer has been set") - // SetTrailer sets the trailer metadata which will be sent with the RPC status -// by the server. This can only be called at most once. Server side only. +// by the server. This can be called multiple times. Server side only. func (s *Stream) SetTrailer(md metadata.MD) error { s.mu.Lock() defer s.mu.Unlock() - if s.trailer != nil { - return ErrIllegalTrailerSet - } - s.trailer = md.Copy() + s.trailer = metadata.Join(s.trailer, md) return nil } @@ -476,16 +469,16 @@ type ServerTransport interface { Drain() } -// StreamErrorf creates an StreamError with the specified error code and description. -func StreamErrorf(c codes.Code, format string, a ...interface{}) StreamError { +// streamErrorf creates an StreamError with the specified error code and description. +func streamErrorf(c codes.Code, format string, a ...interface{}) StreamError { return StreamError{ Code: c, Desc: fmt.Sprintf(format, a...), } } -// ConnectionErrorf creates an ConnectionError with the specified error description. -func ConnectionErrorf(temp bool, e error, format string, a ...interface{}) ConnectionError { +// connectionErrorf creates an ConnectionError with the specified error description. +func connectionErrorf(temp bool, e error, format string, a ...interface{}) ConnectionError { return ConnectionError{ Desc: fmt.Sprintf(format, a...), temp: temp, @@ -522,10 +515,10 @@ func (e ConnectionError) Origin() error { var ( // ErrConnClosing indicates that the transport is closing. - ErrConnClosing = ConnectionError{Desc: "transport is closing", temp: true} + ErrConnClosing = connectionErrorf(true, nil, "transport is closing") // ErrStreamDrain indicates that the stream is rejected by the server because // the server stops accepting new RPCs. - ErrStreamDrain = StreamErrorf(codes.Unavailable, "the server stops accepting new RPCs") + ErrStreamDrain = streamErrorf(codes.Unavailable, "the server stops accepting new RPCs") ) // StreamError is an error that only affects one stream within a connection. @@ -542,9 +535,9 @@ func (e StreamError) Error() string { func ContextErr(err error) StreamError { switch err { case context.DeadlineExceeded: - return StreamErrorf(codes.DeadlineExceeded, "%v", err) + return streamErrorf(codes.DeadlineExceeded, "%v", err) case context.Canceled: - return StreamErrorf(codes.Canceled, "%v", err) + return streamErrorf(codes.Canceled, "%v", err) } panic(fmt.Sprintf("Unexpected error from context packet: %v", err)) }