diff --git a/components/engine/hack/vendor.sh b/components/engine/hack/vendor.sh index 79023252bb..dfa81b42ee 100755 --- a/components/engine/hack/vendor.sh +++ b/components/engine/hack/vendor.sh @@ -60,12 +60,12 @@ clone git golang.org/x/net 2beffdc2e92c8a3027590f898fe88f69af48a3f8 https://gith clone git golang.org/x/sys eb2c74142fd19a79b3f237334c7384d5167b1b46 https://github.com/golang/sys.git clone git github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3 clone git github.com/docker/go-connections fa2850ff103453a9ad190da0df0af134f0314b3d -clone git github.com/docker/engine-api 19b4fb48a86c3318e610e156ec06b684f79ac31d +clone git github.com/docker/engine-api 62043eb79d581a32ea849645277023c550732e52 clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837 clone git github.com/imdario/mergo 0.2.1 #get libnetwork packages -clone git github.com/docker/libnetwork ed311d050fda7821f2e7c53a7e08a0205923aef5 +clone git github.com/docker/libnetwork 377a7337f2387cce3be1df7a4503446147b68ff1 clone git github.com/docker/go-events 39718a26497694185f8fb58a7d6f31947f3dc42d clone git github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec @@ -139,10 +139,10 @@ clone git github.com/docker/docker-credential-helpers v0.3.0 clone git github.com/docker/containerd b93a33be39bc4ef0fb00bfcb79147a28c33d9d43 # cluster -clone git github.com/docker/swarmkit 3f135f206179ea157aeef2d1d401eb795f618da8 +clone git github.com/docker/swarmkit 036a4a1e934bd1bbb35c3ec7f85dea2ba6d4e336 clone git github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9 clone git github.com/gogo/protobuf 43a2e0b1c32252bfbbdf81f7faa7a88fb3fa4028 -clone git github.com/cloudflare/cfssl 92f037e39eb103fb30f9151be40d9ed267fc4ae2 +clone git github.com/cloudflare/cfssl b895b0549c0ff676f92cf09ba971ae02bb41367b clone git github.com/google/certificate-transparency 025a5cab06f6a819c455d9fdc9e2a1b6d0982284 clone git golang.org/x/crypto 3fbbcd23f1cb824e69491a5930cfeff09b12f4d2 https://github.com/golang/crypto.git clone git github.com/mreiferson/go-httpclient 63fe23f7434723dc904c901043af07931f293c47 diff --git a/components/engine/vendor/src/github.com/cloudflare/cfssl/api/api.go b/components/engine/vendor/src/github.com/cloudflare/cfssl/api/api.go new file mode 100644 index 0000000000..f1040caded --- /dev/null +++ b/components/engine/vendor/src/github.com/cloudflare/cfssl/api/api.go @@ -0,0 +1,231 @@ +// Package api implements an HTTP-based API and server for CFSSL. +package api + +import ( + "encoding/json" + "io/ioutil" + "net/http" + + "github.com/cloudflare/cfssl/errors" + "github.com/cloudflare/cfssl/log" +) + +// Handler is an interface providing a generic mechanism for handling HTTP requests. +type Handler interface { + Handle(w http.ResponseWriter, r *http.Request) error +} + +// HTTPHandler is a wrapper that encapsulates Handler interface as http.Handler. +// HTTPHandler also enforces that the Handler only responds to requests with registered HTTP methods. +type HTTPHandler struct { + Handler // CFSSL handler + Methods []string // The associated HTTP methods +} + +// HandlerFunc is similar to the http.HandlerFunc type; it serves as +// an adapter allowing the use of ordinary functions as Handlers. If +// f is a function with the appropriate signature, HandlerFunc(f) is a +// Handler object that calls f. +type HandlerFunc func(http.ResponseWriter, *http.Request) error + +// Handle calls f(w, r) +func (f HandlerFunc) Handle(w http.ResponseWriter, r *http.Request) error { + w.Header().Set("Content-Type", "application/json") + return f(w, r) +} + +// handleError is the centralised error handling and reporting. +func handleError(w http.ResponseWriter, err error) (code int) { + if err == nil { + return http.StatusOK + } + msg := err.Error() + httpCode := http.StatusInternalServerError + + // If it is recognized as HttpError emitted from cfssl, + // we rewrite the status code accordingly. If it is a + // cfssl error, set the http status to StatusBadRequest + switch err := err.(type) { + case *errors.HTTPError: + httpCode = err.StatusCode + code = err.StatusCode + case *errors.Error: + httpCode = http.StatusBadRequest + code = err.ErrorCode + msg = err.Message + } + + response := NewErrorResponse(msg, code) + jsonMessage, err := json.Marshal(response) + if err != nil { + log.Errorf("Failed to marshal JSON: %v", err) + } else { + msg = string(jsonMessage) + } + http.Error(w, msg, httpCode) + return code +} + +// ServeHTTP encapsulates the call to underlying Handler to handle the request +// and return the response with proper HTTP status code +func (h HTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var err error + var match bool + // Throw 405 when requested with an unsupported verb. + for _, m := range h.Methods { + if m == r.Method { + match = true + } + } + if match { + err = h.Handle(w, r) + } else { + err = errors.NewMethodNotAllowed(r.Method) + } + status := handleError(w, err) + log.Infof("%s - \"%s %s\" %d", r.RemoteAddr, r.Method, r.URL, status) +} + +// readRequestBlob takes a JSON-blob-encoded response body in the form +// map[string]string and returns it, the list of keywords presented, +// and any error that occurred. +func readRequestBlob(r *http.Request) (map[string]string, error) { + var blob map[string]string + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, err + } + r.Body.Close() + + err = json.Unmarshal(body, &blob) + if err != nil { + return nil, err + } + return blob, nil +} + +// ProcessRequestOneOf reads a JSON blob for the request and makes +// sure it contains one of a set of keywords. For example, a request +// might have the ('foo' && 'bar') keys, OR it might have the 'baz' +// key. In either case, we want to accept the request; however, if +// none of these sets shows up, the request is a bad request, and it +// should be returned. +func ProcessRequestOneOf(r *http.Request, keywordSets [][]string) (map[string]string, []string, error) { + blob, err := readRequestBlob(r) + if err != nil { + return nil, nil, err + } + + var matched []string + for _, set := range keywordSets { + if matchKeywords(blob, set) { + if matched != nil { + return nil, nil, errors.NewBadRequestString("mismatched parameters") + } + matched = set + } + } + if matched == nil { + return nil, nil, errors.NewBadRequestString("no valid parameter sets found") + } + return blob, matched, nil +} + +// ProcessRequestFirstMatchOf reads a JSON blob for the request and returns +// the first match of a set of keywords. For example, a request +// might have one of the following combinations: (foo=1, bar=2), (foo=1), and (bar=2) +// By giving a specific ordering of those combinations, we could decide how to accept +// the request. +func ProcessRequestFirstMatchOf(r *http.Request, keywordSets [][]string) (map[string]string, []string, error) { + blob, err := readRequestBlob(r) + if err != nil { + return nil, nil, err + } + + for _, set := range keywordSets { + if matchKeywords(blob, set) { + return blob, set, nil + } + } + return nil, nil, errors.NewBadRequestString("no valid parameter sets found") +} + +func matchKeywords(blob map[string]string, keywords []string) bool { + for _, keyword := range keywords { + if _, ok := blob[keyword]; !ok { + return false + } + } + return true +} + +// ResponseMessage implements the standard for response errors and +// messages. A message has a code and a string message. +type ResponseMessage struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// Response implements the CloudFlare standard for API +// responses. +type Response struct { + Success bool `json:"success"` + Result interface{} `json:"result"` + Errors []ResponseMessage `json:"errors"` + Messages []ResponseMessage `json:"messages"` +} + +// NewSuccessResponse is a shortcut for creating new successul API +// responses. +func NewSuccessResponse(result interface{}) Response { + return Response{ + Success: true, + Result: result, + Errors: []ResponseMessage{}, + Messages: []ResponseMessage{}, + } +} + +// NewSuccessResponseWithMessage is a shortcut for creating new successul API +// responses that includes a message. +func NewSuccessResponseWithMessage(result interface{}, message string, code int) Response { + return Response{ + Success: true, + Result: result, + Errors: []ResponseMessage{}, + Messages: []ResponseMessage{{code, message}}, + } +} + +// NewErrorResponse is a shortcut for creating an error response for a +// single error. +func NewErrorResponse(message string, code int) Response { + return Response{ + Success: false, + Result: nil, + Errors: []ResponseMessage{{code, message}}, + Messages: []ResponseMessage{}, + } +} + +// SendResponse builds a response from the result, sets the JSON +// header, and writes to the http.ResponseWriter. +func SendResponse(w http.ResponseWriter, result interface{}) error { + response := NewSuccessResponse(result) + w.Header().Set("Content-Type", "application/json") + enc := json.NewEncoder(w) + err := enc.Encode(response) + return err +} + +// SendResponseWithMessage builds a response from the result and the +// provided message, sets the JSON header, and writes to the +// http.ResponseWriter. +func SendResponseWithMessage(w http.ResponseWriter, result interface{}, message string, code int) error { + response := NewSuccessResponseWithMessage(result, message, code) + w.Header().Set("Content-Type", "application/json") + enc := json.NewEncoder(w) + err := enc.Encode(response) + return err +} diff --git a/components/engine/vendor/src/github.com/cloudflare/cfssl/certdb/README.md b/components/engine/vendor/src/github.com/cloudflare/cfssl/certdb/README.md index 18c6a28030..31eff57ca3 100644 --- a/components/engine/vendor/src/github.com/cloudflare/cfssl/certdb/README.md +++ b/components/engine/vendor/src/github.com/cloudflare/cfssl/certdb/README.md @@ -16,21 +16,26 @@ A database is required for the following: This directory stores [goose](https://bitbucket.org/liamstask/goose/) db migration scripts for various DB backends. Currently supported: - - SQLite in sqlite + - MySQL in mysql - PostgreSQL in pg + - SQLite in sqlite ### Get goose - go get https://bitbucket.org/liamstask/goose/ + go get bitbucket.org/liamstask/goose/cmd/goose -### Use goose to start and terminate a SQLite DB -To start a SQLite DB using goose: +### Use goose to start and terminate a MySQL DB +To start a MySQL using goose: - goose -path $GOPATH/src/github.com/cloudflare/cfssl/certdb/sqlite up' + goose -path $GOPATH/src/github.com/cloudflare/cfssl/certdb/mysql up -To tear down a SQLite DB using goose +To tear down a MySQL DB using goose - goose -path $GOPATH/src/github.com/cloudflare/cfssl/certdb/sqlite down + goose -path $GOPATH/src/github.com/cloudflare/cfssl/certdb/mysql down + +Note: the administration of MySQL DB is not included. We assume +the databases being connected to are already created and access control +is properly handled. ### Use goose to start and terminate a PostgreSQL DB To start a PostgreSQL using goose: @@ -43,7 +48,16 @@ To tear down a PostgreSQL DB using goose Note: the administration of PostgreSQL DB is not included. We assume the databases being connected to are already created and access control -are properly handled. +is properly handled. + +### Use goose to start and terminate a SQLite DB +To start a SQLite DB using goose: + + goose -path $GOPATH/src/github.com/cloudflare/cfssl/certdb/sqlite up + +To tear down a SQLite DB using goose + + goose -path $GOPATH/src/github.com/cloudflare/cfssl/certdb/sqlite down ## CFSSL Configuration @@ -55,4 +69,3 @@ JSON dictionary: or {"driver":"postgres","data_source":"postgres://user:password@host/db"} - diff --git a/components/engine/vendor/src/github.com/cloudflare/cfssl/csr/csr.go b/components/engine/vendor/src/github.com/cloudflare/cfssl/csr/csr.go index 6e3653fba6..4329b79564 100644 --- a/components/engine/vendor/src/github.com/cloudflare/cfssl/csr/csr.go +++ b/components/engine/vendor/src/github.com/cloudflare/cfssl/csr/csr.go @@ -9,6 +9,7 @@ import ( "crypto/rsa" "crypto/x509" "crypto/x509/pkix" + "encoding/asn1" "encoding/pem" "errors" "net" @@ -129,8 +130,9 @@ func (kr *BasicKeyRequest) SigAlgo() x509.SignatureAlgorithm { // CAConfig is a section used in the requests initialising a new CA. type CAConfig struct { - PathLength int `json:"pathlen"` - Expiry string `json:"expiry"` + PathLength int `json:"pathlen"` + PathLenZero bool `json:"pathlenzero"` + Expiry string `json:"expiry"` } // A CertificateRequest encapsulates the API interface to the @@ -175,6 +177,12 @@ func (cr *CertificateRequest) Name() pkix.Name { return name } +// BasicConstraints CSR information RFC 5280, 4.2.1.9 +type BasicConstraints struct { + IsCA bool `asn1:"optional"` + MaxPathLen int `asn1:"optional,default:-1"` +} + // ParseRequest takes a certificate request and generates a key and // CSR from it. It does no validation -- caveat emptor. It will, // however, fail if the key request is not valid (i.e., an unsupported @@ -217,34 +225,11 @@ func ParseRequest(req *CertificateRequest) (csr, key []byte, err error) { panic("Generate should have failed to produce a valid key.") } - var tpl = x509.CertificateRequest{ - Subject: req.Name(), - SignatureAlgorithm: req.KeyRequest.SigAlgo(), - } - - for i := range req.Hosts { - if ip := net.ParseIP(req.Hosts[i]); ip != nil { - tpl.IPAddresses = append(tpl.IPAddresses, ip) - } else if email, err := mail.ParseAddress(req.Hosts[i]); err == nil && email != nil { - tpl.EmailAddresses = append(tpl.EmailAddresses, req.Hosts[i]) - } else { - tpl.DNSNames = append(tpl.DNSNames, req.Hosts[i]) - } - } - - csr, err = x509.CreateCertificateRequest(rand.Reader, &tpl, priv) + csr, err = Generate(priv.(crypto.Signer), req) if err != nil { log.Errorf("failed to generate a CSR: %v", err) err = cferr.Wrap(cferr.CSRError, cferr.BadRequest, err) - return } - block := pem.Block{ - Type: "CERTIFICATE REQUEST", - Bytes: csr, - } - - log.Info("encoded CSR") - csr = pem.EncodeToMemory(&block) return } @@ -265,6 +250,7 @@ func ExtractCertificateRequest(cert *x509.Certificate) *CertificateRequest { // issue date and expiry date. req.CA.Expiry = cert.NotAfter.Sub(cert.NotBefore).String() req.CA.PathLength = cert.MaxPathLen + req.CA.PathLenZero = cert.MaxPathLenZero } return req @@ -377,7 +363,7 @@ func Regenerate(priv crypto.Signer, csr []byte) ([]byte, error) { // Generate creates a new CSR from a CertificateRequest structure and // an existing key. The KeyRequest field is ignored. func Generate(priv crypto.Signer, req *CertificateRequest) (csr []byte, err error) { - sigAlgo := helpers.SignerAlgo(priv, crypto.SHA256) + sigAlgo := helpers.SignerAlgo(priv) if sigAlgo == x509.UnknownSignatureAlgorithm { return nil, cferr.New(cferr.PrivateKeyError, cferr.Unavailable) } @@ -397,6 +383,14 @@ func Generate(priv crypto.Signer, req *CertificateRequest) (csr []byte, err erro } } + if req.CA != nil { + err = appendCAInfoToCSR(req.CA, &tpl) + if err != nil { + err = cferr.Wrap(cferr.CSRError, cferr.GenerationFailed, err) + return + } + } + csr, err = x509.CreateCertificateRequest(rand.Reader, &tpl, priv) if err != nil { log.Errorf("failed to generate a CSR: %v", err) @@ -412,3 +406,26 @@ func Generate(priv crypto.Signer, req *CertificateRequest) (csr []byte, err erro csr = pem.EncodeToMemory(&block) return } + +// appendCAInfoToCSR appends CAConfig BasicConstraint extension to a CSR +func appendCAInfoToCSR(reqConf *CAConfig, csr *x509.CertificateRequest) error { + pathlen := reqConf.PathLength + if pathlen == 0 && !reqConf.PathLenZero { + pathlen = -1 + } + val, err := asn1.Marshal(BasicConstraints{true, pathlen}) + + if err != nil { + return err + } + + csr.ExtraExtensions = []pkix.Extension{ + { + Id: asn1.ObjectIdentifier{2, 5, 29, 19}, + Value: val, + Critical: true, + }, + } + + return nil +} diff --git a/components/engine/vendor/src/github.com/cloudflare/cfssl/helpers/helpers.go b/components/engine/vendor/src/github.com/cloudflare/cfssl/helpers/helpers.go index 74d768134a..85b0d4a314 100644 --- a/components/engine/vendor/src/github.com/cloudflare/cfssl/helpers/helpers.go +++ b/components/engine/vendor/src/github.com/cloudflare/cfssl/helpers/helpers.go @@ -6,6 +6,7 @@ import ( "bytes" "crypto" "crypto/ecdsa" + "crypto/elliptic" "crypto/rsa" "crypto/x509" "encoding/asn1" @@ -410,7 +411,7 @@ func ParseCSR(in []byte) (csr *x509.CertificateRequest, rest []byte, err error) in = bytes.TrimSpace(in) p, rest := pem.Decode(in) if p != nil { - if p.Type != "CERTIFICATE REQUEST" { + if p.Type != "NEW CERTIFICATE REQUEST" && p.Type != "CERTIFICATE REQUEST" { return nil, rest, cferr.New(cferr.CSRError, cferr.BadRequest) } @@ -446,28 +447,28 @@ func ParseCSRPEM(csrPEM []byte) (*x509.CertificateRequest, error) { return csrObject, nil } -// SignerAlgo returns an X.509 signature algorithm corresponding to -// the crypto.Hash provided from a crypto.Signer. -func SignerAlgo(priv crypto.Signer, h crypto.Hash) x509.SignatureAlgorithm { - switch priv.Public().(type) { +// SignerAlgo returns an X.509 signature algorithm from a crypto.Signer. +func SignerAlgo(priv crypto.Signer) x509.SignatureAlgorithm { + switch pub := priv.Public().(type) { case *rsa.PublicKey: - switch h { - case crypto.SHA512: + bitLength := pub.N.BitLen() + switch { + case bitLength >= 4096: return x509.SHA512WithRSA - case crypto.SHA384: + case bitLength >= 3072: return x509.SHA384WithRSA - case crypto.SHA256: + case bitLength >= 2048: return x509.SHA256WithRSA default: return x509.SHA1WithRSA } case *ecdsa.PublicKey: - switch h { - case crypto.SHA512: + switch pub.Curve { + case elliptic.P521(): return x509.ECDSAWithSHA512 - case crypto.SHA384: + case elliptic.P384(): return x509.ECDSAWithSHA384 - case crypto.SHA256: + case elliptic.P256(): return x509.ECDSAWithSHA256 default: return x509.ECDSAWithSHA1 diff --git a/components/engine/vendor/src/github.com/cloudflare/cfssl/initca/initca.go b/components/engine/vendor/src/github.com/cloudflare/cfssl/initca/initca.go index aede763f4e..320ffb70b2 100644 --- a/components/engine/vendor/src/github.com/cloudflare/cfssl/initca/initca.go +++ b/components/engine/vendor/src/github.com/cloudflare/cfssl/initca/initca.go @@ -5,14 +5,10 @@ package initca import ( "crypto" "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" "crypto/rsa" "crypto/x509" - "encoding/pem" "errors" "io/ioutil" - "net" "time" "github.com/cloudflare/cfssl/config" @@ -47,14 +43,18 @@ func validator(req *csr.CertificateRequest) error { // New creates a new root certificate from the certificate request. func New(req *csr.CertificateRequest) (cert, csrPEM, key []byte, err error) { + policy := CAPolicy() if req.CA != nil { if req.CA.Expiry != "" { - CAPolicy.Default.ExpiryString = req.CA.Expiry - CAPolicy.Default.Expiry, err = time.ParseDuration(req.CA.Expiry) + policy.Default.ExpiryString = req.CA.Expiry + policy.Default.Expiry, err = time.ParseDuration(req.CA.Expiry) } - if req.CA.PathLength != 0 { - signer.MaxPathLen = req.CA.PathLength + signer.MaxPathLen = req.CA.PathLength + if req.CA.PathLength != 0 && req.CA.PathLenZero == true { + log.Infof("ignore invalid 'pathlenzero' value") + } else { + signer.MaxPathLenZero = req.CA.PathLenZero } } @@ -77,7 +77,7 @@ func New(req *csr.CertificateRequest) (cert, csrPEM, key []byte, err error) { log.Errorf("failed to create signer: %v", err) return } - s.SetPolicy(CAPolicy) + s.SetPolicy(policy) signReq := signer.SignRequest{Hosts: req.Hosts, Request: string(csrPEM)} cert, err = s.Sign(signReq) @@ -133,92 +133,35 @@ func RenewFromPEM(caFile, keyFile string) ([]byte, error) { // NewFromSigner creates a new root certificate from a crypto.Signer. func NewFromSigner(req *csr.CertificateRequest, priv crypto.Signer) (cert, csrPEM []byte, err error) { + policy := CAPolicy() if req.CA != nil { if req.CA.Expiry != "" { - CAPolicy.Default.ExpiryString = req.CA.Expiry - CAPolicy.Default.Expiry, err = time.ParseDuration(req.CA.Expiry) + policy.Default.ExpiryString = req.CA.Expiry + policy.Default.Expiry, err = time.ParseDuration(req.CA.Expiry) if err != nil { return nil, nil, err } } - if req.CA.PathLength != 0 { - signer.MaxPathLen = req.CA.PathLength - } - } - - var sigAlgo x509.SignatureAlgorithm - switch pub := priv.Public().(type) { - case *rsa.PublicKey: - bitLength := pub.N.BitLen() - switch { - case bitLength >= 4096: - sigAlgo = x509.SHA512WithRSA - case bitLength >= 3072: - sigAlgo = x509.SHA384WithRSA - case bitLength >= 2048: - sigAlgo = x509.SHA256WithRSA - default: - sigAlgo = x509.SHA1WithRSA - } - case *ecdsa.PublicKey: - switch pub.Curve { - case elliptic.P521(): - sigAlgo = x509.ECDSAWithSHA512 - case elliptic.P384(): - sigAlgo = x509.ECDSAWithSHA384 - case elliptic.P256(): - sigAlgo = x509.ECDSAWithSHA256 - default: - sigAlgo = x509.ECDSAWithSHA1 - } - default: - sigAlgo = x509.UnknownSignatureAlgorithm - } - - var tpl = x509.CertificateRequest{ - Subject: req.Name(), - SignatureAlgorithm: sigAlgo, - } - - for i := range req.Hosts { - if ip := net.ParseIP(req.Hosts[i]); ip != nil { - tpl.IPAddresses = append(tpl.IPAddresses, ip) + signer.MaxPathLen = req.CA.PathLength + if req.CA.PathLength != 0 && req.CA.PathLenZero == true { + log.Infof("ignore invalid 'pathlenzero' value") } else { - tpl.DNSNames = append(tpl.DNSNames, req.Hosts[i]) + signer.MaxPathLenZero = req.CA.PathLenZero } } - return signWithCSR(&tpl, priv) -} - -// signWithCSR creates a new root certificate from signing a X509.CertificateRequest -// by a crypto.Signer. -func signWithCSR(tpl *x509.CertificateRequest, priv crypto.Signer) (cert, csrPEM []byte, err error) { - csrPEM, err = x509.CreateCertificateRequest(rand.Reader, tpl, priv) + csrPEM, err = csr.Generate(priv, req) if err != nil { - log.Errorf("failed to generate a CSR: %v", err) - // The use of CertificateError was a matter of some - // debate; it is the one edge case in which a new - // error category specifically for CSRs might be - // useful, but it was deemed that one edge case did - // not a new category justify. - err = cferr.Wrap(cferr.CertificateError, cferr.BadRequest, err) - return + return nil, nil, err } - p := &pem.Block{ - Type: "CERTIFICATE REQUEST", - Bytes: csrPEM, - } - csrPEM = pem.EncodeToMemory(p) - s, err := local.NewSigner(priv, nil, signer.DefaultSigAlgo(priv), nil) if err != nil { log.Errorf("failed to create signer: %v", err) return } - s.SetPolicy(CAPolicy) + s.SetPolicy(policy) signReq := signer.SignRequest{Request: string(csrPEM)} cert, err = s.Sign(signReq) @@ -268,11 +211,13 @@ func RenewFromSigner(ca *x509.Certificate, priv crypto.Signer) ([]byte, error) { } // CAPolicy contains the CA issuing policy as default policy. -var CAPolicy = &config.Signing{ - Default: &config.SigningProfile{ - Usage: []string{"cert sign", "crl sign"}, - ExpiryString: "43800h", - Expiry: 5 * helpers.OneYear, - CA: true, - }, +var CAPolicy = func() *config.Signing { + return &config.Signing{ + Default: &config.SigningProfile{ + Usage: []string{"cert sign", "crl sign"}, + ExpiryString: "43800h", + Expiry: 5 * helpers.OneYear, + CA: true, + }, + } } diff --git a/components/engine/vendor/src/github.com/cloudflare/cfssl/log/log.go b/components/engine/vendor/src/github.com/cloudflare/cfssl/log/log.go index b517c44eea..4ceacc9e2b 100644 --- a/components/engine/vendor/src/github.com/cloudflare/cfssl/log/log.go +++ b/components/engine/vendor/src/github.com/cloudflare/cfssl/log/log.go @@ -45,12 +45,12 @@ var Level = LevelInfo // // SyslogWriter is satisfied by *syslog.Writer. type SyslogWriter interface { - Debug(string) error - Info(string) error - Warning(string) error - Err(string) error - Crit(string) error - Emerg(string) error + Debug(string) + Info(string) + Warning(string) + Err(string) + Crit(string) + Emerg(string) } // syslogWriter stores the SetLogger() parameter. @@ -73,23 +73,19 @@ func init() { func print(l int, msg string) { if l >= Level { if syslogWriter != nil { - var err error switch l { case LevelDebug: - err = syslogWriter.Debug(msg) + syslogWriter.Debug(msg) case LevelInfo: - err = syslogWriter.Info(msg) + syslogWriter.Info(msg) case LevelWarning: - err = syslogWriter.Warning(msg) + syslogWriter.Warning(msg) case LevelError: - err = syslogWriter.Err(msg) + syslogWriter.Err(msg) case LevelCritical: - err = syslogWriter.Crit(msg) + syslogWriter.Crit(msg) case LevelFatal: - err = syslogWriter.Emerg(msg) - } - if err != nil { - log.Printf("Unable to write syslog: %v for msg: %s\n", err, msg) + syslogWriter.Emerg(msg) } } else { log.Printf("[%s] %s", levelPrefix[l], msg) diff --git a/components/engine/vendor/src/github.com/cloudflare/cfssl/signer/local/local.go b/components/engine/vendor/src/github.com/cloudflare/cfssl/signer/local/local.go index ac16ae4a9e..d565bc4f4f 100644 --- a/components/engine/vendor/src/github.com/cloudflare/cfssl/signer/local/local.go +++ b/components/engine/vendor/src/github.com/cloudflare/cfssl/signer/local/local.go @@ -96,7 +96,11 @@ func NewSignerFromFile(caFile, caKeyFile string, policy *config.Signing) (*Signe } func (s *Signer) sign(template *x509.Certificate, profile *config.SigningProfile) (cert []byte, err error) { + var distPoints = template.CRLDistributionPoints err = signer.FillTemplate(template, s.policy.Default, profile) + if distPoints != nil && len(distPoints) > 0 { + template.CRLDistributionPoints = distPoints + } if err != nil { return } @@ -111,9 +115,7 @@ func (s *Signer) sign(template *x509.Certificate, profile *config.SigningProfile template.EmailAddresses = nil s.ca = template initRoot = true - template.MaxPathLen = signer.MaxPathLen } else if template.IsCA { - template.MaxPathLen = 1 template.DNSNames = nil template.EmailAddresses = nil } @@ -203,7 +205,7 @@ func (s *Signer) Sign(req signer.SignRequest) (cert []byte, err error) { return nil, cferr.New(cferr.CSRError, cferr.DecodeFailed) } - if block.Type != "CERTIFICATE REQUEST" { + if block.Type != "NEW CERTIFICATE REQUEST" && block.Type != "CERTIFICATE REQUEST" { return nil, cferr.Wrap(cferr.CSRError, cferr.BadRequest, errors.New("not a certificate or csr")) } @@ -243,6 +245,26 @@ func (s *Signer) Sign(req signer.SignRequest) (cert []byte, err error) { } } + if req.CRLOverride != "" { + safeTemplate.CRLDistributionPoints = []string{req.CRLOverride} + } + + if safeTemplate.IsCA { + if !profile.CA { + return nil, cferr.New(cferr.CertificateError, cferr.InvalidRequest) + } + + if s.ca != nil && s.ca.MaxPathLen > 0 { + if safeTemplate.MaxPathLen >= s.ca.MaxPathLen { + // do not sign a cert with pathlen > current + return nil, cferr.New(cferr.CertificateError, cferr.InvalidRequest) + } + } else if s.ca != nil && s.ca.MaxPathLen == 0 && s.ca.MaxPathLenZero { + // signer has pathlen of 0, do not sign more intermediate CAs + return nil, cferr.New(cferr.CertificateError, cferr.InvalidRequest) + } + } + OverrideHosts(&safeTemplate, req.Hosts) safeTemplate.Subject = PopulateSubjectFromCSR(req.Subject, safeTemplate.Subject) diff --git a/components/engine/vendor/src/github.com/cloudflare/cfssl/signer/signer.go b/components/engine/vendor/src/github.com/cloudflare/cfssl/signer/signer.go index 2911cfc285..6b0c3ea6f1 100644 --- a/components/engine/vendor/src/github.com/cloudflare/cfssl/signer/signer.go +++ b/components/engine/vendor/src/github.com/cloudflare/cfssl/signer/signer.go @@ -26,6 +26,9 @@ import ( // MaxPathLen is the default path length for a new CA certificate. var MaxPathLen = 2 +// MaxPathLenZero indicates whether a new CA certificate has pathlen=0 +var MaxPathLenZero = false + // Subject contains the information that should be used to override the // subject information when signing a certificate. type Subject struct { @@ -50,13 +53,14 @@ type Extension struct { // Extensions requested in the CSR are ignored, except for those processed by // ParseCertificateRequest (mainly subjectAltName). type SignRequest struct { - Hosts []string `json:"hosts"` - Request string `json:"certificate_request"` - Subject *Subject `json:"subject,omitempty"` - Profile string `json:"profile"` - Label string `json:"label"` - Serial *big.Int `json:"serial,omitempty"` - Extensions []Extension `json:"extensions,omitempty"` + Hosts []string `json:"hosts"` + Request string `json:"certificate_request"` + Subject *Subject `json:"subject,omitempty"` + Profile string `json:"profile"` + CRLOverride string `json:"crl_override"` + Label string `json:"label"` + Serial *big.Int `json:"serial,omitempty"` + Extensions []Extension `json:"extensions,omitempty"` } // appendIf appends to a if s is not an empty string. @@ -157,26 +161,46 @@ func DefaultSigAlgo(priv crypto.Signer) x509.SignatureAlgorithm { // ParseCertificateRequest takes an incoming certificate request and // builds a certificate template from it. func ParseCertificateRequest(s Signer, csrBytes []byte) (template *x509.Certificate, err error) { - csr, err := x509.ParseCertificateRequest(csrBytes) + csrv, err := x509.ParseCertificateRequest(csrBytes) if err != nil { err = cferr.Wrap(cferr.CSRError, cferr.ParseFailed, err) return } - err = helpers.CheckSignature(csr, csr.SignatureAlgorithm, csr.RawTBSCertificateRequest, csr.Signature) + err = helpers.CheckSignature(csrv, csrv.SignatureAlgorithm, csrv.RawTBSCertificateRequest, csrv.Signature) if err != nil { err = cferr.Wrap(cferr.CSRError, cferr.KeyMismatch, err) return } template = &x509.Certificate{ - Subject: csr.Subject, - PublicKeyAlgorithm: csr.PublicKeyAlgorithm, - PublicKey: csr.PublicKey, + Subject: csrv.Subject, + PublicKeyAlgorithm: csrv.PublicKeyAlgorithm, + PublicKey: csrv.PublicKey, SignatureAlgorithm: s.SigAlgo(), - DNSNames: csr.DNSNames, - IPAddresses: csr.IPAddresses, - EmailAddresses: csr.EmailAddresses, + DNSNames: csrv.DNSNames, + IPAddresses: csrv.IPAddresses, + EmailAddresses: csrv.EmailAddresses, + } + + for _, val := range csrv.Extensions { + // Check the CSR for the X.509 BasicConstraints (RFC 5280, 4.2.1.9) + // extension and append to template if necessary + if val.Id.Equal(asn1.ObjectIdentifier{2, 5, 29, 19}) { + var constraints csr.BasicConstraints + var rest []byte + + if rest, err = asn1.Unmarshal(val.Value, &constraints); err != nil { + return nil, cferr.Wrap(cferr.CSRError, cferr.ParseFailed, err) + } else if len(rest) != 0 { + return nil, cferr.Wrap(cferr.CSRError, cferr.ParseFailed, errors.New("x509: trailing data after X.509 BasicConstraints")) + } + + template.BasicConstraintsValid = true + template.IsCA = constraints.IsCA + template.MaxPathLen = constraints.MaxPathLen + template.MaxPathLenZero = template.MaxPathLen == 0 + } } return @@ -222,6 +246,7 @@ func FillTemplate(template *x509.Certificate, defaultProfile, profile *config.Si notBefore time.Time notAfter time.Time crlURL, ocspURL string + issuerURL = profile.IssuerURL ) // The third value returned from Usages is a list of unknown key usages. @@ -229,7 +254,7 @@ func FillTemplate(template *x509.Certificate, defaultProfile, profile *config.Si // here. ku, eku, _ = profile.Usages() if profile.IssuerURL == nil { - profile.IssuerURL = defaultProfile.IssuerURL + issuerURL = defaultProfile.IssuerURL } if ku == 0 && len(eku) == 0 { @@ -279,8 +304,8 @@ func FillTemplate(template *x509.Certificate, defaultProfile, profile *config.Si template.CRLDistributionPoints = []string{crlURL} } - if len(profile.IssuerURL) != 0 { - template.IssuingCertificateURL = profile.IssuerURL + if len(issuerURL) != 0 { + template.IssuingCertificateURL = issuerURL } if len(profile.Policies) != 0 { err = addPolicies(template, profile.Policies) diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/container_inspect.go b/components/engine/vendor/src/github.com/docker/engine-api/client/container_inspect.go index bbf560e631..0fa096d38f 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/container_inspect.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/container_inspect.go @@ -27,7 +27,7 @@ func (cli *Client) ContainerInspect(ctx context.Context, containerID string) (ty return response, err } -// ContainerInspectWithRaw returns the container information and it's raw representation. +// ContainerInspectWithRaw returns the container information and its raw representation. func (cli *Client) ContainerInspectWithRaw(ctx context.Context, containerID string, getSize bool) (types.ContainerJSON, []byte, error) { query := url.Values{} if getSize { diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/errors.go b/components/engine/vendor/src/github.com/docker/engine-api/client/errors.go index 6f6a78f97d..e026320bbd 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/errors.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/errors.go @@ -131,6 +131,11 @@ func (e nodeNotFoundError) Error() string { return fmt.Sprintf("Error: No such node: %s", e.nodeID) } +// NoFound indicates that this error type is of NotFound +func (e nodeNotFoundError) NotFound() bool { + return true +} + // IsErrNodeNotFound returns true if the error is caused // when a node is not found. func IsErrNodeNotFound(err error) bool { @@ -148,6 +153,11 @@ func (e serviceNotFoundError) Error() string { return fmt.Sprintf("Error: No such service: %s", e.serviceID) } +// NoFound indicates that this error type is of NotFound +func (e serviceNotFoundError) NotFound() bool { + return true +} + // IsErrServiceNotFound returns true if the error is caused // when a service is not found. func IsErrServiceNotFound(err error) bool { @@ -165,6 +175,11 @@ func (e taskNotFoundError) Error() string { return fmt.Sprintf("Error: No such task: %s", e.taskID) } +// NoFound indicates that this error type is of NotFound +func (e taskNotFoundError) NotFound() bool { + return true +} + // IsErrTaskNotFound returns true if the error is caused // when a task is not found. func IsErrTaskNotFound(err error) bool { diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/interface.go b/components/engine/vendor/src/github.com/docker/engine-api/client/interface.go index f99e88897a..4ad2a5c399 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/interface.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/interface.go @@ -92,7 +92,7 @@ type NetworkAPIClient interface { // NodeAPIClient defines API client methods for the nodes type NodeAPIClient interface { - NodeInspect(ctx context.Context, nodeID string) (swarm.Node, error) + NodeInspectWithRaw(ctx context.Context, nodeID string) (swarm.Node, []byte, error) NodeList(ctx context.Context, options types.NodeListOptions) ([]swarm.Node, error) NodeRemove(ctx context.Context, nodeID string) error NodeUpdate(ctx context.Context, nodeID string, version swarm.Version, node swarm.NodeSpec) error diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/network_inspect.go b/components/engine/vendor/src/github.com/docker/engine-api/client/network_inspect.go index 89469f52c5..e22fcd6710 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/network_inspect.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/network_inspect.go @@ -16,7 +16,7 @@ func (cli *Client) NetworkInspect(ctx context.Context, networkID string) (types. return networkResource, err } -// NetworkInspectWithRaw returns the information for a specific network configured in the docker host and it's raw representation. +// NetworkInspectWithRaw returns the information for a specific network configured in the docker host and its raw representation. func (cli *Client) NetworkInspectWithRaw(ctx context.Context, networkID string) (types.NetworkResource, []byte, error) { var networkResource types.NetworkResource resp, err := cli.get(ctx, "/networks/"+networkID, nil, nil) diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/node_inspect.go b/components/engine/vendor/src/github.com/docker/engine-api/client/node_inspect.go index 91cbfc0bea..5f555bb357 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/node_inspect.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/node_inspect.go @@ -1,25 +1,33 @@ package client import ( + "bytes" "encoding/json" + "io/ioutil" "net/http" "github.com/docker/engine-api/types/swarm" "golang.org/x/net/context" ) -// NodeInspect returns the node information. -func (cli *Client) NodeInspect(ctx context.Context, nodeID string) (swarm.Node, error) { +// NodeInspectWithRaw returns the node information. +func (cli *Client) NodeInspectWithRaw(ctx context.Context, nodeID string) (swarm.Node, []byte, error) { serverResp, err := cli.get(ctx, "/nodes/"+nodeID, nil, nil) if err != nil { if serverResp.statusCode == http.StatusNotFound { - return swarm.Node{}, nodeNotFoundError{nodeID} + return swarm.Node{}, nil, nodeNotFoundError{nodeID} } - return swarm.Node{}, err + return swarm.Node{}, nil, err + } + defer ensureReaderClosed(serverResp) + + body, err := ioutil.ReadAll(serverResp.body) + if err != nil { + return swarm.Node{}, nil, err } var response swarm.Node - err = json.NewDecoder(serverResp.body).Decode(&response) - ensureReaderClosed(serverResp) - return response, err + rdr := bytes.NewReader(body) + err = json.NewDecoder(rdr).Decode(&response) + return response, body, err } diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/plugin_install.go b/components/engine/vendor/src/github.com/docker/engine-api/client/plugin_install.go index 914376f512..3f5e59ff5d 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/plugin_install.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/plugin_install.go @@ -31,6 +31,7 @@ func (cli *Client) PluginInstall(ctx context.Context, name string, options types } var privileges types.PluginPrivileges if err := json.NewDecoder(resp.body).Decode(&privileges); err != nil { + ensureReaderClosed(resp) return err } ensureReaderClosed(resp) diff --git a/components/engine/vendor/src/github.com/docker/engine-api/client/volume_inspect.go b/components/engine/vendor/src/github.com/docker/engine-api/client/volume_inspect.go index 0c50c76bb8..2eaebfafa1 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/client/volume_inspect.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/client/volume_inspect.go @@ -16,7 +16,7 @@ func (cli *Client) VolumeInspect(ctx context.Context, volumeID string) (types.Vo return volume, err } -// VolumeInspectWithRaw returns the information about a specific volume in the docker host and it's raw representation +// VolumeInspectWithRaw returns the information about a specific volume in the docker host and its raw representation func (cli *Client) VolumeInspectWithRaw(ctx context.Context, volumeID string) (types.Volume, []byte, error) { var volume types.Volume resp, err := cli.get(ctx, "/volumes/"+volumeID, nil, nil) diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/swarm/container.go b/components/engine/vendor/src/github.com/docker/engine-api/types/swarm/container.go index ec6587664b..40718b247b 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/swarm/container.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/swarm/container.go @@ -54,7 +54,7 @@ const ( MountPropagationSlave MountPropagation = "slave" ) -// BindOptions define options specific to mounts of type "bind". +// BindOptions defines options specific to mounts of type "bind". type BindOptions struct { Propagation MountPropagation `json:",omitempty"` } diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/swarm/service.go b/components/engine/vendor/src/github.com/docker/engine-api/types/swarm/service.go index c7952c9395..6303c146f6 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/swarm/service.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/swarm/service.go @@ -15,7 +15,7 @@ type ServiceSpec struct { Annotations // TaskTemplate defines how the service should construct new tasks when - // ochestrating this service. + // orchestrating this service. TaskTemplate TaskSpec `json:",omitempty"` Mode ServiceMode `json:",omitempty"` UpdateConfig *UpdateConfig `json:",omitempty"` diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/swarm/swarm.go b/components/engine/vendor/src/github.com/docker/engine-api/types/swarm/swarm.go index 27a0ab6d14..da110fc7e1 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/swarm/swarm.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/swarm/swarm.go @@ -32,7 +32,7 @@ type Policy struct { Secret *string `json:",omitempty"` } -// OrchestrationConfig represents ochestration configuration. +// OrchestrationConfig represents orchestration configuration. type OrchestrationConfig struct { TaskHistoryRetentionLimit int64 `json:",omitempty"` } @@ -54,6 +54,20 @@ type DispatcherConfig struct { // CAConfig represents CA configuration. type CAConfig struct { NodeCertExpiry time.Duration `json:",omitempty"` + ExternalCAs []*ExternalCA `json:",omitempty"` +} + +// ExternalCAProtocol represents type of external CA. +type ExternalCAProtocol string + +// ExternalCAProtocolCFSSL CFSSL +const ExternalCAProtocolCFSSL ExternalCAProtocol = "cfssl" + +// ExternalCA defines external CA to be used by the cluster. +type ExternalCA struct { + Protocol ExternalCAProtocol + URL string + Options map[string]string `json:",omitempty"` } // InitRequest is the request used to init a swarm. diff --git a/components/engine/vendor/src/github.com/docker/engine-api/types/types.go b/components/engine/vendor/src/github.com/docker/engine-api/types/types.go index b91e3c10c6..c70d1c8cbb 100644 --- a/components/engine/vendor/src/github.com/docker/engine-api/types/types.go +++ b/components/engine/vendor/src/github.com/docker/engine-api/types/types.go @@ -504,10 +504,6 @@ type Checkpoint struct { Name string // Name is the name of the checkpoint } -// DefaultRuntimeName is the reserved name/alias used to represent the -// OCI runtime being shipped with the docker daemon package. -var DefaultRuntimeName = "default" - // Runtime describes an OCI runtime type Runtime struct { Path string `json:"path"` diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/agent.go b/components/engine/vendor/src/github.com/docker/libnetwork/agent.go index eaab2893c0..64710ff11d 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/agent.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/agent.go @@ -102,7 +102,7 @@ func (c *controller) handleKeyChange(keys []*types.EncryptionKey) error { deleted = cKey.Key } - if cKey.Subsystem == subsysGossip /* subsysIPSec */ { + if cKey.Subsystem == subsysIPSec { drvEnc.Prune = cKey.Key drvEnc.PruneTag = cKey.LamportTime } @@ -128,7 +128,7 @@ func (c *controller) handleKeyChange(keys []*types.EncryptionKey) error { a.networkDB.SetKey(key.Key) } - if key.Subsystem == subsysGossip /*subsysIPSec*/ { + if key.Subsystem == subsysIPSec { drvEnc.Key = key.Key drvEnc.Tag = key.LamportTime } @@ -138,7 +138,7 @@ func (c *controller) handleKeyChange(keys []*types.EncryptionKey) error { key, tag := c.getPrimaryKeyTag(subsysGossip) a.networkDB.SetPrimaryKey(key) - //key, tag = c.getPrimaryKeyTag(subsysIPSec) + key, tag = c.getPrimaryKeyTag(subsysIPSec) drvEnc.Primary = key drvEnc.PrimaryTag = tag @@ -317,17 +317,12 @@ func (c *controller) agentInit(bindAddrOrInterface string) error { return nil } - drvEnc := discoverapi.DriverEncryptionConfig{} - - keys, tags := c.getKeys(subsysGossip) // getKeys(subsysIPSec) - drvEnc.Keys = keys - drvEnc.Tags = tags - bindAddr, err := resolveAddr(bindAddrOrInterface) if err != nil { return err } + keys, tags := c.getKeys(subsysGossip) hostname, _ := os.Hostname() nDB, err := networkdb.New(&networkdb.Config{ BindAddr: bindAddr, @@ -350,6 +345,11 @@ func (c *controller) agentInit(bindAddrOrInterface string) error { go c.handleTableEvents(ch, c.handleEpTableEvent) + drvEnc := discoverapi.DriverEncryptionConfig{} + keys, tags = c.getKeys(subsysIPSec) + drvEnc.Keys = keys + drvEnc.Tags = tags + c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool { err := driver.DiscoverNew(discoverapi.EncryptionKeysConfig, drvEnc) if err != nil { @@ -380,7 +380,7 @@ func (c *controller) agentDriverNotify(d driverapi.Driver) { }) drvEnc := discoverapi.DriverEncryptionConfig{} - keys, tags := c.getKeys(subsysGossip) // getKeys(subsysIPSec) + keys, tags := c.getKeys(subsysIPSec) drvEnc.Keys = keys drvEnc.Tags = tags diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/controller.go b/components/engine/vendor/src/github.com/docker/libnetwork/controller.go index 80ccf6e6e2..9108d87844 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/controller.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/controller.go @@ -144,7 +144,7 @@ type controller struct { unWatchCh chan *endpoint svcRecords map[string]svcInfo nmap map[string]*netWatch - serviceBindings map[string]*service + serviceBindings map[serviceKey]*service defOsSbox osl.Sandbox ingressSandbox *sandbox sboxOnce sync.Once @@ -167,7 +167,7 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { cfg: config.ParseConfigOptions(cfgOptions...), sandboxes: sandboxTable{}, svcRecords: make(map[string]svcInfo), - serviceBindings: make(map[string]*service), + serviceBindings: make(map[serviceKey]*service), agentInitDone: make(chan struct{}), } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/datastore/datastore.go b/components/engine/vendor/src/github.com/docker/libnetwork/datastore/datastore.go index 63ff717d26..9ba9ebb486 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/datastore/datastore.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/datastore/datastore.go @@ -54,11 +54,12 @@ var ( ) type datastore struct { - scope string - store store.Store - cache *cache - watchCh chan struct{} - active bool + scope string + store store.Store + cache *cache + watchCh chan struct{} + active bool + sequential bool sync.Mutex } @@ -190,6 +191,10 @@ func newClient(scope string, kv string, addr string, config *store.Config, cache if cached && scope != LocalScope { return nil, fmt.Errorf("caching supported only for scope %s", LocalScope) } + sequential := false + if scope == LocalScope { + sequential = true + } if config == nil { config = &store.Config{} @@ -216,7 +221,7 @@ func newClient(scope string, kv string, addr string, config *store.Config, cache return nil, err } - ds := &datastore{scope: scope, store: store, active: true, watchCh: make(chan struct{})} + ds := &datastore{scope: scope, store: store, active: true, watchCh: make(chan struct{}), sequential: sequential} if cached { ds.cache = newCache(ds) } @@ -375,8 +380,10 @@ func (ds *datastore) PutObjectAtomic(kvObject KVObject) error { pair *store.KVPair err error ) - ds.Lock() - defer ds.Unlock() + if ds.sequential { + ds.Lock() + defer ds.Unlock() + } if kvObject == nil { return types.BadRequestErrorf("invalid KV Object : nil") @@ -420,8 +427,10 @@ add_cache: // PutObject adds a new Record based on an object into the datastore func (ds *datastore) PutObject(kvObject KVObject) error { - ds.Lock() - defer ds.Unlock() + if ds.sequential { + ds.Lock() + defer ds.Unlock() + } if kvObject == nil { return types.BadRequestErrorf("invalid KV Object : nil") @@ -456,8 +465,10 @@ func (ds *datastore) putObjectWithKey(kvObject KVObject, key ...string) error { // GetObject returns a record matching the key func (ds *datastore) GetObject(key string, o KVObject) error { - ds.Lock() - defer ds.Unlock() + if ds.sequential { + ds.Lock() + defer ds.Unlock() + } if ds.cache != nil { return ds.cache.get(key, o) @@ -490,8 +501,10 @@ func (ds *datastore) ensureParent(parent string) error { } func (ds *datastore) List(key string, kvObject KVObject) ([]KVObject, error) { - ds.Lock() - defer ds.Unlock() + if ds.sequential { + ds.Lock() + defer ds.Unlock() + } if ds.cache != nil { return ds.cache.list(kvObject) @@ -536,8 +549,10 @@ func (ds *datastore) List(key string, kvObject KVObject) ([]KVObject, error) { // DeleteObject unconditionally deletes a record from the store func (ds *datastore) DeleteObject(kvObject KVObject) error { - ds.Lock() - defer ds.Unlock() + if ds.sequential { + ds.Lock() + defer ds.Unlock() + } // cleaup the cache first if ds.cache != nil { @@ -555,8 +570,10 @@ func (ds *datastore) DeleteObject(kvObject KVObject) error { // DeleteObjectAtomic performs atomic delete on a record func (ds *datastore) DeleteObjectAtomic(kvObject KVObject) error { - ds.Lock() - defer ds.Unlock() + if ds.sequential { + ds.Lock() + defer ds.Unlock() + } if kvObject == nil { return types.BadRequestErrorf("invalid KV Object : nil") @@ -588,8 +605,10 @@ del_cache: // DeleteTree unconditionally deletes a record from the store func (ds *datastore) DeleteTree(kvObject KVObject) error { - ds.Lock() - defer ds.Unlock() + if ds.sequential { + ds.Lock() + defer ds.Unlock() + } // cleaup the cache first if ds.cache != nil { diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go index ce0aec8485..c75ec9f4f9 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go @@ -1020,7 +1020,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, } if err = d.storeUpdate(endpoint); err != nil { - return fmt.Errorf("failed to save bridge endpoint %s to store: %v", ep.id[0:7], err) + return fmt.Errorf("failed to save bridge endpoint %s to store: %v", endpoint.id[0:7], err) } return nil diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/errors.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/errors.go index bda369608f..93960794cb 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/errors.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/bridge/errors.go @@ -183,7 +183,7 @@ func (ieie InvalidEndpointIDError) BadRequest() {} type InvalidSandboxIDError string func (isie InvalidSandboxIDError) Error() string { - return fmt.Sprintf("invalid sanbox id: %s", string(isie)) + return fmt.Sprintf("invalid sandbox id: %s", string(isie)) } // BadRequest denotes the type of this error diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go index c00d4c4d40..78a586e08e 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go @@ -2,7 +2,6 @@ package ovmanager import ( "fmt" - "log" "net" "strconv" "strings" @@ -20,7 +19,7 @@ import ( const ( networkType = "overlay" vxlanIDStart = 256 - vxlanIDEnd = 1000 + vxlanIDEnd = (1 << 24) - 1 ) type networkTable map[string]*network @@ -111,7 +110,8 @@ func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, } if err := n.obtainVxlanID(s); err != nil { - log.Printf("Could not obtain vxlan id for pool %s: %v", s.subnetIP, err) + n.releaseVxlanID() + return nil, fmt.Errorf("could not obtain vxlan id for pool %s: %v", s.subnetIP, err) } n.subnets = append(n.subnets, s) diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/remote/driver.go b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/remote/driver.go index 5383d9ee7a..9fa253bb29 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/drivers/remote/driver.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/drivers/remote/driver.go @@ -294,7 +294,7 @@ func (d *driver) Type() string { // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { if dType != discoverapi.NodeDiscovery { - return fmt.Errorf("Unknown discovery type : %v", dType) + return nil } notif := &api.DiscoveryNotification{ DiscoveryType: dType, @@ -306,7 +306,7 @@ func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) // DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { if dType != discoverapi.NodeDiscovery { - return fmt.Errorf("Unknown discovery type : %v", dType) + return nil } notif := &api.DiscoveryNotification{ DiscoveryType: dType, diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/endpoint.go b/components/engine/vendor/src/github.com/docker/libnetwork/endpoint.go index 3fa1b6dc3e..418017f80d 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/endpoint.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/endpoint.go @@ -722,18 +722,6 @@ func (ep *endpoint) sbLeave(sb *sandbox, force bool, options ...EndpointOption) return nil } -func (n *network) validateForceDelete(locator string) error { - if n.Scope() == datastore.LocalScope { - return nil - } - - if locator == "" { - return fmt.Errorf("invalid endpoint locator identifier") - } - - return nil -} - func (ep *endpoint) Delete(force bool) error { var err error n, err := ep.getNetworkFromStore() @@ -750,15 +738,8 @@ func (ep *endpoint) Delete(force bool) error { epid := ep.id name := ep.name sbid := ep.sandboxID - locator := ep.locator ep.Unlock() - if force { - if err = n.validateForceDelete(locator); err != nil { - return fmt.Errorf("unable to force delete endpoint %s: %v", name, err) - } - } - sb, _ := n.getController().SandboxByID(sbid) if sb != nil && !force { return &ActiveContainerError{name: name, id: epid} diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/osl/route_linux.go b/components/engine/vendor/src/github.com/docker/libnetwork/osl/route_linux.go index 78d1f9a14f..a9ff191b37 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/osl/route_linux.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/osl/route_linux.go @@ -83,17 +83,29 @@ func (n *networkNamespace) programGateway(gw net.IP, isAdd bool) error { return fmt.Errorf("route for the gateway %s could not be found: %v", gw, err) } + var linkIndex int + for _, gwRoute := range gwRoutes { + if gwRoute.Gw == nil { + linkIndex = gwRoute.LinkIndex + break + } + } + + if linkIndex == 0 { + return fmt.Errorf("Direct route for the gateway %s could not be found", gw) + } + if isAdd { return n.nlHandle.RouteAdd(&netlink.Route{ Scope: netlink.SCOPE_UNIVERSE, - LinkIndex: gwRoutes[0].LinkIndex, + LinkIndex: linkIndex, Gw: gw, }) } return n.nlHandle.RouteDel(&netlink.Route{ Scope: netlink.SCOPE_UNIVERSE, - LinkIndex: gwRoutes[0].LinkIndex, + LinkIndex: linkIndex, Gw: gw, }) } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/service.go b/components/engine/vendor/src/github.com/docker/libnetwork/service.go index cdf18bd499..30a17c5056 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/service.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/service.go @@ -1,6 +1,7 @@ package libnetwork import ( + "fmt" "net" "sync" ) @@ -12,6 +13,27 @@ var ( fwMarkCtrMu sync.Mutex ) +type portConfigs []*PortConfig + +func (p portConfigs) String() string { + if len(p) == 0 { + return "" + } + + pc := p[0] + str := fmt.Sprintf("%d:%d/%s", pc.PublishedPort, pc.TargetPort, PortConfig_Protocol_name[int32(pc.Protocol)]) + for _, pc := range p[1:] { + str = str + fmt.Sprintf(",%d:%d/%s", pc.PublishedPort, pc.TargetPort, PortConfig_Protocol_name[int32(pc.Protocol)]) + } + + return str +} + +type serviceKey struct { + id string + ports string +} + type service struct { name string // Service Name id string // Service ID @@ -21,7 +43,7 @@ type service struct { loadBalancers map[string]*loadBalancer // List of ingress ports exposed by the service - ingressPorts []*PortConfig + ingressPorts portConfigs sync.Mutex } diff --git a/components/engine/vendor/src/github.com/docker/libnetwork/service_linux.go b/components/engine/vendor/src/github.com/docker/libnetwork/service_linux.go index d6b4b4cd02..fc8e4fcbe6 100644 --- a/components/engine/vendor/src/github.com/docker/libnetwork/service_linux.go +++ b/components/engine/vendor/src/github.com/docker/libnetwork/service_linux.go @@ -48,13 +48,18 @@ func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, i return err } + skey := serviceKey{ + id: sid, + ports: portConfigs(ingressPorts).String(), + } + c.Lock() - s, ok := c.serviceBindings[sid] + s, ok := c.serviceBindings[skey] if !ok { // Create a new service if we are seeing this service // for the first time. s = newService(name, sid, ingressPorts) - c.serviceBindings[sid] = s + c.serviceBindings[skey] = s } c.Unlock() @@ -121,8 +126,13 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, in return err } + skey := serviceKey{ + id: sid, + ports: portConfigs(ingressPorts).String(), + } + c.Lock() - s, ok := c.serviceBindings[sid] + s, ok := c.serviceBindings[skey] if !ok { c.Unlock() return nil @@ -135,22 +145,19 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, in n.(*network).deleteSvcRecords("tasks."+alias, ip, nil, false) } - // Make sure to remove the right IP since if vip is - // not valid we would have added a DNS RR record. - svcIP := vip - if len(svcIP) == 0 { - svcIP = ip - } - n.(*network).deleteSvcRecords(name, svcIP, nil, false) - for _, alias := range aliases { - n.(*network).deleteSvcRecords(alias, svcIP, nil, false) + // If we are doing DNS RR add the endpoint IP to DNS record + // right away. + if len(vip) == 0 { + n.(*network).deleteSvcRecords(name, ip, nil, false) + for _, alias := range aliases { + n.(*network).deleteSvcRecords(alias, ip, nil, false) + } } s.Lock() - defer s.Unlock() - lb, ok := s.loadBalancers[nid] if !ok { + s.Unlock() return nil } @@ -167,7 +174,7 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, in if len(s.loadBalancers) == 0 { // All loadbalancers for the service removed. Time to // remove the service itself. - delete(c.serviceBindings, sid) + delete(c.serviceBindings, skey) } // Remove loadbalancer service(if needed) and backend in all @@ -175,6 +182,15 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, in if len(vip) != 0 { n.(*network).rmLBBackend(ip, vip, lb.fwMark, ingressPorts, rmService) } + s.Unlock() + + // Remove the DNS record for VIP only if we are removing the service + if rmService && len(vip) != 0 { + n.(*network).deleteSvcRecords(name, vip, nil, false) + for _, alias := range aliases { + n.(*network).deleteSvcRecords(alias, vip, nil, false) + } + } return nil } @@ -314,7 +330,7 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P if addService { var iPorts []*PortConfig if sb.ingress { - iPorts = ingressPorts + iPorts = filterPortConfigs(ingressPorts, false) if err := programIngress(gwIP, iPorts, false); err != nil { logrus.Errorf("Failed to add ingress: %v", err) return @@ -383,7 +399,7 @@ func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*Po var iPorts []*PortConfig if sb.ingress { - iPorts = ingressPorts + iPorts = filterPortConfigs(ingressPorts, true) if err := programIngress(gwIP, iPorts, true); err != nil { logrus.Errorf("Failed to delete ingress: %v", err) } @@ -401,8 +417,47 @@ var ( ingressOnce sync.Once ingressProxyMu sync.Mutex ingressProxyTbl = make(map[string]io.Closer) + portConfigMu sync.Mutex + portConfigTbl = make(map[PortConfig]int) ) +func filterPortConfigs(ingressPorts []*PortConfig, isDelete bool) []*PortConfig { + portConfigMu.Lock() + iPorts := make([]*PortConfig, 0, len(ingressPorts)) + for _, pc := range ingressPorts { + if isDelete { + if cnt, ok := portConfigTbl[*pc]; ok { + // This is the last reference to this + // port config. Delete the port config + // and add it to filtered list to be + // plumbed. + if cnt == 1 { + delete(portConfigTbl, *pc) + iPorts = append(iPorts, pc) + continue + } + + portConfigTbl[*pc] = cnt - 1 + } + + continue + } + + if cnt, ok := portConfigTbl[*pc]; ok { + portConfigTbl[*pc] = cnt + 1 + continue + } + + // We are adding it for the first time. Add it to the + // filter list to be plumbed. + portConfigTbl[*pc] = 1 + iPorts = append(iPorts, pc) + } + portConfigMu.Unlock() + + return iPorts +} + func programIngress(gwIP net.IP, ingressPorts []*PortConfig, isDelete bool) error { addDelOpt := "-I" if isDelete { diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/agent/agent.go b/components/engine/vendor/src/github.com/docker/swarmkit/agent/agent.go index bb77289be9..db349f1e36 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/agent/agent.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/agent/agent.go @@ -4,7 +4,6 @@ import ( "fmt" "math/rand" "reflect" - "sync" "time" "github.com/docker/swarmkit/api" @@ -37,7 +36,6 @@ type Agent struct { stopped chan struct{} // requests shutdown closed chan struct{} // only closed in run err error // read only after closed is closed - mu sync.Mutex } // New returns a new agent, ready for task dispatch. diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/agent/exec/controller.go b/components/engine/vendor/src/github.com/docker/swarmkit/agent/exec/controller.go index b691a74b3b..bbd2a5db15 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/agent/exec/controller.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/agent/exec/controller.go @@ -55,7 +55,7 @@ type ContainerStatuser interface { // correct status depending on the tasks current state according to the result. // // Unlike Do, if an error is returned, the status should still be reported. The -// error merely reports the +// error merely reports the failure at getting the controller. func Resolve(ctx context.Context, task *api.Task, executor Executor) (Controller, *api.TaskStatus, error) { status := task.Status.Copy() diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/agent/exec/executor.go b/components/engine/vendor/src/github.com/docker/swarmkit/agent/exec/executor.go index d4f3da58f8..58c7001813 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/agent/exec/executor.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/agent/exec/executor.go @@ -10,7 +10,7 @@ type Executor interface { // Describe returns the underlying node description. Describe(ctx context.Context) (*api.NodeDescription, error) - // Configure uses the node object state to propogate node + // Configure uses the node object state to propagate node // state to the underlying executor. Configure(ctx context.Context, node *api.Node) error diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/agent/node.go b/components/engine/vendor/src/github.com/docker/swarmkit/agent/node.go index 1f9fca4fcf..fd50e4b740 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/agent/node.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/agent/node.go @@ -49,6 +49,10 @@ type NodeConfig struct { // Secret to be used on the first certificate request. Secret string + // ExternalCAs is a list of CAs to which a manager node + // will make certificate signing requests for node certificates. + ExternalCAs []*api.ExternalCA + // ForceNewCluster creates a new cluster from current raft state. ForceNewCluster bool @@ -81,7 +85,6 @@ type Node struct { config *NodeConfig remotes *persistentRemotes role string - roleCond *sync.Cond conn *grpc.ClientConn connCond *sync.Cond nodeID string @@ -95,6 +98,7 @@ type Node struct { agent *Agent manager *manager.Manager roleChangeReq chan api.NodeRole // used to send role updates from the dispatcher api on promotion/demotion + managerRoleCh chan struct{} } // NewNode returns new Node instance. @@ -124,8 +128,8 @@ func NewNode(c *NodeConfig) (*Node, error) { ready: make(chan struct{}), certificateRequested: make(chan struct{}), roleChangeReq: make(chan api.NodeRole, 1), + managerRoleCh: make(chan struct{}, 32), // 32 just for the case } - n.roleCond = sync.NewCond(n.RLocker()) n.connCond = sync.NewCond(n.RLocker()) if err := n.loadCertificates(); err != nil { return nil, err @@ -174,6 +178,8 @@ func (n *Node) run(ctx context.Context) (err error) { } }() + // NOTE: When this node is created by NewNode(), our nodeID is set if + // n.loadCertificates() succeeded in loading TLS credentials. if n.config.JoinAddr == "" && n.nodeID == "" { if err := n.bootstrapCA(); err != nil { return err @@ -234,6 +240,10 @@ func (n *Node) run(ctx context.Context) (err error) { return err } + if n.role == ca.ManagerRole { + n.managerRoleCh <- struct{}{} + } + forceCertRenewal := make(chan struct{}) go func() { n.RLock() @@ -270,7 +280,9 @@ func (n *Node) run(ctx context.Context) (err error) { } n.Lock() n.role = certUpdate.Role - n.roleCond.Broadcast() + if n.role == ca.ManagerRole { + n.managerRoleCh <- struct{}{} + } n.Unlock() case <-ctx.Done(): return @@ -419,34 +431,6 @@ func (n *Node) CertificateRequested() <-chan struct{} { return n.certificateRequested } -func (n *Node) waitRole(ctx context.Context, role string) <-chan struct{} { - c := make(chan struct{}) - n.roleCond.L.Lock() - if role == n.role { - close(c) - n.roleCond.L.Unlock() - return c - } - go func() { - select { - case <-ctx.Done(): - n.roleCond.Broadcast() - case <-c: - } - }() - go func() { - defer n.roleCond.L.Unlock() - defer close(c) - for role != n.role { - n.roleCond.Wait() - if ctx.Err() != nil { - return - } - } - }() - return c -} - func (n *Node) setControlSocket(conn *grpc.ClientConn) { n.Lock() n.conn = conn @@ -549,7 +533,6 @@ func (n *Node) loadCertificates() error { n.role = clientTLSCreds.Role() n.nodeID = clientTLSCreds.NodeID() n.nodeMembership = api.NodeMembershipAccepted - n.roleCond.Broadcast() n.Unlock() return nil @@ -599,10 +582,17 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig select { case <-ctx.Done(): return ctx.Err() - case <-n.waitRole(ctx, ca.ManagerRole): + case <-n.managerRoleCh: if ctx.Err() != nil { return ctx.Err() } + n.Lock() + // in case if we missed some notifications + if n.role != ca.ManagerRole { + n.Unlock() + continue + } + n.Unlock() remoteAddr, _ := n.remotes.Select(n.nodeID) m, err := manager.New(&manager.Config{ ForceNewCluster: n.config.ForceNewCluster, @@ -611,6 +601,7 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig "unix": n.config.ListenControlAPI, }, SecurityConfig: securityConfig, + ExternalCAs: n.config.ExternalCAs, JoinRaft: remoteAddr.Addr, StateDir: n.config.StateDir, HeartbeatTick: n.config.HeartbeatTick, @@ -629,17 +620,21 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig n.manager = m n.Unlock() - go n.initManagerConnection(ctx, ready) + connCtx, connCancel := context.WithCancel(ctx) + go n.initManagerConnection(connCtx, ready) - go func() { - select { - case <-ready: - case <-ctx.Done(): - } - if ctx.Err() == nil { - n.remotes.Observe(api.Peer{NodeID: n.nodeID, Addr: n.config.ListenRemoteAPI}, 5) - } - }() + // this happens only on initial start + if ready != nil { + go func(ready chan struct{}) { + select { + case <-ready: + n.remotes.Observe(api.Peer{NodeID: n.nodeID, Addr: n.config.ListenRemoteAPI}, 5) + case <-connCtx.Done(): + } + }(ready) + } + + ready = nil select { case <-ctx.Done(): @@ -648,8 +643,8 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig // in case of demotion manager will stop itself case <-done: } + connCancel() - ready = nil // ready event happens once, even on multiple starts n.Lock() n.manager = nil if n.conn != nil { @@ -669,7 +664,6 @@ type persistentRemotes struct { c *sync.Cond picker.Remotes storePath string - ch []chan api.Peer lastSavedState []api.Peer } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/agent/session.go b/components/engine/vendor/src/github.com/docker/swarmkit/agent/session.go index 8fde4bdfec..39fc64e444 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/agent/session.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/agent/session.go @@ -26,7 +26,6 @@ var ( // agent through errs, messages and tasks. type session struct { agent *Agent - nodeID string sessionID string session api.Dispatcher_SessionClient errs chan error diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/agent/storage.go b/components/engine/vendor/src/github.com/docker/swarmkit/agent/storage.go index e9cb19934a..5d0c420e0c 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/agent/storage.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/agent/storage.go @@ -1,8 +1,6 @@ package agent import ( - "bytes" - "github.com/boltdb/bolt" "github.com/docker/swarmkit/api" "github.com/gogo/protobuf/proto" @@ -22,12 +20,6 @@ var ( bucketKeyStatus = []byte("status") ) -type bucketKeyPath [][]byte - -func (bk bucketKeyPath) String() string { - return string(bytes.Join([][]byte(bk), []byte("/"))) -} - // InitDB prepares a database for writing task data. // // Proper buckets will be created if they don't already exist. diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/README.md b/components/engine/vendor/src/github.com/docker/swarmkit/api/README.md new file mode 100644 index 0000000000..78eee22678 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/README.md @@ -0,0 +1,8 @@ +### Notice + +Do not change .pb.go files directly. You need to change the corresponding .proto files and run the following command to regenerate the .pb.go files. +``` +$ make generate +``` + +Click [here](https://github.com/google/protobuf) for more information about protobuf. diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/gen.go b/components/engine/vendor/src/github.com/docker/swarmkit/api/gen.go index 46eaf9e4f6..1a9b0109d8 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/gen.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/gen.go @@ -1,3 +1,3 @@ package api -//go:generate protoc -I.:../protobuf:../vendor:../vendor/github.com/gogo/protobuf --gogoswarm_out=plugins=grpc+deepcopy+raftproxy+authenticatedwrapper,import_path=github.com/docker/swarmkit/api,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto,Mtimestamp/timestamp.proto=github.com/docker/swarmkit/api/timestamp,Mduration/duration.proto=github.com/docker/swarmkit/api/duration,Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/protoc-gen-gogo/descriptor,Mplugin/plugin.proto=github.com/docker/swarmkit/protobuf/plugin:. types.proto specs.proto objects.proto control.proto dispatcher.proto ca.proto snapshot.proto raft.proto +//go:generate protoc -I.:../protobuf:../vendor:../vendor/github.com/gogo/protobuf --gogoswarm_out=plugins=grpc+deepcopy+raftproxy+authenticatedwrapper,import_path=github.com/docker/swarmkit/api,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto,Mtimestamp/timestamp.proto=github.com/docker/swarmkit/api/timestamp,Mduration/duration.proto=github.com/docker/swarmkit/api/duration,Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/protoc-gen-gogo/descriptor,Mplugin/plugin.proto=github.com/docker/swarmkit/protobuf/plugin:. types.proto specs.proto objects.proto control.proto dispatcher.proto ca.proto snapshot.proto raft.proto health.proto diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/health.pb.go b/components/engine/vendor/src/github.com/docker/swarmkit/api/health.pb.go new file mode 100644 index 0000000000..c80119d62e --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/health.pb.go @@ -0,0 +1,714 @@ +// Code generated by protoc-gen-gogo. +// source: health.proto +// DO NOT EDIT! + +package api + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/gogo/protobuf/gogoproto" +import _ "github.com/docker/swarmkit/protobuf/plugin" + +import strings "strings" +import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" +import sort "sort" +import strconv "strconv" +import reflect "reflect" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +import raftpicker "github.com/docker/swarmkit/manager/raftpicker" +import codes "google.golang.org/grpc/codes" +import metadata "google.golang.org/grpc/metadata" +import transport "google.golang.org/grpc/transport" + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type HealthCheckResponse_ServingStatus int32 + +const ( + HealthCheckResponse_UNKNOWN HealthCheckResponse_ServingStatus = 0 + HealthCheckResponse_SERVING HealthCheckResponse_ServingStatus = 1 + HealthCheckResponse_NOT_SERVING HealthCheckResponse_ServingStatus = 2 +) + +var HealthCheckResponse_ServingStatus_name = map[int32]string{ + 0: "UNKNOWN", + 1: "SERVING", + 2: "NOT_SERVING", +} +var HealthCheckResponse_ServingStatus_value = map[string]int32{ + "UNKNOWN": 0, + "SERVING": 1, + "NOT_SERVING": 2, +} + +func (x HealthCheckResponse_ServingStatus) String() string { + return proto.EnumName(HealthCheckResponse_ServingStatus_name, int32(x)) +} +func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) { + return fileDescriptorHealth, []int{1, 0} +} + +type HealthCheckRequest struct { + Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"` +} + +func (m *HealthCheckRequest) Reset() { *m = HealthCheckRequest{} } +func (*HealthCheckRequest) ProtoMessage() {} +func (*HealthCheckRequest) Descriptor() ([]byte, []int) { return fileDescriptorHealth, []int{0} } + +type HealthCheckResponse struct { + Status HealthCheckResponse_ServingStatus `protobuf:"varint,1,opt,name=status,proto3,enum=docker.swarmkit.v1.HealthCheckResponse_ServingStatus" json:"status,omitempty"` +} + +func (m *HealthCheckResponse) Reset() { *m = HealthCheckResponse{} } +func (*HealthCheckResponse) ProtoMessage() {} +func (*HealthCheckResponse) Descriptor() ([]byte, []int) { return fileDescriptorHealth, []int{1} } + +func init() { + proto.RegisterType((*HealthCheckRequest)(nil), "docker.swarmkit.v1.HealthCheckRequest") + proto.RegisterType((*HealthCheckResponse)(nil), "docker.swarmkit.v1.HealthCheckResponse") + proto.RegisterEnum("docker.swarmkit.v1.HealthCheckResponse_ServingStatus", HealthCheckResponse_ServingStatus_name, HealthCheckResponse_ServingStatus_value) +} + +type authenticatedWrapperHealthServer struct { + local HealthServer + authorize func(context.Context, []string) error +} + +func NewAuthenticatedWrapperHealthServer(local HealthServer, authorize func(context.Context, []string) error) HealthServer { + return &authenticatedWrapperHealthServer{ + local: local, + authorize: authorize, + } +} + +func (p *authenticatedWrapperHealthServer) Check(ctx context.Context, r *HealthCheckRequest) (*HealthCheckResponse, error) { + + if err := p.authorize(ctx, []string{"swarm-manager"}); err != nil { + return nil, err + } + return p.local.Check(ctx, r) +} + +func (m *HealthCheckRequest) Copy() *HealthCheckRequest { + if m == nil { + return nil + } + + o := &HealthCheckRequest{ + Service: m.Service, + } + + return o +} + +func (m *HealthCheckResponse) Copy() *HealthCheckResponse { + if m == nil { + return nil + } + + o := &HealthCheckResponse{ + Status: m.Status, + } + + return o +} + +func (this *HealthCheckRequest) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 5) + s = append(s, "&api.HealthCheckRequest{") + s = append(s, "Service: "+fmt.Sprintf("%#v", this.Service)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *HealthCheckResponse) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 5) + s = append(s, "&api.HealthCheckResponse{") + s = append(s, "Status: "+fmt.Sprintf("%#v", this.Status)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func valueToGoStringHealth(v interface{}, typ string) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) +} +func extensionToGoStringHealth(e map[int32]github_com_gogo_protobuf_proto.Extension) string { + if e == nil { + return "nil" + } + s := "map[int32]proto.Extension{" + keys := make([]int, 0, len(e)) + for k := range e { + keys = append(keys, int(k)) + } + sort.Ints(keys) + ss := []string{} + for _, k := range keys { + ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString()) + } + s += strings.Join(ss, ",") + "}" + return s +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion2 + +// Client API for Health service + +type HealthClient interface { + Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) +} + +type healthClient struct { + cc *grpc.ClientConn +} + +func NewHealthClient(cc *grpc.ClientConn) HealthClient { + return &healthClient{cc} +} + +func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { + out := new(HealthCheckResponse) + err := grpc.Invoke(ctx, "/docker.swarmkit.v1.Health/Check", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for Health service + +type HealthServer interface { + Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) +} + +func RegisterHealthServer(s *grpc.Server, srv HealthServer) { + s.RegisterService(&_Health_serviceDesc, srv) +} + +func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HealthCheckRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HealthServer).Check(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/docker.swarmkit.v1.Health/Check", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Health_serviceDesc = grpc.ServiceDesc{ + ServiceName: "docker.swarmkit.v1.Health", + HandlerType: (*HealthServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Check", + Handler: _Health_Check_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, +} + +func (m *HealthCheckRequest) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *HealthCheckRequest) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Service) > 0 { + data[i] = 0xa + i++ + i = encodeVarintHealth(data, i, uint64(len(m.Service))) + i += copy(data[i:], m.Service) + } + return i, nil +} + +func (m *HealthCheckResponse) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *HealthCheckResponse) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Status != 0 { + data[i] = 0x8 + i++ + i = encodeVarintHealth(data, i, uint64(m.Status)) + } + return i, nil +} + +func encodeFixed64Health(data []byte, offset int, v uint64) int { + data[offset] = uint8(v) + data[offset+1] = uint8(v >> 8) + data[offset+2] = uint8(v >> 16) + data[offset+3] = uint8(v >> 24) + data[offset+4] = uint8(v >> 32) + data[offset+5] = uint8(v >> 40) + data[offset+6] = uint8(v >> 48) + data[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Health(data []byte, offset int, v uint32) int { + data[offset] = uint8(v) + data[offset+1] = uint8(v >> 8) + data[offset+2] = uint8(v >> 16) + data[offset+3] = uint8(v >> 24) + return offset + 4 +} +func encodeVarintHealth(data []byte, offset int, v uint64) int { + for v >= 1<<7 { + data[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + data[offset] = uint8(v) + return offset + 1 +} + +type raftProxyHealthServer struct { + local HealthServer + connSelector *raftpicker.ConnSelector + cluster raftpicker.RaftCluster + ctxMods []func(context.Context) (context.Context, error) +} + +func NewRaftProxyHealthServer(local HealthServer, connSelector *raftpicker.ConnSelector, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) HealthServer { + redirectChecker := func(ctx context.Context) (context.Context, error) { + s, ok := transport.StreamFromContext(ctx) + if !ok { + return ctx, grpc.Errorf(codes.InvalidArgument, "remote addr is not found in context") + } + addr := s.ServerTransport().RemoteAddr().String() + md, ok := metadata.FromContext(ctx) + if ok && len(md["redirect"]) != 0 { + return ctx, grpc.Errorf(codes.ResourceExhausted, "more than one redirect to leader from: %s", md["redirect"]) + } + if !ok { + md = metadata.New(map[string]string{}) + } + md["redirect"] = append(md["redirect"], addr) + return metadata.NewContext(ctx, md), nil + } + mods := []func(context.Context) (context.Context, error){redirectChecker} + mods = append(mods, ctxMod) + + return &raftProxyHealthServer{ + local: local, + cluster: cluster, + connSelector: connSelector, + ctxMods: mods, + } +} +func (p *raftProxyHealthServer) runCtxMods(ctx context.Context) (context.Context, error) { + var err error + for _, mod := range p.ctxMods { + ctx, err = mod(ctx) + if err != nil { + return ctx, err + } + } + return ctx, nil +} + +func (p *raftProxyHealthServer) Check(ctx context.Context, r *HealthCheckRequest) (*HealthCheckResponse, error) { + + if p.cluster.IsLeader() { + return p.local.Check(ctx, r) + } + ctx, err := p.runCtxMods(ctx) + if err != nil { + return nil, err + } + conn, err := p.connSelector.Conn() + if err != nil { + return nil, err + } + return NewHealthClient(conn).Check(ctx, r) +} + +func (m *HealthCheckRequest) Size() (n int) { + var l int + _ = l + l = len(m.Service) + if l > 0 { + n += 1 + l + sovHealth(uint64(l)) + } + return n +} + +func (m *HealthCheckResponse) Size() (n int) { + var l int + _ = l + if m.Status != 0 { + n += 1 + sovHealth(uint64(m.Status)) + } + return n +} + +func sovHealth(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozHealth(x uint64) (n int) { + return sovHealth(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *HealthCheckRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&HealthCheckRequest{`, + `Service:` + fmt.Sprintf("%v", this.Service) + `,`, + `}`, + }, "") + return s +} +func (this *HealthCheckResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&HealthCheckResponse{`, + `Status:` + fmt.Sprintf("%v", this.Status) + `,`, + `}`, + }, "") + return s +} +func valueToStringHealth(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *HealthCheckRequest) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHealth + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: HealthCheckRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: HealthCheckRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Service", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHealth + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthHealth + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Service = string(data[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipHealth(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthHealth + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *HealthCheckResponse) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHealth + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: HealthCheckResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: HealthCheckResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + m.Status = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHealth + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + m.Status |= (HealthCheckResponse_ServingStatus(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipHealth(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthHealth + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipHealth(data []byte) (n int, err error) { + l := len(data) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowHealth + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowHealth + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if data[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowHealth + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthHealth + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowHealth + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipHealth(data[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthHealth = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowHealth = fmt.Errorf("proto: integer overflow") +) + +var fileDescriptorHealth = []byte{ + // 284 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xc9, 0x48, 0x4d, 0xcc, + 0x29, 0xc9, 0xd0, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x4a, 0xc9, 0x4f, 0xce, 0x4e, 0x2d, + 0xd2, 0x2b, 0x2e, 0x4f, 0x2c, 0xca, 0xcd, 0xce, 0x2c, 0xd1, 0x2b, 0x33, 0x94, 0x12, 0x49, 0xcf, + 0x4f, 0xcf, 0x07, 0x4b, 0xeb, 0x83, 0x58, 0x10, 0x95, 0x52, 0xc2, 0x05, 0x39, 0xa5, 0xe9, 0x99, + 0x79, 0xfa, 0x10, 0x0a, 0x22, 0xa8, 0xa4, 0xc7, 0x25, 0xe4, 0x01, 0x36, 0xce, 0x39, 0x23, 0x35, + 0x39, 0x3b, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x48, 0x82, 0x8b, 0xbd, 0x38, 0xb5, 0xa8, + 0x2c, 0x33, 0x39, 0x55, 0x82, 0x51, 0x81, 0x51, 0x83, 0x33, 0x08, 0xc6, 0x55, 0x5a, 0xc0, 0xc8, + 0x25, 0x8c, 0xa2, 0xa1, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0x55, 0xc8, 0x97, 0x8b, 0xad, 0xb8, 0x24, + 0xb1, 0xa4, 0xb4, 0x18, 0xac, 0x81, 0xcf, 0xc8, 0x54, 0x0f, 0xd3, 0x5d, 0x7a, 0x58, 0x34, 0xea, + 0x05, 0x83, 0x0c, 0xce, 0x4b, 0x0f, 0x06, 0x6b, 0x0e, 0x82, 0x1a, 0xa2, 0x64, 0xc5, 0xc5, 0x8b, + 0x22, 0x21, 0xc4, 0xcd, 0xc5, 0x1e, 0xea, 0xe7, 0xed, 0xe7, 0x1f, 0xee, 0x27, 0xc0, 0x00, 0xe2, + 0x04, 0xbb, 0x06, 0x85, 0x79, 0xfa, 0xb9, 0x0b, 0x30, 0x0a, 0xf1, 0x73, 0x71, 0xfb, 0xf9, 0x87, + 0xc4, 0xc3, 0x04, 0x98, 0x8c, 0x2a, 0xb9, 0xd8, 0x20, 0x16, 0x09, 0xe5, 0x73, 0xb1, 0x82, 0x2d, + 0x13, 0x52, 0x23, 0xe8, 0x1a, 0xb0, 0xbf, 0xa5, 0xd4, 0x89, 0x74, 0xb5, 0x92, 0xe8, 0xa9, 0x75, + 0xef, 0x66, 0x30, 0xf1, 0x73, 0xf1, 0x82, 0x15, 0xea, 0xe6, 0x26, 0xe6, 0x25, 0xa6, 0xa7, 0x16, + 0x39, 0xc9, 0x9c, 0x78, 0x28, 0xc7, 0x70, 0x03, 0x88, 0x3f, 0x3c, 0x94, 0x63, 0x6c, 0x78, 0x24, + 0xc7, 0x78, 0x02, 0x88, 0x2f, 0x00, 0xf1, 0x03, 0x20, 0x4e, 0x62, 0x03, 0x07, 0xb9, 0x31, 0x20, + 0x00, 0x00, 0xff, 0xff, 0xf7, 0x14, 0x7c, 0x23, 0xc1, 0x01, 0x00, 0x00, +} diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/health.proto b/components/engine/vendor/src/github.com/docker/swarmkit/api/health.proto new file mode 100644 index 0000000000..5edb38bde4 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/health.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +// See: https://github.com/grpc/grpc-go/blob/master/health/grpc_health_v1/health.proto +// +// We use the same health check service proto description defined in the gRPC documentation, +// including the authorization check. This requires our own implementation of the health +// package located in `manager/health`. +// +// For more infos, refer to: +// https://github.com/grpc/grpc/blob/master/doc/health-checking.md + +package docker.swarmkit.v1; + +import "gogoproto/gogo.proto"; +import "plugin/plugin.proto"; + +service Health { + rpc Check(HealthCheckRequest) returns (HealthCheckResponse) { + option (docker.protobuf.plugin.tls_authorization) = { roles: "swarm-manager" }; + }; +} + +message HealthCheckRequest { + string service = 1; +} + +message HealthCheckResponse { + enum ServingStatus { + UNKNOWN = 0; + SERVING = 1; + NOT_SERVING = 2; + } + ServingStatus status = 1; +} diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/raft.pb.go b/components/engine/vendor/src/github.com/docker/swarmkit/api/raft.pb.go index 2fed8ffea8..a805663036 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/raft.pb.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/raft.pb.go @@ -90,8 +90,13 @@ func (*JoinRequest) ProtoMessage() {} func (*JoinRequest) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{1} } type JoinResponse struct { - RaftID uint64 `protobuf:"varint,1,opt,name=raft_id,json=raftId,proto3" json:"raft_id,omitempty"` + // RaftID is the ID assigned to the new member. + RaftID uint64 `protobuf:"varint,1,opt,name=raft_id,json=raftId,proto3" json:"raft_id,omitempty"` + // Members is the membership set of the cluster. Members []*RaftMember `protobuf:"bytes,2,rep,name=members" json:"members,omitempty"` + // RemovedMembers is a list of members that have been removed from + // the cluster, so the new node can avoid communicating with them. + RemovedMembers []uint64 `protobuf:"varint,3,rep,name=removed_members,json=removedMembers" json:"removed_members,omitempty"` } func (m *JoinResponse) Reset() { *m = JoinResponse{} } @@ -489,6 +494,13 @@ func (m *JoinResponse) Copy() *JoinResponse { } } + if m.RemovedMembers != nil { + o.RemovedMembers = make([]uint64, 0, len(m.RemovedMembers)) + for _, v := range m.RemovedMembers { + o.RemovedMembers = append(o.RemovedMembers, v) + } + } + return o } @@ -639,12 +651,13 @@ func (this *JoinResponse) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 6) + s := make([]string, 0, 7) s = append(s, "&api.JoinResponse{") s = append(s, "RaftID: "+fmt.Sprintf("%#v", this.RaftID)+",\n") if this.Members != nil { s = append(s, "Members: "+fmt.Sprintf("%#v", this.Members)+",\n") } + s = append(s, "RemovedMembers: "+fmt.Sprintf("%#v", this.RemovedMembers)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -1111,6 +1124,13 @@ func (m *JoinResponse) MarshalTo(data []byte) (int, error) { i += n } } + if len(m.RemovedMembers) > 0 { + for _, num := range m.RemovedMembers { + data[i] = 0x18 + i++ + i = encodeVarintRaft(data, i, uint64(num)) + } + } return i, nil } @@ -1611,6 +1631,11 @@ func (m *JoinResponse) Size() (n int) { n += 1 + l + sovRaft(uint64(l)) } } + if len(m.RemovedMembers) > 0 { + for _, e := range m.RemovedMembers { + n += 1 + sovRaft(uint64(e)) + } + } return n } @@ -1781,6 +1806,7 @@ func (this *JoinResponse) String() string { s := strings.Join([]string{`&JoinResponse{`, `RaftID:` + fmt.Sprintf("%v", this.RaftID) + `,`, `Members:` + strings.Replace(fmt.Sprintf("%v", this.Members), "RaftMember", "RaftMember", 1) + `,`, + `RemovedMembers:` + fmt.Sprintf("%v", this.RemovedMembers) + `,`, `}`, }, "") return s @@ -2238,6 +2264,26 @@ func (m *JoinResponse) Unmarshal(data []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RemovedMembers", wireType) + } + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.RemovedMembers = append(m.RemovedMembers, v) default: iNdEx = preIndex skippy, err := skipRaft(data[iNdEx:]) @@ -3108,58 +3154,59 @@ var ( ) var fileDescriptorRaft = []byte{ - // 833 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x95, 0x4d, 0x53, 0xdb, 0x46, - 0x18, 0xc7, 0x2d, 0x59, 0xc8, 0xed, 0x9a, 0xb7, 0x59, 0x0a, 0x35, 0x2a, 0x63, 0x40, 0x74, 0xa6, - 0x85, 0x29, 0xf2, 0xd4, 0x3d, 0xb4, 0xd3, 0xf6, 0x62, 0x1b, 0xcf, 0xd4, 0x05, 0x6c, 0x46, 0xd8, - 0x2d, 0x37, 0x2a, 0x4b, 0x8b, 0x51, 0x6d, 0x6b, 0x5d, 0xed, 0xda, 0x4c, 0x2f, 0x19, 0x8e, 0x19, - 0xae, 0x39, 0x24, 0x97, 0x9c, 0x92, 0x33, 0x1f, 0x20, 0x9f, 0x80, 0xc9, 0x29, 0xb7, 0xe4, 0x44, - 0x02, 0x1f, 0x20, 0xc9, 0x47, 0xc8, 0xae, 0x5e, 0x0c, 0x31, 0xb2, 0xf1, 0x41, 0xb0, 0xec, 0xfe, - 0xfe, 0xcf, 0xff, 0xd9, 0x67, 0xf7, 0x59, 0x00, 0x70, 0x8d, 0x23, 0xaa, 0x75, 0x5c, 0x4c, 0x31, - 0x84, 0x16, 0x36, 0x9b, 0xc8, 0xd5, 0xc8, 0x89, 0xe1, 0xb6, 0x9b, 0x36, 0xd5, 0x7a, 0x3f, 0x2a, - 0x53, 0xb8, 0xfe, 0x2f, 0x32, 0x29, 0xf1, 0x11, 0x25, 0x49, 0xff, 0xef, 0xa0, 0xf0, 0x8f, 0xcd, - 0x86, 0x4d, 0x8f, 0xbb, 0x75, 0xcd, 0xc4, 0xed, 0x8c, 0x89, 0x5d, 0x84, 0x49, 0x06, 0x51, 0xd3, - 0xca, 0xf0, 0x90, 0xde, 0x8f, 0x4e, 0x3d, 0x73, 0x13, 0x5e, 0xf9, 0xaa, 0x81, 0x1b, 0xd8, 0x1b, - 0x66, 0xf8, 0x28, 0x98, 0x9d, 0xeb, 0xb4, 0xba, 0x0d, 0xdb, 0xc9, 0xf8, 0xbf, 0xfc, 0x49, 0xf5, - 0x5c, 0x00, 0x40, 0x67, 0xca, 0x5d, 0xd4, 0xae, 0x23, 0x17, 0xae, 0x81, 0x04, 0x8f, 0x73, 0x68, - 0x5b, 0x29, 0x61, 0x45, 0xf8, 0x5e, 0xca, 0x83, 0xeb, 0xcb, 0x65, 0x99, 0x03, 0xa5, 0x2d, 0x5d, - 0xe6, 0x4b, 0x25, 0x8b, 0x43, 0x0e, 0xb6, 0x10, 0x87, 0x44, 0x06, 0x7d, 0xe9, 0x43, 0x65, 0x36, - 0xc5, 0x21, 0xbe, 0xc4, 0x20, 0x08, 0x24, 0xc3, 0xb2, 0xdc, 0x54, 0x9c, 0x13, 0xba, 0x37, 0x86, - 0x79, 0x20, 0x13, 0x6a, 0xd0, 0x2e, 0x49, 0x49, 0x6c, 0x36, 0x99, 0xfd, 0x56, 0xbb, 0x5b, 0x07, - 0xed, 0x26, 0x9b, 0x7d, 0x8f, 0xcd, 0x4b, 0x17, 0x97, 0xcb, 0x31, 0x3d, 0x50, 0xaa, 0xab, 0x20, - 0xf9, 0x27, 0xb6, 0x1d, 0x1d, 0xfd, 0xd7, 0x45, 0x84, 0xf6, 0x6d, 0x84, 0x1b, 0x1b, 0xb5, 0x0d, - 0x26, 0x7d, 0x84, 0x74, 0xb0, 0x43, 0xd0, 0x78, 0x9b, 0xfa, 0x05, 0x24, 0xda, 0x9e, 0x2b, 0x61, - 0x9b, 0x8a, 0xb3, 0xe4, 0xd2, 0xa3, 0x93, 0xd3, 0x43, 0x5c, 0xcd, 0x83, 0xc9, 0x1d, 0x64, 0xf4, - 0x50, 0x98, 0x52, 0x16, 0x48, 0xbc, 0x06, 0x9e, 0xd7, 0xfd, 0x61, 0x3c, 0x56, 0x9d, 0x01, 0x53, - 0x41, 0x0c, 0x3f, 0x67, 0x75, 0x07, 0x2c, 0xee, 0xb9, 0xd8, 0x44, 0x84, 0xf8, 0x2c, 0x21, 0x46, - 0xa3, 0xef, 0xb0, 0xce, 0x73, 0xf5, 0x66, 0x02, 0x93, 0x19, 0xcd, 0xbf, 0x04, 0x5a, 0x08, 0x86, - 0xeb, 0xbf, 0x4a, 0xa7, 0x8f, 0xd5, 0x98, 0xba, 0x04, 0x94, 0xa8, 0x68, 0x81, 0xd7, 0xef, 0x60, - 0x9e, 0x8d, 0x71, 0xab, 0x87, 0x72, 0xac, 0x7c, 0x1c, 0x0a, 0x7c, 0xc6, 0x29, 0x9c, 0xfa, 0x03, - 0x58, 0x18, 0x54, 0x07, 0x75, 0x8f, 0x3a, 0x9b, 0x23, 0x30, 0x57, 0x72, 0x28, 0x72, 0x1d, 0xa3, - 0xc5, 0xe3, 0x84, 0x4e, 0x0b, 0x40, 0xec, 0x9b, 0xc8, 0xcc, 0x44, 0x64, 0x06, 0x6c, 0x06, 0xfe, - 0x0c, 0x64, 0xc3, 0xa4, 0x36, 0x76, 0x82, 0x43, 0x59, 0x8e, 0xaa, 0xe6, 0x3e, 0x65, 0x2d, 0x91, - 0xf3, 0x30, 0x3d, 0xc0, 0xd5, 0xb7, 0x22, 0x48, 0xde, 0x9a, 0x87, 0xbf, 0xf5, 0x03, 0x71, 0x93, - 0xe9, 0xec, 0xda, 0x3d, 0x81, 0xb6, 0x6d, 0xc7, 0x0a, 0x83, 0x41, 0x2d, 0x38, 0x51, 0xd1, 0x2b, - 0x76, 0x2a, 0x4a, 0xca, 0x6f, 0xff, 0x1f, 0x31, 0xff, 0x34, 0x59, 0xd6, 0x09, 0x82, 0xdc, 0x9e, - 0x6d, 0x22, 0xef, 0xfa, 0x27, 0xb3, 0xdf, 0x44, 0xba, 0xf9, 0x08, 0x53, 0x85, 0x34, 0x37, 0xa2, - 0x06, 0x69, 0x06, 0xed, 0x11, 0x69, 0x54, 0x65, 0xeb, 0xdc, 0x88, 0x73, 0xdc, 0xc8, 0x41, 0xf4, - 0x04, 0xbb, 0xcd, 0xd4, 0xc4, 0x70, 0xa3, 0xb2, 0x8f, 0x70, 0xa3, 0x80, 0xe6, 0x42, 0xb3, 0xd5, - 0x25, 0xec, 0x20, 0x52, 0xf2, 0x70, 0x61, 0xc1, 0x47, 0xb8, 0x30, 0xa0, 0xf3, 0x5f, 0x00, 0x99, - 0x1a, 0x6e, 0x03, 0xd1, 0x8d, 0x0f, 0x02, 0x98, 0x19, 0x28, 0x18, 0xfc, 0x0e, 0x24, 0x6a, 0xe5, - 0xed, 0x72, 0xe5, 0xef, 0xf2, 0x6c, 0x4c, 0x51, 0xce, 0x9e, 0xae, 0x2c, 0x0c, 0x10, 0x35, 0xa7, - 0xe9, 0xe0, 0x13, 0x87, 0xf5, 0xc8, 0xdc, 0x7e, 0xb5, 0xa2, 0x17, 0x0f, 0x73, 0x85, 0x6a, 0xa9, - 0x52, 0x3e, 0x2c, 0xe8, 0xc5, 0x5c, 0xb5, 0x38, 0x2b, 0x28, 0x8b, 0x4c, 0x34, 0x3f, 0x20, 0x2a, - 0xb8, 0xc8, 0xa0, 0xe8, 0x8e, 0xa6, 0xb6, 0xb7, 0xc5, 0x35, 0x62, 0xa4, 0xa6, 0xd6, 0xb1, 0xa2, - 0x34, 0x7a, 0x71, 0xb7, 0xf2, 0x57, 0x71, 0x36, 0x1e, 0xa9, 0xd1, 0x51, 0x1b, 0xf7, 0x90, 0xf2, - 0xf5, 0xc3, 0x67, 0xe9, 0xd8, 0x8b, 0xe7, 0xe9, 0xc1, 0xdd, 0x65, 0x1f, 0x89, 0x40, 0xe2, 0x97, - 0x16, 0x9e, 0x09, 0x00, 0xde, 0xed, 0x27, 0xb8, 0x19, 0x55, 0xc3, 0xa1, 0x5d, 0xac, 0x68, 0xe3, - 0xe2, 0x41, 0x9b, 0xce, 0xbf, 0x3c, 0x7f, 0xff, 0x44, 0x64, 0x2f, 0x85, 0xc7, 0x6f, 0xb6, 0x0d, - 0x87, 0xad, 0xba, 0xf0, 0x01, 0x98, 0xfe, 0xbc, 0xff, 0xe0, 0x7a, 0xe4, 0x93, 0x13, 0xd5, 0xe1, - 0xca, 0xc6, 0x38, 0xe8, 0x48, 0xff, 0xec, 0x6b, 0x81, 0x25, 0xd0, 0x7f, 0xcf, 0xc8, 0xb1, 0xdd, - 0x81, 0xff, 0x00, 0x89, 0x3f, 0xc0, 0x30, 0xb2, 0x5b, 0x6f, 0xbd, 0xde, 0xca, 0xca, 0x70, 0x60, - 0xf4, 0xa6, 0x4d, 0x30, 0xe1, 0xbd, 0x97, 0x30, 0x32, 0xc2, 0xed, 0xe7, 0x58, 0x59, 0x1d, 0x41, - 0x8c, 0x34, 0xc9, 0x2f, 0x5d, 0x5c, 0xa5, 0x63, 0x6f, 0xd8, 0xf7, 0xf1, 0x2a, 0x2d, 0x9c, 0x5e, - 0xa7, 0x85, 0x0b, 0xf6, 0xbd, 0x62, 0xdf, 0x3b, 0xf6, 0x1d, 0xc4, 0x0f, 0xa4, 0xba, 0xec, 0xfd, - 0x13, 0xfd, 0xe9, 0x53, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa4, 0xfb, 0x14, 0x74, 0xdc, 0x07, 0x00, - 0x00, + // 852 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x95, 0x4f, 0x53, 0x23, 0x45, + 0x18, 0xc6, 0x33, 0x93, 0x61, 0xa2, 0x1d, 0x20, 0x54, 0x23, 0x18, 0x46, 0x2a, 0xc0, 0x60, 0x95, + 0x42, 0xc9, 0xa4, 0x8c, 0x07, 0x2d, 0xf5, 0x92, 0x84, 0x54, 0x19, 0x81, 0x84, 0x1a, 0x12, 0xe5, + 0x16, 0x27, 0x33, 0x4d, 0x18, 0x93, 0x4c, 0xc7, 0xe9, 0x4e, 0x28, 0x2f, 0x16, 0x47, 0x8b, 0xab, + 0x55, 0xea, 0xc5, 0x93, 0x9e, 0xf9, 0x00, 0x7e, 0x02, 0x6a, 0x4f, 0x7b, 0xdb, 0x3d, 0xb1, 0x0b, + 0x1f, 0x60, 0x77, 0x3f, 0xc2, 0x76, 0xcf, 0x9f, 0xc0, 0x86, 0x49, 0xc8, 0xa1, 0xa1, 0x79, 0xfb, + 0xf7, 0xbc, 0x4f, 0xf7, 0xdb, 0xf3, 0x36, 0x00, 0xb8, 0xc6, 0x09, 0xd5, 0x7a, 0x2e, 0xa6, 0x18, + 0x42, 0x0b, 0x9b, 0x6d, 0xe4, 0x6a, 0xe4, 0xcc, 0x70, 0xbb, 0x6d, 0x9b, 0x6a, 0x83, 0xcf, 0x95, + 0x39, 0xdc, 0xfc, 0x19, 0x99, 0x94, 0xf8, 0x88, 0x92, 0xa4, 0xbf, 0xf6, 0x50, 0xf8, 0xc7, 0x4e, + 0xcb, 0xa6, 0xa7, 0xfd, 0xa6, 0x66, 0xe2, 0x6e, 0xd6, 0xc4, 0x2e, 0xc2, 0x24, 0x8b, 0xa8, 0x69, + 0x65, 0x79, 0x4a, 0xef, 0x47, 0xaf, 0x99, 0xbd, 0x4b, 0xaf, 0x7c, 0xd0, 0xc2, 0x2d, 0xec, 0x4d, + 0xb3, 0x7c, 0x16, 0x44, 0x17, 0x7b, 0x9d, 0x7e, 0xcb, 0x76, 0xb2, 0xfe, 0x2f, 0x3f, 0xa8, 0x5e, + 0x0a, 0x00, 0xe8, 0x4c, 0x79, 0x80, 0xba, 0x4d, 0xe4, 0xc2, 0x4d, 0x90, 0xe0, 0x79, 0x1a, 0xb6, + 0x95, 0x16, 0xd6, 0x85, 0x4f, 0xa5, 0x02, 0xb8, 0xbd, 0x5e, 0x93, 0x39, 0x50, 0xde, 0xd5, 0x65, + 0xbe, 0x54, 0xb6, 0x38, 0xe4, 0x60, 0x0b, 0x71, 0x48, 0x64, 0xd0, 0xfb, 0x3e, 0x54, 0x61, 0x21, + 0x0e, 0xf1, 0x25, 0x06, 0x41, 0x20, 0x19, 0x96, 0xe5, 0xa6, 0xe3, 0x9c, 0xd0, 0xbd, 0x39, 0x2c, + 0x00, 0x99, 0x50, 0x83, 0xf6, 0x49, 0x5a, 0x62, 0xd1, 0x64, 0xee, 0x63, 0xed, 0x61, 0x1d, 0xb4, + 0xbb, 0xdd, 0x1c, 0x79, 0x6c, 0x41, 0xba, 0xba, 0x5e, 0x8b, 0xe9, 0x81, 0x52, 0xdd, 0x00, 0xc9, + 0xef, 0xb1, 0xed, 0xe8, 0xe8, 0x97, 0x3e, 0x22, 0x74, 0x68, 0x23, 0xdc, 0xd9, 0xa8, 0x7f, 0x0a, + 0x60, 0xd6, 0x67, 0x48, 0x0f, 0x3b, 0x04, 0x4d, 0x77, 0xaa, 0xaf, 0x40, 0xa2, 0xeb, 0xd9, 0x12, + 0x76, 0xaa, 0x38, 0xdb, 0x5d, 0x66, 0xf2, 0xee, 0xf4, 0x10, 0x87, 0x9f, 0x80, 0x94, 0x8b, 0xba, + 0x78, 0x80, 0xac, 0x46, 0x98, 0x21, 0xce, 0x32, 0x48, 0xfa, 0x7c, 0x10, 0xf6, 0x05, 0x44, 0x2d, + 0x80, 0xd9, 0x7d, 0x64, 0x0c, 0x50, 0xb8, 0xf9, 0x1c, 0x90, 0x78, 0xb5, 0xbc, 0x4d, 0x3d, 0xee, + 0xe7, 0xb1, 0x6a, 0x0a, 0xcc, 0x05, 0x39, 0xfc, 0xc3, 0xa9, 0xfb, 0x60, 0xe5, 0xd0, 0xc5, 0x26, + 0x22, 0xc4, 0x67, 0x09, 0x31, 0x5a, 0x43, 0x87, 0x2d, 0x7e, 0x28, 0x2f, 0x12, 0x98, 0xa4, 0x34, + 0xff, 0x73, 0xd1, 0x42, 0x30, 0x5c, 0xff, 0x5a, 0x3a, 0xff, 0x4b, 0x8d, 0xa9, 0xab, 0x40, 0x89, + 0xca, 0x16, 0x78, 0x7d, 0x0b, 0x96, 0xd8, 0x1c, 0x77, 0x06, 0x28, 0xcf, 0x0a, 0xcd, 0xa1, 0xc0, + 0x67, 0x9a, 0x0a, 0xab, 0x9f, 0x81, 0xe5, 0x51, 0x75, 0x70, 0x41, 0x51, 0xb7, 0x78, 0x02, 0x16, + 0xcb, 0x0e, 0x45, 0xae, 0x63, 0x74, 0x78, 0x9e, 0xd0, 0x69, 0x19, 0x88, 0x43, 0x13, 0x99, 0x99, + 0x88, 0xcc, 0x80, 0x45, 0xe0, 0x97, 0x40, 0x36, 0x4c, 0x6a, 0x63, 0x27, 0xb8, 0xbd, 0xb5, 0xa8, + 0x6a, 0x1e, 0x51, 0xd6, 0x3c, 0x79, 0x0f, 0xd3, 0x03, 0x5c, 0x7d, 0x21, 0x82, 0xe4, 0xbd, 0x38, + 0xfc, 0x66, 0x98, 0x88, 0x9b, 0xcc, 0xe7, 0x36, 0x1f, 0x49, 0xb4, 0x67, 0x3b, 0x56, 0x98, 0x0c, + 0x6a, 0xc1, 0x8d, 0x8a, 0x5e, 0xb1, 0xd3, 0x51, 0x52, 0xde, 0x27, 0xdf, 0xc5, 0xfc, 0xdb, 0x64, + 0xbb, 0x4e, 0x10, 0xe4, 0x0e, 0x6c, 0x13, 0x79, 0x8d, 0x92, 0xcc, 0x7d, 0x14, 0xe9, 0xe6, 0x23, + 0x4c, 0x15, 0xd2, 0xdc, 0x88, 0x1a, 0xa4, 0x1d, 0x34, 0x52, 0xa4, 0x51, 0x8d, 0xad, 0x73, 0x23, + 0xce, 0x71, 0x23, 0x07, 0xd1, 0x33, 0xec, 0xb6, 0xd3, 0x33, 0xe3, 0x8d, 0x2a, 0x3e, 0xc2, 0x8d, + 0x02, 0x9a, 0x0b, 0xcd, 0x4e, 0x9f, 0xb0, 0x8b, 0x48, 0xcb, 0xe3, 0x85, 0x45, 0x1f, 0xe1, 0xc2, + 0x80, 0x2e, 0xbc, 0x07, 0x64, 0x6a, 0xb8, 0x2d, 0x44, 0xb7, 0x5f, 0x0b, 0x20, 0x35, 0x52, 0x30, + 0xd6, 0x33, 0x89, 0x7a, 0x65, 0xaf, 0x52, 0xfd, 0xb1, 0xb2, 0x10, 0x53, 0x94, 0x8b, 0x7f, 0xd6, + 0x97, 0x47, 0x88, 0xba, 0xd3, 0x76, 0xf0, 0x99, 0xc3, 0x7a, 0x64, 0xf1, 0xa8, 0x56, 0xd5, 0x4b, + 0x8d, 0x7c, 0xb1, 0x56, 0xae, 0x56, 0x1a, 0x45, 0xbd, 0x94, 0xaf, 0x95, 0x16, 0x04, 0x65, 0x85, + 0x89, 0x96, 0x46, 0x44, 0x45, 0x17, 0x19, 0x14, 0x3d, 0xd0, 0xd4, 0x0f, 0x77, 0xb9, 0x46, 0x8c, + 0xd4, 0xd4, 0x7b, 0x56, 0x94, 0x46, 0x2f, 0x1d, 0x54, 0x7f, 0x28, 0x2d, 0xc4, 0x23, 0x35, 0xba, + 0xd7, 0xd7, 0xca, 0x87, 0xbf, 0xff, 0x9b, 0x89, 0xfd, 0xff, 0x5f, 0x66, 0xf4, 0x74, 0xb9, 0x3f, + 0x44, 0x20, 0xf1, 0x8f, 0x16, 0x5e, 0x08, 0x00, 0x3e, 0xec, 0x27, 0xb8, 0x13, 0x55, 0xc3, 0xb1, + 0x5d, 0xac, 0x68, 0xd3, 0xe2, 0x41, 0x9b, 0x2e, 0x3d, 0xb9, 0x7c, 0xf5, 0xb7, 0xc8, 0x5e, 0x0a, + 0x8f, 0xdf, 0xe9, 0x1a, 0x0e, 0x5b, 0x75, 0xe1, 0x6f, 0x60, 0xfe, 0xdd, 0xfe, 0x83, 0x5b, 0x91, + 0x4f, 0x4e, 0x54, 0x87, 0x2b, 0xdb, 0xd3, 0xa0, 0x13, 0xfd, 0x73, 0xcf, 0x04, 0xb6, 0x81, 0xe1, + 0x7b, 0x46, 0x4e, 0xed, 0x1e, 0xfc, 0x09, 0x48, 0xfc, 0xa5, 0x86, 0x91, 0xdd, 0x7a, 0xef, 0x9d, + 0x57, 0xd6, 0xc7, 0x03, 0x93, 0x0f, 0x6d, 0x82, 0x19, 0xef, 0xbd, 0x84, 0x91, 0x19, 0xee, 0x3f, + 0xc7, 0xca, 0xc6, 0x04, 0x62, 0xa2, 0x49, 0x61, 0xf5, 0xea, 0x26, 0x13, 0x7b, 0xce, 0xc6, 0x9b, + 0x9b, 0x8c, 0x70, 0x7e, 0x9b, 0x11, 0xae, 0xd8, 0x78, 0xca, 0xc6, 0x4b, 0x36, 0x8e, 0xe3, 0xc7, + 0x52, 0x53, 0xf6, 0xfe, 0xdd, 0x7e, 0xf1, 0x36, 0x00, 0x00, 0xff, 0xff, 0xd7, 0x61, 0x3c, 0x43, + 0x06, 0x08, 0x00, 0x00, } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/raft.proto b/components/engine/vendor/src/github.com/docker/swarmkit/api/raft.proto index 26b1cf00f7..911de323f4 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/raft.proto +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/raft.proto @@ -58,8 +58,15 @@ message JoinRequest { } message JoinResponse { + // RaftID is the ID assigned to the new member. uint64 raft_id = 1 [(gogoproto.customname) = "RaftID"]; + + // Members is the membership set of the cluster. repeated RaftMember members = 2; + + // RemovedMembers is a list of members that have been removed from + // the cluster, so the new node can avoid communicating with them. + repeated uint64 removed_members = 3; } message LeaveRequest { diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.pb.go b/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.pb.go index 7622551e76..c8f45105e8 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.pb.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.pb.go @@ -268,7 +268,7 @@ func _ServiceSpec_OneofSizer(msg proto.Message) (n int) { // instructing Swarm on how this service should work on the particular // network. type ServiceSpec_NetworkAttachmentConfig struct { - // Target specifies the target network for attachement. This value may be a + // Target specifies the target network for attachment. This value may be a // network name or identifier. Only identifiers are supported at this time. Target string `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` // Aliases specifies a list of discoverable alternate names for the service on this Target. @@ -281,7 +281,7 @@ func (*ServiceSpec_NetworkAttachmentConfig) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{1, 0} } -// ReplicatedService set the reconcilation target to certain number of replicas. +// ReplicatedService sets the reconciliation target to certain number of replicas. type ReplicatedService struct { Replicas uint64 `protobuf:"varint,1,opt,name=replicas,proto3" json:"replicas,omitempty"` } @@ -290,7 +290,7 @@ func (m *ReplicatedService) Reset() { *m = ReplicatedService{ func (*ReplicatedService) ProtoMessage() {} func (*ReplicatedService) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{2} } -// GlobalService represent global service. +// GlobalService represents global service. type GlobalService struct { } @@ -415,9 +415,12 @@ type ContainerSpec struct { // executable and the following elements are treated as arguments. // // If command is empty, execution will fall back to the image's entrypoint. + // + // Command should only be used when overriding entrypoint. Command []string `protobuf:"bytes,3,rep,name=command" json:"command,omitempty"` // Args specifies arguments provided to the image's entrypoint. - // Ignored if command is specified. + // + // If Command and Args are provided, Args will be appended to Command. Args []string `protobuf:"bytes,4,rep,name=args" json:"args,omitempty"` // Env specifies the environment variables for the container in NAME=VALUE // format. These must be compliant with [IEEE Std diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.proto b/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.proto index 195bcacc0f..b012ec6611 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.proto +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/specs.proto @@ -78,7 +78,7 @@ message ServiceSpec { // instructing Swarm on how this service should work on the particular // network. message NetworkAttachmentConfig { - // Target specifies the target network for attachement. This value may be a + // Target specifies the target network for attachment. This value may be a // network name or identifier. Only identifiers are supported at this time. string target = 1; // Aliases specifies a list of discoverable alternate names for the service on this Target. @@ -91,12 +91,12 @@ message ServiceSpec { EndpointSpec endpoint = 8; } -// ReplicatedService set the reconcilation target to certain number of replicas. +// ReplicatedService sets the reconciliation target to certain number of replicas. message ReplicatedService { uint64 replicas = 1; } -// GlobalService represent global service. +// GlobalService represents global service. message GlobalService { // Empty message for now. } @@ -138,10 +138,13 @@ message ContainerSpec { // executable and the following elements are treated as arguments. // // If command is empty, execution will fall back to the image's entrypoint. + // + // Command should only be used when overriding entrypoint. repeated string command = 3; // Args specifies arguments provided to the image's entrypoint. - // Ignored if command is specified. + // + // If Command and Args are provided, Args will be appended to Command. repeated string args = 4; // Env specifies the environment variables for the container in NAME=VALUE diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/types.pb.go b/components/engine/vendor/src/github.com/docker/swarmkit/api/types.pb.go index 242e108f95..065a58c217 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/types.pb.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/types.pb.go @@ -14,6 +14,7 @@ ca.proto snapshot.proto raft.proto + health.proto It has these top-level messages: Version @@ -40,6 +41,7 @@ WeightedPeer IssuanceStatus AcceptancePolicy + ExternalCA CAConfig OrchestrationConfig DispatcherConfig @@ -132,6 +134,8 @@ ResolveAddressResponse InternalRaftRequest StoreAction + HealthCheckRequest + HealthCheckResponse */ package api @@ -462,6 +466,26 @@ func (x IssuanceStatus_State) String() string { } func (IssuanceStatus_State) EnumDescriptor() ([]byte, []int) { return fileDescriptorTypes, []int{22, 0} } +type ExternalCA_CAProtocol int32 + +const ( + ExternalCA_CAProtocolCFSSL ExternalCA_CAProtocol = 0 +) + +var ExternalCA_CAProtocol_name = map[int32]string{ + 0: "CFSSL", +} +var ExternalCA_CAProtocol_value = map[string]int32{ + "CFSSL": 0, +} + +func (x ExternalCA_CAProtocol) String() string { + return proto.EnumName(ExternalCA_CAProtocol_name, int32(x)) +} +func (ExternalCA_CAProtocol) EnumDescriptor() ([]byte, []int) { + return fileDescriptorTypes, []int{24, 0} +} + // Encryption algorithm that can implemented using this key type EncryptionKey_Algorithm int32 @@ -480,7 +504,7 @@ func (x EncryptionKey_Algorithm) String() string { return proto.EnumName(EncryptionKey_Algorithm_name, int32(x)) } func (EncryptionKey_Algorithm) EnumDescriptor() ([]byte, []int) { - return fileDescriptorTypes, []int{31, 0} + return fileDescriptorTypes, []int{32, 0} } // Version tracks the last time an object in the store was updated. @@ -955,14 +979,31 @@ func (*AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Descriptor() ([]byte, return fileDescriptorTypes, []int{23, 0, 0} } +type ExternalCA struct { + // Protocol is the protocol used by this external CA. + Protocol ExternalCA_CAProtocol `protobuf:"varint,1,opt,name=protocol,proto3,enum=docker.swarmkit.v1.ExternalCA_CAProtocol" json:"protocol,omitempty"` + // URL is the URL where the external CA can be reached. + URL string `protobuf:"bytes,2,opt,name=url,proto3" json:"url,omitempty"` + // Options is a set of additional key/value pairs whose interpretation + // depends on the specified CA type. + Options map[string]string `protobuf:"bytes,3,rep,name=options" json:"options,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (m *ExternalCA) Reset() { *m = ExternalCA{} } +func (*ExternalCA) ProtoMessage() {} +func (*ExternalCA) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{24} } + type CAConfig struct { // NodeCertExpiry is the duration certificates should be issued for NodeCertExpiry *docker_swarmkit_v11.Duration `protobuf:"bytes,1,opt,name=node_cert_expiry,json=nodeCertExpiry" json:"node_cert_expiry,omitempty"` + // ExternalCAs is a list of CAs to which a manager node will make + // certificate signing requests for node certificates. + ExternalCAs []*ExternalCA `protobuf:"bytes,2,rep,name=external_cas,json=externalCas" json:"external_cas,omitempty"` } func (m *CAConfig) Reset() { *m = CAConfig{} } func (*CAConfig) ProtoMessage() {} -func (*CAConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{24} } +func (*CAConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{25} } // OrchestrationConfig defines cluster-level orchestration settings. type OrchestrationConfig struct { @@ -973,7 +1014,7 @@ type OrchestrationConfig struct { func (m *OrchestrationConfig) Reset() { *m = OrchestrationConfig{} } func (*OrchestrationConfig) ProtoMessage() {} -func (*OrchestrationConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{25} } +func (*OrchestrationConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{26} } // DispatcherConfig defines cluster-level dispatcher settings. type DispatcherConfig struct { @@ -984,7 +1025,7 @@ type DispatcherConfig struct { func (m *DispatcherConfig) Reset() { *m = DispatcherConfig{} } func (*DispatcherConfig) ProtoMessage() {} -func (*DispatcherConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{26} } +func (*DispatcherConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{27} } // RaftConfig defines raft settings for the cluster. type RaftConfig struct { @@ -1006,7 +1047,7 @@ type RaftConfig struct { func (m *RaftConfig) Reset() { *m = RaftConfig{} } func (*RaftConfig) ProtoMessage() {} -func (*RaftConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{27} } +func (*RaftConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{28} } // Placement specifies task distribution constraints. type Placement struct { @@ -1016,7 +1057,7 @@ type Placement struct { func (m *Placement) Reset() { *m = Placement{} } func (*Placement) ProtoMessage() {} -func (*Placement) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{28} } +func (*Placement) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{29} } type RootCA struct { // CAKey is the root CA private key. @@ -1029,7 +1070,7 @@ type RootCA struct { func (m *RootCA) Reset() { *m = RootCA{} } func (*RootCA) ProtoMessage() {} -func (*RootCA) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{29} } +func (*RootCA) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{30} } type Certificate struct { Role NodeRole `protobuf:"varint,1,opt,name=role,proto3,enum=docker.swarmkit.v1.NodeRole" json:"role,omitempty"` @@ -1042,7 +1083,7 @@ type Certificate struct { func (m *Certificate) Reset() { *m = Certificate{} } func (*Certificate) ProtoMessage() {} -func (*Certificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{30} } +func (*Certificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{31} } // Symmetric keys to encrypt inter-agent communication. type EncryptionKey struct { @@ -1058,7 +1099,7 @@ type EncryptionKey struct { func (m *EncryptionKey) Reset() { *m = EncryptionKey{} } func (*EncryptionKey) ProtoMessage() {} -func (*EncryptionKey) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{31} } +func (*EncryptionKey) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{32} } // ManagerStatus provides informations about the state of a manager in the cluster. type ManagerStatus struct { @@ -1075,7 +1116,7 @@ type ManagerStatus struct { func (m *ManagerStatus) Reset() { *m = ManagerStatus{} } func (*ManagerStatus) ProtoMessage() {} -func (*ManagerStatus) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{32} } +func (*ManagerStatus) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{33} } func init() { proto.RegisterType((*Version)(nil), "docker.swarmkit.v1.Version") @@ -1106,6 +1147,7 @@ func init() { proto.RegisterType((*AcceptancePolicy)(nil), "docker.swarmkit.v1.AcceptancePolicy") proto.RegisterType((*AcceptancePolicy_RoleAdmissionPolicy)(nil), "docker.swarmkit.v1.AcceptancePolicy.RoleAdmissionPolicy") proto.RegisterType((*AcceptancePolicy_RoleAdmissionPolicy_HashedSecret)(nil), "docker.swarmkit.v1.AcceptancePolicy.RoleAdmissionPolicy.HashedSecret") + proto.RegisterType((*ExternalCA)(nil), "docker.swarmkit.v1.ExternalCA") proto.RegisterType((*CAConfig)(nil), "docker.swarmkit.v1.CAConfig") proto.RegisterType((*OrchestrationConfig)(nil), "docker.swarmkit.v1.OrchestrationConfig") proto.RegisterType((*DispatcherConfig)(nil), "docker.swarmkit.v1.DispatcherConfig") @@ -1125,6 +1167,7 @@ func init() { proto.RegisterEnum("docker.swarmkit.v1.IPAMConfig_AddressFamily", IPAMConfig_AddressFamily_name, IPAMConfig_AddressFamily_value) proto.RegisterEnum("docker.swarmkit.v1.PortConfig_Protocol", PortConfig_Protocol_name, PortConfig_Protocol_value) proto.RegisterEnum("docker.swarmkit.v1.IssuanceStatus_State", IssuanceStatus_State_name, IssuanceStatus_State_value) + proto.RegisterEnum("docker.swarmkit.v1.ExternalCA_CAProtocol", ExternalCA_CAProtocol_name, ExternalCA_CAProtocol_value) proto.RegisterEnum("docker.swarmkit.v1.EncryptionKey_Algorithm", EncryptionKey_Algorithm_name, EncryptionKey_Algorithm_value) } @@ -1564,6 +1607,26 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Copy() *AcceptancePo return o } +func (m *ExternalCA) Copy() *ExternalCA { + if m == nil { + return nil + } + + o := &ExternalCA{ + Protocol: m.Protocol, + URL: m.URL, + } + + if m.Options != nil { + o.Options = make(map[string]string) + for k, v := range m.Options { + o.Options[k] = v + } + } + + return o +} + func (m *CAConfig) Copy() *CAConfig { if m == nil { return nil @@ -1573,6 +1636,13 @@ func (m *CAConfig) Copy() *CAConfig { NodeCertExpiry: m.NodeCertExpiry.Copy(), } + if m.ExternalCAs != nil { + o.ExternalCAs = make([]*ExternalCA, 0, len(m.ExternalCAs)) + for _, v := range m.ExternalCAs { + o.ExternalCAs = append(o.ExternalCAs, v.Copy()) + } + } + return o } @@ -2122,15 +2192,42 @@ func (this *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) GoString() string s = append(s, "}") return strings.Join(s, "") } +func (this *ExternalCA) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 7) + s = append(s, "&api.ExternalCA{") + s = append(s, "Protocol: "+fmt.Sprintf("%#v", this.Protocol)+",\n") + s = append(s, "URL: "+fmt.Sprintf("%#v", this.URL)+",\n") + keysForOptions := make([]string, 0, len(this.Options)) + for k, _ := range this.Options { + keysForOptions = append(keysForOptions, k) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForOptions) + mapStringForOptions := "map[string]string{" + for _, k := range keysForOptions { + mapStringForOptions += fmt.Sprintf("%#v: %#v,", k, this.Options[k]) + } + mapStringForOptions += "}" + if this.Options != nil { + s = append(s, "Options: "+mapStringForOptions+",\n") + } + s = append(s, "}") + return strings.Join(s, "") +} func (this *CAConfig) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 5) + s := make([]string, 0, 6) s = append(s, "&api.CAConfig{") if this.NodeCertExpiry != nil { s = append(s, "NodeCertExpiry: "+fmt.Sprintf("%#v", this.NodeCertExpiry)+",\n") } + if this.ExternalCAs != nil { + s = append(s, "ExternalCAs: "+fmt.Sprintf("%#v", this.ExternalCAs)+",\n") + } s = append(s, "}") return strings.Join(s, "") } @@ -3341,6 +3438,52 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) MarshalTo(data []byt return i, nil } +func (m *ExternalCA) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *ExternalCA) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Protocol != 0 { + data[i] = 0x8 + i++ + i = encodeVarintTypes(data, i, uint64(m.Protocol)) + } + if len(m.URL) > 0 { + data[i] = 0x12 + i++ + i = encodeVarintTypes(data, i, uint64(len(m.URL))) + i += copy(data[i:], m.URL) + } + if len(m.Options) > 0 { + for k, _ := range m.Options { + data[i] = 0x1a + i++ + v := m.Options[k] + mapSize := 1 + len(k) + sovTypes(uint64(len(k))) + 1 + len(v) + sovTypes(uint64(len(v))) + i = encodeVarintTypes(data, i, uint64(mapSize)) + data[i] = 0xa + i++ + i = encodeVarintTypes(data, i, uint64(len(k))) + i += copy(data[i:], k) + data[i] = 0x12 + i++ + i = encodeVarintTypes(data, i, uint64(len(v))) + i += copy(data[i:], v) + } + } + return i, nil +} + func (m *CAConfig) Marshal() (data []byte, err error) { size := m.Size() data = make([]byte, size) @@ -3366,6 +3509,18 @@ func (m *CAConfig) MarshalTo(data []byte) (int, error) { } i += n18 } + if len(m.ExternalCAs) > 0 { + for _, msg := range m.ExternalCAs { + data[i] = 0x12 + i++ + i = encodeVarintTypes(data, i, uint64(msg.Size())) + n, err := msg.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n + } + } return i, nil } @@ -4161,6 +4316,27 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Size() (n int) { return n } +func (m *ExternalCA) Size() (n int) { + var l int + _ = l + if m.Protocol != 0 { + n += 1 + sovTypes(uint64(m.Protocol)) + } + l = len(m.URL) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + if len(m.Options) > 0 { + for k, v := range m.Options { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovTypes(uint64(len(k))) + 1 + len(v) + sovTypes(uint64(len(v))) + n += mapEntrySize + 1 + sovTypes(uint64(mapEntrySize)) + } + } + return n +} + func (m *CAConfig) Size() (n int) { var l int _ = l @@ -4168,6 +4344,12 @@ func (m *CAConfig) Size() (n int) { l = m.NodeCertExpiry.Size() n += 1 + l + sovTypes(uint64(l)) } + if len(m.ExternalCAs) > 0 { + for _, e := range m.ExternalCAs { + l = e.Size() + n += 1 + l + sovTypes(uint64(l)) + } + } return n } @@ -4701,12 +4883,35 @@ func (this *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) String() string { }, "") return s } +func (this *ExternalCA) String() string { + if this == nil { + return "nil" + } + keysForOptions := make([]string, 0, len(this.Options)) + for k, _ := range this.Options { + keysForOptions = append(keysForOptions, k) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForOptions) + mapStringForOptions := "map[string]string{" + for _, k := range keysForOptions { + mapStringForOptions += fmt.Sprintf("%v: %v,", k, this.Options[k]) + } + mapStringForOptions += "}" + s := strings.Join([]string{`&ExternalCA{`, + `Protocol:` + fmt.Sprintf("%v", this.Protocol) + `,`, + `URL:` + fmt.Sprintf("%v", this.URL) + `,`, + `Options:` + mapStringForOptions + `,`, + `}`, + }, "") + return s +} func (this *CAConfig) String() string { if this == nil { return "nil" } s := strings.Join([]string{`&CAConfig{`, `NodeCertExpiry:` + strings.Replace(fmt.Sprintf("%v", this.NodeCertExpiry), "Duration", "docker_swarmkit_v11.Duration", 1) + `,`, + `ExternalCAs:` + strings.Replace(fmt.Sprintf("%v", this.ExternalCAs), "ExternalCA", "ExternalCA", 1) + `,`, `}`, }, "") return s @@ -8574,6 +8779,215 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Unmarshal(data []byt } return nil } +func (m *ExternalCA) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ExternalCA: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ExternalCA: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Protocol", wireType) + } + m.Protocol = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + m.Protocol |= (ExternalCA_CAProtocol(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field URL", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.URL = string(data[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Options", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + var keykey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + keykey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthTypes + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey := string(data[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + var valuekey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + valuekey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLenmapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return ErrInvalidLengthTypes + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue := string(data[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + if m.Options == nil { + m.Options = make(map[string]string) + } + m.Options[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *CAConfig) Unmarshal(data []byte) error { l := len(data) iNdEx := 0 @@ -8636,6 +9050,37 @@ func (m *CAConfig) Unmarshal(data []byte) error { return err } iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ExternalCAs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ExternalCAs = append(m.ExternalCAs, &ExternalCA{}) + if err := m.ExternalCAs[len(m.ExternalCAs)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipTypes(data[iNdEx:]) @@ -9755,188 +10200,195 @@ var ( ) var fileDescriptorTypes = []byte{ - // 2925 bytes of a gzipped FileDescriptorProto + // 3030 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x58, 0x4d, 0x6c, 0x1b, 0xc7, - 0xf5, 0x37, 0x3f, 0x45, 0x0e, 0x29, 0x89, 0x5e, 0x3b, 0x8e, 0xcc, 0xe8, 0x2f, 0xfb, 0xbf, 0x89, - 0x1b, 0x37, 0x49, 0x99, 0x58, 0x49, 0x0b, 0x37, 0x41, 0x9b, 0x2c, 0x3f, 0x64, 0xb1, 0x96, 0x28, - 0x62, 0x48, 0xc9, 0x08, 0x8a, 0x96, 0x58, 0x2d, 0x47, 0xe2, 0x46, 0xcb, 0x5d, 0x76, 0x77, 0x29, - 0x99, 0x28, 0x0a, 0x38, 0xbd, 0xb4, 0xc8, 0xa9, 0xf7, 0x22, 0x28, 0x8a, 0xf6, 0xda, 0x73, 0x81, - 0x9e, 0x7c, 0xf4, 0xb1, 0x45, 0x81, 0x22, 0xa7, 0xa0, 0x49, 0x0f, 0xbd, 0x16, 0x68, 0xd1, 0x1c, - 0xda, 0x43, 0xdf, 0x9b, 0x8f, 0xe5, 0x87, 0xd7, 0x8a, 0xd3, 0xf4, 0x40, 0x70, 0xe7, 0xcd, 0xef, - 0xbd, 0x99, 0x37, 0xf3, 0xe6, 0xfd, 0xde, 0x0c, 0x29, 0x84, 0x93, 0x11, 0x0b, 0x2a, 0x23, 0xdf, - 0x0b, 0x3d, 0x4d, 0xeb, 0x7b, 0xd6, 0x09, 0xf3, 0x2b, 0xc1, 0x99, 0xe9, 0x0f, 0x4f, 0xec, 0xb0, - 0x72, 0x7a, 0xab, 0x7c, 0x35, 0xb4, 0x87, 0x2c, 0x08, 0xcd, 0xe1, 0xe8, 0xd5, 0xe8, 0x4b, 0xc0, - 0xcb, 0xcf, 0xf6, 0xc7, 0xbe, 0x19, 0xda, 0x9e, 0xfb, 0xaa, 0xfa, 0x90, 0x1d, 0x97, 0x8f, 0xbd, - 0x63, 0x8f, 0x7f, 0xbe, 0x8a, 0x5f, 0x42, 0xaa, 0x5f, 0x23, 0x4b, 0x07, 0xcc, 0x0f, 0x00, 0xa6, - 0x5d, 0x26, 0x19, 0xdb, 0xed, 0xb3, 0xfb, 0x6b, 0x89, 0xeb, 0x89, 0x9b, 0x69, 0x2a, 0x1a, 0xfa, - 0x2f, 0x13, 0xa4, 0x60, 0xb8, 0xae, 0x17, 0x72, 0x5b, 0x81, 0xa6, 0x91, 0xb4, 0x6b, 0x0e, 0x19, - 0x07, 0xe5, 0x29, 0xff, 0xd6, 0x6a, 0x24, 0xeb, 0x98, 0x87, 0xcc, 0x09, 0xd6, 0x92, 0xd7, 0x53, - 0x37, 0x0b, 0x9b, 0x2f, 0x57, 0x1e, 0x9f, 0x73, 0x65, 0xc6, 0x48, 0x65, 0x87, 0xa3, 0x1b, 0x6e, - 0xe8, 0x4f, 0xa8, 0x54, 0x2d, 0x7f, 0x93, 0x14, 0x66, 0xc4, 0x5a, 0x89, 0xa4, 0x4e, 0xd8, 0x44, - 0x0e, 0x83, 0x9f, 0x38, 0xbf, 0x53, 0xd3, 0x19, 0x33, 0x18, 0x04, 0x65, 0xa2, 0xf1, 0x66, 0xf2, - 0x76, 0x42, 0x7f, 0x97, 0xe4, 0x29, 0x0b, 0xbc, 0xb1, 0x6f, 0xb1, 0x40, 0xfb, 0x2a, 0xc9, 0xbb, - 0xa6, 0xeb, 0xf5, 0xac, 0xd1, 0x38, 0xe0, 0xea, 0xa9, 0x6a, 0xf1, 0xd3, 0x8f, 0xaf, 0xe5, 0x5a, - 0x20, 0xac, 0xb5, 0xf7, 0x03, 0x9a, 0xc3, 0xee, 0x1a, 0xf4, 0x6a, 0xff, 0x4f, 0x8a, 0x43, 0x36, - 0xf4, 0xfc, 0x49, 0xef, 0x70, 0x12, 0xb2, 0x80, 0x1b, 0x4e, 0xd1, 0x82, 0x90, 0x55, 0x51, 0xa4, - 0xff, 0x2c, 0x41, 0x2e, 0x2b, 0xdb, 0x94, 0xfd, 0x60, 0x6c, 0xfb, 0x6c, 0xc8, 0xdc, 0x30, 0xd0, - 0xbe, 0x0e, 0x3e, 0xdb, 0x43, 0x3b, 0x14, 0x63, 0x14, 0x36, 0xff, 0x2f, 0xce, 0xe7, 0x68, 0x56, - 0x54, 0x82, 0x35, 0x83, 0x14, 0x7d, 0x16, 0x30, 0xff, 0x54, 0xac, 0x04, 0x1f, 0xf2, 0x73, 0x95, - 0xe7, 0x54, 0xf4, 0x2d, 0x92, 0x6b, 0x3b, 0x66, 0x78, 0xe4, 0xf9, 0x43, 0x4d, 0x27, 0x45, 0xd3, - 0xb7, 0x06, 0x76, 0xc8, 0xac, 0x70, 0xec, 0xab, 0x5d, 0x99, 0x93, 0x69, 0x57, 0x48, 0xd2, 0x13, - 0x03, 0xe5, 0xab, 0x59, 0x58, 0x89, 0xe4, 0x5e, 0x87, 0x82, 0x44, 0x7f, 0x8b, 0x5c, 0x6c, 0x3b, - 0xe3, 0x63, 0xdb, 0xad, 0xb3, 0xc0, 0xf2, 0xed, 0x11, 0x5a, 0xc7, 0xed, 0xc5, 0xe0, 0x53, 0xdb, - 0x8b, 0xdf, 0xd1, 0x96, 0x27, 0xa7, 0x5b, 0xae, 0xff, 0x24, 0x49, 0x2e, 0x36, 0x5c, 0x50, 0x66, - 0xb3, 0xda, 0x37, 0xc8, 0x0a, 0xe3, 0xc2, 0xde, 0xa9, 0x08, 0x2a, 0x69, 0x67, 0x59, 0x48, 0x55, - 0xa4, 0x35, 0x17, 0xe2, 0xe5, 0x56, 0x9c, 0xfb, 0x8f, 0x59, 0x8f, 0x8b, 0x1a, 0xad, 0x41, 0x96, - 0x46, 0xdc, 0x89, 0x60, 0x2d, 0xc5, 0x6d, 0xdd, 0x88, 0xb3, 0xf5, 0x98, 0x9f, 0xd5, 0xf4, 0xa3, - 0x8f, 0xaf, 0x5d, 0xa0, 0x4a, 0xf7, 0xcb, 0x04, 0xdf, 0x5f, 0x12, 0x64, 0xb5, 0xe5, 0xf5, 0xe7, - 0xd6, 0xa1, 0x4c, 0x72, 0x03, 0x2f, 0x08, 0x67, 0x0e, 0x4a, 0xd4, 0xd6, 0x6e, 0x93, 0xdc, 0x48, - 0x6e, 0x9f, 0xdc, 0xfd, 0xf5, 0xf8, 0x29, 0x0b, 0x0c, 0x8d, 0xd0, 0xda, 0x5b, 0x24, 0xef, 0xab, - 0x98, 0x00, 0x6f, 0x9f, 0x22, 0x70, 0xa6, 0x78, 0xed, 0x5b, 0x24, 0x2b, 0x36, 0x61, 0x2d, 0xcd, - 0x35, 0x6f, 0x3c, 0xd5, 0x9a, 0x53, 0xa9, 0xa4, 0x7f, 0x94, 0x20, 0x25, 0x6a, 0x1e, 0x85, 0xbb, - 0x6c, 0x78, 0xc8, 0xfc, 0x0e, 0x1c, 0x64, 0x38, 0x3f, 0x57, 0x60, 0x1f, 0x99, 0xd9, 0x67, 0x3e, - 0x77, 0x32, 0x47, 0x65, 0x4b, 0xdb, 0xc7, 0x20, 0x37, 0xad, 0x81, 0x79, 0x68, 0x3b, 0x76, 0x38, - 0xe1, 0x6e, 0xae, 0xc4, 0xef, 0xf2, 0xa2, 0x4d, 0x98, 0xfc, 0x54, 0x91, 0xce, 0x99, 0xd1, 0xd6, - 0xc8, 0x12, 0xe4, 0xba, 0xc0, 0x3c, 0x66, 0xdc, 0xfb, 0x3c, 0x55, 0x4d, 0x08, 0xe5, 0xe2, 0xac, - 0x9e, 0x56, 0x20, 0x4b, 0xfb, 0xad, 0xbb, 0xad, 0xbd, 0x7b, 0xad, 0xd2, 0x05, 0x6d, 0x95, 0x14, - 0xf6, 0x5b, 0xb4, 0x61, 0xd4, 0xb6, 0x8d, 0xea, 0x4e, 0xa3, 0x94, 0xd0, 0x96, 0x21, 0x5d, 0x44, - 0xcd, 0xa4, 0xfe, 0x8b, 0x04, 0x21, 0xb8, 0x81, 0xd2, 0xa9, 0x37, 0x49, 0x06, 0xf2, 0x69, 0x28, - 0x36, 0x6e, 0x65, 0xf3, 0x85, 0xb8, 0x59, 0x4f, 0xe1, 0x15, 0xfc, 0x63, 0x54, 0xa8, 0xcc, 0xce, - 0x30, 0xb9, 0x38, 0xc3, 0x0c, 0x47, 0xce, 0x4f, 0x2d, 0x47, 0xd2, 0x75, 0xfc, 0x4a, 0x68, 0x79, - 0x92, 0x81, 0x39, 0xd5, 0xdf, 0x2d, 0x25, 0x21, 0xf8, 0x8a, 0xf5, 0x66, 0xa7, 0xb6, 0xd7, 0x6a, - 0x35, 0x6a, 0xdd, 0x46, 0xbd, 0x94, 0xd2, 0x6f, 0x90, 0x4c, 0x73, 0x08, 0x56, 0xb4, 0x75, 0x8c, - 0x80, 0x23, 0xe6, 0x33, 0xd7, 0x52, 0x81, 0x35, 0x15, 0xe8, 0xff, 0x5c, 0x22, 0x99, 0x5d, 0x6f, - 0xec, 0x86, 0xda, 0xe6, 0xcc, 0x29, 0x5e, 0xd9, 0xdc, 0x88, 0x73, 0x81, 0x03, 0x2b, 0x5d, 0x40, - 0xc9, 0x53, 0x0e, 0x9b, 0x29, 0x62, 0x45, 0x4e, 0x5d, 0xb6, 0x50, 0x1e, 0x9a, 0xfe, 0x31, 0x0b, - 0xe5, 0xa2, 0xcb, 0x16, 0xc6, 0xf8, 0x99, 0x6f, 0x87, 0xe6, 0xa1, 0x23, 0x42, 0x2a, 0x47, 0xa3, - 0xb6, 0xb6, 0x4d, 0x8a, 0x87, 0x40, 0x1f, 0x3d, 0x6f, 0x24, 0xb2, 0x5c, 0xe6, 0xc9, 0x21, 0x27, - 0xe6, 0x51, 0x05, 0xf4, 0x9e, 0x00, 0xd3, 0xc2, 0xe1, 0xb4, 0xa1, 0xb5, 0xc8, 0xca, 0xa9, 0xe7, - 0x8c, 0x87, 0x2c, 0xb2, 0x95, 0xe5, 0xb6, 0x5e, 0x7c, 0xb2, 0xad, 0x03, 0x8e, 0x57, 0xd6, 0x96, - 0x4f, 0x67, 0x9b, 0xe5, 0x1f, 0xa7, 0x48, 0x61, 0x66, 0x30, 0xad, 0x43, 0x0a, 0x40, 0x84, 0x23, - 0xf3, 0x98, 0x27, 0x57, 0xb9, 0x60, 0xb7, 0x9e, 0x6a, 0xa2, 0x95, 0xf6, 0x54, 0x91, 0xce, 0x5a, - 0xd1, 0x3f, 0x4c, 0x92, 0xc2, 0x4c, 0xa7, 0xf6, 0x12, 0xc9, 0xd1, 0x36, 0x6d, 0x1e, 0x18, 0xdd, - 0x46, 0xe9, 0x42, 0x79, 0xfd, 0x83, 0x0f, 0xaf, 0xaf, 0x71, 0x6b, 0xb3, 0x06, 0xda, 0xbe, 0x7d, - 0x8a, 0xf1, 0x71, 0x93, 0x2c, 0x29, 0x68, 0xa2, 0xfc, 0x1c, 0x40, 0x9f, 0x5d, 0x84, 0xce, 0x20, - 0x69, 0x67, 0xdb, 0xa0, 0x10, 0x22, 0xc9, 0x78, 0x24, 0xed, 0x0c, 0x4c, 0x9f, 0xf5, 0xb5, 0xaf, - 0x90, 0xac, 0x04, 0xa6, 0xca, 0x65, 0x00, 0x5e, 0x59, 0x04, 0x4e, 0x71, 0xb4, 0xb3, 0x63, 0x1c, - 0x34, 0x4a, 0xe9, 0x78, 0x1c, 0xed, 0x38, 0xe6, 0x29, 0xd3, 0x5e, 0x80, 0x60, 0xe6, 0xb0, 0x4c, - 0xf9, 0x2a, 0xc0, 0x9e, 0x79, 0xcc, 0x1c, 0xa2, 0xca, 0x6b, 0x3f, 0xfd, 0xd5, 0xc6, 0x85, 0xdf, - 0xfd, 0x7a, 0xa3, 0xb4, 0xd8, 0x5d, 0xfe, 0x47, 0x82, 0x2c, 0xcf, 0xed, 0x12, 0x06, 0xd3, 0xc8, - 0x1b, 0x8d, 0x1d, 0x75, 0xee, 0x20, 0x98, 0x54, 0x5b, 0xbb, 0xbb, 0xc0, 0x16, 0xaf, 0x3f, 0xe5, - 0xd6, 0xc7, 0xf2, 0xc5, 0xdb, 0x64, 0xb9, 0x0f, 0xeb, 0xc7, 0xfc, 0x9e, 0xe5, 0xb9, 0x47, 0xf6, - 0xb1, 0xcc, 0xa3, 0xe5, 0x38, 0x9b, 0x75, 0x0e, 0xa4, 0x45, 0xa1, 0x50, 0xe3, 0xf8, 0x2f, 0xc3, - 0x14, 0xf7, 0x48, 0x1a, 0xcf, 0x9b, 0xf6, 0x1c, 0x49, 0x57, 0x9b, 0xad, 0x3a, 0x84, 0xc2, 0x45, - 0x58, 0xbd, 0x65, 0x3e, 0x75, 0xec, 0xc0, 0xd8, 0xd2, 0xae, 0x91, 0xec, 0xc1, 0xde, 0xce, 0xfe, - 0x2e, 0x6e, 0xff, 0x25, 0xe8, 0x5e, 0x8d, 0xba, 0x85, 0x73, 0xe5, 0x8b, 0x72, 0x59, 0xf3, 0x51, - 0x87, 0xfe, 0xaf, 0x24, 0x59, 0xa6, 0x58, 0x05, 0xfa, 0x61, 0xdb, 0x73, 0x6c, 0x6b, 0xa2, 0xb5, - 0x49, 0x1e, 0xfc, 0xeb, 0xdb, 0x33, 0x41, 0xbd, 0xf9, 0x04, 0xaa, 0x98, 0x6a, 0xa9, 0x56, 0x4d, - 0x69, 0xd2, 0xa9, 0x11, 0x48, 0x29, 0x99, 0x3e, 0x73, 0xcc, 0xc9, 0x79, 0x9c, 0x55, 0x97, 0x15, - 0x27, 0x15, 0x50, 0x5e, 0x5f, 0x99, 0xf7, 0x7b, 0x66, 0x18, 0xb2, 0xe1, 0x28, 0x14, 0x9c, 0x95, - 0x86, 0xfa, 0xca, 0xbc, 0x6f, 0x48, 0x91, 0xf6, 0x06, 0xc9, 0x9e, 0x81, 0xdb, 0xde, 0x99, 0xa4, - 0xa5, 0xf3, 0xed, 0x4a, 0xac, 0xfe, 0x01, 0xb2, 0xd1, 0xc2, 0x64, 0x71, 0x59, 0x5b, 0x7b, 0xad, - 0x86, 0x5a, 0x56, 0xd9, 0xbf, 0xe7, 0xb6, 0x3c, 0x17, 0x43, 0x96, 0xec, 0xb5, 0x7a, 0x5b, 0x46, - 0x73, 0x67, 0x9f, 0xe2, 0xd2, 0x5e, 0x06, 0x48, 0x29, 0x82, 0x6c, 0x99, 0xb6, 0x83, 0xa5, 0xd2, - 0x55, 0x92, 0x32, 0x5a, 0x90, 0x83, 0xcb, 0x25, 0xe8, 0x2e, 0x46, 0xdd, 0x86, 0x3b, 0x99, 0x46, - 0xf3, 0xe2, 0xb8, 0xfa, 0x7b, 0xa4, 0xb8, 0x3f, 0xea, 0x43, 0xa4, 0x8a, 0x08, 0xd1, 0xae, 0x43, - 0x4a, 0x31, 0x7d, 0xd3, 0x71, 0x98, 0x63, 0x07, 0x43, 0x59, 0x4d, 0xcf, 0x8a, 0xa0, 0x04, 0x78, - 0xfa, 0xb5, 0x94, 0x95, 0x8a, 0x50, 0xd0, 0x7f, 0x44, 0x56, 0x61, 0x94, 0xd0, 0x04, 0x4a, 0x56, - 0x24, 0xbc, 0x49, 0x8a, 0x96, 0x12, 0xf5, 0xec, 0xbe, 0x08, 0xc5, 0xea, 0x2a, 0x14, 0x7a, 0x85, - 0x08, 0xda, 0xac, 0xd3, 0x42, 0x04, 0x6a, 0xf6, 0xd1, 0xcf, 0x11, 0x40, 0x71, 0xf8, 0x4c, 0x75, - 0x09, 0xa0, 0xa9, 0x36, 0x40, 0x50, 0x06, 0xab, 0x98, 0x67, 0xf7, 0xed, 0x10, 0x8e, 0x47, 0x5f, - 0xd0, 0x6c, 0x86, 0xe6, 0x50, 0x50, 0x83, 0xb6, 0xfe, 0x7e, 0x92, 0x90, 0xae, 0x19, 0x9c, 0xc8, - 0xa1, 0xa1, 0x20, 0x89, 0xae, 0x1f, 0xe7, 0x95, 0xc1, 0x5d, 0x05, 0xa2, 0x53, 0xbc, 0xf6, 0xba, - 0xe2, 0x59, 0x51, 0x1d, 0xc4, 0x2b, 0xca, 0xb1, 0xe2, 0x08, 0x76, 0xbe, 0x04, 0xc0, 0x83, 0xc8, - 0x7c, 0x9f, 0x47, 0x11, 0x1c, 0x44, 0xf8, 0x84, 0x5b, 0x49, 0x3e, 0xf2, 0x59, 0x32, 0xd0, 0xf3, - 0x71, 0x83, 0x2c, 0x2c, 0xe8, 0xf6, 0x05, 0x3a, 0xd5, 0xab, 0x96, 0xc8, 0x8a, 0x0f, 0xc7, 0x0c, - 0x66, 0xdd, 0x0b, 0x78, 0xb7, 0xfe, 0x47, 0x58, 0x83, 0x66, 0xdb, 0xd8, 0x95, 0xbb, 0x5d, 0x27, - 0xd9, 0x23, 0x73, 0x68, 0x3b, 0x13, 0x79, 0xcc, 0x5e, 0x89, 0x1b, 0x62, 0x8a, 0xaf, 0x18, 0xfd, - 0x3e, 0x14, 0x65, 0xc1, 0x16, 0xd7, 0xa1, 0x52, 0x97, 0x93, 0xef, 0xf8, 0xd0, 0x05, 0x92, 0x55, - 0xe4, 0xcb, 0x5b, 0x98, 0x4c, 0x7c, 0xd3, 0x8d, 0xbc, 0x15, 0x0d, 0x5c, 0x05, 0xc8, 0xa4, 0xec, - 0x0c, 0x22, 0x48, 0xf8, 0xab, 0x9a, 0x40, 0xbc, 0x39, 0x71, 0x57, 0x60, 0x7d, 0x70, 0x19, 0xb3, - 0xe5, 0xe7, 0xcd, 0x87, 0x4a, 0xb8, 0x48, 0x93, 0x91, 0x76, 0xf9, 0x2d, 0x9e, 0x52, 0xa6, 0x5d, - 0x5f, 0x28, 0xd3, 0xbd, 0x46, 0x96, 0xe7, 0xfc, 0x7c, 0xac, 0xea, 0x69, 0xb6, 0x0f, 0xde, 0x28, - 0xa5, 0xe5, 0xd7, 0x37, 0x4a, 0x59, 0xfd, 0xef, 0x50, 0x84, 0xb5, 0x3d, 0x7e, 0xac, 0x70, 0x55, - 0xe3, 0x6f, 0x99, 0x39, 0x7e, 0x67, 0xb5, 0x3c, 0x47, 0xc6, 0x4c, 0x6c, 0x11, 0x30, 0xb5, 0x82, - 0x04, 0xcd, 0xe1, 0x34, 0x52, 0x84, 0xf4, 0x5a, 0x10, 0xf5, 0x4b, 0x6f, 0x04, 0x38, 0xbe, 0xac, - 0xcb, 0x94, 0x08, 0x11, 0x6a, 0xe2, 0x15, 0x66, 0x34, 0x3e, 0x84, 0x63, 0x3a, 0x60, 0x7d, 0x81, - 0x49, 0x73, 0xcc, 0x72, 0x24, 0x45, 0x98, 0x5e, 0x87, 0x4b, 0x98, 0xb2, 0xb9, 0x46, 0x52, 0xdd, - 0x5a, 0x1b, 0xf2, 0xce, 0x2a, 0x64, 0x8d, 0x82, 0x12, 0x83, 0x08, 0x7b, 0xf6, 0xeb, 0x6d, 0x48, - 0x37, 0x73, 0x3d, 0x20, 0x2a, 0xa7, 0x31, 0x9d, 0xe8, 0x3f, 0x4f, 0x90, 0xac, 0x60, 0x99, 0x58, - 0x8f, 0x0d, 0xb2, 0xa4, 0xaa, 0x1e, 0x41, 0x7d, 0x2f, 0x3e, 0x99, 0xa6, 0x2a, 0x92, 0xf5, 0xc4, - 0x3e, 0x2a, 0xbd, 0xf2, 0x9b, 0xa4, 0x38, 0xdb, 0xf1, 0x85, 0x76, 0xf1, 0x87, 0xa4, 0x80, 0x81, - 0xa2, 0x38, 0x7a, 0x93, 0x64, 0x05, 0x13, 0xca, 0xa3, 0x7e, 0x1e, 0x67, 0x4a, 0x24, 0x64, 0xba, - 0x25, 0xc1, 0xb3, 0xea, 0x7a, 0xb6, 0x71, 0x7e, 0x38, 0x52, 0x05, 0xd7, 0xdf, 0x26, 0xe9, 0x36, - 0x03, 0x0b, 0xcf, 0x93, 0x25, 0x17, 0x52, 0xcf, 0x34, 0xb3, 0x11, 0x48, 0x57, 0x59, 0x2c, 0xc0, - 0x21, 0x63, 0x65, 0xb1, 0x0b, 0xf2, 0x19, 0x2c, 0x9e, 0x09, 0xf1, 0xa6, 0x6e, 0xa8, 0xf8, 0xad, - 0x77, 0x49, 0xf1, 0x1e, 0xb3, 0x8f, 0x07, 0x21, 0xec, 0x18, 0x1a, 0x7a, 0x85, 0xa4, 0x47, 0x2c, - 0x9a, 0xfc, 0x5a, 0x6c, 0xe8, 0x40, 0x3f, 0xe5, 0x28, 0x3c, 0x90, 0x67, 0x5c, 0x5b, 0x3e, 0x0a, - 0xc8, 0x96, 0xfe, 0x9b, 0x24, 0x59, 0x69, 0x06, 0xc1, 0xd8, 0x84, 0x82, 0x5b, 0x66, 0xc1, 0x6f, - 0xcf, 0x5f, 0x18, 0x6e, 0xc6, 0x7a, 0x38, 0xa7, 0x32, 0x7f, 0x69, 0x90, 0x99, 0x2b, 0x19, 0x65, - 0x2e, 0xfd, 0x51, 0x42, 0xdd, 0x16, 0x6e, 0xcc, 0x9c, 0x9b, 0xf2, 0x1a, 0x04, 0xd1, 0xe5, 0x59, - 0x4b, 0x6c, 0xdf, 0x3d, 0x71, 0xbd, 0x33, 0x17, 0x88, 0x16, 0x6e, 0x0f, 0xad, 0xc6, 0x3d, 0x88, - 0xb4, 0x2b, 0x00, 0xd2, 0xe6, 0x40, 0x94, 0xb9, 0xec, 0x0c, 0x2d, 0xb5, 0x1b, 0xad, 0x7a, 0xb3, - 0x75, 0x07, 0xe8, 0xed, 0x71, 0x4b, 0x6d, 0x06, 0x74, 0xe6, 0x1e, 0xc3, 0x72, 0x67, 0x9b, 0x9d, - 0xce, 0x3e, 0x2f, 0x15, 0x9f, 0x05, 0xd4, 0xa5, 0x39, 0x14, 0x36, 0xa0, 0x4e, 0x04, 0x10, 0x32, - 0x29, 0x80, 0xd2, 0x31, 0x20, 0x24, 0x53, 0x48, 0x20, 0x22, 0xc2, 0xff, 0x9a, 0x24, 0x25, 0xc3, - 0xb2, 0xd8, 0x28, 0xc4, 0x7e, 0x59, 0x9d, 0x74, 0xb1, 0xda, 0x83, 0x2f, 0x9b, 0xe1, 0xeb, 0x09, - 0x86, 0xc5, 0xed, 0xd8, 0x17, 0xa3, 0x05, 0xbd, 0x0a, 0xf5, 0x1c, 0x66, 0xf4, 0x87, 0x76, 0x80, - 0xaf, 0x08, 0x42, 0x46, 0x23, 0x4b, 0xe5, 0x7f, 0x27, 0xc8, 0xa5, 0x18, 0x84, 0xf6, 0x1a, 0x49, - 0xfb, 0x20, 0x96, 0xdb, 0xb3, 0xfe, 0xa4, 0xfb, 0x1c, 0xaa, 0x52, 0x8e, 0xd4, 0x36, 0x08, 0x31, - 0xc7, 0xa1, 0x67, 0xf2, 0xf1, 0xf9, 0xc6, 0xe4, 0xe8, 0x8c, 0x44, 0xfb, 0x1e, 0x64, 0x6b, 0x66, - 0xf9, 0xf2, 0x4a, 0x54, 0xd8, 0x6c, 0xfc, 0xb7, 0xb3, 0xaf, 0x6c, 0x9b, 0x98, 0x51, 0x3a, 0xdc, - 0x18, 0x95, 0x46, 0xcb, 0x6f, 0x90, 0xe2, 0xac, 0x1c, 0xa3, 0x1b, 0xca, 0x0b, 0x93, 0x3b, 0x50, - 0xa4, 0xfc, 0x1b, 0x83, 0xc6, 0x74, 0x8e, 0x55, 0xd0, 0xc0, 0xa7, 0x4e, 0x49, 0xae, 0x66, 0xc8, - 0xf4, 0xb9, 0x45, 0x4a, 0xfc, 0xd0, 0x58, 0xcc, 0x0f, 0x7b, 0xec, 0xfe, 0xc8, 0xf6, 0x27, 0x32, - 0xee, 0xcf, 0xaf, 0xaf, 0x56, 0x50, 0xab, 0x06, 0x4a, 0x0d, 0xae, 0xa3, 0x1f, 0x90, 0x4b, 0x7b, - 0xbe, 0x35, 0x00, 0xc6, 0x16, 0x00, 0x69, 0xfe, 0x6d, 0xb2, 0x1e, 0x02, 0x33, 0xf7, 0x06, 0x76, - 0x10, 0xe2, 0xeb, 0x19, 0x4c, 0x92, 0xb9, 0xd8, 0xdf, 0xe3, 0xaf, 0x5c, 0xe2, 0xd5, 0x8d, 0x5e, - 0x45, 0xcc, 0xb6, 0x80, 0x50, 0x85, 0xd8, 0x41, 0x80, 0xfe, 0x5d, 0x52, 0xaa, 0xdb, 0xc1, 0xc8, - 0x0c, 0xc1, 0xb6, 0x2c, 0xac, 0xb5, 0x3b, 0xa4, 0x34, 0x60, 0x50, 0x58, 0x1d, 0x32, 0x13, 0x92, - 0x33, 0xf3, 0x6d, 0xaf, 0xff, 0x54, 0x73, 0x5e, 0x8d, 0xb4, 0xda, 0x5c, 0x49, 0xff, 0x0c, 0xa8, - 0x04, 0x9f, 0x15, 0xa4, 0xdd, 0x97, 0xc9, 0xc5, 0xc0, 0x35, 0x47, 0xc1, 0xc0, 0x0b, 0x7b, 0xb6, - 0x1b, 0xe2, 0x3b, 0x9a, 0x23, 0x8b, 0xb2, 0x92, 0xea, 0x68, 0x4a, 0x39, 0x24, 0x09, 0xed, 0x84, - 0xb1, 0x51, 0xcf, 0x73, 0xfa, 0x3d, 0xd5, 0x29, 0xde, 0xce, 0x00, 0x8d, 0x3d, 0x7b, 0x4e, 0xbf, - 0xa3, 0xe4, 0x5a, 0x95, 0x6c, 0x38, 0xde, 0x71, 0x0f, 0x3c, 0xf3, 0x21, 0x00, 0x7b, 0x47, 0x9e, - 0xdf, 0x0b, 0x1c, 0xef, 0x0c, 0x3e, 0x1c, 0xf8, 0x63, 0xbe, 0xaa, 0x78, 0xcb, 0x80, 0x6a, 0x08, - 0xd0, 0x96, 0xe7, 0x77, 0xa0, 0x6f, 0x4b, 0x21, 0x90, 0x6f, 0xa6, 0x6e, 0x87, 0xb6, 0x75, 0xa2, - 0xf8, 0x26, 0x92, 0x76, 0x41, 0x08, 0x47, 0x6e, 0x99, 0x39, 0xcc, 0xe2, 0x8b, 0xcc, 0x51, 0x19, - 0x8e, 0x2a, 0x2a, 0x21, 0x82, 0xf4, 0xaf, 0x91, 0x7c, 0xdb, 0x31, 0x2d, 0xfe, 0x42, 0x89, 0x65, - 0x28, 0xe4, 0x52, 0xdc, 0x39, 0xf0, 0x5a, 0x9c, 0xb3, 0x3c, 0x9d, 0x15, 0xe9, 0xef, 0x03, 0xfb, - 0x50, 0xcf, 0x0b, 0x6b, 0x06, 0x80, 0xb3, 0x96, 0xd9, 0x53, 0xcc, 0x50, 0xac, 0xe6, 0x21, 0xc9, - 0x66, 0x6a, 0xc6, 0x5d, 0x36, 0xa1, 0x19, 0xcb, 0x84, 0x3f, 0xcc, 0xc3, 0x80, 0xc0, 0x80, 0xe2, - 0xcb, 0x51, 0x14, 0x79, 0x18, 0x22, 0x0e, 0x24, 0x14, 0x94, 0xf1, 0x1f, 0x8e, 0x5a, 0x51, 0x82, - 0x7a, 0x03, 0x88, 0x60, 0x51, 0xb5, 0x54, 0x57, 0x00, 0x49, 0x04, 0x12, 0xe3, 0x9a, 0x12, 0x81, - 0xc6, 0x6f, 0xfd, 0x4f, 0x09, 0x52, 0xc0, 0x86, 0x7d, 0x64, 0x5b, 0x98, 0xf0, 0xbe, 0xf8, 0x61, - 0x85, 0x5a, 0xd6, 0x0a, 0x7c, 0x39, 0x29, 0x5e, 0xcb, 0xd6, 0x3a, 0x94, 0xa2, 0x4c, 0x7b, 0x07, - 0xce, 0x29, 0x4f, 0xb8, 0xf2, 0x9c, 0xea, 0x9f, 0x9f, 0x9a, 0x65, 0xb9, 0x2d, 0xf5, 0xf8, 0x22, - 0x4e, 0x67, 0xc7, 0xb7, 0xa6, 0x48, 0x67, 0x45, 0xf8, 0xba, 0x6a, 0xb9, 0x7c, 0x37, 0xe4, 0xeb, - 0x6a, 0xad, 0x45, 0x41, 0xa2, 0xff, 0x01, 0xee, 0xb8, 0x0d, 0xd7, 0xf2, 0x27, 0x9c, 0x3d, 0x71, - 0x05, 0xd7, 0x49, 0x1e, 0xaa, 0xba, 0x60, 0x12, 0xc0, 0xd5, 0x47, 0x3d, 0xde, 0x44, 0x02, 0xad, - 0x49, 0xf2, 0x70, 0x8a, 0x3d, 0xdf, 0x0e, 0x07, 0x43, 0x59, 0xde, 0xbc, 0x1c, 0xff, 0x44, 0x37, - 0x63, 0xb3, 0x62, 0x28, 0x15, 0x3a, 0xd5, 0x56, 0x1c, 0x9f, 0xe2, 0x93, 0xe5, 0x1c, 0x0f, 0x17, - 0x31, 0x07, 0x6a, 0x6e, 0x28, 0x5c, 0x7a, 0x58, 0xca, 0x72, 0x3f, 0xe0, 0x4e, 0x22, 0x65, 0x58, - 0x9e, 0xeb, 0x3a, 0xc9, 0x47, 0xc6, 0xf0, 0xc9, 0xcc, 0x68, 0x74, 0x7a, 0xb7, 0x36, 0x6f, 0xf7, - 0xee, 0xd4, 0x76, 0x81, 0x7a, 0x44, 0x32, 0xff, 0x2d, 0xf8, 0xb4, 0x6b, 0xba, 0x50, 0x6e, 0xab, - 0xcb, 0x07, 0x44, 0x85, 0x0f, 0x47, 0x4d, 0xb1, 0x73, 0x5a, 0x44, 0x05, 0x9e, 0x3e, 0x64, 0x67, - 0xec, 0x8a, 0x67, 0xe7, 0x99, 0xa7, 0xc3, 0xd4, 0xb9, 0x4f, 0x87, 0xe9, 0xff, 0xc9, 0xd3, 0xe1, - 0x4b, 0x9f, 0xa5, 0x48, 0x3e, 0xba, 0x4c, 0x60, 0xc8, 0x20, 0x59, 0x5e, 0x10, 0xd7, 0xbc, 0x48, - 0xde, 0xe2, 0x34, 0x99, 0x37, 0x76, 0x76, 0xf6, 0x6a, 0x06, 0xbe, 0xbc, 0xbd, 0x23, 0xd8, 0x34, - 0x02, 0x18, 0x70, 0x68, 0x71, 0xd3, 0xfb, 0x9a, 0x3e, 0x65, 0xd3, 0x07, 0xf2, 0x32, 0x19, 0xa1, - 0x14, 0x95, 0xbe, 0x40, 0x72, 0x46, 0xa7, 0xd3, 0xbc, 0xd3, 0x02, 0x4b, 0x0f, 0x13, 0xe5, 0x67, - 0x00, 0x74, 0x71, 0x6a, 0x0a, 0x58, 0xe0, 0xd8, 0x05, 0x4b, 0x88, 0xaa, 0xd5, 0x1a, 0x6d, 0x1c, - 0xef, 0x41, 0x72, 0x11, 0xc5, 0x39, 0x84, 0xbf, 0xcc, 0xe4, 0xdb, 0xb4, 0xd1, 0x36, 0x28, 0x8e, - 0xf8, 0x30, 0xb9, 0x30, 0xaf, 0xb6, 0xcf, 0xe0, 0x7a, 0x89, 0x63, 0x6e, 0xa8, 0x67, 0xc4, 0x07, - 0xa9, 0xb2, 0x06, 0x98, 0x95, 0xe9, 0x0d, 0x0a, 0xd6, 0x77, 0x82, 0xa3, 0x75, 0xba, 0x06, 0xed, - 0x72, 0x33, 0xa9, 0x85, 0xd1, 0x3a, 0x78, 0xaf, 0x45, 0x2b, 0xe0, 0x1d, 0xdd, 0x6f, 0xb5, 0xb8, - 0x77, 0xe9, 0x05, 0xef, 0xe8, 0xd8, 0x75, 0x11, 0x73, 0x03, 0xe8, 0x66, 0x6f, 0xb7, 0xbd, 0xd3, - 0xe8, 0x36, 0x4a, 0x0f, 0xd3, 0x0b, 0x13, 0xaa, 0x79, 0xc3, 0x91, 0xc3, 0x42, 0xe1, 0x5e, 0x67, - 0x7b, 0xbf, 0xcb, 0x5f, 0x39, 0x1f, 0x64, 0x16, 0x07, 0x1c, 0x8c, 0xc3, 0x3e, 0xd6, 0x2f, 0xd7, - 0xa3, 0x82, 0xe2, 0x61, 0x46, 0xbc, 0x7a, 0x44, 0x18, 0x51, 0x4d, 0xa0, 0x1d, 0xda, 0xf8, 0x8e, - 0x78, 0x10, 0x7d, 0x90, 0x5d, 0xb0, 0x43, 0xd9, 0x7b, 0x90, 0x05, 0xa1, 0xe6, 0x88, 0xde, 0x46, - 0xa2, 0xae, 0x97, 0xbe, 0x4f, 0x72, 0x2a, 0x61, 0xc0, 0xea, 0x64, 0xef, 0xed, 0xd1, 0xbb, 0x0d, - 0x0a, 0x5b, 0xcf, 0x57, 0x47, 0xf5, 0xdc, 0xf3, 0x7c, 0x88, 0x2e, 0x98, 0xc6, 0xd2, 0xae, 0xd1, - 0x32, 0xee, 0x00, 0x40, 0x3e, 0xbe, 0x28, 0x80, 0x8c, 0xfa, 0x72, 0x49, 0x0e, 0x10, 0xd9, 0xac, - 0xae, 0x3f, 0xfa, 0x64, 0xe3, 0xc2, 0x47, 0xf0, 0xfb, 0xdb, 0x27, 0x1b, 0x89, 0x07, 0x9f, 0x6e, - 0x24, 0x1e, 0xc1, 0xef, 0xf7, 0xf0, 0xfb, 0x33, 0xfc, 0x0e, 0xb3, 0xfc, 0xe2, 0xf1, 0xfa, 0x7f, - 0x02, 0x00, 0x00, 0xff, 0xff, 0x35, 0xa0, 0x81, 0xdd, 0xd2, 0x1b, 0x00, 0x00, + 0xf5, 0x17, 0x3f, 0x45, 0x0e, 0x29, 0x89, 0x5e, 0x3b, 0x8e, 0xcc, 0xe8, 0x2f, 0xfb, 0xbf, 0x89, + 0x1b, 0xe7, 0xa3, 0x4c, 0xac, 0xa4, 0x85, 0x9b, 0xa0, 0x4d, 0x96, 0x1f, 0xb2, 0x58, 0x4b, 0x14, + 0x31, 0x94, 0x64, 0x04, 0x45, 0x4b, 0xac, 0x96, 0x23, 0x71, 0xa3, 0xe5, 0x2e, 0xbb, 0xbb, 0x94, + 0x4c, 0x14, 0x05, 0x9c, 0x5e, 0x5a, 0xe4, 0xd4, 0x7b, 0x11, 0x04, 0x45, 0x8b, 0xde, 0x7a, 0x2e, + 0xd0, 0x93, 0x8f, 0x3e, 0xb6, 0x28, 0x50, 0xe4, 0x14, 0x34, 0xe9, 0xa1, 0xd7, 0x02, 0x2d, 0x9a, + 0x43, 0x7b, 0xe8, 0x7b, 0xf3, 0xb1, 0xfc, 0xf0, 0x5a, 0x71, 0x9a, 0x1c, 0x08, 0xee, 0xbc, 0xf9, + 0xbd, 0x37, 0xf3, 0xde, 0xbc, 0x79, 0x1f, 0x43, 0x0a, 0xe1, 0x78, 0xc8, 0x82, 0xca, 0xd0, 0xf7, + 0x42, 0x4f, 0xd3, 0x7a, 0x9e, 0x75, 0xc2, 0xfc, 0x4a, 0x70, 0x66, 0xfa, 0x83, 0x13, 0x3b, 0xac, + 0x9c, 0xde, 0x2c, 0x5f, 0x09, 0xed, 0x01, 0x0b, 0x42, 0x73, 0x30, 0x7c, 0x25, 0xfa, 0x12, 0xf0, + 0xf2, 0xd3, 0xbd, 0x91, 0x6f, 0x86, 0xb6, 0xe7, 0xbe, 0xa2, 0x3e, 0xe4, 0xc4, 0xa5, 0x63, 0xef, + 0xd8, 0xe3, 0x9f, 0xaf, 0xe0, 0x97, 0xa0, 0xea, 0x57, 0xc9, 0xe2, 0x01, 0xf3, 0x03, 0x80, 0x69, + 0x97, 0x48, 0xc6, 0x76, 0x7b, 0xec, 0xde, 0x6a, 0xe2, 0x5a, 0xe2, 0x46, 0x9a, 0x8a, 0x81, 0xfe, + 0xcb, 0x04, 0x29, 0x18, 0xae, 0xeb, 0x85, 0x5c, 0x56, 0xa0, 0x69, 0x24, 0xed, 0x9a, 0x03, 0xc6, + 0x41, 0x79, 0xca, 0xbf, 0xb5, 0x1a, 0xc9, 0x3a, 0xe6, 0x21, 0x73, 0x82, 0xd5, 0xe4, 0xb5, 0xd4, + 0x8d, 0xc2, 0xc6, 0x4b, 0x95, 0x47, 0xf7, 0x5c, 0x99, 0x12, 0x52, 0xd9, 0xe6, 0xe8, 0x86, 0x1b, + 0xfa, 0x63, 0x2a, 0x59, 0xcb, 0xdf, 0x22, 0x85, 0x29, 0xb2, 0x56, 0x22, 0xa9, 0x13, 0x36, 0x96, + 0xcb, 0xe0, 0x27, 0xee, 0xef, 0xd4, 0x74, 0x46, 0x0c, 0x16, 0x41, 0x9a, 0x18, 0xbc, 0x91, 0xbc, + 0x95, 0xd0, 0xdf, 0x21, 0x79, 0xca, 0x02, 0x6f, 0xe4, 0x5b, 0x2c, 0xd0, 0x5e, 0x20, 0x79, 0xd7, + 0x74, 0xbd, 0xae, 0x35, 0x1c, 0x05, 0x9c, 0x3d, 0x55, 0x2d, 0x7e, 0xfa, 0xf1, 0xd5, 0x5c, 0x0b, + 0x88, 0xb5, 0xf6, 0x7e, 0x40, 0x73, 0x38, 0x5d, 0x83, 0x59, 0xed, 0xff, 0x49, 0x71, 0xc0, 0x06, + 0x9e, 0x3f, 0xee, 0x1e, 0x8e, 0x43, 0x16, 0x70, 0xc1, 0x29, 0x5a, 0x10, 0xb4, 0x2a, 0x92, 0xf4, + 0x9f, 0x27, 0xc8, 0x25, 0x25, 0x9b, 0xb2, 0x1f, 0x8e, 0x6c, 0x9f, 0x0d, 0x98, 0x1b, 0x06, 0xda, + 0x37, 0x40, 0x67, 0x7b, 0x60, 0x87, 0x62, 0x8d, 0xc2, 0xc6, 0xff, 0xc5, 0xe9, 0x1c, 0xed, 0x8a, + 0x4a, 0xb0, 0x66, 0x90, 0xa2, 0xcf, 0x02, 0xe6, 0x9f, 0x0a, 0x4b, 0xf0, 0x25, 0x3f, 0x97, 0x79, + 0x86, 0x45, 0xdf, 0x24, 0xb9, 0xb6, 0x63, 0x86, 0x47, 0x9e, 0x3f, 0xd0, 0x74, 0x52, 0x34, 0x7d, + 0xab, 0x6f, 0x87, 0xcc, 0x0a, 0x47, 0xbe, 0x3a, 0x95, 0x19, 0x9a, 0x76, 0x99, 0x24, 0x3d, 0xb1, + 0x50, 0xbe, 0x9a, 0x05, 0x4b, 0x24, 0x77, 0x3b, 0x14, 0x28, 0xfa, 0x9b, 0xe4, 0x42, 0xdb, 0x19, + 0x1d, 0xdb, 0x6e, 0x9d, 0x05, 0x96, 0x6f, 0x0f, 0x51, 0x3a, 0x1e, 0x2f, 0x3a, 0x9f, 0x3a, 0x5e, + 0xfc, 0x8e, 0x8e, 0x3c, 0x39, 0x39, 0x72, 0xfd, 0xa7, 0x49, 0x72, 0xa1, 0xe1, 0x02, 0x33, 0x9b, + 0xe6, 0xbe, 0x4e, 0x96, 0x19, 0x27, 0x76, 0x4f, 0x85, 0x53, 0x49, 0x39, 0x4b, 0x82, 0xaa, 0x3c, + 0xad, 0x39, 0xe7, 0x2f, 0x37, 0xe3, 0xd4, 0x7f, 0x44, 0x7a, 0x9c, 0xd7, 0x68, 0x0d, 0xb2, 0x38, + 0xe4, 0x4a, 0x04, 0xab, 0x29, 0x2e, 0xeb, 0x7a, 0x9c, 0xac, 0x47, 0xf4, 0xac, 0xa6, 0x1f, 0x7e, + 0x7c, 0x75, 0x81, 0x2a, 0xde, 0x2f, 0xe3, 0x7c, 0x7f, 0x4d, 0x90, 0x95, 0x96, 0xd7, 0x9b, 0xb1, + 0x43, 0x99, 0xe4, 0xfa, 0x5e, 0x10, 0x4e, 0x5d, 0x94, 0x68, 0xac, 0xdd, 0x22, 0xb9, 0xa1, 0x3c, + 0x3e, 0x79, 0xfa, 0x6b, 0xf1, 0x5b, 0x16, 0x18, 0x1a, 0xa1, 0xb5, 0x37, 0x49, 0xde, 0x57, 0x3e, + 0x01, 0xda, 0x3e, 0x81, 0xe3, 0x4c, 0xf0, 0xda, 0xb7, 0x49, 0x56, 0x1c, 0xc2, 0x6a, 0x9a, 0x73, + 0x5e, 0x7f, 0x22, 0x9b, 0x53, 0xc9, 0xa4, 0x7f, 0x94, 0x20, 0x25, 0x6a, 0x1e, 0x85, 0x3b, 0x6c, + 0x70, 0xc8, 0xfc, 0x0e, 0x5c, 0x64, 0xb8, 0x3f, 0x97, 0xe1, 0x1c, 0x99, 0xd9, 0x63, 0x3e, 0x57, + 0x32, 0x47, 0xe5, 0x48, 0xdb, 0x47, 0x27, 0x37, 0xad, 0xbe, 0x79, 0x68, 0x3b, 0x76, 0x38, 0xe6, + 0x6a, 0x2e, 0xc7, 0x9f, 0xf2, 0xbc, 0x4c, 0xd8, 0xfc, 0x84, 0x91, 0xce, 0x88, 0xd1, 0x56, 0xc9, + 0x22, 0xc4, 0xba, 0xc0, 0x3c, 0x66, 0x5c, 0xfb, 0x3c, 0x55, 0x43, 0x70, 0xe5, 0xe2, 0x34, 0x9f, + 0x56, 0x20, 0x8b, 0xfb, 0xad, 0x3b, 0xad, 0xdd, 0xbb, 0xad, 0xd2, 0x82, 0xb6, 0x42, 0x0a, 0xfb, + 0x2d, 0xda, 0x30, 0x6a, 0x5b, 0x46, 0x75, 0xbb, 0x51, 0x4a, 0x68, 0x4b, 0x10, 0x2e, 0xa2, 0x61, + 0x52, 0xff, 0x30, 0x41, 0x08, 0x1e, 0xa0, 0x54, 0xea, 0x0d, 0x92, 0x81, 0x78, 0x1a, 0x8a, 0x83, + 0x5b, 0xde, 0x78, 0x2e, 0x6e, 0xd7, 0x13, 0x78, 0x05, 0xff, 0x18, 0x15, 0x2c, 0xd3, 0x3b, 0x4c, + 0xce, 0xef, 0x30, 0xc3, 0x91, 0xb3, 0x5b, 0xcb, 0x91, 0x74, 0x1d, 0xbf, 0x12, 0x5a, 0x9e, 0x64, + 0x60, 0x4f, 0xf5, 0x77, 0x4a, 0x49, 0x70, 0xbe, 0x62, 0xbd, 0xd9, 0xa9, 0xed, 0xb6, 0x5a, 0x8d, + 0xda, 0x5e, 0xa3, 0x5e, 0x4a, 0xe9, 0xd7, 0x49, 0xa6, 0x39, 0x00, 0x29, 0xda, 0x1a, 0x7a, 0xc0, + 0x11, 0xf3, 0x99, 0x6b, 0x29, 0xc7, 0x9a, 0x10, 0xf4, 0x7f, 0x2d, 0x92, 0xcc, 0x8e, 0x37, 0x72, + 0x43, 0x6d, 0x63, 0xea, 0x16, 0x2f, 0x6f, 0xac, 0xc7, 0xa9, 0xc0, 0x81, 0x95, 0x3d, 0x40, 0xc9, + 0x5b, 0x0e, 0x87, 0x29, 0x7c, 0x45, 0x6e, 0x5d, 0x8e, 0x90, 0x1e, 0x9a, 0xfe, 0x31, 0x0b, 0xa5, + 0xd1, 0xe5, 0x08, 0x7d, 0xfc, 0xcc, 0xb7, 0x43, 0xf3, 0xd0, 0x11, 0x2e, 0x95, 0xa3, 0xd1, 0x58, + 0xdb, 0x22, 0xc5, 0x43, 0x48, 0x1f, 0x5d, 0x6f, 0x28, 0xa2, 0x5c, 0xe6, 0xf1, 0x2e, 0x27, 0xf6, + 0x51, 0x05, 0xf4, 0xae, 0x00, 0xd3, 0xc2, 0xe1, 0x64, 0xa0, 0xb5, 0xc8, 0xf2, 0xa9, 0xe7, 0x8c, + 0x06, 0x2c, 0x92, 0x95, 0xe5, 0xb2, 0x9e, 0x7f, 0xbc, 0xac, 0x03, 0x8e, 0x57, 0xd2, 0x96, 0x4e, + 0xa7, 0x87, 0xe5, 0x9f, 0xa4, 0x48, 0x61, 0x6a, 0x31, 0xad, 0x43, 0x0a, 0x90, 0x08, 0x87, 0xe6, + 0x31, 0x0f, 0xae, 0xd2, 0x60, 0x37, 0x9f, 0x68, 0xa3, 0x95, 0xf6, 0x84, 0x91, 0x4e, 0x4b, 0xd1, + 0x3f, 0x48, 0x92, 0xc2, 0xd4, 0xa4, 0xf6, 0x22, 0xc9, 0xd1, 0x36, 0x6d, 0x1e, 0x18, 0x7b, 0x8d, + 0xd2, 0x42, 0x79, 0xed, 0xfd, 0x0f, 0xae, 0xad, 0x72, 0x69, 0xd3, 0x02, 0xda, 0xbe, 0x7d, 0x8a, + 0xfe, 0x71, 0x83, 0x2c, 0x2a, 0x68, 0xa2, 0xfc, 0x0c, 0x40, 0x9f, 0x9e, 0x87, 0x4e, 0x21, 0x69, + 0x67, 0xcb, 0xa0, 0xe0, 0x22, 0xc9, 0x78, 0x24, 0xed, 0xf4, 0x4d, 0x9f, 0xf5, 0xb4, 0xaf, 0x91, + 0xac, 0x04, 0xa6, 0xca, 0x65, 0x00, 0x5e, 0x9e, 0x07, 0x4e, 0x70, 0xb4, 0xb3, 0x6d, 0x1c, 0x34, + 0x4a, 0xe9, 0x78, 0x1c, 0xed, 0x38, 0xe6, 0x29, 0xd3, 0x9e, 0x03, 0x67, 0xe6, 0xb0, 0x4c, 0xf9, + 0x0a, 0xc0, 0x9e, 0x7a, 0x44, 0x1c, 0xa2, 0xca, 0xab, 0x3f, 0xfb, 0xd5, 0xfa, 0xc2, 0xef, 0x7f, + 0xbd, 0x5e, 0x9a, 0x9f, 0x2e, 0xff, 0x33, 0x41, 0x96, 0x66, 0x4e, 0x09, 0x9d, 0x69, 0xe8, 0x0d, + 0x47, 0x8e, 0xba, 0x77, 0xe0, 0x4c, 0x6a, 0xac, 0xdd, 0x99, 0xcb, 0x16, 0xaf, 0x3d, 0xe1, 0xd1, + 0xc7, 0xe6, 0x8b, 0xb7, 0xc8, 0x52, 0x0f, 0xec, 0xc7, 0xfc, 0xae, 0xe5, 0xb9, 0x47, 0xf6, 0xb1, + 0x8c, 0xa3, 0xe5, 0x38, 0x99, 0x75, 0x0e, 0xa4, 0x45, 0xc1, 0x50, 0xe3, 0xf8, 0x2f, 0x93, 0x29, + 0xee, 0x92, 0x34, 0xde, 0x37, 0xed, 0x19, 0x92, 0xae, 0x36, 0x5b, 0x75, 0x70, 0x85, 0x0b, 0x60, + 0xbd, 0x25, 0xbe, 0x75, 0x9c, 0x40, 0xdf, 0xd2, 0xae, 0x92, 0xec, 0xc1, 0xee, 0xf6, 0xfe, 0x0e, + 0x1e, 0xff, 0x45, 0x98, 0x5e, 0x89, 0xa6, 0x85, 0x72, 0xe5, 0x0b, 0xd2, 0xac, 0xf9, 0x68, 0x42, + 0xff, 0x77, 0x92, 0x2c, 0x51, 0xac, 0x02, 0xfd, 0xb0, 0xed, 0x39, 0xb6, 0x35, 0xd6, 0xda, 0x24, + 0x0f, 0xfa, 0xf5, 0xec, 0x29, 0xa7, 0xde, 0x78, 0x4c, 0xaa, 0x98, 0x70, 0xa9, 0x51, 0x4d, 0x71, + 0xd2, 0x89, 0x10, 0x08, 0x29, 0x99, 0x1e, 0x73, 0xcc, 0xf1, 0x79, 0x39, 0xab, 0x2e, 0x2b, 0x4e, + 0x2a, 0xa0, 0xbc, 0xbe, 0x32, 0xef, 0x75, 0xcd, 0x30, 0x64, 0x83, 0x61, 0x28, 0x72, 0x56, 0x1a, + 0xea, 0x2b, 0xf3, 0x9e, 0x21, 0x49, 0xda, 0xeb, 0x24, 0x7b, 0x06, 0x6a, 0x7b, 0x67, 0x32, 0x2d, + 0x9d, 0x2f, 0x57, 0x62, 0xf5, 0xf7, 0x31, 0x1b, 0xcd, 0x6d, 0x16, 0xcd, 0xda, 0xda, 0x6d, 0x35, + 0x94, 0x59, 0xe5, 0xfc, 0xae, 0xdb, 0xf2, 0x5c, 0x74, 0x59, 0xb2, 0xdb, 0xea, 0x6e, 0x1a, 0xcd, + 0xed, 0x7d, 0x8a, 0xa6, 0xbd, 0x04, 0x90, 0x52, 0x04, 0xd9, 0x34, 0x6d, 0x07, 0x4b, 0xa5, 0x2b, + 0x24, 0x65, 0xb4, 0x20, 0x06, 0x97, 0x4b, 0x30, 0x5d, 0x8c, 0xa6, 0x0d, 0x77, 0x3c, 0xf1, 0xe6, + 0xf9, 0x75, 0xf5, 0x77, 0x49, 0x71, 0x7f, 0xd8, 0x03, 0x4f, 0x15, 0x1e, 0xa2, 0x5d, 0x83, 0x90, + 0x62, 0xfa, 0xa6, 0xe3, 0x30, 0xc7, 0x0e, 0x06, 0xb2, 0x9a, 0x9e, 0x26, 0x41, 0x09, 0xf0, 0xe4, + 0xb6, 0x94, 0x95, 0x8a, 0x60, 0xd0, 0x7f, 0x4c, 0x56, 0x60, 0x95, 0xd0, 0x84, 0x94, 0xac, 0x92, + 0xf0, 0x06, 0x29, 0x5a, 0x8a, 0xd4, 0xb5, 0x7b, 0xc2, 0x15, 0xab, 0x2b, 0x50, 0xe8, 0x15, 0x22, + 0x68, 0xb3, 0x4e, 0x0b, 0x11, 0xa8, 0xd9, 0x43, 0x3d, 0x87, 0x00, 0xc5, 0xe5, 0x33, 0xd5, 0x45, + 0x80, 0xa6, 0xda, 0x00, 0x41, 0x1a, 0x58, 0x31, 0xcf, 0xee, 0xd9, 0x21, 0x5c, 0x8f, 0x9e, 0x48, + 0xb3, 0x19, 0x9a, 0x43, 0x42, 0x0d, 0xc6, 0xfa, 0x7b, 0x49, 0x42, 0xf6, 0xcc, 0xe0, 0x44, 0x2e, + 0x0d, 0x05, 0x49, 0xd4, 0x7e, 0x9c, 0x57, 0x06, 0xef, 0x29, 0x10, 0x9d, 0xe0, 0xb5, 0xd7, 0x54, + 0x9e, 0x15, 0xd5, 0x41, 0x3c, 0xa3, 0x5c, 0x2b, 0x2e, 0xc1, 0xce, 0x96, 0x00, 0x78, 0x11, 0x99, + 0xef, 0x73, 0x2f, 0x82, 0x8b, 0x08, 0x9f, 0xd0, 0x95, 0xe4, 0x23, 0x9d, 0x65, 0x06, 0x7a, 0x36, + 0x6e, 0x91, 0x39, 0x83, 0x6e, 0x2d, 0xd0, 0x09, 0x5f, 0xb5, 0x44, 0x96, 0x7d, 0xb8, 0x66, 0xb0, + 0xeb, 0x6e, 0xc0, 0xa7, 0xf5, 0x3f, 0x81, 0x0d, 0x9a, 0x6d, 0x63, 0x47, 0x9e, 0x76, 0x9d, 0x64, + 0x8f, 0xcc, 0x81, 0xed, 0x8c, 0xe5, 0x35, 0x7b, 0x39, 0x6e, 0x89, 0x09, 0xbe, 0x62, 0xf4, 0x7a, + 0x50, 0x94, 0x05, 0x9b, 0x9c, 0x87, 0x4a, 0x5e, 0x9e, 0x7c, 0x47, 0x87, 0x2e, 0x24, 0x59, 0x95, + 0x7c, 0xf9, 0x08, 0x83, 0x89, 0x6f, 0xba, 0x91, 0xb6, 0x62, 0x80, 0x56, 0x80, 0x48, 0xca, 0xce, + 0xc0, 0x83, 0x84, 0xbe, 0x6a, 0x08, 0x89, 0x37, 0x27, 0x7a, 0x05, 0xd6, 0x03, 0x95, 0x31, 0x5a, + 0x7e, 0xde, 0x7e, 0xa8, 0x84, 0x8b, 0x30, 0x19, 0x71, 0x97, 0xdf, 0xe4, 0x21, 0x65, 0x32, 0xf5, + 0x85, 0x22, 0xdd, 0xab, 0x64, 0x69, 0x46, 0xcf, 0x47, 0xaa, 0x9e, 0x66, 0xfb, 0xe0, 0xf5, 0x52, + 0x5a, 0x7e, 0x7d, 0xb3, 0x94, 0xd5, 0xff, 0x01, 0x45, 0x58, 0xdb, 0xe3, 0xd7, 0x0a, 0xad, 0x1a, + 0xdf, 0x65, 0xe6, 0x78, 0xcf, 0x6a, 0x79, 0x8e, 0xf4, 0x99, 0xd8, 0x22, 0x60, 0x22, 0x05, 0x13, + 0x34, 0x87, 0xd3, 0x88, 0x11, 0xc2, 0x6b, 0x41, 0xd4, 0x2f, 0xdd, 0x21, 0xe0, 0xb8, 0x59, 0x97, + 0x28, 0x11, 0x24, 0xe4, 0xc4, 0x16, 0x66, 0x38, 0x3a, 0x84, 0x6b, 0xda, 0x67, 0x3d, 0x81, 0x49, + 0x73, 0xcc, 0x52, 0x44, 0x45, 0x98, 0x5e, 0x87, 0x26, 0x4c, 0xc9, 0x5c, 0x25, 0xa9, 0xbd, 0x5a, + 0x1b, 0xe2, 0xce, 0x0a, 0x44, 0x8d, 0x82, 0x22, 0x03, 0x09, 0x67, 0xf6, 0xeb, 0x6d, 0x08, 0x37, + 0x33, 0x33, 0x40, 0x2a, 0xa7, 0x31, 0x9c, 0xe8, 0xbf, 0x48, 0x90, 0xac, 0xc8, 0x32, 0xb1, 0x1a, + 0x1b, 0x64, 0x51, 0x55, 0x3d, 0x22, 0xf5, 0x3d, 0xff, 0xf8, 0x34, 0x55, 0x91, 0x59, 0x4f, 0x9c, + 0xa3, 0xe2, 0x2b, 0xbf, 0x41, 0x8a, 0xd3, 0x13, 0x5f, 0xe8, 0x14, 0x7f, 0x44, 0x0a, 0xe8, 0x28, + 0x2a, 0x47, 0x6f, 0x90, 0xac, 0xc8, 0x84, 0xf2, 0xaa, 0x9f, 0x97, 0x33, 0x25, 0x12, 0x22, 0xdd, + 0xa2, 0xc8, 0xb3, 0xaa, 0x3d, 0x5b, 0x3f, 0xdf, 0x1d, 0xa9, 0x82, 0xeb, 0x6f, 0x91, 0x74, 0x9b, + 0x81, 0x84, 0x67, 0xc9, 0xa2, 0x0b, 0xa1, 0x67, 0x12, 0xd9, 0x08, 0x84, 0xab, 0x2c, 0x16, 0xe0, + 0x10, 0xb1, 0xb2, 0x38, 0x05, 0xf1, 0x0c, 0x8c, 0x67, 0x82, 0xbf, 0xa9, 0x0e, 0x15, 0xbf, 0xf5, + 0x3d, 0x52, 0xbc, 0xcb, 0xec, 0xe3, 0x7e, 0x08, 0x27, 0x86, 0x82, 0x5e, 0x26, 0xe9, 0x21, 0x8b, + 0x36, 0xbf, 0x1a, 0xeb, 0x3a, 0x30, 0x4f, 0x39, 0x0a, 0x2f, 0xe4, 0x19, 0xe7, 0x96, 0x8f, 0x02, + 0x72, 0xa4, 0xff, 0x36, 0x49, 0x96, 0x9b, 0x41, 0x30, 0x32, 0xa1, 0xe0, 0x96, 0x51, 0xf0, 0x3b, + 0xb3, 0x0d, 0xc3, 0x8d, 0x58, 0x0d, 0x67, 0x58, 0x66, 0x9b, 0x06, 0x19, 0xb9, 0x92, 0x51, 0xe4, + 0xd2, 0x1f, 0x26, 0x54, 0xb7, 0x70, 0x7d, 0xea, 0xde, 0x94, 0x57, 0xc1, 0x89, 0x2e, 0x4d, 0x4b, + 0x62, 0xfb, 0xee, 0x89, 0xeb, 0x9d, 0xb9, 0x90, 0x68, 0xa1, 0x7b, 0x68, 0x35, 0xee, 0x82, 0xa7, + 0x5d, 0x06, 0x90, 0x36, 0x03, 0xa2, 0xcc, 0x65, 0x67, 0x28, 0xa9, 0xdd, 0x68, 0xd5, 0x9b, 0xad, + 0xdb, 0x90, 0xde, 0x1e, 0x95, 0xd4, 0x66, 0x90, 0xce, 0xdc, 0x63, 0x30, 0x77, 0xb6, 0xd9, 0xe9, + 0xec, 0xf3, 0x52, 0xf1, 0x69, 0x40, 0x5d, 0x9c, 0x41, 0xe1, 0x00, 0xea, 0x44, 0x00, 0x61, 0x26, + 0x05, 0x50, 0x3a, 0x06, 0x84, 0xc9, 0x14, 0x02, 0x88, 0xf0, 0xf0, 0xbf, 0x25, 0x49, 0xc9, 0xb0, + 0x2c, 0x36, 0x0c, 0x71, 0x5e, 0x56, 0x27, 0x7b, 0x58, 0xed, 0xc1, 0x97, 0xcd, 0xf0, 0xf5, 0x04, + 0xdd, 0xe2, 0x56, 0xec, 0x8b, 0xd1, 0x1c, 0x5f, 0x85, 0x7a, 0x0e, 0x33, 0x7a, 0x03, 0x3b, 0xc0, + 0x57, 0x04, 0x41, 0xa3, 0x91, 0xa4, 0xf2, 0x7f, 0x12, 0xe4, 0x62, 0x0c, 0x42, 0x7b, 0x95, 0xa4, + 0x7d, 0x20, 0xcb, 0xe3, 0x59, 0x7b, 0x5c, 0x3f, 0x87, 0xac, 0x94, 0x23, 0xb5, 0x75, 0x42, 0xcc, + 0x51, 0xe8, 0x99, 0x7c, 0x7d, 0x7e, 0x30, 0x39, 0x3a, 0x45, 0xd1, 0xbe, 0x0f, 0xd1, 0x9a, 0x59, + 0xbe, 0x6c, 0x89, 0x0a, 0x1b, 0x8d, 0xff, 0x75, 0xf7, 0x95, 0x2d, 0x13, 0x23, 0x4a, 0x87, 0x0b, + 0xa3, 0x52, 0x68, 0xf9, 0x75, 0x52, 0x9c, 0xa6, 0xa3, 0x77, 0x43, 0x79, 0x61, 0x72, 0x05, 0x8a, + 0x94, 0x7f, 0xa3, 0xd3, 0x98, 0xce, 0xb1, 0x72, 0x1a, 0xf8, 0xd4, 0x3f, 0x84, 0xbc, 0xd4, 0xb8, + 0x17, 0x32, 0xdf, 0x35, 0x9d, 0x9a, 0xa1, 0x35, 0xa6, 0xa2, 0xa5, 0xd0, 0xfc, 0x85, 0xd8, 0x8e, + 0x3f, 0xe2, 0xa8, 0xd4, 0x8c, 0x98, 0x78, 0x09, 0x95, 0xc2, 0xc8, 0x77, 0xe4, 0xeb, 0x11, 0xaf, + 0x14, 0xf6, 0xe9, 0x36, 0x45, 0x1a, 0x3e, 0xbd, 0xa8, 0xe8, 0x94, 0x7a, 0xfc, 0xb3, 0xdf, 0xd4, + 0x02, 0x5f, 0x7d, 0x84, 0x7a, 0x99, 0x90, 0xc9, 0xae, 0xe1, 0xd8, 0x32, 0xb5, 0xcd, 0x4e, 0x67, + 0x1b, 0xae, 0x0a, 0xaf, 0x9c, 0x27, 0x53, 0x9c, 0xac, 0xff, 0x26, 0x41, 0x72, 0x35, 0x43, 0x66, + 0x98, 0x4d, 0x52, 0xe2, 0x71, 0xc5, 0x62, 0x7e, 0xd8, 0x65, 0xf7, 0x86, 0xb6, 0x3f, 0x96, 0xa1, + 0xe1, 0xfc, 0x12, 0x74, 0x19, 0xb9, 0x6a, 0xc0, 0xd4, 0xe0, 0x3c, 0x1a, 0x25, 0x45, 0x26, 0x55, + 0xec, 0x5a, 0xa6, 0x0a, 0xd4, 0xeb, 0xe7, 0x9b, 0x42, 0x94, 0x67, 0x93, 0x31, 0x34, 0xbd, 0x4a, + 0x48, 0xcd, 0x0c, 0xf4, 0x03, 0x72, 0x71, 0xd7, 0xb7, 0xfa, 0x50, 0x28, 0x89, 0x45, 0xe5, 0x96, + 0xdf, 0x22, 0x6b, 0x21, 0x14, 0x44, 0xdd, 0xbe, 0x1d, 0x84, 0xf8, 0x68, 0x09, 0xbe, 0xc1, 0x5c, + 0x9c, 0xef, 0xf2, 0xc7, 0x45, 0xf1, 0xd8, 0x49, 0xaf, 0x20, 0x66, 0x4b, 0x40, 0xa8, 0x42, 0x6c, + 0x23, 0x40, 0xff, 0x1e, 0x29, 0xd5, 0xed, 0x60, 0x68, 0x86, 0x20, 0x5b, 0xf6, 0x33, 0xda, 0x6d, + 0x52, 0xea, 0x33, 0xa8, 0x67, 0x0f, 0x99, 0x09, 0x39, 0x91, 0xf9, 0xb6, 0xd7, 0x7b, 0x22, 0x3b, + 0xac, 0x44, 0x5c, 0x6d, 0xce, 0xa4, 0x7f, 0x06, 0x19, 0x1c, 0x5f, 0x73, 0xa4, 0xdc, 0x97, 0xc8, + 0x85, 0xc0, 0x35, 0x87, 0x41, 0xdf, 0x0b, 0xbb, 0xb6, 0x1b, 0xe2, 0xf3, 0xa5, 0x23, 0x6b, 0xe1, + 0x92, 0x9a, 0x68, 0x4a, 0x3a, 0xc4, 0x66, 0xed, 0x84, 0xb1, 0x61, 0xd7, 0x73, 0x7a, 0x5d, 0x35, + 0x29, 0x9e, 0x2c, 0x01, 0x8d, 0x33, 0xbb, 0x4e, 0xaf, 0xa3, 0xe8, 0x5a, 0x95, 0xac, 0x3b, 0xde, + 0x71, 0x17, 0x34, 0xf3, 0xe1, 0xde, 0x77, 0x8f, 0x3c, 0xbf, 0x1b, 0x38, 0xde, 0x19, 0x7c, 0x38, + 0xf0, 0xc7, 0x7c, 0xd5, 0x68, 0x94, 0x01, 0xd5, 0x10, 0xa0, 0x4d, 0xcf, 0xef, 0xc0, 0xdc, 0xa6, + 0x42, 0x60, 0x9a, 0x9f, 0xa8, 0x1d, 0xda, 0xd6, 0x89, 0x4a, 0xf3, 0x11, 0x75, 0x0f, 0x88, 0x10, + 0xe9, 0x96, 0x98, 0xc3, 0x2c, 0x6e, 0x64, 0x8e, 0xca, 0x70, 0x54, 0x51, 0x11, 0x11, 0xa4, 0x7f, + 0x9d, 0xe4, 0xdb, 0x8e, 0x69, 0xf1, 0x87, 0x61, 0xac, 0xfe, 0x21, 0x85, 0xe1, 0xc9, 0x81, 0xd6, + 0x22, 0xbc, 0xe5, 0xe9, 0x34, 0x49, 0x7f, 0x0f, 0x92, 0x3e, 0xf5, 0xbc, 0x10, 0x2e, 0xe9, 0x35, + 0x92, 0xb5, 0xcc, 0xae, 0x72, 0xf7, 0x62, 0x35, 0x0f, 0x6e, 0x91, 0xa9, 0x19, 0x77, 0xd8, 0x98, + 0x66, 0x2c, 0x13, 0xfe, 0x30, 0xfd, 0x01, 0x02, 0x9d, 0x94, 0x9b, 0xa3, 0x28, 0xd2, 0x1f, 0x78, + 0x31, 0x50, 0x28, 0x30, 0xe3, 0x3f, 0x44, 0xb8, 0xa2, 0x04, 0x75, 0xfb, 0x10, 0x38, 0x44, 0xb1, + 0x58, 0x5d, 0x06, 0x24, 0x11, 0x48, 0x0c, 0x27, 0x94, 0x08, 0x34, 0x7e, 0xeb, 0x7f, 0x4e, 0x90, + 0x02, 0x0e, 0xec, 0x23, 0xdb, 0xc2, 0x3c, 0xf3, 0xc5, 0x63, 0x24, 0x04, 0x06, 0x2b, 0xf0, 0xe5, + 0xa6, 0x78, 0x60, 0xa8, 0x75, 0x28, 0x45, 0x9a, 0xf6, 0x36, 0x84, 0x47, 0x9e, 0xe7, 0x64, 0x78, + 0xd4, 0x3f, 0x3f, 0x23, 0xca, 0x2e, 0x47, 0xf2, 0x71, 0x23, 0x4e, 0x76, 0xc7, 0x8f, 0xa6, 0x48, + 0xa7, 0x49, 0xf8, 0xa8, 0x6d, 0xb9, 0xfc, 0x34, 0xe4, 0xa3, 0x76, 0xad, 0x45, 0x81, 0xa2, 0xff, + 0x31, 0x41, 0x96, 0x1a, 0xae, 0xe5, 0x8f, 0x79, 0x48, 0x41, 0x0b, 0xae, 0x91, 0x3c, 0x14, 0xd3, + 0xc1, 0x38, 0x80, 0x8e, 0x53, 0xbd, 0x99, 0x45, 0x04, 0xad, 0x49, 0xf2, 0x10, 0x3c, 0x3d, 0xdf, + 0x0e, 0xfb, 0x03, 0x59, 0x55, 0xc6, 0x87, 0xb1, 0x69, 0x99, 0x15, 0x43, 0xb1, 0xd0, 0x09, 0xb7, + 0x0a, 0x5c, 0x29, 0xbe, 0x59, 0x1e, 0xb8, 0xa0, 0xff, 0x75, 0xa0, 0xd5, 0x81, 0x7a, 0xb1, 0x8b, + 0x1d, 0x04, 0xd7, 0x03, 0x5a, 0x41, 0x49, 0xc3, 0xae, 0x48, 0xd7, 0x49, 0x3e, 0x12, 0x86, 0x2f, + 0x95, 0x46, 0xa3, 0xd3, 0xbd, 0xb9, 0x71, 0xab, 0x7b, 0xbb, 0xb6, 0x03, 0x61, 0x4c, 0xe4, 0xd0, + 0xdf, 0x81, 0x4e, 0x3b, 0xa6, 0x0b, 0x5d, 0x8e, 0xea, 0xf9, 0xc0, 0x2b, 0x7c, 0xb8, 0x6a, 0xaa, + 0x28, 0x4a, 0x0b, 0xaf, 0xc0, 0xdb, 0x87, 0x45, 0x11, 0x4e, 0xc5, 0x17, 0x45, 0x53, 0x2f, 0xb6, + 0xa9, 0x73, 0x5f, 0x6c, 0xd3, 0x5f, 0xc9, 0x8b, 0xed, 0x8b, 0x9f, 0xa5, 0x48, 0x3e, 0xea, 0xe1, + 0xd0, 0x65, 0xb0, 0x46, 0x59, 0x10, 0xdd, 0x75, 0x44, 0x6f, 0xf1, 0xea, 0x24, 0x6f, 0x6c, 0x6f, + 0xef, 0xd6, 0x0c, 0x7c, 0xf0, 0x7c, 0x5b, 0x14, 0x31, 0x11, 0xc0, 0x80, 0x4b, 0x8b, 0x87, 0xde, + 0xd3, 0xf4, 0x49, 0x11, 0x73, 0x5f, 0xf6, 0xf0, 0x11, 0x4a, 0x55, 0x30, 0xcf, 0x91, 0x9c, 0xd1, + 0xe9, 0x34, 0x6f, 0xb7, 0x40, 0xd2, 0x83, 0x44, 0xf9, 0x29, 0x00, 0x5d, 0x98, 0x88, 0x82, 0xe4, + 0x7b, 0xec, 0x82, 0x24, 0x44, 0xd5, 0x6a, 0x8d, 0x36, 0xae, 0x77, 0x3f, 0x39, 0x8f, 0xe2, 0xa9, + 0x9b, 0x3f, 0x88, 0xe5, 0xdb, 0xb4, 0xd1, 0x36, 0x28, 0xae, 0xf8, 0x20, 0x39, 0xb7, 0xaf, 0xb6, + 0xcf, 0xa0, 0xab, 0xc7, 0x35, 0xd7, 0xd5, 0xeb, 0xed, 0xfd, 0x54, 0x59, 0x03, 0xcc, 0xf2, 0xa4, + 0x71, 0x05, 0xfb, 0x8e, 0x71, 0xb5, 0xce, 0x9e, 0x41, 0xf7, 0xb8, 0x98, 0xd4, 0xdc, 0x6a, 0x1d, + 0x7c, 0x4e, 0x40, 0x29, 0xa0, 0x1d, 0xdd, 0x6f, 0xb5, 0xb8, 0x76, 0xe9, 0x39, 0xed, 0xe8, 0xc8, + 0x75, 0x11, 0x73, 0x1d, 0x52, 0xd8, 0xee, 0x4e, 0x7b, 0xbb, 0xb1, 0xd7, 0x28, 0x3d, 0x48, 0xcf, + 0x6d, 0xa8, 0xe6, 0x0d, 0x86, 0x0e, 0x0b, 0x85, 0x7a, 0x9d, 0xad, 0xfd, 0x3d, 0xfe, 0xb8, 0x7c, + 0x3f, 0x33, 0xbf, 0x60, 0x7f, 0x14, 0xf6, 0xb0, 0x6c, 0xbc, 0x16, 0xd5, 0x71, 0x0f, 0x32, 0x22, + 0x65, 0x46, 0x18, 0x51, 0xc4, 0xa1, 0x1c, 0xda, 0xf8, 0xae, 0x78, 0x87, 0xbe, 0x9f, 0x9d, 0x93, + 0x43, 0xd9, 0xbb, 0x10, 0x05, 0xa1, 0xd4, 0x8b, 0x9e, 0xa4, 0xa2, 0xa9, 0x17, 0x7f, 0x40, 0x72, + 0x2a, 0x60, 0x80, 0x75, 0xb2, 0x77, 0x77, 0xe9, 0x9d, 0x06, 0x85, 0xa3, 0xe7, 0xd6, 0x51, 0x33, + 0x77, 0x3d, 0x1f, 0xbc, 0x0b, 0xb6, 0xb1, 0xb8, 0x63, 0xb4, 0x8c, 0xdb, 0x00, 0x90, 0x6f, 0x5e, + 0x0a, 0x20, 0xbd, 0xbe, 0x5c, 0x92, 0x0b, 0x44, 0x32, 0xab, 0x6b, 0x0f, 0x3f, 0x59, 0x5f, 0xf8, + 0x08, 0x7e, 0x7f, 0xff, 0x64, 0x3d, 0x71, 0xff, 0xd3, 0xf5, 0xc4, 0x43, 0xf8, 0xfd, 0x01, 0x7e, + 0x7f, 0x81, 0xdf, 0x61, 0x96, 0xd7, 0x2f, 0xaf, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff, 0x2c, 0xf4, + 0xf4, 0x16, 0x49, 0x1d, 0x00, 0x00, } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/api/types.proto b/components/engine/vendor/src/github.com/docker/swarmkit/api/types.proto index f92320325f..487a02a7d7 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/api/types.proto +++ b/components/engine/vendor/src/github.com/docker/swarmkit/api/types.proto @@ -447,9 +447,29 @@ message AcceptancePolicy { } +message ExternalCA { + enum CAProtocol { + CFSSL = 0 [(gogoproto.enumvalue_customname) = "CAProtocolCFSSL"]; + } + + // Protocol is the protocol used by this external CA. + CAProtocol protocol = 1; + + // URL is the URL where the external CA can be reached. + string url = 2 [(gogoproto.customname) = "URL"]; + + // Options is a set of additional key/value pairs whose interpretation + // depends on the specified CA type. + map options = 3; +} + message CAConfig { // NodeCertExpiry is the duration certificates should be issued for Duration node_cert_expiry = 1; + + // ExternalCAs is a list of CAs to which a manager node will make + // certificate signing requests for node certificates. + repeated ExternalCA external_cas = 2 [(gogoproto.customname) = "ExternalCAs"]; } // OrchestrationConfig defines cluster-level orchestration settings. diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/ca/certificates.go b/components/engine/vendor/src/github.com/docker/swarmkit/ca/certificates.go index d39813b305..42019e5838 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/ca/certificates.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/ca/certificates.go @@ -167,7 +167,16 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert } // Get the remote manager to issue a CA signed certificate for this node - signedCert, err := GetRemoteSignedCertificate(ctx, csr, role, secret, rca.Pool, picker, transport, nodeInfo) + // Retry up to 5 times in case the manager we first try to contact isn't + // responding properly (for example, it may have just been demoted). + var signedCert []byte + for i := 0; i != 5; i++ { + signedCert, err = GetRemoteSignedCertificate(ctx, csr, role, secret, rca.Pool, picker, transport, nodeInfo) + if err == nil { + break + } + log.Warningf("error fetching signed node certificate: %v", err) + } if err != nil { return nil, err } @@ -192,6 +201,12 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert return nil, err } + // Create a valid TLSKeyPair out of the PEM encoded private key and certificate + tlsKeyPair, err := tls.X509KeyPair(signedCert, key) + if err != nil { + return nil, err + } + log.Infof("Downloaded new TLS credentials with role: %s.", role) // Ensure directory exists @@ -210,13 +225,27 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert return nil, err } - // Create a valid TLSKeyPair out of the PEM encoded private key and certificate - tlsKeyPair, err := tls.X509KeyPair(signedCert, key) - if err != nil { - return nil, err + return &tlsKeyPair, nil +} + +// PrepareCSR creates a CFSSL Sign Request based on the given raw CSR and +// overrides the Subject and Hosts with the given extra args. +func PrepareCSR(csrBytes []byte, cn, ou, org string) cfsigner.SignRequest { + // All managers get added the subject-alt-name of CA, so they can be + // used for cert issuance. + hosts := []string{ou} + if ou == ManagerRole { + hosts = append(hosts, CARole) } - return &tlsKeyPair, nil + return cfsigner.SignRequest{ + Request: string(csrBytes), + // OU is used for Authentication of the node type. The CN has the random + // node ID. + Subject: &cfsigner.Subject{CN: cn, Names: []cfcsr.Name{{OU: ou, O: org}}}, + // Adding ou as DNS alt name, so clients can connect to ManagerRole and CARole + Hosts: hosts, + } } // ParseValidateAndSignCSR returns a signed certificate from a particular rootCA and a CSR. @@ -225,25 +254,21 @@ func (rca *RootCA) ParseValidateAndSignCSR(csrBytes []byte, cn, ou, org string) return nil, ErrNoValidSigner } - // All managers get added the subject-alt-name of CA, so they can be used for cert issuance - hosts := []string{ou} - if ou == ManagerRole { - hosts = append(hosts, CARole) - } + signRequest := PrepareCSR(csrBytes, cn, ou, org) - cert, err := rca.Signer.Sign(cfsigner.SignRequest{ - Request: string(csrBytes), - // OU is used for Authentication of the node type. The CN has the random - // node ID. - Subject: &cfsigner.Subject{CN: cn, Names: []cfcsr.Name{{OU: ou, O: org}}}, - // Adding ou as DNS alt name, so clients can connect to ManagerRole and CARole - Hosts: hosts, - }) + cert, err := rca.Signer.Sign(signRequest) if err != nil { log.Debugf("failed to sign node certificate: %v", err) return nil, err } + return rca.AppendFirstRootPEM(cert) +} + +// AppendFirstRootPEM appends the first certificate from this RootCA's cert +// bundle to the given cert bundle (which should already be encoded as a series +// of PEM-encoded certificate blocks). +func (rca *RootCA) AppendFirstRootPEM(cert []byte) ([]byte, error) { // Append the first root CA Cert to the certificate, to create a valid chain // Get the first Root CA Cert on the bundle firstRootCA, _, err := helpers.ParseOneCertificateFromPEM(rca.Cert) @@ -390,7 +415,7 @@ func GetLocalRootCA(baseDir string) (RootCA, error) { rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration) if err == nil { - log.Debugf("successfully loaded the signer for the Root CA: %s", paths.RootCA.Cert) + log.Debugf("successfully loaded the Root CA: %s", paths.RootCA.Cert) } return rootCA, err @@ -602,7 +627,7 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, role, secret st } defer conn.Close() - // Create a CAClient to retreive a new Certificate + // Create a CAClient to retrieve a new Certificate caClient := api.NewNodeCAClient(conn) // Convert our internal string roles into an API role @@ -644,7 +669,15 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, role, secret st if statusResponse.Certificate == nil { return nil, fmt.Errorf("no certificate in CertificateStatus response") } - return statusResponse.Certificate.Certificate, nil + + // The certificate in the response must match the CSR + // we submitted. If we are getting a response for a + // certificate that was previously issued, we need to + // retry until the certificate gets updated per our + // current request. + if bytes.Equal(statusResponse.Certificate.CSR, csr) { + return statusResponse.Certificate.Certificate, nil + } } // If we're still pending, the issuance failed, or the state is unknown diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/ca/config.go b/components/engine/vendor/src/github.com/docker/swarmkit/ca/config.go index 64eb6cf1fb..0de5cbd8c9 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/ca/config.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/ca/config.go @@ -45,7 +45,8 @@ const ( type SecurityConfig struct { mu sync.Mutex - rootCA *RootCA + rootCA *RootCA + externalCA *ExternalCA ServerTLSCreds *MutableTLSCreds ClientTLSCreds *MutableTLSCreds @@ -60,8 +61,19 @@ type CertificateUpdate struct { // NewSecurityConfig initializes and returns a new SecurityConfig. func NewSecurityConfig(rootCA *RootCA, clientTLSCreds, serverTLSCreds *MutableTLSCreds) *SecurityConfig { + // Make a new TLS config for the external CA client without a + // ServerName value set. + clientTLSConfig := clientTLSCreds.Config() + + externalCATLSConfig := &tls.Config{ + Certificates: clientTLSConfig.Certificates, + RootCAs: clientTLSConfig.RootCAs, + MinVersion: tls.VersionTLS12, + } + return &SecurityConfig{ rootCA: rootCA, + externalCA: NewExternalCA(rootCA, externalCATLSConfig), ClientTLSCreds: clientTLSCreds, ServerTLSCreds: serverTLSCreds, } @@ -164,8 +176,18 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, caHash, secret return nil, err } - // Get the remote CA certificate, verify integrity with the hash provided - rootCA, err = GetRemoteCA(ctx, d, picker) + // Get the remote CA certificate, verify integrity with the + // hash provided. Retry up to 5 times, in case the manager we + // first try to contact is not responding properly (it may have + // just been demoted, for example). + + for i := 0; i != 5; i++ { + rootCA, err = GetRemoteCA(ctx, d, picker) + if err == nil { + break + } + log.Warningf("failed to retrieve remote root CA certificate: %v", err) + } if err != nil { return nil, err } @@ -180,9 +202,9 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, caHash, secret return nil, err } - // At this point we've successfully loaded the CA details from disk, or successfully - // downloaded them remotely. - // The next step is to try to load our certificates. + // At this point we've successfully loaded the CA details from disk, or + // successfully downloaded them remotely. The next step is to try to + // load our certificates. clientTLSCreds, serverTLSCreds, err = LoadTLSCreds(rootCA, paths.Node) if err != nil { log.Debugf("no valid local TLS credentials found: %v", err) @@ -204,6 +226,9 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, caHash, secret } } tlsKeyPair, err = rootCA.IssueAndSaveNewCertificates(paths.Node, cn, proposedRole, org) + if err != nil { + return nil, err + } } else { // There was an error loading our Credentials, let's get a new certificate issued // Last argument is nil because at this point we don't have any valid TLS creds @@ -211,7 +236,6 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, caHash, secret if err != nil { return nil, err } - } // Create the Server TLS Credentials for this node. These will not be used by agents. serverTLSCreds, err = rootCA.NewServerTLSCredentials(tlsKeyPair) @@ -236,12 +260,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, caHash, secret log.Debugf("loaded local TLS credentials: %s.", paths.Node.Cert) } - return &SecurityConfig{ - rootCA: &rootCA, - - ServerTLSCreds: serverTLSCreds, - ClientTLSCreds: clientTLSCreds, - }, nil + return NewSecurityConfig(&rootCA, clientTLSCreds, serverTLSCreds), nil } // RenewTLSConfig will continuously monitor for the necessity of renewing the local certificates, either by @@ -317,6 +336,14 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string, updates <- CertificateUpdate{Err: err} } + // Update the external CA to use the new client TLS + // config using a copy without a serverName specified. + s.externalCA.UpdateTLSConfig(&tls.Config{ + Certificates: clientTLSConfig.Certificates, + RootCAs: clientTLSConfig.RootCAs, + MinVersion: tls.VersionTLS12, + }) + err = s.ServerTLSCreds.LoadNewTLSConfig(serverTLSConfig) if err != nil { log.Debugf("failed to update the server TLS credentials: %v", err) @@ -405,7 +432,7 @@ func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLS } keyPair, newErr = tls.X509KeyPair(cert, key) - if err != nil { + if newErr != nil { return nil, nil, err } } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/ca/external.go b/components/engine/vendor/src/github.com/docker/swarmkit/ca/external.go new file mode 100644 index 0000000000..cd34478b09 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/swarmkit/ca/external.go @@ -0,0 +1,141 @@ +package ca + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "sync" + + log "github.com/Sirupsen/logrus" + "github.com/cloudflare/cfssl/api" + "github.com/cloudflare/cfssl/signer" +) + +// ErrNoExternalCAURLs is an error used it indicate that an ExternalCA is +// configured with no URLs to which it can proxy certificate signing requests. +var ErrNoExternalCAURLs = errors.New("no external CA URLs") + +// ExternalCA is able to make certificate signing requests to one of a list +// remote CFSSL API endpoints. +type ExternalCA struct { + mu sync.Mutex + rootCA *RootCA + urls []string + client *http.Client +} + +// NewExternalCA creates a new ExternalCA which uses the given tlsConfig to +// authenticate to any of the given URLS of CFSSL API endpoints. +func NewExternalCA(rootCA *RootCA, tlsConfig *tls.Config, urls ...string) *ExternalCA { + return &ExternalCA{ + rootCA: rootCA, + urls: urls, + client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + }, + } +} + +// UpdateTLSConfig updates the HTTP Client for this ExternalCA by creating +// a new client which uses the given tlsConfig. +func (eca *ExternalCA) UpdateTLSConfig(tlsConfig *tls.Config) { + eca.mu.Lock() + defer eca.mu.Unlock() + + eca.client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + } +} + +// UpdateURLs updates the list of CSR API endpoints by setting it to the given +// urls. +func (eca *ExternalCA) UpdateURLs(urls ...string) { + eca.mu.Lock() + defer eca.mu.Unlock() + + eca.urls = urls +} + +// Sign signs a new certificate by proxying the given certificate signing +// request to an external CFSSL API server. +func (eca *ExternalCA) Sign(req signer.SignRequest) (cert []byte, err error) { + // Get the current HTTP client and list of URLs in a small critical + // section. We will use these to make certificate signing requests. + eca.mu.Lock() + urls := eca.urls + client := eca.client + eca.mu.Unlock() + + if len(urls) == 0 { + return nil, ErrNoExternalCAURLs + } + + csrJSON, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("unable to JSON-encode CFSSL signing request: %s", err) + } + + // Try each configured proxy URL. Return after the first success. If + // all fail then the last error will be returned. + for _, url := range urls { + cert, err = makeExternalSignRequest(client, url, csrJSON) + if err == nil { + return eca.rootCA.AppendFirstRootPEM(cert) + } + + log.Debugf("unable to proxy certificate signing request to %s: %s", url, err) + } + + return nil, err +} + +func makeExternalSignRequest(client *http.Client, url string, csrJSON []byte) (cert []byte, err error) { + resp, err := client.Post(url, "application/json", bytes.NewReader(csrJSON)) + if err != nil { + return nil, fmt.Errorf("unable to perform certificate signing request: %s", err) + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("unable to read CSR response body: %s", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code in CSR response: %d - %s", resp.StatusCode, string(body)) + } + + var apiResponse api.Response + if err := json.Unmarshal(body, &apiResponse); err != nil { + log.Debugf("unable to JSON-parse CFSSL API response body: %s", string(body)) + return nil, fmt.Errorf("unable to parse JSON response: %s", err) + } + + if !apiResponse.Success || apiResponse.Result == nil { + if len(apiResponse.Errors) > 0 { + return nil, fmt.Errorf("response errors: %v", apiResponse.Errors) + } + + return nil, fmt.Errorf("certificate signing request failed") + } + + result, ok := apiResponse.Result.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid result type: %T", apiResponse.Result) + } + + certPEM, ok := result["certificate"].(string) + if !ok { + return nil, fmt.Errorf("invalid result certificate field type: %T", result["certificate"]) + } + + return []byte(certPEM), nil +} diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/ca/server.go b/components/engine/vendor/src/github.com/docker/swarmkit/ca/server.go index 9d118cdad8..ee6325375c 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/ca/server.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/ca/server.go @@ -355,7 +355,7 @@ func (s *Server) Run(ctx context.Context) error { s.mu.Lock() if s.isRunning() { s.mu.Unlock() - return fmt.Errorf("CA signer is stopped") + return fmt.Errorf("CA signer is already running") } s.wg.Add(1) defer s.wg.Done() @@ -443,12 +443,14 @@ func (s *Server) Run(ctx context.Context) error { func (s *Server) Stop() error { s.mu.Lock() if !s.isRunning() { + s.mu.Unlock() return fmt.Errorf("CA signer is already stopped") } s.cancel() s.mu.Unlock() // wait for all handlers to finish their CA deals, s.wg.Wait() + s.started = make(chan struct{}) return nil } @@ -530,6 +532,21 @@ func (s *Server) updateCluster(ctx context.Context, cluster *api.Cluster) { }).Debugf("Root CA updated successfully") } } + + // Update our security config with the list of External CA URLs + // from the new cluster state. + + // TODO(aaronl): In the future, this will be abstracted with an + // ExternalCA interface that has different implementations for + // different CA types. At the moment, only CFSSL is supported. + var cfsslURLs []string + for _, ca := range cluster.Spec.CAConfig.ExternalCAs { + if ca.Protocol == api.ExternalCA_CAProtocolCFSSL { + cfsslURLs = append(cfsslURLs, ca.URL) + } + } + + s.securityConfig.externalCA.UpdateURLs(cfsslURLs...) } // evaluateAndSignNodeCert implements the logic of which certificates to sign @@ -555,13 +572,8 @@ func (s *Server) evaluateAndSignNodeCert(ctx context.Context, node *api.Node) { // signNodeCert does the bulk of the work for signing a certificate func (s *Server) signNodeCert(ctx context.Context, node *api.Node) { - if !s.securityConfig.RootCA().CanSign() { - log.G(ctx).WithFields(logrus.Fields{ - "node.id": node.ID, - "method": "(*Server).signNodeCert", - }).Errorf("no valid signer found") - return - } + rootCA := s.securityConfig.RootCA() + externalCA := s.securityConfig.externalCA node = node.Copy() nodeID := node.ID @@ -576,7 +588,20 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) { } // Attempt to sign the CSR - cert, err := s.securityConfig.RootCA().ParseValidateAndSignCSR(node.Certificate.CSR, node.Certificate.CN, role, s.securityConfig.ClientTLSCreds.Organization()) + var ( + rawCSR = node.Certificate.CSR + cn = node.Certificate.CN + ou = role + org = s.securityConfig.ClientTLSCreds.Organization() + ) + + // Try using the external CA first. + cert, err := externalCA.Sign(PrepareCSR(rawCSR, cn, ou, org)) + if err == ErrNoExternalCAURLs { + // No external CA servers configured. Try using the local CA. + cert, err = rootCA.ParseValidateAndSignCSR(rawCSR, cn, ou, org) + } + if err != nil { log.G(ctx).WithFields(logrus.Fields{ "node.id": node.ID, diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/ca/transport.go b/components/engine/vendor/src/github.com/docker/swarmkit/ca/transport.go index d39dc64af7..10d678e379 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/ca/transport.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/ca/transport.go @@ -124,6 +124,14 @@ func (c *MutableTLSCreds) LoadNewTLSConfig(newConfig *tls.Config) error { return nil } +// Config returns the current underlying TLS config. +func (c *MutableTLSCreds) Config() *tls.Config { + c.Lock() + defer c.Unlock() + + return c.config +} + // Role returns the OU for the certificate encapsulated in this TransportAuthenticator func (c *MutableTLSCreds) Role() string { c.Lock() diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/network.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/network.go index e9a55b0798..46b90505fa 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/network.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/network.go @@ -52,6 +52,16 @@ type networkContext struct { // A table of unallocated tasks which will be revisited if any thing // changes in system state that might help task allocation. unallocatedTasks map[string]*api.Task + + // A table of unallocated services which will be revisited if + // any thing changes in system state that might help service + // allocation. + unallocatedServices map[string]*api.Service + + // A table of unallocated networks which will be revisited if + // any thing changes in system state that might help network + // allocation. + unallocatedNetworks map[string]*api.Network } func (a *Allocator) doNetworkInit(ctx context.Context) error { @@ -61,8 +71,10 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error { } nc := &networkContext{ - nwkAllocator: na, - unallocatedTasks: make(map[string]*api.Task), + nwkAllocator: na, + unallocatedTasks: make(map[string]*api.Task), + unallocatedServices: make(map[string]*api.Service), + unallocatedNetworks: make(map[string]*api.Network), } // Check if we have the ingress network. If not found create @@ -326,6 +338,8 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) { case state.EventCreateTask, state.EventUpdateTask, state.EventDeleteTask: a.doTaskAlloc(ctx, nc, ev) case state.EventCommit: + a.procUnallocatedNetworks(ctx, nc) + a.procUnallocatedServices(ctx, nc) a.procUnallocatedTasksNetwork(ctx, nc) return } @@ -554,29 +568,34 @@ func (a *Allocator) allocateNode(ctx context.Context, nc *networkContext, node * } func (a *Allocator) allocateService(ctx context.Context, nc *networkContext, s *api.Service) error { - // The service is trying to expose ports to the external - // world. Automatically attach the service to the ingress - // network only if it is not already done. - if s.Spec.Endpoint != nil && len(s.Spec.Endpoint.Ports) != 0 { + if s.Spec.Endpoint != nil { if s.Endpoint == nil { - s.Endpoint = &api.Endpoint{} - } - - var found bool - for _, vip := range s.Endpoint.VirtualIPs { - if vip.NetworkID == ingressNetwork.ID { - found = true - break + s.Endpoint = &api.Endpoint{ + Spec: s.Spec.Endpoint.Copy(), } } - if !found { - s.Endpoint.VirtualIPs = append(s.Endpoint.VirtualIPs, - &api.Endpoint_VirtualIP{NetworkID: ingressNetwork.ID}) + // The service is trying to expose ports to the external + // world. Automatically attach the service to the ingress + // network only if it is not already done. + if len(s.Spec.Endpoint.Ports) != 0 { + var found bool + for _, vip := range s.Endpoint.VirtualIPs { + if vip.NetworkID == ingressNetwork.ID { + found = true + break + } + } + + if !found { + s.Endpoint.VirtualIPs = append(s.Endpoint.VirtualIPs, + &api.Endpoint_VirtualIP{NetworkID: ingressNetwork.ID}) + } } } if err := nc.nwkAllocator.ServiceAllocate(s); err != nil { + nc.unallocatedServices[s.ID] = s return err } @@ -611,6 +630,7 @@ func (a *Allocator) allocateService(ctx context.Context, nc *networkContext, s * func (a *Allocator) allocateNetwork(ctx context.Context, nc *networkContext, n *api.Network) error { if err := nc.nwkAllocator.Allocate(n); err != nil { + nc.unallocatedNetworks[n.ID] = n return fmt.Errorf("failed during network allocation for network %s: %v", n.ID, err) } @@ -666,6 +686,8 @@ func (a *Allocator) allocateTask(ctx context.Context, nc *networkContext, tx sto if !nc.nwkAllocator.IsAllocated(n) { return nil, fmt.Errorf("network %s attached to task %s not allocated yet", n.ID, t.ID) } + + na.Network = n } if err := nc.nwkAllocator.AllocateTask(t); err != nil { @@ -696,6 +718,32 @@ func (a *Allocator) allocateTask(ctx context.Context, nc *networkContext, tx sto return storeT, nil } +func (a *Allocator) procUnallocatedNetworks(ctx context.Context, nc *networkContext) { + for _, n := range nc.unallocatedNetworks { + if !nc.nwkAllocator.IsAllocated(n) { + if err := a.allocateNetwork(ctx, nc, n); err != nil { + log.G(ctx).Debugf("Failed allocation of unallocated network %s: %v", n.ID, err) + continue + } + } + + delete(nc.unallocatedNetworks, n.ID) + } +} + +func (a *Allocator) procUnallocatedServices(ctx context.Context, nc *networkContext) { + for _, s := range nc.unallocatedServices { + if serviceAllocationNeeded(s, nc) { + if err := a.allocateService(ctx, nc, s); err != nil { + log.G(ctx).Debugf("Failed allocation of unallocated service %s: %v", s.ID, err) + continue + } + } + + delete(nc.unallocatedServices, s.ID) + } +} + func (a *Allocator) procUnallocatedTasksNetwork(ctx context.Context, nc *networkContext) { tasks := make([]*api.Task, 0, len(nc.unallocatedTasks)) diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go index 4a054b67ff..c9c32d48a3 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go @@ -14,7 +14,10 @@ import ( ) const ( - defaultDriver = "overlay" + // DefaultDriver defines the name of the driver to be used by + // default if a network without any driver name specified is + // created. + DefaultDriver = "overlay" ) var ( @@ -69,7 +72,7 @@ func New() (*NetworkAllocator, error) { } // Add the manager component of overlay driver to the registry. - if err := reg.AddDriver(defaultDriver, defaultDriverInitFunc, nil); err != nil { + if err := reg.AddDriver(DefaultDriver, defaultDriverInitFunc, nil); err != nil { return nil, err } @@ -96,6 +99,7 @@ func (na *NetworkAllocator) Allocate(n *api.Network) error { } if err := na.allocateDriverState(n); err != nil { + na.freePools(n, pools) return fmt.Errorf("failed while allocating driver state for network %s: %v", n.ID, err) } @@ -146,7 +150,9 @@ func (na *NetworkAllocator) ServiceAllocate(s *api.Service) (err error) { } if s.Endpoint == nil { - s.Endpoint = &api.Endpoint{} + s.Endpoint = &api.Endpoint{ + Spec: s.Spec.Endpoint.Copy(), + } } // First allocate VIPs for all the pre-populated endpoint attachments @@ -520,7 +526,7 @@ func (na *NetworkAllocator) allocateDriverState(n *api.Network) error { // Resolve network driver func (na *NetworkAllocator) resolveDriver(n *api.Network) (driverapi.Driver, string, error) { - dName := defaultDriver + dName := DefaultDriver if n.Spec.DriverConfig != nil && n.Spec.DriverConfig.Name != "" { dName = n.Spec.DriverConfig.Name } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/network.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/network.go index 35fcd70633..bdda1e6e02 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/network.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/network.go @@ -3,8 +3,10 @@ package controlapi import ( "net" + "github.com/docker/libnetwork/ipamapi" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/identity" + "github.com/docker/swarmkit/manager/allocator/networkallocator" "github.com/docker/swarmkit/manager/state/store" "golang.org/x/net/context" "google.golang.org/grpc" @@ -57,6 +59,10 @@ func validateIPAM(ipam *api.IPAMOptions) error { return err } + if ipam.Driver != nil && ipam.Driver.Name != ipamapi.DefaultIPAM { + return grpc.Errorf(codes.InvalidArgument, "invalid IPAM specified") + } + for _, ipamConf := range ipam.Configs { if err := validateIPAMConfiguration(ipamConf); err != nil { return err @@ -79,6 +85,10 @@ func validateNetworkSpec(spec *api.NetworkSpec) error { return err } + if spec.DriverConfig != nil && spec.DriverConfig.Name != networkallocator.DefaultDriver { + return grpc.Errorf(codes.InvalidArgument, "invalid driver specified") + } + if err := validateIPAM(spec.IPAM); err != nil { return err } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go index d8d162dbaf..253f71b724 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go @@ -8,6 +8,7 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/manager/state/store" + "github.com/docker/swarmkit/protobuf/ptypes" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -15,6 +16,7 @@ import ( var ( errNetworkUpdateNotSupported = errors.New("changing network in service is not supported") + errModeChangeNotAllowed = errors.New("service mode change is not allowed") ) func validateResources(r *api.Resources) error { @@ -45,21 +47,70 @@ func validateResourceRequirements(r *api.ResourceRequirements) error { return nil } -func validateServiceSpecTemplate(spec *api.ServiceSpec) error { - if err := validateResourceRequirements(spec.Task.Resources); err != nil { +func validateRestartPolicy(rp *api.RestartPolicy) error { + if rp == nil { + return nil + } + + if rp.Delay != nil { + delay, err := ptypes.Duration(rp.Delay) + if err != nil { + return err + } + if delay < 0 { + return grpc.Errorf(codes.InvalidArgument, "TaskSpec: restart-delay cannot be negative") + } + } + + if rp.Window != nil { + win, err := ptypes.Duration(rp.Window) + if err != nil { + return err + } + if win < 0 { + return grpc.Errorf(codes.InvalidArgument, "TaskSpec: restart-window cannot be negative") + } + } + + return nil +} + +func validateUpdate(uc *api.UpdateConfig) error { + if uc == nil { + return nil + } + + delay, err := ptypes.Duration(&uc.Delay) + if err != nil { return err } - if spec.Task.GetRuntime() == nil { + if delay < 0 { + return grpc.Errorf(codes.InvalidArgument, "TaskSpec: update-delay cannot be negative") + } + + return nil +} + +func validateTask(taskSpec api.TaskSpec) error { + if err := validateResourceRequirements(taskSpec.Resources); err != nil { + return err + } + + if err := validateRestartPolicy(taskSpec.Restart); err != nil { + return err + } + + if taskSpec.GetRuntime() == nil { return grpc.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime") } - _, ok := spec.Task.GetRuntime().(*api.TaskSpec_Container) + _, ok := taskSpec.GetRuntime().(*api.TaskSpec_Container) if !ok { return grpc.Errorf(codes.Unimplemented, "RuntimeSpec: unimplemented runtime in service spec") } - container := spec.Task.GetContainer() + container := taskSpec.GetContainer() if container == nil { return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: missing in service spec") } @@ -99,7 +150,13 @@ func validateServiceSpec(spec *api.ServiceSpec) error { if err := validateAnnotations(spec.Annotations); err != nil { return err } - if err := validateServiceSpecTemplate(spec); err != nil { + if err := validateTask(spec.Task); err != nil { + return err + } + if err := validateUpdate(spec.Update); err != nil { + return err + } + if err := validateEndpointSpec(spec.Endpoint); err != nil { return err } return nil @@ -179,6 +236,12 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe return errNetworkUpdateNotSupported } + // orchestrator is designed to be stateless, so it should not deal + // with service mode change (comparing current config with previous config). + // proper way to change service mode is to delete and re-add. + if request.Spec != nil && reflect.TypeOf(service.Spec.Mode) != reflect.TypeOf(request.Spec.Mode) { + return errModeChangeNotAllowed + } service.Meta.Version = *request.ServiceVersion service.Spec = *request.Spec.Copy() return store.UpdateService(tx, service) diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go index b91643bdd0..4f040b50dd 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go @@ -29,6 +29,7 @@ const ( DefaultHeartBeatPeriod = 5 * time.Second defaultHeartBeatEpsilon = 500 * time.Millisecond defaultGracePeriodMultiplier = 3 + defaultRateLimitPeriod = 16 * time.Second // maxBatchItems is the threshold of queued writes that should // trigger an actual transaction to commit them to the shared store. @@ -59,9 +60,12 @@ var ( // DefautConfig. type Config struct { // Addr configures the address the dispatcher reports to agents. - Addr string - HeartbeatPeriod time.Duration - HeartbeatEpsilon time.Duration + Addr string + HeartbeatPeriod time.Duration + HeartbeatEpsilon time.Duration + // RateLimitPeriod specifies how often node with same ID can try to register + // new session. + RateLimitPeriod time.Duration GracePeriodMultiplier int } @@ -70,6 +74,7 @@ func DefaultConfig() *Config { return &Config{ HeartbeatPeriod: DefaultHeartBeatPeriod, HeartbeatEpsilon: defaultHeartBeatEpsilon, + RateLimitPeriod: defaultRateLimitPeriod, GracePeriodMultiplier: defaultGracePeriodMultiplier, } } @@ -116,12 +121,11 @@ func (b weightedPeerByNodeID) Swap(i, j int) { b[i], b[j] = b[j], b[i] } func New(cluster Cluster, c *Config) *Dispatcher { return &Dispatcher{ addr: c.Addr, - nodes: newNodeStore(c.HeartbeatPeriod, c.HeartbeatEpsilon, c.GracePeriodMultiplier), + nodes: newNodeStore(c.HeartbeatPeriod, c.HeartbeatEpsilon, c.GracePeriodMultiplier, c.RateLimitPeriod), store: cluster.MemoryStore(), cluster: cluster, mgrQueue: watch.NewQueue(16), keyMgrQueue: watch.NewQueue(16), - lastSeenManagers: getWeightedPeers(cluster), taskUpdates: make(map[string]*api.TaskStatus), processTaskUpdatesTrigger: make(chan struct{}, 1), config: c, @@ -149,12 +153,12 @@ func (d *Dispatcher) Run(ctx context.Context) error { d.mu.Lock() if d.isRunning() { d.mu.Unlock() - return fmt.Errorf("dispatcher is stopped") + return fmt.Errorf("dispatcher is already running") } logger := log.G(ctx).WithField("module", "dispatcher") ctx = log.WithLogger(ctx, logger) if err := d.markNodesUnknown(ctx); err != nil { - logger.Errorf("failed to mark all nodes unknown: %v", err) + logger.Errorf(`failed to move all nodes to "unknown" state: %v`, err) } configWatcher, cancel, err := store.ViewAndWatch( d.store, @@ -177,6 +181,7 @@ func (d *Dispatcher) Run(ctx context.Context) error { state.EventUpdateCluster{}, ) if err != nil { + d.mu.Unlock() return err } defer cancel() @@ -238,6 +243,7 @@ func (d *Dispatcher) Run(ctx context.Context) error { func (d *Dispatcher) Stop() error { d.mu.Lock() if !d.isRunning() { + d.mu.Unlock() return fmt.Errorf("dispatcher is already stopped") } d.cancel() @@ -280,20 +286,20 @@ func (d *Dispatcher) markNodesUnknown(ctx context.Context) error { } node.Status = api.NodeStatus{ State: api.NodeStatus_UNKNOWN, - Message: "Node marked as unknown due to leadership change in cluster", + Message: `Node moved to "unknown" state due to leadership change in cluster`, } nodeID := node.ID expireFunc := func() { log := log.WithField("node", nodeID) - nodeStatus := api.NodeStatus{State: api.NodeStatus_DOWN, Message: "heartbeat failure for unknown node"} + nodeStatus := api.NodeStatus{State: api.NodeStatus_DOWN, Message: `heartbeat failure for node in "unknown" state`} log.Debugf("heartbeat expiration for unknown node") if err := d.nodeRemove(nodeID, nodeStatus); err != nil { - log.WithError(err).Errorf("failed deregistering node after heartbeat expiration for unknown node") + log.WithError(err).Errorf(`failed deregistering node after heartbeat expiration for node in "unknown" state`) } } if err := d.nodes.AddUnknown(node, expireFunc); err != nil { - return fmt.Errorf("add unknown node failed: %v", err) + return fmt.Errorf(`adding node in "unknown" state to node store failed: %v`, err) } if err := store.UpdateNode(tx, node); err != nil { return fmt.Errorf("update failed %v", err) @@ -301,7 +307,7 @@ func (d *Dispatcher) markNodesUnknown(ctx context.Context) error { return nil }) if err != nil { - log.WithField("node", n.ID).WithError(err).Errorf("failed to mark node as unknown") + log.WithField("node", n.ID).WithError(err).Errorf(`failed to move node to "unknown" state`) } } return nil @@ -328,6 +334,10 @@ func (d *Dispatcher) register(ctx context.Context, nodeID string, description *a return "", "", err } + if err := d.nodes.CheckRateLimit(nodeID); err != nil { + return "", "", err + } + // create or update node in store // TODO(stevvooe): Validate node specification. var node *api.Node diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/dispatcher/nodes.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/dispatcher/nodes.go index 5982db7b90..b69b4be2ad 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/dispatcher/nodes.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/dispatcher/nodes.go @@ -15,6 +15,7 @@ import ( type registeredNode struct { SessionID string Heartbeat *heartbeat.Heartbeat + Registered time.Time Node *api.Node Disconnect chan struct{} // signal to disconnect mu sync.Mutex @@ -41,15 +42,17 @@ func (rn *registeredNode) checkSessionID(sessionID string) error { type nodeStore struct { periodChooser *periodChooser gracePeriodMultiplier time.Duration + rateLimitPeriod time.Duration nodes map[string]*registeredNode mu sync.RWMutex } -func newNodeStore(hbPeriod, hbEpsilon time.Duration, graceMultiplier int) *nodeStore { +func newNodeStore(hbPeriod, hbEpsilon time.Duration, graceMultiplier int, rateLimitPeriod time.Duration) *nodeStore { return &nodeStore{ nodes: make(map[string]*registeredNode), periodChooser: newPeriodChooser(hbPeriod, hbEpsilon), gracePeriodMultiplier: time.Duration(graceMultiplier), + rateLimitPeriod: rateLimitPeriod, } } @@ -77,6 +80,19 @@ func (s *nodeStore) AddUnknown(n *api.Node, expireFunc func()) error { return nil } +// CheckRateLimit returs error if node with specified id is allowed to re-register +// again. +func (s *nodeStore) CheckRateLimit(id string) error { + s.mu.Lock() + defer s.mu.Unlock() + if existRn, ok := s.nodes[id]; ok { + if time.Since(existRn.Registered) < s.rateLimitPeriod { + return grpc.Errorf(codes.Unavailable, "node %s attempted registration too recently", id) + } + } + return nil +} + // Add adds new node and returns it, it replaces existing without notification. func (s *nodeStore) Add(n *api.Node, expireFunc func()) *registeredNode { s.mu.Lock() @@ -88,6 +104,7 @@ func (s *nodeStore) Add(n *api.Node, expireFunc func()) *registeredNode { rn := ®isteredNode{ SessionID: identity.NewID(), // session ID is local to the dispatcher. Node: n, + Registered: time.Now(), Disconnect: make(chan struct{}), } s.nodes[n.ID] = rn diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/health/health.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/health/health.go new file mode 100644 index 0000000000..bf220bdc18 --- /dev/null +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/health/health.go @@ -0,0 +1,58 @@ +// Package health provides some utility functions to health-check a server. The implementation +// is based on protobuf. Users need to write their own implementations if other IDLs are used. +// +// See original source: https://github.com/grpc/grpc-go/blob/master/health/health.go +// +// We use our own implementation of grpc server health check to include the authorization +// wrapper necessary for the Managers. +package health + +import ( + "sync" + + "github.com/docker/swarmkit/api" + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +) + +// Server represents a Health Check server to check +// if a service is running or not on some host. +type Server struct { + mu sync.Mutex + // statusMap stores the serving status of the services this HealthServer monitors. + statusMap map[string]api.HealthCheckResponse_ServingStatus +} + +// NewHealthServer creates a new health check server for grpc services. +func NewHealthServer() *Server { + return &Server{ + statusMap: make(map[string]api.HealthCheckResponse_ServingStatus), + } +} + +// Check checks if the grpc server is healthy and running. +func (s *Server) Check(ctx context.Context, in *api.HealthCheckRequest) (*api.HealthCheckResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + if in.Service == "" { + // check the server overall health status. + return &api.HealthCheckResponse{ + Status: api.HealthCheckResponse_SERVING, + }, nil + } + if status, ok := s.statusMap[in.Service]; ok { + return &api.HealthCheckResponse{ + Status: status, + }, nil + } + return nil, grpc.Errorf(codes.NotFound, "unknown service") +} + +// SetServingStatus is called when need to reset the serving status of a service +// or insert a new service entry into the statusMap. +func (s *Server) SetServingStatus(service string, status api.HealthCheckResponse_ServingStatus) { + s.mu.Lock() + s.statusMap[service] = status + s.mu.Unlock() +} diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/keymanager/keymanager.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/keymanager/keymanager.go index 80d027a684..b761035a08 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/keymanager/keymanager.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/keymanager/keymanager.go @@ -32,6 +32,8 @@ const ( // DefaultSubsystem is gossip DefaultSubsystem = SubsystemGossip + // number of keys to mainrain in the key ring. + keyringSize = 3 ) // map of subsystems and corresponding encryption algorithm. Initially only @@ -59,7 +61,6 @@ type KeyManager struct { config *Config store *store.MemoryStore keyRing *keyRing - ticker *time.Ticker ctx context.Context cancel context.CancelFunc @@ -72,7 +73,7 @@ func DefaultConfig() *Config { ClusterName: store.DefaultClusterName, Keylen: DefaultKeyLen, RotationInterval: DefaultKeyRotationInterval, - Subsystems: []string{DefaultSubsystem}, + Subsystems: []string{SubsystemGossip, SubsystemIPSec}, } } @@ -148,7 +149,7 @@ func (k *KeyManager) rotateKey(ctx context.Context) error { // We maintain the latest key and the one before in the key ring to allow // agents to communicate without disruption on key change. for subsys, keys := range subsysKeys { - if len(keys) > 1 { + if len(keys) == keyringSize { min := 0 for i, key := range keys[1:] { if key.LamportTime < keys[min].LamportTime { @@ -189,7 +190,9 @@ func (k *KeyManager) Run(ctx context.Context) error { cluster := clusters[0] if len(cluster.NetworkBootstrapKeys) == 0 { for _, subsys := range k.config.Subsystems { - k.keyRing.keys = append(k.keyRing.keys, k.allocateKey(ctx, subsys)) + for i := 0; i < keyringSize; i++ { + k.keyRing.keys = append(k.keyRing.keys, k.allocateKey(ctx, subsys)) + } } if err := k.updateKey(cluster); err != nil { log.Errorf("store update failed %v", err) diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/manager.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/manager.go index 08ce94dd00..179ed1d93f 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/manager.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/manager.go @@ -19,6 +19,7 @@ import ( "github.com/docker/swarmkit/manager/allocator" "github.com/docker/swarmkit/manager/controlapi" "github.com/docker/swarmkit/manager/dispatcher" + "github.com/docker/swarmkit/manager/health" "github.com/docker/swarmkit/manager/keymanager" "github.com/docker/swarmkit/manager/orchestrator" "github.com/docker/swarmkit/manager/raftpicker" @@ -39,6 +40,10 @@ const ( type Config struct { SecurityConfig *ca.SecurityConfig + // ExternalCAs is a list of initial CAs to which a manager node + // will make certificate signing requests for node certificates. + ExternalCAs []*api.ExternalCA + ProtoAddr map[string]string // ProtoListener will be used for grpc serving if it's not nil, // ProtoAddr fields will be used to create listeners otherwise. @@ -83,8 +88,7 @@ type Manager struct { localserver *grpc.Server RaftNode *raft.Node - mu sync.Mutex - once sync.Once + mu sync.Mutex stopped chan struct{} } @@ -202,13 +206,7 @@ func New(config *Config) (*Manager, error) { ForceNewCluster: config.ForceNewCluster, TLSCredentials: config.SecurityConfig.ClientTLSCreds, } - RaftNode, err := raft.NewNode(context.TODO(), newNodeOpts) - if err != nil { - for _, lis := range listeners { - lis.Close() - } - return nil, fmt.Errorf("can't create raft node: %v", err) - } + RaftNode := raft.NewNode(context.TODO(), newNodeOpts) opts := []grpc.ServerOption{ grpc.Creds(config.SecurityConfig.ServerTLSCreds)} @@ -275,6 +273,10 @@ func (m *Manager) Run(parent context.Context) error { raftCfg.HeartbeatTick = uint32(m.RaftNode.Config.HeartbeatTick) clusterID := m.config.SecurityConfig.ClientTLSCreds.Organization() + + initialCAConfig := ca.DefaultCAConfig() + initialCAConfig.ExternalCAs = m.config.ExternalCAs + s.Update(func(tx store.Tx) error { // Add a default cluster object to the // store. Don't check the error because @@ -294,7 +296,7 @@ func (m *Manager) Run(parent context.Context) error { HeartbeatPeriod: ptypes.DurationProto(dispatcher.DefaultHeartBeatPeriod), }, Raft: raftCfg, - CAConfig: ca.DefaultCAConfig(), + CAConfig: initialCAConfig, }, RootCA: api.RootCA{ CAKey: rootCA.Key, @@ -327,7 +329,7 @@ func (m *Manager) Run(parent context.Context) error { log.G(ctx).WithError(err).Error("root key-encrypting-key rotation failed") } - m.replicatedOrchestrator = orchestrator.New(s) + m.replicatedOrchestrator = orchestrator.NewReplicatedOrchestrator(s) m.globalOrchestrator = orchestrator.NewGlobalOrchestrator(s) m.taskReaper = orchestrator.NewTaskReaper(s) m.scheduler = scheduler.New(s) @@ -421,14 +423,6 @@ func (m *Manager) Run(parent context.Context) error { } }() - go func() { - err := m.RaftNode.Run(ctx) - if err != nil { - log.G(ctx).Error(err) - m.Stop(ctx) - } - }() - proxyOpts := []grpc.DialOption{ grpc.WithBackoffMaxDelay(2 * time.Second), grpc.WithTransportCredentials(m.config.SecurityConfig.ClientTLSCreds), @@ -443,12 +437,14 @@ func (m *Manager) Run(parent context.Context) error { } baseControlAPI := controlapi.NewServer(m.RaftNode.MemoryStore(), m.RaftNode) + healthServer := health.NewHealthServer() authenticatedControlAPI := api.NewAuthenticatedWrapperControlServer(baseControlAPI, authorize) authenticatedDispatcherAPI := api.NewAuthenticatedWrapperDispatcherServer(m.Dispatcher, authorize) authenticatedCAAPI := api.NewAuthenticatedWrapperCAServer(m.caserver, authorize) authenticatedNodeCAAPI := api.NewAuthenticatedWrapperNodeCAServer(m.caserver, authorize) authenticatedRaftAPI := api.NewAuthenticatedWrapperRaftServer(m.RaftNode, authorize) + authenticatedHealthAPI := api.NewAuthenticatedWrapperHealthServer(healthServer, authorize) authenticatedRaftMembershipAPI := api.NewAuthenticatedWrapperRaftMembershipServer(m.RaftNode, authorize) proxyDispatcherAPI := api.NewRaftProxyDispatcherServer(authenticatedDispatcherAPI, cs, m.RaftNode, ca.WithMetadataForwardTLSInfo) @@ -470,6 +466,7 @@ func (m *Manager) Run(parent context.Context) error { api.RegisterCAServer(m.server, proxyCAAPI) api.RegisterNodeCAServer(m.server, proxyNodeCAAPI) api.RegisterRaftServer(m.server, authenticatedRaftAPI) + api.RegisterHealthServer(m.server, authenticatedHealthAPI) api.RegisterRaftMembershipServer(m.server, proxyRaftMembershipAPI) api.RegisterControlServer(m.localserver, localProxyControlAPI) api.RegisterControlServer(m.server, authenticatedControlAPI) @@ -492,6 +489,24 @@ func (m *Manager) Run(parent context.Context) error { }(proto, l) } + // Set the raft server as serving for the health server + healthServer.SetServingStatus("Raft", api.HealthCheckResponse_SERVING) + + if err := m.RaftNode.JoinAndStart(); err != nil { + for _, lis := range m.listeners { + lis.Close() + } + return fmt.Errorf("can't initialize raft node: %v", err) + } + + go func() { + err := m.RaftNode.Run(ctx) + if err != nil { + log.G(ctx).Error(err) + m.Stop(ctx) + } + }() + if err := raft.WaitForLeader(ctx, m.RaftNode); err != nil { m.server.Stop() return err diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/global.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/global.go index be1a8f3b8a..bc6a0075e8 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/global.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/global.go @@ -255,7 +255,7 @@ func (g *GlobalOrchestrator) reconcileOneNode(ctx context.Context, node *api.Nod return } // typically there are only a few global services on a node - // iterate thru all of them one by one. If raft store visits become a concern, + // iterate through all of them one by one. If raft store visits become a concern, // it can be optimized. for _, service := range g.globalServices { g.reconcileServiceOneNode(ctx, service.ID, node.ID) diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/replicated.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/replicated.go index cd73860c43..485e4ef914 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/replicated.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/replicated.go @@ -29,8 +29,8 @@ type ReplicatedOrchestrator struct { restarts *RestartSupervisor } -// New creates a new ReplicatedOrchestrator. -func New(store *store.MemoryStore) *ReplicatedOrchestrator { +// NewReplicatedOrchestrator creates a new ReplicatedOrchestrator. +func NewReplicatedOrchestrator(store *store.MemoryStore) *ReplicatedOrchestrator { restartSupervisor := NewRestartSupervisor(store) updater := NewUpdateSupervisor(store, restartSupervisor) return &ReplicatedOrchestrator{ @@ -114,6 +114,9 @@ func newTask(service *api.Service, instance uint64) *api.Task { Timestamp: ptypes.MustTimestampProto(time.Now()), Message: "created", }, + Endpoint: &api.Endpoint{ + Spec: service.Spec.Endpoint.Copy(), + }, DesiredState: api.TaskStateRunning, } } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go index 07643ee322..6dd7e0d66f 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/services.go @@ -1,6 +1,8 @@ package orchestrator import ( + "sort" + "github.com/docker/go-events" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" @@ -68,6 +70,27 @@ func (r *ReplicatedOrchestrator) resolveService(ctx context.Context, task *api.T return service } +type taskWithIndex struct { + task *api.Task + + // index is a counter that counts this task as the nth instance of + // the service on its node. This is used for sorting the tasks so that + // when scaling down we leave tasks more evenly balanced. + index int +} + +type tasksByIndex []taskWithIndex + +func (ts tasksByIndex) Len() int { return len(ts) } +func (ts tasksByIndex) Swap(i, j int) { ts[i], ts[j] = ts[j], ts[i] } + +func (ts tasksByIndex) Less(i, j int) bool { + if ts[i].index < 0 { + return false + } + return ts[i].index < ts[j].index +} + func (r *ReplicatedOrchestrator) reconcile(ctx context.Context, service *api.Service) { var ( tasks []*api.Task @@ -97,8 +120,6 @@ func (r *ReplicatedOrchestrator) reconcile(ctx context.Context, service *api.Ser deploy := service.Spec.GetMode().(*api.ServiceSpec_Replicated) specifiedInstances := int(deploy.Replicated.Replicas) - // TODO(aaronl): Add support for restart delays. - switch { case specifiedInstances > numTasks: log.G(ctx).Debugf("Service %s was scaled up from %d to %d instances", service.ID, numTasks, specifiedInstances) @@ -115,9 +136,35 @@ func (r *ReplicatedOrchestrator) reconcile(ctx context.Context, service *api.Ser case specifiedInstances < numTasks: // Update up to N tasks then remove the extra log.G(ctx).Debugf("Service %s was scaled down from %d to %d instances", service.ID, numTasks, specifiedInstances) - r.updater.Update(ctx, service, runningTasks[:specifiedInstances]) + + // Preferentially remove tasks on the nodes that have the most + // copies of this service, to leave a more balanced result. + // Assign each task an index that counts it as the nth copy of + // of the service on its node (1, 2, 3, ...), and sort the + // tasks by this counter value. + + instancesByNode := make(map[string]int) + tasksWithIndices := make(tasksByIndex, 0, numTasks) + + for _, t := range runningTasks { + if t.NodeID != "" { + instancesByNode[t.NodeID]++ + tasksWithIndices = append(tasksWithIndices, taskWithIndex{task: t, index: instancesByNode[t.NodeID]}) + } else { + tasksWithIndices = append(tasksWithIndices, taskWithIndex{task: t, index: -1}) + } + } + + sort.Sort(tasksWithIndices) + + sortedTasks := make([]*api.Task, 0, numTasks) + for _, t := range tasksWithIndices { + sortedTasks = append(sortedTasks, t.task) + } + + r.updater.Update(ctx, service, sortedTasks[:specifiedInstances]) _, err = r.store.Batch(func(batch *store.Batch) error { - r.removeTasks(ctx, batch, service, runningTasks[specifiedInstances:]) + r.removeTasks(ctx, batch, service, sortedTasks[specifiedInstances:]) return nil }) if err != nil { diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/updater.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/updater.go index 7a895c4568..ba1047c954 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/updater.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/orchestrator/updater.go @@ -104,7 +104,8 @@ func (u *Updater) Run(ctx context.Context, service *api.Service, tasks []*api.Ta dirtyTasks := []*api.Task{} for _, t := range tasks { if !reflect.DeepEqual(service.Spec.Task, t.Spec) || - !reflect.DeepEqual(service.Endpoint, t.Endpoint) { + (t.Endpoint != nil && + !reflect.DeepEqual(service.Spec.Endpoint, t.Endpoint.Spec)) { dirtyTasks = append(dirtyTasks, t) } } @@ -191,6 +192,9 @@ func (u *Updater) updateTask(ctx context.Context, service *api.Service, original if t == nil { return fmt.Errorf("task %s not found while trying to update it", original.ID) } + if t.DesiredState > api.TaskStateRunning { + return fmt.Errorf("task %s was already shut down when reached by updater", original.ID) + } t.DesiredState = api.TaskStateShutdown if err := store.UpdateTask(tx, t); err != nil { return err diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/indexed_node_heap.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/indexed_node_heap.go index 90c44f85be..8732595e38 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/indexed_node_heap.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/scheduler/indexed_node_heap.go @@ -48,13 +48,6 @@ func (nh *nodeHeap) alloc(n int) { nh.index = make(map[string]int, n) } -func (nh *nodeHeap) peek() *NodeInfo { - if len(nh.heap) == 0 { - return nil - } - return &nh.heap[0] -} - // nodeInfo returns the NodeInfo struct for a given node identified by its ID. func (nh *nodeHeap) nodeInfo(nodeID string) NodeInfo { index, ok := nh.index[nodeID] @@ -95,9 +88,7 @@ func (nh *nodeHeap) updateNode(n NodeInfo) { func (nh *nodeHeap) remove(nodeID string) { index, ok := nh.index[nodeID] if ok { - nh.heap[index].Tasks = nil - heap.Fix(nh, index) - heap.Pop(nh) + heap.Remove(nh, index) } } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go index 5ffb943f09..89c91b8a7e 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/membership/cluster.go @@ -27,8 +27,6 @@ var ( // Cluster represents a set of active // raft Members type Cluster struct { - id uint64 - mu sync.RWMutex members map[uint64]*Member @@ -103,17 +101,15 @@ func (c *Cluster) RemoveMember(id uint64) error { c.mu.Lock() defer c.mu.Unlock() - if c.members[id] == nil { - return ErrIDNotFound - } - - conn := c.members[id].Conn - if conn != nil { - _ = conn.Close() + if c.members[id] != nil { + conn := c.members[id].Conn + if conn != nil { + _ = conn.Close() + } + delete(c.members, id) } c.removed[id] = true - delete(c.members, id) return nil } diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go index d474701520..a7875d1ec6 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go @@ -31,6 +31,11 @@ import ( ) var ( + // ErrHealthCheckFailure is returned when there is an issue with the initial handshake which means + // that the address provided must be invalid or there is ongoing connectivity issues at join time. + ErrHealthCheckFailure = errors.New("raft: could not connect to prospective new cluster member using its advertised address") + // ErrNoRaftMember is thrown when the node is not yet part of a raft cluster + ErrNoRaftMember = errors.New("raft: node is not yet part of a raft cluster") // ErrConfChangeRefused is returned when there is an issue with the configuration change ErrConfChangeRefused = errors.New("raft: propose configuration change refused") // ErrApplyNotSpecified is returned during the creation of a raft node when no apply method was provided @@ -83,12 +88,13 @@ type Node struct { raftStore *raft.MemoryStorage memoryStore *store.MemoryStore Config *raft.Config + opts NewNodeOptions reqIDGen *idutil.Generator wait *wait wal *wal.WAL snapshotter *snap.Snapshotter wasLeader bool - removed uint32 + isMember uint32 joinAddr string // waitProp waits for all the proposals to be terminated before @@ -103,14 +109,15 @@ type Node struct { appliedIndex uint64 snapshotIndex uint64 - ticker clock.Ticker - sendTimeout time.Duration - stopCh chan struct{} - doneCh chan struct{} + ticker clock.Ticker + sendTimeout time.Duration + stopCh chan struct{} + doneCh chan struct{} + // removeRaftCh notifies about node deletion from raft cluster + removeRaftCh chan struct{} + removeRaftOnce sync.Once leadershipBroadcast *events.Broadcaster - startNodePeers []raft.Peer - // used to coordinate shutdown stopMu sync.RWMutex // used for membership management checks @@ -153,7 +160,7 @@ func init() { } // NewNode generates a new Raft node -func NewNode(ctx context.Context, opts NewNodeOptions) (*Node, error) { +func NewNode(ctx context.Context, opts NewNodeOptions) *Node { cfg := opts.Config if cfg == nil { cfg = DefaultNodeConfig() @@ -173,6 +180,7 @@ func NewNode(ctx context.Context, opts NewNodeOptions) (*Node, error) { tlsCredentials: opts.TLSCredentials, raftStore: raftStore, Address: opts.Addr, + opts: opts, Config: &raft.Config{ ElectionTick: cfg.ElectionTick, HeartbeatTick: cfg.HeartbeatTick, @@ -184,6 +192,7 @@ func NewNode(ctx context.Context, opts NewNodeOptions) (*Node, error) { forceNewCluster: opts.ForceNewCluster, stopCh: make(chan struct{}), doneCh: make(chan struct{}), + removeRaftCh: make(chan struct{}), StateDir: opts.StateDir, joinAddr: opts.JoinAddr, sendTimeout: 2 * time.Second, @@ -200,13 +209,21 @@ func NewNode(ctx context.Context, opts NewNodeOptions) (*Node, error) { n.sendTimeout = opts.SendTimeout } - loadAndStartErr := n.loadAndStart(ctx, opts.ForceNewCluster) + n.reqIDGen = idutil.NewGenerator(uint16(n.Config.ID), time.Now()) + n.wait = newWait() + + return n +} + +// JoinAndStart joins and starts the raft server +func (n *Node) JoinAndStart() error { + loadAndStartErr := n.loadAndStart(n.Ctx, n.opts.ForceNewCluster) if loadAndStartErr != nil && loadAndStartErr != errNoWAL { n.ticker.Stop() - return nil, loadAndStartErr + return loadAndStartErr } - snapshot, err := raftStore.Snapshot() + snapshot, err := n.raftStore.Snapshot() // Snapshot never returns an error if err != nil { panic("could not get snapshot of raft store") @@ -215,14 +232,12 @@ func NewNode(ctx context.Context, opts NewNodeOptions) (*Node, error) { n.confState = snapshot.Metadata.ConfState n.appliedIndex = snapshot.Metadata.Index n.snapshotIndex = snapshot.Metadata.Index - n.reqIDGen = idutil.NewGenerator(uint16(n.Config.ID), time.Now()) - n.wait = newWait() if loadAndStartErr == errNoWAL { if n.joinAddr != "" { c, err := n.ConnectToMember(n.joinAddr, 10*time.Second) if err != nil { - return nil, err + return err } client := api.NewRaftMembershipClient(c.Conn) defer func() { @@ -235,40 +250,42 @@ func NewNode(ctx context.Context, opts NewNodeOptions) (*Node, error) { Addr: n.Address, }) if err != nil { - return nil, err + return err } n.Config.ID = resp.RaftID - if _, err := n.createWAL(opts.ID); err != nil { - return nil, err + if _, err := n.createWAL(n.opts.ID); err != nil { + return err } n.Node = raft.StartNode(n.Config, []raft.Peer{}) if err := n.registerNodes(resp.Members); err != nil { - return nil, err + return err } } else { // First member in the cluster, self-assign ID n.Config.ID = uint64(rand.Int63()) + 1 - peer, err := n.createWAL(opts.ID) + peer, err := n.createWAL(n.opts.ID) if err != nil { - return nil, err + return err } n.Node = raft.StartNode(n.Config, []raft.Peer{peer}) if err := n.Campaign(n.Ctx); err != nil { - return nil, err + return err } } - return n, nil + atomic.StoreUint32(&n.isMember, 1) + return nil } if n.joinAddr != "" { n.Config.Logger.Warning("ignoring request to join cluster, because raft state already exists") } n.Node = raft.RestartNode(n.Config) - return n, nil + atomic.StoreUint32(&n.isMember, 1) + return nil } // DefaultNodeConfig returns the default config for a @@ -377,21 +394,6 @@ func (n *Node) Run(ctx context.Context) error { } } - // If the node was removed from other members, - // send back an error to the caller to start - // the shutdown process. - if n.mustStop() { - n.stop() - - // Move WAL and snapshot out of the way, since - // they are no longer usable. - if err := n.moveWALAndSnap(); err != nil { - n.Config.Logger.Error(err) - } - - return ErrMemberRemoved - } - // Advance the state machine n.Advance() @@ -400,6 +402,19 @@ func (n *Node) Run(ctx context.Context) error { n.snapshotIndex = snapshotIndex } n.snapshotInProgress = nil + case <-n.removeRaftCh: + // If the node was removed from other members, + // send back an error to the caller to start + // the shutdown process. + n.stop() + + // Move WAL and snapshot out of the way, since + // they are no longer usable. + if err := n.moveWALAndSnap(); err != nil { + n.Config.Logger.Error(err) + } + + return ErrMemberRemoved case <-n.stopCh: n.stop() return nil @@ -434,6 +449,7 @@ func (n *Node) stop() { } } n.Stop() + n.ticker.Stop() if err := n.wal.Close(); err != nil { n.Config.Logger.Errorf("raft: error closing WAL: %v", err) } @@ -442,6 +458,10 @@ func (n *Node) stop() { // IsLeader checks if we are the leader or not func (n *Node) IsLeader() bool { + if !n.IsMember() { + return false + } + if n.Node.Status().Lead == n.Config.ID { return true } @@ -450,6 +470,9 @@ func (n *Node) IsLeader() bool { // Leader returns the id of the leader func (n *Node) Leader() uint64 { + if !n.IsMember() { + return 0 + } return n.Node.Status().Lead } @@ -479,7 +502,11 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons n.membershipLock.Lock() defer n.membershipLock.Unlock() - if n.Node == nil { + if !n.IsMember() { + return nil, ErrNoRaftMember + } + + if n.IsStopped() { log.WithError(ErrStopped).Errorf(ErrStopped.Error()) return nil, ErrStopped } @@ -497,6 +524,12 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons } } + // We do not bother submitting a configuration change for the + // new member if we can't contact it back using its address + if err := n.checkHealth(ctx, req.Addr, 5*time.Second); err != nil { + return nil, err + } + err = n.addMember(ctx, req.Addr, raftID, nodeInfo.NodeID) if err != nil { log.WithError(err).Errorf("failed to add member") @@ -516,6 +549,28 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons return &api.JoinResponse{Members: nodes, RaftID: raftID}, nil } +// checkHealth tries to contact an aspiring member through its advertised address +// and checks if its raft server is running. +func (n *Node) checkHealth(ctx context.Context, addr string, timeout time.Duration) error { + conn, err := dial(addr, "tcp", n.tlsCredentials, timeout) + if err != nil { + return err + } + + client := api.NewHealthClient(conn) + defer conn.Close() + + resp, err := client.Check(ctx, &api.HealthCheckRequest{Service: "Raft"}) + if err != nil { + return ErrHealthCheckFailure + } + if resp != nil && resp.Status != api.HealthCheckResponse_SERVING { + return ErrHealthCheckFailure + } + + return nil +} + // addMember submits a configuration change to add a new member on the raft cluster. func (n *Node) addMember(ctx context.Context, addr string, raftID uint64, nodeID string) error { node := api.RaftMember{ @@ -563,7 +618,11 @@ func (n *Node) Leave(ctx context.Context, req *api.LeaveRequest) (*api.LeaveResp n.stopMu.RLock() defer n.stopMu.RUnlock() - if n.Node == nil { + if !n.IsMember() { + return nil, ErrNoRaftMember + } + + if n.IsStopped() { return nil, ErrStopped } @@ -612,7 +671,12 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa // can't stop the raft node while an async RPC is in progress n.stopMu.RLock() defer n.stopMu.RUnlock() - if n.Node == nil { + + if !n.IsMember() { + return nil, ErrNoRaftMember + } + + if n.IsStopped() { return nil, ErrStopped } @@ -625,6 +689,10 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa // ResolveAddress returns the address reaching for a given node ID. func (n *Node) ResolveAddress(ctx context.Context, msg *api.ResolveAddressRequest) (*api.ResolveAddressResponse, error) { + if !n.IsMember() { + return nil, ErrNoRaftMember + } + nodeInfo, err := ca.RemoteNode(ctx) if err != nil { return nil, err @@ -656,7 +724,7 @@ func (n *Node) LeaderAddr() (string, error) { if err := WaitForLeader(ctx, n); err != nil { return "", ErrNoClusterLeader } - if n.Node == nil { + if n.IsStopped() { return "", ErrStopped } ms := n.cluster.Members() @@ -671,7 +739,7 @@ func (n *Node) LeaderAddr() (string, error) { func (n *Node) registerNode(node *api.RaftMember) error { member := &membership.Member{} - if n.cluster.GetMember(node.RaftID) != nil { + if n.cluster.GetMember(node.RaftID) != nil || n.cluster.IsIDRemoved(node.RaftID) { // member already exists return nil } @@ -760,11 +828,18 @@ func (n *Node) GetMemberlist() map[uint64]*api.RaftMember { return memberlist } -// mustStop checks if the raft node must be stopped -// because it was removed from the cluster from -// other members -func (n *Node) mustStop() bool { - return atomic.LoadUint32(&n.removed) == 1 +// IsMember checks if the raft node has effectively joined +// a cluster of existing members. +func (n *Node) IsMember() bool { + return atomic.LoadUint32(&n.isMember) == 1 +} + +// IsStopped checks if the raft node is stopped or not +func (n *Node) IsStopped() bool { + if n.Node == nil { + return true + } + return false } // canSubmitProposal defines if any more proposals @@ -882,12 +957,14 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess _, err := conn.ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m}) if err != nil { if grpc.ErrorDesc(err) == ErrMemberRemoved.Error() { - atomic.StoreUint32(&n.removed, 1) + n.removeRaftOnce.Do(func() { + close(n.removeRaftCh) + }) } if m.Type == raftpb.MsgSnap { n.ReportSnapshot(m.To, raft.SnapshotFailure) } - if n.Node == nil { + if n.IsStopped() { panic("node is nil") } n.ReportUnreachable(m.To) diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go index c74c1e4edd..c0d4b454aa 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/raft/storage.go @@ -162,6 +162,22 @@ func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewC } n.Config.ID = raftNode.RaftID + // All members that are no longer part of the cluster must be added to + // the removed list right away, so that we don't try to connect to them + // before processing the configuration change entries, which could make + // us get stuck. + for _, ent := range ents { + if ent.Index <= st.Commit && ent.Type == raftpb.EntryConfChange { + var cc raftpb.ConfChange + if err := cc.Unmarshal(ent.Data); err != nil { + return fmt.Errorf("error unmarshalling config change: %v", err) + } + if cc.Type == raftpb.ConfChangeRemoveNode { + n.cluster.RemoveMember(cc.NodeID) + } + } + } + if forceNewCluster { // discard the previously uncommitted entries for i, ent := range ents { @@ -174,6 +190,23 @@ func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewC // force append the configuration change entries toAppEnts := createConfigChangeEnts(getIDs(snapshot, ents), uint64(n.Config.ID), st.Term, st.Commit) + + // All members that are being removed as part of the + // force-new-cluster process must be added to the + // removed list right away, so that we don't try to + // connect to them before processing the configuration + // change entries, which could make us get stuck. + for _, ccEnt := range toAppEnts { + if ccEnt.Type == raftpb.EntryConfChange { + var cc raftpb.ConfChange + if err := cc.Unmarshal(ccEnt.Data); err != nil { + return fmt.Errorf("error unmarshalling force-new-cluster config change: %v", err) + } + if cc.Type == raftpb.ConfChangeRemoveNode { + n.cluster.RemoveMember(cc.NodeID) + } + } + } ents = append(ents, toAppEnts...) // force commit newly appended entries @@ -347,9 +380,10 @@ func (n *Node) restoreFromSnapshot(data []byte, forceNewCluster bool) error { return err } } - for _, removedMember := range snapshot.Membership.Removed { - n.cluster.RemoveMember(removedMember) - } + } + + for _, removedMember := range snapshot.Membership.Removed { + n.cluster.RemoveMember(removedMember) } return nil diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/store/memory.go b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/store/memory.go index ec3b4e6fe6..35d47682a2 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/store/memory.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/manager/state/store/memory.go @@ -23,7 +23,6 @@ const ( indexID = "id" indexName = "name" indexServiceID = "serviceid" - indexServiceMode = "servicemode" indexNodeID = "nodeid" indexSlot = "slot" indexCN = "cn" diff --git a/components/engine/vendor/src/github.com/docker/swarmkit/picker/picker.go b/components/engine/vendor/src/github.com/docker/swarmkit/picker/picker.go index 21d143a70b..34923e0ef0 100644 --- a/components/engine/vendor/src/github.com/docker/swarmkit/picker/picker.go +++ b/components/engine/vendor/src/github.com/docker/swarmkit/picker/picker.go @@ -272,7 +272,7 @@ func (p *Picker) PickAddr() (string, error) { p.mu.Lock() p.peer = peer p.mu.Unlock() - return p.peer.Addr, err + return peer.Addr, err } // State returns the connectivity state of the underlying connections.