From c2f1681e6f1116dc7082f8e618464dabfbacc2b7 Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Mon, 13 Mar 2017 14:45:06 -0700 Subject: [PATCH] Vendoring swarmkit @9fdea50 Signed-off-by: Alessandro Boch Upstream-commit: 434e9695e42f9399c627d433972dbd73f7c8502f Component: engine --- components/engine/vendor.conf | 2 +- .../docker/swarmkit/api/specs.pb.go | 250 ++++++++------- .../docker/swarmkit/api/specs.proto | 6 + .../docker/swarmkit/ca/certificates.go | 65 +++- .../github.com/docker/swarmkit/ca/config.go | 2 +- .../github.com/docker/swarmkit/ca/external.go | 2 +- .../swarmkit/manager/allocator/network.go | 286 +++++++++++------- .../swarmkit/manager/controlapi/network.go | 109 ++++--- .../swarmkit/manager/controlapi/service.go | 45 ++- .../docker/swarmkit/manager/dirty.go | 2 +- .../docker/swarmkit/manager/manager.go | 39 ++- .../swarmkit/manager/scheduler/nodeset.go | 4 + .../swarmkit/manager/scheduler/scheduler.go | 26 +- 13 files changed, 552 insertions(+), 286 deletions(-) diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index e7d98d2285..bf9769c8b8 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -105,7 +105,7 @@ github.com/docker/containerd 422e31ce907fd9c3833a38d7b8fdd023e5a76e73 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4 # cluster -github.com/docker/swarmkit 0e2d9ebcea9d5bbd4a06b3b964fb96356801f880 +github.com/docker/swarmkit 9fdea50c14492b6e1f472813849794d36bfef217 github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9 github.com/gogo/protobuf 8d70fb3182befc465c4a1eac8ad4d38ff49778e2 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/specs.pb.go b/components/engine/vendor/github.com/docker/swarmkit/api/specs.pb.go index b20e67607a..851227b858 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/specs.pb.go +++ b/components/engine/vendor/github.com/docker/swarmkit/api/specs.pb.go @@ -591,6 +591,11 @@ type NetworkSpec struct { // enabled(default case) no manual attachment to this network // can happen. Attachable bool `protobuf:"varint,6,opt,name=attachable,proto3" json:"attachable,omitempty"` + // Ingress indicates this network will provide the routing-mesh. + // In older versions, the network providing the routing mesh was + // swarm internally created only and it was identified by the name + // "ingress" and the label "com.docker.swarm.internal": "true". + Ingress bool `protobuf:"varint,7,opt,name=ingress,proto3" json:"ingress,omitempty"` } func (m *NetworkSpec) Reset() { *m = NetworkSpec{} } @@ -1795,6 +1800,16 @@ func (m *NetworkSpec) MarshalTo(dAtA []byte) (int, error) { } i++ } + if m.Ingress { + dAtA[i] = 0x38 + i++ + if m.Ingress { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -2255,6 +2270,9 @@ func (m *NetworkSpec) Size() (n int) { if m.Attachable { n += 2 } + if m.Ingress { + n += 2 + } return n } @@ -2502,6 +2520,7 @@ func (this *NetworkSpec) String() string { `Internal:` + fmt.Sprintf("%v", this.Internal) + `,`, `IPAM:` + strings.Replace(fmt.Sprintf("%v", this.IPAM), "IPAMOptions", "IPAMOptions", 1) + `,`, `Attachable:` + fmt.Sprintf("%v", this.Attachable) + `,`, + `Ingress:` + fmt.Sprintf("%v", this.Ingress) + `,`, `}`, }, "") return s @@ -4688,6 +4707,26 @@ func (m *NetworkSpec) Unmarshal(dAtA []byte) error { } } m.Attachable = bool(v != 0) + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Ingress", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpecs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Ingress = bool(v != 0) default: iNdEx = preIndex skippy, err := skipSpecs(dAtA[iNdEx:]) @@ -5218,112 +5257,113 @@ var ( func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) } var fileDescriptorSpecs = []byte{ - // 1707 bytes of a gzipped FileDescriptorProto + // 1717 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x41, 0x73, 0x1b, 0xb7, - 0x15, 0x26, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x1a, 0x75, 0xd2, 0x35, 0xdd, 0x90, 0x34, 0xe3, + 0x15, 0x16, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x1a, 0x75, 0xd2, 0x35, 0xdd, 0x50, 0x34, 0xe3, 0xa6, 0x4a, 0x33, 0xa5, 0xa6, 0x6a, 0x27, 0x75, 0xea, 0x66, 0x5a, 0x52, 0x64, 0x65, 0x55, 0x95, 0xcc, 0x01, 0x15, 0x77, 0x7c, 0xe2, 0x80, 0xbb, 0x10, 0xb9, 0xa3, 0xe5, 0x62, 0x0b, 0x60, 0x99, - 0xe1, 0xad, 0xc7, 0x8c, 0x0f, 0x3d, 0xf5, 0xaa, 0xe9, 0xa1, 0xbf, 0xa1, 0xff, 0xc1, 0xc7, 0x1e, - 0x7b, 0xd2, 0x34, 0xfc, 0x0b, 0xfd, 0x01, 0xed, 0x00, 0x0b, 0x92, 0xcb, 0x64, 0x15, 0x7b, 0x26, - 0xbe, 0xe1, 0xbd, 0xfd, 0xbe, 0x07, 0xe0, 0xe1, 0xc3, 0xc3, 0x5b, 0xb0, 0x45, 0x44, 0x5d, 0xd1, - 0x8a, 0x38, 0x93, 0x0c, 0x21, 0x8f, 0xb9, 0x57, 0x94, 0xb7, 0xc4, 0x97, 0x84, 0x4f, 0xaf, 0x7c, - 0xd9, 0x9a, 0xfd, 0xbc, 0x6a, 0xcb, 0x79, 0x44, 0x0d, 0xa0, 0x7a, 0x7f, 0xcc, 0xc6, 0x4c, 0x0f, - 0x0f, 0xd4, 0xc8, 0x78, 0x6b, 0x63, 0xc6, 0xc6, 0x01, 0x3d, 0xd0, 0xd6, 0x28, 0xbe, 0x3c, 0xf0, - 0x62, 0x4e, 0xa4, 0xcf, 0xc2, 0xe4, 0x7b, 0xf3, 0xba, 0x00, 0xd6, 0x39, 0xf3, 0xe8, 0x20, 0xa2, - 0x2e, 0x3a, 0x06, 0x9b, 0x84, 0x21, 0x93, 0x1a, 0x20, 0x9c, 0x7c, 0x23, 0xbf, 0x6f, 0x1f, 0xd6, - 0x5b, 0xdf, 0x9e, 0xb9, 0xd5, 0x5e, 0xc3, 0x3a, 0x85, 0xd7, 0x37, 0xf5, 0x1c, 0x4e, 0x33, 0xd1, - 0x6f, 0xa1, 0xec, 0x51, 0xe1, 0x73, 0xea, 0x0d, 0x39, 0x0b, 0xa8, 0xb3, 0xd5, 0xc8, 0xef, 0xdf, - 0x39, 0xfc, 0x51, 0x56, 0x24, 0x35, 0x39, 0x66, 0x01, 0xc5, 0xb6, 0x61, 0x28, 0x03, 0x1d, 0x03, - 0x4c, 0xe9, 0x74, 0x44, 0xb9, 0x98, 0xf8, 0x91, 0xb3, 0xad, 0xe9, 0x3f, 0xb9, 0x8d, 0xae, 0xd6, - 0xde, 0x3a, 0x5b, 0xc1, 0x71, 0x8a, 0x8a, 0xce, 0xa0, 0x4c, 0x66, 0xc4, 0x0f, 0xc8, 0xc8, 0x0f, - 0x7c, 0x39, 0x77, 0x0a, 0x3a, 0xd4, 0xc7, 0xdf, 0x19, 0xaa, 0x9d, 0x22, 0xe0, 0x0d, 0x7a, 0xd3, - 0x03, 0x58, 0x4f, 0x84, 0x3e, 0x82, 0xdd, 0x7e, 0xef, 0xbc, 0x7b, 0x72, 0x7e, 0x5c, 0xc9, 0x55, - 0x1f, 0xbc, 0xba, 0x6e, 0xbc, 0xa7, 0x62, 0xac, 0x01, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0xfb, - 0x60, 0xb5, 0x8f, 0x8e, 0x7a, 0xfd, 0x8b, 0x5e, 0xb7, 0x92, 0xaf, 0x56, 0x5f, 0x5d, 0x37, 0xde, - 0xdf, 0x04, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xd5, 0xc2, 0x57, 0xff, 0xa8, 0xe5, 0x9a, 0x5f, - 0xe5, 0xa1, 0x9c, 0x5e, 0x04, 0xfa, 0x08, 0x8a, 0xed, 0xa3, 0x8b, 0x93, 0x17, 0xbd, 0x4a, 0x6e, - 0x4d, 0x4f, 0x23, 0xda, 0xae, 0xf4, 0x67, 0x14, 0x3d, 0x86, 0x9d, 0x7e, 0xfb, 0x8b, 0x41, 0xaf, - 0x92, 0x5f, 0x2f, 0x27, 0x0d, 0xeb, 0x93, 0x58, 0x68, 0x54, 0x17, 0xb7, 0x4f, 0xce, 0x2b, 0x5b, - 0xd9, 0xa8, 0x2e, 0x27, 0x7e, 0x68, 0x96, 0xf2, 0xf7, 0x02, 0xd8, 0x03, 0xca, 0x67, 0xbe, 0xfb, - 0x8e, 0x25, 0xf2, 0x29, 0x14, 0x24, 0x11, 0x57, 0x5a, 0x1a, 0x76, 0xb6, 0x34, 0x2e, 0x88, 0xb8, - 0x52, 0x93, 0x1a, 0xba, 0xc6, 0x2b, 0x65, 0x70, 0x1a, 0x05, 0xbe, 0x4b, 0x24, 0xf5, 0xb4, 0x32, - 0xec, 0xc3, 0x1f, 0x67, 0xb1, 0xf1, 0x0a, 0x65, 0xd6, 0xff, 0x2c, 0x87, 0x53, 0x54, 0xf4, 0x14, - 0x8a, 0xe3, 0x80, 0x8d, 0x48, 0xa0, 0x35, 0x61, 0x1f, 0x3e, 0xca, 0x0a, 0x72, 0xac, 0x11, 0xeb, - 0x00, 0x86, 0x82, 0x9e, 0x40, 0x31, 0x8e, 0x3c, 0x22, 0xa9, 0x53, 0xd4, 0xe4, 0x46, 0x16, 0xf9, - 0x0b, 0x8d, 0x38, 0x62, 0xe1, 0xa5, 0x3f, 0xc6, 0x06, 0x8f, 0x4e, 0xc1, 0x0a, 0xa9, 0xfc, 0x92, - 0xf1, 0x2b, 0xe1, 0xec, 0x36, 0xb6, 0xf7, 0xed, 0xc3, 0x4f, 0x32, 0xc5, 0x98, 0x60, 0xda, 0x52, - 0x12, 0x77, 0x32, 0xa5, 0xa1, 0x4c, 0xc2, 0x74, 0xb6, 0x9c, 0x3c, 0x5e, 0x05, 0x40, 0xbf, 0x01, - 0x8b, 0x86, 0x5e, 0xc4, 0xfc, 0x50, 0x3a, 0xd6, 0xed, 0x0b, 0xe9, 0x19, 0x8c, 0x4a, 0x26, 0x5e, - 0x31, 0x14, 0x9b, 0xb3, 0x20, 0x18, 0x11, 0xf7, 0xca, 0x29, 0xbd, 0xe5, 0x36, 0x56, 0x8c, 0x4e, - 0x11, 0x0a, 0x53, 0xe6, 0xd1, 0xe6, 0x01, 0xdc, 0xfb, 0x56, 0xaa, 0x51, 0x15, 0x2c, 0x93, 0xea, - 0x44, 0x23, 0x05, 0xbc, 0xb2, 0x9b, 0x77, 0x61, 0x6f, 0x23, 0xad, 0xcd, 0xbf, 0x16, 0xc0, 0x5a, - 0x9e, 0x35, 0x6a, 0x43, 0xc9, 0x65, 0xa1, 0x24, 0x7e, 0x48, 0xb9, 0x91, 0x57, 0xe6, 0xc9, 0x1c, - 0x2d, 0x41, 0x8a, 0xf5, 0x2c, 0x87, 0xd7, 0x2c, 0xf4, 0x7b, 0x28, 0x71, 0x2a, 0x58, 0xcc, 0x5d, - 0x2a, 0x8c, 0xbe, 0xf6, 0xb3, 0x15, 0x92, 0x80, 0x30, 0xfd, 0x73, 0xec, 0x73, 0xaa, 0xb2, 0x2c, - 0xf0, 0x9a, 0x8a, 0x9e, 0xc2, 0x2e, 0xa7, 0x42, 0x12, 0x2e, 0xbf, 0x4b, 0x22, 0x38, 0x81, 0xf4, - 0x59, 0xe0, 0xbb, 0x73, 0xbc, 0x64, 0xa0, 0xa7, 0x50, 0x8a, 0x02, 0xe2, 0xea, 0xa8, 0xce, 0x8e, - 0xa6, 0x7f, 0x90, 0x45, 0xef, 0x2f, 0x41, 0x78, 0x8d, 0x47, 0x9f, 0x01, 0x04, 0x6c, 0x3c, 0xf4, - 0xb8, 0x3f, 0xa3, 0xdc, 0x48, 0xac, 0x9a, 0xc5, 0xee, 0x6a, 0x04, 0x2e, 0x05, 0x6c, 0x9c, 0x0c, - 0xd1, 0xf1, 0xf7, 0xd2, 0x57, 0x4a, 0x5b, 0xa7, 0x00, 0x64, 0xf5, 0xd5, 0xa8, 0xeb, 0xe3, 0xb7, - 0x0a, 0x65, 0x4e, 0x24, 0x45, 0x47, 0x8f, 0xa0, 0x7c, 0xc9, 0xb8, 0x4b, 0x87, 0xe6, 0xd6, 0x94, - 0xb4, 0x26, 0x6c, 0xed, 0x4b, 0xf4, 0xd5, 0x29, 0xc1, 0x2e, 0x8f, 0x43, 0xe9, 0x4f, 0x69, 0xf3, - 0x14, 0xde, 0xcb, 0x0c, 0x8a, 0x0e, 0xa1, 0xbc, 0x3a, 0xe6, 0xa1, 0xef, 0x69, 0x7d, 0x94, 0x3a, - 0x77, 0x17, 0x37, 0x75, 0x7b, 0xa5, 0x87, 0x93, 0x2e, 0xb6, 0x57, 0xa0, 0x13, 0xaf, 0xf9, 0x37, - 0x0b, 0xf6, 0x36, 0xc4, 0x82, 0xee, 0xc3, 0x8e, 0x3f, 0x25, 0x63, 0x9a, 0xd0, 0x71, 0x62, 0xa0, - 0x1e, 0x14, 0x03, 0x32, 0xa2, 0x81, 0x92, 0x8c, 0x4a, 0xdb, 0xcf, 0xde, 0xa8, 0xba, 0xd6, 0x1f, - 0x35, 0xbe, 0x17, 0x4a, 0x3e, 0xc7, 0x86, 0x8c, 0x1c, 0xd8, 0x75, 0xd9, 0x74, 0x4a, 0x42, 0x55, - 0x9c, 0xb6, 0xf7, 0x4b, 0x78, 0x69, 0x22, 0x04, 0x05, 0xc2, 0xc7, 0xc2, 0x29, 0x68, 0xb7, 0x1e, - 0xa3, 0x0a, 0x6c, 0xd3, 0x70, 0xe6, 0xec, 0x68, 0x97, 0x1a, 0x2a, 0x8f, 0xe7, 0x27, 0x67, 0x5e, - 0xc2, 0x6a, 0xa8, 0x78, 0xb1, 0xa0, 0xdc, 0xd9, 0xd5, 0x2e, 0x3d, 0x46, 0xbf, 0x82, 0xe2, 0x94, - 0xc5, 0xa1, 0x14, 0x8e, 0xa5, 0x17, 0xfb, 0x20, 0x6b, 0xb1, 0x67, 0x0a, 0x61, 0x8a, 0xa7, 0x81, - 0xa3, 0x1e, 0xdc, 0x13, 0x92, 0x45, 0xc3, 0x31, 0x27, 0x2e, 0x1d, 0x46, 0x94, 0xfb, 0xcc, 0x33, - 0x97, 0xff, 0x41, 0x2b, 0xe9, 0x15, 0x5a, 0xcb, 0x5e, 0xa1, 0xd5, 0x35, 0xbd, 0x02, 0xbe, 0xab, - 0x38, 0xc7, 0x8a, 0xd2, 0xd7, 0x0c, 0xd4, 0x87, 0x72, 0x14, 0x07, 0xc1, 0x90, 0x45, 0xc9, 0x3b, - 0x00, 0x3a, 0xc2, 0x5b, 0xa4, 0xac, 0x1f, 0x07, 0xc1, 0xf3, 0x84, 0x84, 0xed, 0x68, 0x6d, 0xa0, - 0xf7, 0xa1, 0x38, 0xe6, 0x2c, 0x8e, 0x84, 0x63, 0xeb, 0x64, 0x18, 0x0b, 0x7d, 0x0e, 0xbb, 0x82, - 0xba, 0x9c, 0x4a, 0xe1, 0x94, 0xf5, 0x56, 0x3f, 0xcc, 0x9a, 0x64, 0xa0, 0x21, 0x98, 0x5e, 0x52, - 0x4e, 0x43, 0x97, 0xe2, 0x25, 0x07, 0x3d, 0x80, 0x6d, 0x29, 0xe7, 0xce, 0x5e, 0x23, 0xbf, 0x6f, - 0x75, 0x76, 0x17, 0x37, 0xf5, 0xed, 0x8b, 0x8b, 0x97, 0x58, 0xf9, 0x54, 0x8d, 0x9a, 0x30, 0x21, - 0x43, 0x32, 0xa5, 0xce, 0x1d, 0x9d, 0xdb, 0x95, 0x8d, 0x5e, 0x02, 0x78, 0xa1, 0x18, 0xba, 0xfa, - 0x52, 0x38, 0x77, 0xf5, 0xee, 0x3e, 0x79, 0xf3, 0xee, 0xba, 0xe7, 0x03, 0x53, 0xa7, 0xf7, 0x16, - 0x37, 0xf5, 0xd2, 0xca, 0xc4, 0x25, 0x2f, 0x14, 0xc9, 0x10, 0x75, 0xc0, 0x9e, 0x50, 0x12, 0xc8, - 0x89, 0x3b, 0xa1, 0xee, 0x95, 0x53, 0xb9, 0xbd, 0xf0, 0x3e, 0xd3, 0x30, 0x13, 0x21, 0x4d, 0x52, - 0x0a, 0x56, 0x4b, 0x15, 0xce, 0x3d, 0x9d, 0xab, 0xc4, 0x40, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x43, - 0x21, 0x3d, 0x3f, 0x74, 0x90, 0xda, 0x32, 0x2e, 0x29, 0xcf, 0x40, 0x39, 0xd0, 0x43, 0x55, 0x16, - 0x89, 0x37, 0x64, 0x61, 0x30, 0x77, 0x7e, 0xa0, 0xbf, 0x5a, 0xca, 0xf1, 0x3c, 0x0c, 0xe6, 0xa8, - 0x0e, 0xb6, 0xd6, 0x85, 0xf0, 0xc7, 0x21, 0x09, 0x9c, 0xfb, 0x3a, 0x1f, 0xa0, 0x5c, 0x03, 0xed, - 0xa9, 0x7e, 0x06, 0x76, 0x4a, 0xee, 0x4a, 0xa6, 0x57, 0x74, 0x6e, 0x6e, 0x90, 0x1a, 0xaa, 0x35, - 0xcd, 0x48, 0x10, 0x27, 0xcd, 0x5e, 0x09, 0x27, 0xc6, 0xaf, 0xb7, 0x9e, 0xe4, 0xab, 0x87, 0x60, - 0xa7, 0x8e, 0x1d, 0x7d, 0x08, 0x7b, 0x9c, 0x8e, 0x7d, 0x21, 0xf9, 0x7c, 0x48, 0x62, 0x39, 0x71, - 0x7e, 0xa7, 0x09, 0xe5, 0xa5, 0xb3, 0x1d, 0xcb, 0x49, 0x75, 0x08, 0xeb, 0xec, 0xa1, 0x06, 0xd8, - 0xea, 0x54, 0x04, 0xe5, 0x33, 0xca, 0xd5, 0x83, 0xa2, 0x36, 0x9d, 0x76, 0x29, 0xf5, 0x08, 0x4a, - 0xb8, 0x3b, 0xd1, 0x97, 0xb7, 0x84, 0x8d, 0xa5, 0x6e, 0xe3, 0x52, 0xa2, 0xe6, 0x36, 0x1a, 0xb3, - 0xf9, 0xdf, 0x3c, 0x94, 0xd3, 0xef, 0x22, 0x3a, 0x4a, 0xde, 0x33, 0xbd, 0xa5, 0x3b, 0x87, 0x07, - 0x6f, 0x7a, 0x47, 0xf5, 0xeb, 0x11, 0xc4, 0x2a, 0xd8, 0x99, 0x6a, 0x61, 0x35, 0x19, 0xfd, 0x12, - 0x76, 0x22, 0xc6, 0xe5, 0xb2, 0x86, 0xd4, 0x32, 0x2b, 0x3e, 0xe3, 0xcb, 0x6a, 0x9b, 0x80, 0x9b, - 0x13, 0xb8, 0xb3, 0x19, 0x0d, 0x3d, 0x86, 0xed, 0x17, 0x27, 0xfd, 0x4a, 0xae, 0xfa, 0xf0, 0xd5, - 0x75, 0xe3, 0x87, 0x9b, 0x1f, 0x5f, 0xf8, 0x5c, 0xc6, 0x24, 0x38, 0xe9, 0xa3, 0x9f, 0xc2, 0x4e, - 0xf7, 0x7c, 0x80, 0x71, 0x25, 0x5f, 0xad, 0xbf, 0xba, 0x6e, 0x3c, 0xdc, 0xc4, 0xa9, 0x4f, 0x2c, - 0x0e, 0x3d, 0xcc, 0x46, 0xab, 0x76, 0xee, 0x9f, 0x5b, 0x60, 0x9b, 0xd2, 0xfa, 0xae, 0x3b, 0xfe, - 0xbd, 0xe4, 0xb5, 0x5a, 0xde, 0x99, 0xad, 0x37, 0x3e, 0x5a, 0xe5, 0x84, 0x60, 0xce, 0xf8, 0x11, - 0x94, 0xfd, 0x68, 0xf6, 0xe9, 0x90, 0x86, 0x64, 0x14, 0x98, 0xce, 0xce, 0xc2, 0xb6, 0xf2, 0xf5, - 0x12, 0x97, 0xba, 0xb0, 0x7e, 0x28, 0x29, 0x0f, 0x4d, 0xcf, 0x66, 0xe1, 0x95, 0x8d, 0x3e, 0x87, - 0x82, 0x1f, 0x91, 0xa9, 0x79, 0x69, 0x33, 0x77, 0x70, 0xd2, 0x6f, 0x9f, 0x19, 0x0d, 0x76, 0xac, - 0xc5, 0x4d, 0xbd, 0xa0, 0x1c, 0x58, 0xd3, 0x50, 0x6d, 0xf9, 0xd8, 0xa9, 0x99, 0x74, 0xf1, 0xb5, - 0x70, 0xca, 0xd3, 0xfc, 0x5f, 0x01, 0xec, 0xa3, 0x20, 0x16, 0xd2, 0x3c, 0x21, 0xef, 0x2c, 0x6f, - 0x2f, 0xe1, 0x1e, 0xd1, 0xcd, 0x3f, 0x09, 0x55, 0x3d, 0xd6, 0x4d, 0x84, 0xc9, 0xdd, 0xe3, 0xcc, - 0x70, 0x2b, 0x70, 0xd2, 0x70, 0x74, 0x8a, 0x2a, 0xa6, 0x93, 0xc7, 0x15, 0xf2, 0x8d, 0x2f, 0x68, - 0x00, 0x7b, 0x8c, 0xbb, 0x13, 0x2a, 0x64, 0x52, 0xc5, 0x4d, 0xb3, 0x9c, 0xf9, 0x1b, 0xf5, 0x3c, - 0x0d, 0x34, 0x25, 0x2c, 0x59, 0xed, 0x66, 0x0c, 0xf4, 0x04, 0x0a, 0x9c, 0x5c, 0x2e, 0x1b, 0xa2, - 0x4c, 0x7d, 0x63, 0x72, 0x29, 0x37, 0x42, 0x68, 0x06, 0xfa, 0x03, 0x80, 0xe7, 0x8b, 0x88, 0x48, - 0x77, 0x42, 0xb9, 0x39, 0xa7, 0xcc, 0x2d, 0x76, 0x57, 0xa8, 0x8d, 0x28, 0x29, 0x36, 0x3a, 0x85, - 0x92, 0x4b, 0x96, 0x4a, 0x2b, 0xde, 0xfe, 0x07, 0x71, 0xd4, 0x36, 0x21, 0x2a, 0x2a, 0xc4, 0xe2, - 0xa6, 0x6e, 0x2d, 0x3d, 0xd8, 0x72, 0x89, 0x51, 0xde, 0x29, 0xec, 0xa9, 0x3f, 0x8b, 0xa1, 0x47, - 0x2f, 0x49, 0x1c, 0x48, 0xa1, 0x1f, 0xda, 0x5b, 0x4a, 0xb2, 0x6a, 0x53, 0xbb, 0x06, 0x67, 0xd6, - 0x55, 0x96, 0x29, 0x1f, 0xfa, 0x13, 0xdc, 0xa3, 0xa1, 0xcb, 0xe7, 0x5a, 0x67, 0xcb, 0x15, 0x5a, - 0xb7, 0x6f, 0xb6, 0xb7, 0x02, 0x6f, 0x6c, 0xb6, 0x42, 0xbf, 0xe1, 0x6f, 0xfa, 0x00, 0xc9, 0x23, - 0xf7, 0x6e, 0xf5, 0x87, 0xa0, 0xe0, 0x11, 0x49, 0xb4, 0xe4, 0xca, 0x58, 0x8f, 0x3b, 0xce, 0xeb, - 0xaf, 0x6b, 0xb9, 0x7f, 0x7f, 0x5d, 0xcb, 0xfd, 0x65, 0x51, 0xcb, 0xbf, 0x5e, 0xd4, 0xf2, 0xff, - 0x5a, 0xd4, 0xf2, 0xff, 0x59, 0xd4, 0xf2, 0xa3, 0xa2, 0x6e, 0x0d, 0x7e, 0xf1, 0xff, 0x00, 0x00, - 0x00, 0xff, 0xff, 0xed, 0xbe, 0x26, 0xe6, 0x9a, 0x10, 0x00, 0x00, + 0xe1, 0xad, 0xc7, 0x8c, 0x0f, 0x3d, 0xf5, 0xaa, 0xe9, 0xa1, 0x7f, 0xc6, 0xb7, 0xf6, 0xd8, 0x93, + 0xa6, 0xe1, 0x5f, 0xe8, 0x0f, 0x68, 0x07, 0x58, 0x2c, 0xb9, 0x4c, 0x56, 0xb1, 0x67, 0xe2, 0x1b, + 0xde, 0xdb, 0xef, 0x7b, 0x00, 0x1e, 0x3e, 0x3c, 0xbc, 0x05, 0x5b, 0x44, 0xd4, 0x15, 0xad, 0x88, + 0x33, 0xc9, 0x10, 0xf2, 0x98, 0x7b, 0x45, 0x79, 0x4b, 0x7c, 0x49, 0xf8, 0xf4, 0xca, 0x97, 0xad, + 0xd9, 0xcf, 0x6b, 0xb6, 0x9c, 0x47, 0xd4, 0x00, 0x6a, 0xf7, 0xc7, 0x6c, 0xcc, 0xf4, 0xf0, 0x40, + 0x8d, 0x8c, 0xb7, 0x3e, 0x66, 0x6c, 0x1c, 0xd0, 0x03, 0x6d, 0x8d, 0xe2, 0xcb, 0x03, 0x2f, 0xe6, + 0x44, 0xfa, 0x2c, 0x4c, 0xbe, 0x37, 0xaf, 0x8b, 0x60, 0x9d, 0x33, 0x8f, 0x0e, 0x22, 0xea, 0xa2, + 0x63, 0xb0, 0x49, 0x18, 0x32, 0xa9, 0x01, 0xc2, 0x29, 0x34, 0x0a, 0xfb, 0xf6, 0xe1, 0x5e, 0xeb, + 0xdb, 0x33, 0xb7, 0xda, 0x2b, 0x58, 0xa7, 0xf8, 0xfa, 0x66, 0x6f, 0x03, 0x67, 0x99, 0xe8, 0xb7, + 0x50, 0xf1, 0xa8, 0xf0, 0x39, 0xf5, 0x86, 0x9c, 0x05, 0xd4, 0xd9, 0x6c, 0x14, 0xf6, 0xef, 0x1c, + 0xfe, 0x28, 0x2f, 0x92, 0x9a, 0x1c, 0xb3, 0x80, 0x62, 0xdb, 0x30, 0x94, 0x81, 0x8e, 0x01, 0xa6, + 0x74, 0x3a, 0xa2, 0x5c, 0x4c, 0xfc, 0xc8, 0xd9, 0xd2, 0xf4, 0x9f, 0xdc, 0x46, 0x57, 0x6b, 0x6f, + 0x9d, 0x2d, 0xe1, 0x38, 0x43, 0x45, 0x67, 0x50, 0x21, 0x33, 0xe2, 0x07, 0x64, 0xe4, 0x07, 0xbe, + 0x9c, 0x3b, 0x45, 0x1d, 0xea, 0xe3, 0xef, 0x0c, 0xd5, 0xce, 0x10, 0xf0, 0x1a, 0xbd, 0xe9, 0x01, + 0xac, 0x26, 0x42, 0x1f, 0xc1, 0x4e, 0xbf, 0x77, 0xde, 0x3d, 0x39, 0x3f, 0xae, 0x6e, 0xd4, 0x1e, + 0xbc, 0xba, 0x6e, 0xbc, 0xa7, 0x62, 0xac, 0x00, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0xfb, 0x60, + 0xb5, 0x8f, 0x8e, 0x7a, 0xfd, 0x8b, 0x5e, 0xb7, 0x5a, 0xa8, 0xd5, 0x5e, 0x5d, 0x37, 0xde, 0x5f, + 0x07, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xb5, 0xe2, 0x57, 0xff, 0xa8, 0x6f, 0x34, 0xbf, 0x2a, + 0x40, 0x25, 0xbb, 0x08, 0xf4, 0x11, 0x94, 0xda, 0x47, 0x17, 0x27, 0x2f, 0x7a, 0xd5, 0x8d, 0x15, + 0x3d, 0x8b, 0x68, 0xbb, 0xd2, 0x9f, 0x51, 0xf4, 0x18, 0xb6, 0xfb, 0xed, 0x2f, 0x06, 0xbd, 0x6a, + 0x61, 0xb5, 0x9c, 0x2c, 0xac, 0x4f, 0x62, 0xa1, 0x51, 0x5d, 0xdc, 0x3e, 0x39, 0xaf, 0x6e, 0xe6, + 0xa3, 0xba, 0x9c, 0xf8, 0xa1, 0x59, 0xca, 0xdf, 0x8b, 0x60, 0x0f, 0x28, 0x9f, 0xf9, 0xee, 0x3b, + 0x96, 0xc8, 0xa7, 0x50, 0x94, 0x44, 0x5c, 0x69, 0x69, 0xd8, 0xf9, 0xd2, 0xb8, 0x20, 0xe2, 0x4a, + 0x4d, 0x6a, 0xe8, 0x1a, 0xaf, 0x94, 0xc1, 0x69, 0x14, 0xf8, 0x2e, 0x91, 0xd4, 0xd3, 0xca, 0xb0, + 0x0f, 0x7f, 0x9c, 0xc7, 0xc6, 0x4b, 0x94, 0x59, 0xff, 0xb3, 0x0d, 0x9c, 0xa1, 0xa2, 0xa7, 0x50, + 0x1a, 0x07, 0x6c, 0x44, 0x02, 0xad, 0x09, 0xfb, 0xf0, 0x51, 0x5e, 0x90, 0x63, 0x8d, 0x58, 0x05, + 0x30, 0x14, 0xf4, 0x04, 0x4a, 0x71, 0xe4, 0x11, 0x49, 0x9d, 0x92, 0x26, 0x37, 0xf2, 0xc8, 0x5f, + 0x68, 0xc4, 0x11, 0x0b, 0x2f, 0xfd, 0x31, 0x36, 0x78, 0x74, 0x0a, 0x56, 0x48, 0xe5, 0x97, 0x8c, + 0x5f, 0x09, 0x67, 0xa7, 0xb1, 0xb5, 0x6f, 0x1f, 0x7e, 0x92, 0x2b, 0xc6, 0x04, 0xd3, 0x96, 0x92, + 0xb8, 0x93, 0x29, 0x0d, 0x65, 0x12, 0xa6, 0xb3, 0xe9, 0x14, 0xf0, 0x32, 0x00, 0xfa, 0x0d, 0x58, + 0x34, 0xf4, 0x22, 0xe6, 0x87, 0xd2, 0xb1, 0x6e, 0x5f, 0x48, 0xcf, 0x60, 0x54, 0x32, 0xf1, 0x92, + 0xa1, 0xd8, 0x9c, 0x05, 0xc1, 0x88, 0xb8, 0x57, 0x4e, 0xf9, 0x2d, 0xb7, 0xb1, 0x64, 0x74, 0x4a, + 0x50, 0x9c, 0x32, 0x8f, 0x36, 0x0f, 0xe0, 0xde, 0xb7, 0x52, 0x8d, 0x6a, 0x60, 0x99, 0x54, 0x27, + 0x1a, 0x29, 0xe2, 0xa5, 0xdd, 0xbc, 0x0b, 0xbb, 0x6b, 0x69, 0x6d, 0xfe, 0xb5, 0x08, 0x56, 0x7a, + 0xd6, 0xa8, 0x0d, 0x65, 0x97, 0x85, 0x92, 0xf8, 0x21, 0xe5, 0x46, 0x5e, 0xb9, 0x27, 0x73, 0x94, + 0x82, 0x14, 0xeb, 0xd9, 0x06, 0x5e, 0xb1, 0xd0, 0xef, 0xa1, 0xcc, 0xa9, 0x60, 0x31, 0x77, 0xa9, + 0x30, 0xfa, 0xda, 0xcf, 0x57, 0x48, 0x02, 0xc2, 0xf4, 0xcf, 0xb1, 0xcf, 0xa9, 0xca, 0xb2, 0xc0, + 0x2b, 0x2a, 0x7a, 0x0a, 0x3b, 0x9c, 0x0a, 0x49, 0xb8, 0xfc, 0x2e, 0x89, 0xe0, 0x04, 0xd2, 0x67, + 0x81, 0xef, 0xce, 0x71, 0xca, 0x40, 0x4f, 0xa1, 0x1c, 0x05, 0xc4, 0xd5, 0x51, 0x9d, 0x6d, 0x4d, + 0xff, 0x20, 0x8f, 0xde, 0x4f, 0x41, 0x78, 0x85, 0x47, 0x9f, 0x01, 0x04, 0x6c, 0x3c, 0xf4, 0xb8, + 0x3f, 0xa3, 0xdc, 0x48, 0xac, 0x96, 0xc7, 0xee, 0x6a, 0x04, 0x2e, 0x07, 0x6c, 0x9c, 0x0c, 0xd1, + 0xf1, 0xf7, 0xd2, 0x57, 0x46, 0x5b, 0xa7, 0x00, 0x64, 0xf9, 0xd5, 0xa8, 0xeb, 0xe3, 0xb7, 0x0a, + 0x65, 0x4e, 0x24, 0x43, 0x47, 0x8f, 0xa0, 0x72, 0xc9, 0xb8, 0x4b, 0x87, 0xe6, 0xd6, 0x94, 0xb5, + 0x26, 0x6c, 0xed, 0x4b, 0xf4, 0xd5, 0x29, 0xc3, 0x0e, 0x8f, 0x43, 0xe9, 0x4f, 0x69, 0xf3, 0x14, + 0xde, 0xcb, 0x0d, 0x8a, 0x0e, 0xa1, 0xb2, 0x3c, 0xe6, 0xa1, 0xef, 0x69, 0x7d, 0x94, 0x3b, 0x77, + 0x17, 0x37, 0x7b, 0xf6, 0x52, 0x0f, 0x27, 0x5d, 0x6c, 0x2f, 0x41, 0x27, 0x5e, 0xf3, 0x6f, 0x16, + 0xec, 0xae, 0x89, 0x05, 0xdd, 0x87, 0x6d, 0x7f, 0x4a, 0xc6, 0x34, 0xa1, 0xe3, 0xc4, 0x40, 0x3d, + 0x28, 0x05, 0x64, 0x44, 0x03, 0x25, 0x19, 0x95, 0xb6, 0x9f, 0xbd, 0x51, 0x75, 0xad, 0x3f, 0x6a, + 0x7c, 0x2f, 0x94, 0x7c, 0x8e, 0x0d, 0x19, 0x39, 0xb0, 0xe3, 0xb2, 0xe9, 0x94, 0x84, 0xaa, 0x38, + 0x6d, 0xed, 0x97, 0x71, 0x6a, 0x22, 0x04, 0x45, 0xc2, 0xc7, 0xc2, 0x29, 0x6a, 0xb7, 0x1e, 0xa3, + 0x2a, 0x6c, 0xd1, 0x70, 0xe6, 0x6c, 0x6b, 0x97, 0x1a, 0x2a, 0x8f, 0xe7, 0x27, 0x67, 0x5e, 0xc6, + 0x6a, 0xa8, 0x78, 0xb1, 0xa0, 0xdc, 0xd9, 0xd1, 0x2e, 0x3d, 0x46, 0xbf, 0x82, 0xd2, 0x94, 0xc5, + 0xa1, 0x14, 0x8e, 0xa5, 0x17, 0xfb, 0x20, 0x6f, 0xb1, 0x67, 0x0a, 0x61, 0x8a, 0xa7, 0x81, 0xa3, + 0x1e, 0xdc, 0x13, 0x92, 0x45, 0xc3, 0x31, 0x27, 0x2e, 0x1d, 0x46, 0x94, 0xfb, 0xcc, 0x33, 0x97, + 0xff, 0x41, 0x2b, 0xe9, 0x15, 0x5a, 0x69, 0xaf, 0xd0, 0xea, 0x9a, 0x5e, 0x01, 0xdf, 0x55, 0x9c, + 0x63, 0x45, 0xe9, 0x6b, 0x06, 0xea, 0x43, 0x25, 0x8a, 0x83, 0x60, 0xc8, 0xa2, 0xe4, 0x1d, 0x00, + 0x1d, 0xe1, 0x2d, 0x52, 0xd6, 0x8f, 0x83, 0xe0, 0x79, 0x42, 0xc2, 0x76, 0xb4, 0x32, 0xd0, 0xfb, + 0x50, 0x1a, 0x73, 0x16, 0x47, 0xc2, 0xb1, 0x75, 0x32, 0x8c, 0x85, 0x3e, 0x87, 0x1d, 0x41, 0x5d, + 0x4e, 0xa5, 0x70, 0x2a, 0x7a, 0xab, 0x1f, 0xe6, 0x4d, 0x32, 0xd0, 0x10, 0x4c, 0x2f, 0x29, 0xa7, + 0xa1, 0x4b, 0x71, 0xca, 0x41, 0x0f, 0x60, 0x4b, 0xca, 0xb9, 0xb3, 0xdb, 0x28, 0xec, 0x5b, 0x9d, + 0x9d, 0xc5, 0xcd, 0xde, 0xd6, 0xc5, 0xc5, 0x4b, 0xac, 0x7c, 0xaa, 0x46, 0x4d, 0x98, 0x90, 0x21, + 0x99, 0x52, 0xe7, 0x8e, 0xce, 0xed, 0xd2, 0x46, 0x2f, 0x01, 0xbc, 0x50, 0x0c, 0x5d, 0x7d, 0x29, + 0x9c, 0xbb, 0x7a, 0x77, 0x9f, 0xbc, 0x79, 0x77, 0xdd, 0xf3, 0x81, 0xa9, 0xd3, 0xbb, 0x8b, 0x9b, + 0xbd, 0xf2, 0xd2, 0xc4, 0x65, 0x2f, 0x14, 0xc9, 0x10, 0x75, 0xc0, 0x9e, 0x50, 0x12, 0xc8, 0x89, + 0x3b, 0xa1, 0xee, 0x95, 0x53, 0xbd, 0xbd, 0xf0, 0x3e, 0xd3, 0x30, 0x13, 0x21, 0x4b, 0x52, 0x0a, + 0x56, 0x4b, 0x15, 0xce, 0x3d, 0x9d, 0xab, 0xc4, 0x40, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x43, 0x21, + 0x3d, 0x3f, 0x74, 0x90, 0xda, 0x32, 0x2e, 0x2b, 0xcf, 0x40, 0x39, 0xd0, 0x43, 0x55, 0x16, 0x89, + 0x37, 0x64, 0x61, 0x30, 0x77, 0x7e, 0xa0, 0xbf, 0x5a, 0xca, 0xf1, 0x3c, 0x0c, 0xe6, 0x68, 0x0f, + 0x6c, 0xad, 0x0b, 0xe1, 0x8f, 0x43, 0x12, 0x38, 0xf7, 0x75, 0x3e, 0x40, 0xb9, 0x06, 0xda, 0x53, + 0xfb, 0x0c, 0xec, 0x8c, 0xdc, 0x95, 0x4c, 0xaf, 0xe8, 0xdc, 0xdc, 0x20, 0x35, 0x54, 0x6b, 0x9a, + 0x91, 0x20, 0x4e, 0x9a, 0xbd, 0x32, 0x4e, 0x8c, 0x5f, 0x6f, 0x3e, 0x29, 0xd4, 0x0e, 0xc1, 0xce, + 0x1c, 0x3b, 0xfa, 0x10, 0x76, 0x39, 0x1d, 0xfb, 0x42, 0xf2, 0xf9, 0x90, 0xc4, 0x72, 0xe2, 0xfc, + 0x4e, 0x13, 0x2a, 0xa9, 0xb3, 0x1d, 0xcb, 0x49, 0x6d, 0x08, 0xab, 0xec, 0xa1, 0x06, 0xd8, 0xea, + 0x54, 0x04, 0xe5, 0x33, 0xca, 0xd5, 0x83, 0xa2, 0x36, 0x9d, 0x75, 0x29, 0xf5, 0x08, 0x4a, 0xb8, + 0x3b, 0xd1, 0x97, 0xb7, 0x8c, 0x8d, 0xa5, 0x6e, 0x63, 0x2a, 0x51, 0x73, 0x1b, 0x8d, 0xd9, 0xfc, + 0x6f, 0x01, 0x2a, 0xd9, 0x77, 0x11, 0x1d, 0x25, 0xef, 0x99, 0xde, 0xd2, 0x9d, 0xc3, 0x83, 0x37, + 0xbd, 0xa3, 0xfa, 0xf5, 0x08, 0x62, 0x15, 0xec, 0x4c, 0xb5, 0xb0, 0x9a, 0x8c, 0x7e, 0x09, 0xdb, + 0x11, 0xe3, 0x32, 0xad, 0x21, 0xf5, 0xdc, 0x8a, 0xcf, 0x78, 0x5a, 0x6d, 0x13, 0x70, 0x73, 0x02, + 0x77, 0xd6, 0xa3, 0xa1, 0xc7, 0xb0, 0xf5, 0xe2, 0xa4, 0x5f, 0xdd, 0xa8, 0x3d, 0x7c, 0x75, 0xdd, + 0xf8, 0xe1, 0xfa, 0xc7, 0x17, 0x3e, 0x97, 0x31, 0x09, 0x4e, 0xfa, 0xe8, 0xa7, 0xb0, 0xdd, 0x3d, + 0x1f, 0x60, 0x5c, 0x2d, 0xd4, 0xf6, 0x5e, 0x5d, 0x37, 0x1e, 0xae, 0xe3, 0xd4, 0x27, 0x16, 0x87, + 0x1e, 0x66, 0xa3, 0x65, 0x3b, 0xf7, 0xcf, 0x4d, 0xb0, 0x4d, 0x69, 0x7d, 0xd7, 0x1d, 0xff, 0x6e, + 0xf2, 0x5a, 0xa5, 0x77, 0x66, 0xf3, 0x8d, 0x8f, 0x56, 0x25, 0x21, 0x98, 0x33, 0x7e, 0x04, 0x15, + 0x3f, 0x9a, 0x7d, 0x3a, 0xa4, 0x21, 0x19, 0x05, 0xa6, 0xb3, 0xb3, 0xb0, 0xad, 0x7c, 0xbd, 0xc4, + 0xa5, 0x2e, 0xac, 0x1f, 0x4a, 0xca, 0x43, 0xd3, 0xb3, 0x59, 0x78, 0x69, 0xa3, 0xcf, 0xa1, 0xe8, + 0x47, 0x64, 0x6a, 0x5e, 0xda, 0xdc, 0x1d, 0x9c, 0xf4, 0xdb, 0x67, 0x46, 0x83, 0x1d, 0x6b, 0x71, + 0xb3, 0x57, 0x54, 0x0e, 0xac, 0x69, 0xa8, 0x9e, 0x3e, 0x76, 0x6a, 0x26, 0x5d, 0x7c, 0x2d, 0x9c, + 0xf1, 0x28, 0x1d, 0xf9, 0xe1, 0x98, 0x53, 0x21, 0x74, 0x19, 0xb6, 0x70, 0x6a, 0x36, 0xff, 0x57, + 0x04, 0xfb, 0x28, 0x88, 0x85, 0x34, 0x8f, 0xcb, 0x3b, 0xcb, 0xe8, 0x4b, 0xb8, 0x47, 0xf4, 0x6f, + 0x01, 0x09, 0x55, 0xa5, 0xd6, 0xed, 0x85, 0xc9, 0xea, 0xe3, 0xdc, 0x70, 0x4b, 0x70, 0xd2, 0x8a, + 0x74, 0x4a, 0x2a, 0xa6, 0x53, 0xc0, 0x55, 0xf2, 0x8d, 0x2f, 0x68, 0x00, 0xbb, 0x8c, 0xbb, 0x13, + 0x2a, 0x64, 0x52, 0xdf, 0x4d, 0x1b, 0x9d, 0xfb, 0x83, 0xf5, 0x3c, 0x0b, 0x34, 0xc5, 0x2d, 0x59, + 0xed, 0x7a, 0x0c, 0xf4, 0x04, 0x8a, 0x9c, 0x5c, 0xa6, 0xad, 0x52, 0xae, 0xf2, 0x31, 0xb9, 0x94, + 0x6b, 0x21, 0x34, 0x03, 0xfd, 0x01, 0xc0, 0xf3, 0x45, 0x44, 0xa4, 0x3b, 0xa1, 0xdc, 0x9c, 0x60, + 0xee, 0x16, 0xbb, 0x4b, 0xd4, 0x5a, 0x94, 0x0c, 0x1b, 0x9d, 0x42, 0xd9, 0x25, 0xa9, 0x06, 0x4b, + 0xb7, 0xff, 0x5b, 0x1c, 0xb5, 0x4d, 0x88, 0xaa, 0x0a, 0xb1, 0xb8, 0xd9, 0xb3, 0x52, 0x0f, 0xb6, + 0x5c, 0x62, 0x34, 0x79, 0x0a, 0xbb, 0xea, 0x9f, 0x63, 0xe8, 0xd1, 0x4b, 0x12, 0x07, 0x32, 0x39, + 0xfb, 0x5b, 0x8a, 0xb5, 0x6a, 0x60, 0xbb, 0x06, 0x67, 0xd6, 0x55, 0x91, 0x19, 0x1f, 0xfa, 0x13, + 0xdc, 0xa3, 0xa1, 0xcb, 0xe7, 0x5a, 0x81, 0xe9, 0x0a, 0xad, 0xdb, 0x37, 0xdb, 0x5b, 0x82, 0xd7, + 0x36, 0x5b, 0xa5, 0xdf, 0xf0, 0x37, 0x7d, 0x80, 0xe4, 0xf9, 0x7b, 0xb7, 0xfa, 0x43, 0x50, 0xf4, + 0x88, 0x24, 0x5a, 0x72, 0x15, 0xac, 0xc7, 0x1d, 0xe7, 0xf5, 0xd7, 0xf5, 0x8d, 0x7f, 0x7f, 0x5d, + 0xdf, 0xf8, 0xcb, 0xa2, 0x5e, 0x78, 0xbd, 0xa8, 0x17, 0xfe, 0xb5, 0xa8, 0x17, 0xfe, 0xb3, 0xa8, + 0x17, 0x46, 0x25, 0xdd, 0x34, 0xfc, 0xe2, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x64, 0x44, 0x1e, + 0x4f, 0xb4, 0x10, 0x00, 0x00, } diff --git a/components/engine/vendor/github.com/docker/swarmkit/api/specs.proto b/components/engine/vendor/github.com/docker/swarmkit/api/specs.proto index 4350483f8e..c1ed35f9b7 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/api/specs.proto +++ b/components/engine/vendor/github.com/docker/swarmkit/api/specs.proto @@ -321,6 +321,12 @@ message NetworkSpec { // enabled(default case) no manual attachment to this network // can happen. bool attachable = 6; + + // Ingress indicates this network will provide the routing-mesh. + // In older versions, the network providing the routing mesh was + // swarm internally created only and it was identified by the name + // "ingress" and the label "com.docker.swarm.internal": "true". + bool ingress = 7; } // ClusterSpec specifies global cluster settings. diff --git a/components/engine/vendor/github.com/docker/swarmkit/ca/certificates.go b/components/engine/vendor/github.com/docker/swarmkit/ca/certificates.go index 7a7b9abc73..da60839636 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/ca/certificates.go +++ b/components/engine/vendor/github.com/docker/swarmkit/ca/certificates.go @@ -113,12 +113,39 @@ type LocalSigner struct { Key []byte } -// RootCA is the representation of everything we need to sign certificates +// RootCA is the representation of everything we need to sign certificates and/or to verify certificates +// +// RootCA.Cert: [signing CA cert][CA cert1][CA cert2] +// RootCA.Intermediates: [intermediate CA1][intermediate CA2][intermediate CA3] +// RootCA.Signer.Key: [signing CA key] +// +// Requirements: +// +// - [signing CA key] must be the private key for [signing CA cert] +// - [signing CA cert] must be the first cert in RootCA.Cert +// +// - [intermediate CA1] must have the same public key and subject as [signing CA cert], because otherwise when +// appended to a leaf certificate, the intermediates will not form a chain (because [intermediate CA1] won't because +// the signer of the leaf certificate) +// - [intermediate CA1] must be signed by [intermediate CA2], which must be signed by [intermediate CA3] +// +// - When we issue a certificate, the intermediates will be appended so that the certificate looks like: +// [leaf signed by signing CA cert][intermediate CA1][intermediate CA2][intermediate CA3] +// - [leaf signed by signing CA cert][intermediate CA1][intermediate CA2][intermediate CA3] is guaranteed to form a +// valid chain from [leaf signed by signing CA cert] to one of the root certs ([signing CA cert], [CA cert1], [CA cert2]) +// using zero or more of the intermediate certs ([intermediate CA1][intermediate CA2][intermediate CA3]) as intermediates +// type RootCA struct { // Cert contains a bundle of PEM encoded Certificate for the Root CA, the first one of which // must correspond to the key in the local signer, if provided Cert []byte + // Intermediates contains a bundle of PEM encoded intermediate CA certificates to append to any + // issued TLS (leaf) certificates. The first one must have the same public key and subject as the + // signing root certificate, and the rest must form a chain, each one certifying the one above it, + // as per RFC5246 section 7.4.2. + Intermediates []byte + // Pool is the root pool used to validate TLS certificates Pool *x509.CertPool @@ -306,7 +333,7 @@ func (rca *RootCA) ParseValidateAndSignCSR(csrBytes []byte, cn, ou, org string) return nil, errors.Wrap(err, "failed to sign node certificate") } - return cert, nil + return append(cert, rca.Intermediates...), nil } // CrossSignCACertificate takes a CA root certificate and generates an intermediate CA from it signed with the current root signer @@ -348,7 +375,7 @@ func (rca *RootCA) CrossSignCACertificate(otherCAPEM []byte) ([]byte, error) { // NewRootCA creates a new RootCA object from unparsed PEM cert bundle and key byte // slices. key may be nil, and in this case NewRootCA will return a RootCA // without a signer. -func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, error) { +func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration, intermediates []byte) (RootCA, error) { // Parse all the certificates in the cert bundle parsedCerts, err := helpers.ParseCertificatesPEM(certBytes) if err != nil { @@ -368,7 +395,6 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er default: return RootCA{}, fmt.Errorf("unsupported signature algorithm: %s", cert.SignatureAlgorithm.String()) } - // Check to see if all of the certificates are valid, self-signed root CA certs selfpool := x509.NewCertPool() selfpool.AddCert(cert) @@ -381,9 +407,28 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er // Calculate the digest for our Root CA bundle digest := digest.FromBytes(certBytes) + // We do not yet support arbitrary chains of intermediates (e.g. the case of an offline root, and the swarm CA is an + // intermediate CA). We currently only intermediates for which the first intermediate is cross-signed version of the + // CA signing cert (the first cert of the root certs) for the purposes of root rotation. If we wanted to support + // offline roots, we'd have to separate the CA signing cert from the self-signed root certs, but this intermediate + // validation logic should remain the same. Either the first intermediate would BE the intermediate CA we sign with + // (in which case it'd have the same subject and public key), or it would be a cross-signed intermediate with the + // same subject and public key as our signing cert (which could be either an intermediate cert or a self-signed root + // cert). + if len(intermediates) > 0 { + parsedIntermediates, err := ValidateCertChain(pool, intermediates, false) + if err != nil { + return RootCA{}, errors.Wrap(err, "invalid intermediate chain") + } + if !bytes.Equal(parsedIntermediates[0].RawSubject, parsedCerts[0].RawSubject) || + !bytes.Equal(parsedIntermediates[0].RawSubjectPublicKeyInfo, parsedCerts[0].RawSubjectPublicKeyInfo) { + return RootCA{}, errors.New("invalid intermediate chain - the first intermediate must have the same subject and public key as the root") + } + } + if len(keyBytes) == 0 { - // This RootCA does not have a valid signer. - return RootCA{Cert: certBytes, Digest: digest, Pool: pool}, nil + // This RootCA does not have a valid signer + return RootCA{Cert: certBytes, Intermediates: intermediates, Digest: digest, Pool: pool}, nil } var ( @@ -434,7 +479,7 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er } } - return RootCA{Signer: &LocalSigner{Signer: signer, Key: keyBytes}, Digest: digest, Cert: certBytes, Pool: pool}, nil + return RootCA{Signer: &LocalSigner{Signer: signer, Key: keyBytes}, Intermediates: intermediates, Digest: digest, Cert: certBytes, Pool: pool}, nil } // ValidateCertChain checks checks that the certificates provided chain up to the root pool provided. In addition @@ -586,7 +631,7 @@ func GetLocalRootCA(paths CertPaths) (RootCA, error) { key = nil } - return NewRootCA(cert, key, DefaultNodeCertExpiration) + return NewRootCA(cert, key, DefaultNodeCertExpiration, nil) } func getGRPCConnection(creds credentials.TransportCredentials, connBroker *connectionbroker.Broker, forceRemote bool) (*connectionbroker.Conn, error) { @@ -641,7 +686,7 @@ func GetRemoteCA(ctx context.Context, d digest.Digest, connBroker *connectionbro // NewRootCA will validate that the certificates are otherwise valid and create a RootCA object. // Since there is no key, the certificate expiry does not matter and will not be used. - return NewRootCA(response.Certificate, nil, DefaultNodeCertExpiration) + return NewRootCA(response.Certificate, nil, DefaultNodeCertExpiration, nil) } // CreateRootCA creates a Certificate authority for a new Swarm Cluster, potentially @@ -660,7 +705,7 @@ func CreateRootCA(rootCN string, paths CertPaths) (RootCA, error) { return RootCA{}, err } - rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration) + rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration, nil) if err != nil { return RootCA{}, err } diff --git a/components/engine/vendor/github.com/docker/swarmkit/ca/config.go b/components/engine/vendor/github.com/docker/swarmkit/ca/config.go index 75b0c95863..8a10ebadfb 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/ca/config.go +++ b/components/engine/vendor/github.com/docker/swarmkit/ca/config.go @@ -132,7 +132,7 @@ func (s *SecurityConfig) UpdateRootCA(cert, key []byte, certExpiry time.Duration s.mu.Lock() defer s.mu.Unlock() - rootCA, err := NewRootCA(cert, key, certExpiry) + rootCA, err := NewRootCA(cert, key, certExpiry, nil) if err != nil { return err } diff --git a/components/engine/vendor/github.com/docker/swarmkit/ca/external.go b/components/engine/vendor/github.com/docker/swarmkit/ca/external.go index f53078f8f4..7dd8a7c8be 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/ca/external.go +++ b/components/engine/vendor/github.com/docker/swarmkit/ca/external.go @@ -96,7 +96,7 @@ func (eca *ExternalCA) Sign(ctx context.Context, req signer.SignRequest) (cert [ for _, url := range urls { cert, err = makeExternalSignRequest(ctx, client, url, csrJSON) if err == nil { - return cert, err + return append(cert, eca.rootCA.Intermediates...), err } logrus.Debugf("unable to proxy certificate signing request to %s: %s", url, err) } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/allocator/network.go b/components/engine/vendor/github.com/docker/swarmkit/manager/allocator/network.go index 13e0235aac..3525506951 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/allocator/network.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/allocator/network.go @@ -6,7 +6,6 @@ import ( "github.com/docker/go-events" "github.com/docker/swarmkit/api" - "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/allocator/networkallocator" "github.com/docker/swarmkit/manager/state" @@ -18,42 +17,18 @@ import ( const ( // Network allocator Voter ID for task allocation vote. - networkVoter = "network" - - ingressNetworkName = "ingress" - ingressSubnet = "10.255.0.0/16" - + networkVoter = "network" allocatedStatusMessage = "pending task scheduling" ) var ( + // ErrNoIngress is returned when no ingress network is found in store + ErrNoIngress = errors.New("no ingress network found") errNoChanges = errors.New("task unchanged") retryInterval = 5 * time.Minute ) -func newIngressNetwork() *api.Network { - return &api.Network{ - Spec: api.NetworkSpec{ - Annotations: api.Annotations{ - Name: ingressNetworkName, - Labels: map[string]string{ - "com.docker.swarm.internal": "true", - }, - }, - DriverConfig: &api.Driver{}, - IPAM: &api.IPAMOptions{ - Driver: &api.Driver{}, - Configs: []*api.IPAMConfig{ - { - Subnet: ingressSubnet, - }, - }, - }, - }, - } -} - // Network context information which is used throughout the network allocation code. type networkContext struct { ingressNetwork *api.Network @@ -97,7 +72,6 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) { unallocatedTasks: make(map[string]*api.Task), unallocatedServices: make(map[string]*api.Service), unallocatedNetworks: make(map[string]*api.Network), - ingressNetwork: newIngressNetwork(), lastRetry: time.Now(), } a.netCtx = nc @@ -108,63 +82,38 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) { } }() - // Check if we have the ingress network. If not found create - // it before reading all network objects for allocation. - var networks []*api.Network - a.store.View(func(tx store.ReadTx) { - networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName)) - if len(networks) > 0 { - nc.ingressNetwork = networks[0] - } - }) - if err != nil { - return errors.Wrap(err, "failed to find ingress network during init") - } - - // If ingress network is not found, create one right away - // using the predefined template. - if len(networks) == 0 { - if err := a.store.Update(func(tx store.Tx) error { - nc.ingressNetwork.ID = identity.NewID() - if err := store.CreateNetwork(tx, nc.ingressNetwork); err != nil { - return err - } - - return nil - }); err != nil { - return errors.Wrap(err, "failed to create ingress network") - } - - a.store.View(func(tx store.ReadTx) { - networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName)) - if len(networks) > 0 { - nc.ingressNetwork = networks[0] - } - }) - if err != nil { - return errors.Wrap(err, "failed to find ingress network after creating it") - } - - } - - // Try to complete ingress network allocation before anything else so - // that the we can get the preferred subnet for ingress - // network. - if !na.IsAllocated(nc.ingressNetwork) { - if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil { - log.G(ctx).WithError(err).Error("failed allocating ingress network during init") - } else if _, err := a.store.Batch(func(batch *store.Batch) error { - if err := a.commitAllocatedNetwork(ctx, batch, nc.ingressNetwork); err != nil { + // Ingress network is now created at cluster's first time creation. + // Check if we have the ingress network. If found, make sure it is + // allocated, before reading all network objects for allocation. + // If not found, it means it was removed by user, nothing to do here. + ingressNetwork, err := GetIngressNetwork(a.store) + switch err { + case nil: + // Try to complete ingress network allocation before anything else so + // that the we can get the preferred subnet for ingress network. + nc.ingressNetwork = ingressNetwork + if !na.IsAllocated(nc.ingressNetwork) { + if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil { + log.G(ctx).WithError(err).Error("failed allocating ingress network during init") + } else if _, err := a.store.Batch(func(batch *store.Batch) error { + if err := a.commitAllocatedNetwork(ctx, batch, nc.ingressNetwork); err != nil { + log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init") + } + return nil + }); err != nil { log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init") } - return nil - }); err != nil { - log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init") } + case ErrNoIngress: + // Ingress network is not present in store, It means user removed it + // and did not create a new one. + default: + return errors.Wrap(err, "failure while looking for ingress network during init") } // Allocate networks in the store so far before we started // watching. + var networks []*api.Network a.store.View(func(tx store.ReadTx) { networks, err = store.FindNetworks(tx, store.All) }) @@ -196,43 +145,12 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) { log.G(ctx).WithError(err).Error("failed committing allocation of networks during init") } - // Allocate nodes in the store so far before we process watched events. - var nodes []*api.Node - a.store.View(func(tx store.ReadTx) { - nodes, err = store.FindNodes(tx, store.All) - }) - if err != nil { - return errors.Wrap(err, "error listing all nodes in store while trying to allocate during init") - } - - var allocatedNodes []*api.Node - for _, node := range nodes { - if na.IsNodeAllocated(node) { - continue + // Allocate nodes in the store so far before we process watched events, + // if the ingress network is present. + if nc.ingressNetwork != nil { + if err := a.allocateNodes(ctx); err != nil { + return err } - - if node.Attachment == nil { - node.Attachment = &api.NetworkAttachment{} - } - - node.Attachment.Network = nc.ingressNetwork.Copy() - if err := a.allocateNode(ctx, node); err != nil { - log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s during init", node.ID) - continue - } - - allocatedNodes = append(allocatedNodes, node) - } - - if _, err := a.store.Batch(func(batch *store.Batch) error { - for _, node := range allocatedNodes { - if err := a.commitAllocatedNode(ctx, batch, node); err != nil { - log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s during init", node.ID) - } - } - return nil - }); err != nil { - log.G(ctx).WithError(err).Error("Failed to commit allocation of network resources for nodes during init") } // Allocate services in the store so far before we process watched events. @@ -346,6 +264,12 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) { break } + if IsIngressNetwork(n) && nc.ingressNetwork != nil { + log.G(ctx).Errorf("Cannot allocate ingress network %s (%s) because another ingress network is already present: %s (%s)", + n.ID, n.Spec.Annotations.Name, nc.ingressNetwork.ID, nc.ingressNetwork.Spec.Annotations) + break + } + if err := a.allocateNetwork(ctx, n); err != nil { log.G(ctx).WithError(err).Errorf("Failed allocation for network %s", n.ID) break @@ -356,9 +280,24 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) { }); err != nil { log.G(ctx).WithError(err).Errorf("Failed to commit allocation for network %s", n.ID) } + + if IsIngressNetwork(n) { + nc.ingressNetwork = n + err := a.allocateNodes(ctx) + if err != nil { + log.G(ctx).WithError(err).Error(err) + } + } case state.EventDeleteNetwork: n := v.Network.Copy() + if IsIngressNetwork(n) && nc.ingressNetwork.ID == n.ID { + nc.ingressNetwork = nil + if err := a.deallocateNodes(ctx); err != nil { + log.G(ctx).WithError(err).Error(err) + } + } + // The assumption here is that all dependent objects // have been cleaned up when we are here so the only // thing that needs to happen is free the network @@ -467,7 +406,7 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) { return } - if !nc.nwkAllocator.IsNodeAllocated(node) { + if !nc.nwkAllocator.IsNodeAllocated(node) && nc.ingressNetwork != nil { if node.Attachment == nil { node.Attachment = &api.NetworkAttachment{} } @@ -486,6 +425,85 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) { } } +func (a *Allocator) allocateNodes(ctx context.Context) error { + // Allocate nodes in the store so far before we process watched events. + var ( + allocatedNodes []*api.Node + nodes []*api.Node + err error + nc = a.netCtx + ) + + a.store.View(func(tx store.ReadTx) { + nodes, err = store.FindNodes(tx, store.All) + }) + if err != nil { + return errors.Wrap(err, "error listing all nodes in store while trying to allocate network resources") + } + + for _, node := range nodes { + if nc.nwkAllocator.IsNodeAllocated(node) { + continue + } + + if node.Attachment == nil { + node.Attachment = &api.NetworkAttachment{} + } + + node.Attachment.Network = nc.ingressNetwork.Copy() + if err := a.allocateNode(ctx, node); err != nil { + log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s", node.ID) + continue + } + + allocatedNodes = append(allocatedNodes, node) + } + + if _, err := a.store.Batch(func(batch *store.Batch) error { + for _, node := range allocatedNodes { + if err := a.commitAllocatedNode(ctx, batch, node); err != nil { + log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s", node.ID) + } + } + return nil + }); err != nil { + log.G(ctx).WithError(err).Error("Failed to commit allocation of network resources for nodes") + } + + return nil +} + +func (a *Allocator) deallocateNodes(ctx context.Context) error { + var ( + nodes []*api.Node + nc = a.netCtx + err error + ) + + a.store.View(func(tx store.ReadTx) { + nodes, err = store.FindNodes(tx, store.All) + }) + if err != nil { + return fmt.Errorf("error listing all nodes in store while trying to free network resources") + } + + for _, node := range nodes { + if nc.nwkAllocator.IsNodeAllocated(node) { + if err := nc.nwkAllocator.DeallocateNode(node); err != nil { + log.G(ctx).WithError(err).Errorf("Failed freeing network resources for node %s", node.ID) + } + node.Attachment = nil + if _, err := a.store.Batch(func(batch *store.Batch) error { + return a.commitAllocatedNode(ctx, batch, node) + }); err != nil { + log.G(ctx).WithError(err).Errorf("Failed to commit deallocation of network resources for node %s", node.ID) + } + } + } + + return nil +} + // taskReadyForNetworkVote checks if the task is ready for a network // vote to move it to PENDING state. func taskReadyForNetworkVote(t *api.Task, s *api.Service, nc *networkContext) bool { @@ -711,6 +729,9 @@ func (a *Allocator) allocateService(ctx context.Context, s *api.Service) error { // world. Automatically attach the service to the ingress // network only if it is not already done. if isIngressNetworkNeeded(s) { + if nc.ingressNetwork == nil { + return fmt.Errorf("ingress network is missing") + } var found bool for _, vip := range s.Endpoint.VirtualIPs { if vip.NetworkID == nc.ingressNetwork.ID { @@ -1022,3 +1043,36 @@ func updateTaskStatus(t *api.Task, newStatus api.TaskState, message string) { t.Status.Message = message t.Status.Timestamp = ptypes.MustTimestampProto(time.Now()) } + +// IsIngressNetwork returns whether the passed network is an ingress network. +func IsIngressNetwork(nw *api.Network) bool { + if nw.Spec.Ingress { + return true + } + // Check if legacy defined ingress network + _, ok := nw.Spec.Annotations.Labels["com.docker.swarm.internal"] + return ok && nw.Spec.Annotations.Name == "ingress" +} + +// GetIngressNetwork fetches the ingress network from store. +// ErrNoIngress will be returned if the ingress network is not present, +// nil otherwise. In case of any other failure in accessing the store, +// the respective error will be reported as is. +func GetIngressNetwork(s *store.MemoryStore) (*api.Network, error) { + var ( + networks []*api.Network + err error + ) + s.View(func(tx store.ReadTx) { + networks, err = store.FindNetworks(tx, store.All) + }) + if err != nil { + return nil, err + } + for _, n := range networks { + if IsIngressNetwork(n) { + return n, nil + } + } + return nil, ErrNoIngress +} diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/network.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/network.go index b5cd25fd07..c99a738cf5 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/network.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/network.go @@ -1,7 +1,6 @@ package controlapi import ( - "fmt" "net" "github.com/docker/docker/pkg/plugingetter" @@ -9,6 +8,7 @@ import ( "github.com/docker/libnetwork/ipamapi" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/identity" + "github.com/docker/swarmkit/manager/allocator" "github.com/docker/swarmkit/manager/state/store" "golang.org/x/net/context" "google.golang.org/grpc" @@ -75,6 +75,14 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } + if spec.Ingress && spec.DriverConfig != nil && spec.DriverConfig.Name != "overlay" { + return grpc.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network") + } + + if spec.Attachable && spec.Ingress { + return grpc.Errorf(codes.InvalidArgument, "ingress network cannot be attachable") + } + if err := validateAnnotations(spec.Annotations); err != nil { return err } @@ -94,16 +102,10 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er // - Returns `InvalidArgument` if the NetworkSpec is malformed. // - Returns an error if the creation fails. func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRequest) (*api.CreateNetworkResponse, error) { - // if you change this function, you have to change createInternalNetwork in - // the tests to match it (except the part where we check the label). if err := validateNetworkSpec(request.Spec, s.pg); err != nil { return nil, err } - if _, ok := request.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok { - return nil, grpc.Errorf(codes.PermissionDenied, "label com.docker.swarm.internal is for predefined internal networks and cannot be applied by users") - } - // TODO(mrjana): Consider using `Name` as a primary key to handle // duplicate creations. See #65 n := &api.Network{ @@ -112,6 +114,13 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe } err := s.store.Update(func(tx store.Tx) error { + if request.Spec.Ingress { + if n, err := allocator.GetIngressNetwork(s.store); err == nil { + return grpc.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID) + } else if err != allocator.ErrNoIngress { + return grpc.Errorf(codes.Internal, "failed ingress network presence check: %v", err) + } + } return store.CreateNetwork(tx, n) }) if err != nil { @@ -152,38 +161,23 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) } - err := s.store.Update(func(tx store.Tx) error { - services, err := store.FindServices(tx, store.ByReferencedNetworkID(request.NetworkID)) - if err != nil { - return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", request.NetworkID, err) - } + var ( + n *api.Network + rm = s.removeNetwork + ) - if len(services) != 0 { - return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", request.NetworkID, services[0].ID) - } - - tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(request.NetworkID)) - if err != nil { - return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", request.NetworkID, err) - } - - for _, t := range tasks { - if t.DesiredState <= api.TaskStateRunning && t.Status.State <= api.TaskStateRunning { - return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", request.NetworkID, t.ID) - } - } - - nw := store.GetNetwork(tx, request.NetworkID) - if _, ok := nw.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok { - networkDescription := nw.ID - if nw.Spec.Annotations.Name != "" { - networkDescription = fmt.Sprintf("%s (%s)", nw.Spec.Annotations.Name, nw.ID) - } - return grpc.Errorf(codes.PermissionDenied, "%s is a pre-defined network and cannot be removed", networkDescription) - } - return store.DeleteNetwork(tx, request.NetworkID) + s.store.View(func(tx store.ReadTx) { + n = store.GetNetwork(tx, request.NetworkID) }) - if err != nil { + if n == nil { + return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) + } + + if allocator.IsIngressNetwork(n) { + rm = s.removeIngressNetwork + } + + if err := rm(n.ID); err != nil { if err == store.ErrNotExist { return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) } @@ -192,6 +186,47 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe return &api.RemoveNetworkResponse{}, nil } +func (s *Server) removeNetwork(id string) error { + return s.store.Update(func(tx store.Tx) error { + services, err := store.FindServices(tx, store.ByReferencedNetworkID(id)) + if err != nil { + return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) + } + + if len(services) != 0 { + return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID) + } + + tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(id)) + if err != nil { + return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err) + } + + for _, t := range tasks { + if t.DesiredState <= api.TaskStateRunning && t.Status.State <= api.TaskStateRunning { + return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID) + } + } + + return store.DeleteNetwork(tx, id) + }) +} + +func (s *Server) removeIngressNetwork(id string) error { + return s.store.Update(func(tx store.Tx) error { + services, err := store.FindServices(tx, store.All) + if err != nil { + return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) + } + for _, srv := range services { + if doesServiceNeedIngress(srv) { + return grpc.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID) + } + } + return store.DeleteNetwork(tx, id) + }) +} + func filterNetworks(candidates []*api.Network, filters ...func(*api.Network) bool) []*api.Network { result := []*api.Network{} diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/service.go b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/service.go index 47a6ac26f1..b5e8710a4b 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/service.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/controlapi/service.go @@ -11,6 +11,7 @@ import ( "github.com/docker/distribution/reference" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/identity" + "github.com/docker/swarmkit/manager/allocator" "github.com/docker/swarmkit/manager/constraint" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/protobuf/ptypes" @@ -288,7 +289,7 @@ func (s *Server) validateNetworks(networks []*api.NetworkAttachmentConfig) error if network == nil { continue } - if _, ok := network.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok { + if network.Spec.Internal { return grpc.Errorf(codes.InvalidArgument, "Service cannot be explicitly attached to %q network which is a swarm internal network", network.Spec.Annotations.Name) @@ -424,6 +425,36 @@ func (s *Server) checkSecretExistence(tx store.Tx, spec *api.ServiceSpec) error return nil } +func doesServiceNeedIngress(srv *api.Service) bool { + // Only VIP mode with target ports needs routing mesh. + // If no endpoint is specified, it defaults to VIP mode but no target ports + // are specified, so the service does not need the routing mesh. + if srv.Spec.Endpoint == nil || srv.Spec.Endpoint.Mode != api.ResolutionModeVirtualIP { + return false + } + // Go through the ports' config + for _, p := range srv.Spec.Endpoint.Ports { + if p.PublishMode != api.PublishModeIngress { + continue + } + if p.PublishedPort != 0 { + return true + } + } + // Go through the ports' state + if srv.Endpoint != nil { + for _, p := range srv.Endpoint.Ports { + if p.PublishMode != api.PublishModeIngress { + continue + } + if p.PublishedPort != 0 { + return true + } + } + } + return false +} + // CreateService creates and returns a Service based on the provided ServiceSpec. // - Returns `InvalidArgument` if the ServiceSpec is malformed. // - Returns `Unimplemented` if the ServiceSpec references unimplemented features. @@ -449,6 +480,12 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe Spec: *request.Spec, } + if doesServiceNeedIngress(service) { + if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress { + return nil, grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") + } + } + err := s.store.Update(func(tx store.Tx) error { // Check to see if all the secrets being added exist as objects // in our datastore @@ -578,6 +615,12 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe service.UpdateStatus = nil } + if doesServiceNeedIngress(service) { + if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress { + return grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") + } + } + return store.UpdateService(tx, service) }) if err != nil { diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/dirty.go b/components/engine/vendor/github.com/docker/swarmkit/manager/dirty.go index 606dc626e9..c989ec2c86 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/dirty.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/dirty.go @@ -47,7 +47,7 @@ func (m *Manager) IsStateDirty() (bool, error) { if structField.Type.Kind() != reflect.Slice { panic("unexpected field type in StoreSnapshot") } - if structField.Name != "Nodes" && structField.Name != "Clusters" && field.Len() != 0 { + if structField.Name != "Nodes" && structField.Name != "Clusters" && structField.Name != "Networks" && field.Len() != 0 { // One of the other data types has an entry return true, nil } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go b/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go index 1f2cdf93f3..9733311815 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/manager.go @@ -19,6 +19,7 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/connectionbroker" + "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/allocator" "github.com/docker/swarmkit/manager/controlapi" @@ -892,7 +893,18 @@ func (m *Manager) becomeLeader(ctx context.Context) { rootCA)) // Add Node entry for ourself, if one // doesn't exist already. - store.CreateNode(tx, managerNode(nodeID, m.config.Availability)) + freshCluster := nil == store.CreateNode(tx, managerNode(nodeID, m.config.Availability)) + + if freshCluster { + // This is a fresh swarm cluster. Add to store now any initial + // cluster resource, like the default ingress network which + // provides the routing mesh for this cluster. + log.G(ctx).Info("Creating default ingress network") + if err := store.CreateNetwork(tx, newIngressNetwork()); err != nil { + log.G(ctx).WithError(err).Error("failed to create default ingress network") + } + } + return nil }) @@ -1084,3 +1096,28 @@ func managerNode(nodeID string, availability api.NodeSpec_Availability) *api.Nod }, } } + +// newIngressNetwork returns the network object for the default ingress +// network, the network which provides the routing mesh. Caller will save to +// store this object once, at fresh cluster creation. It is expected to +// call this function inside a store update transaction. +func newIngressNetwork() *api.Network { + return &api.Network{ + ID: identity.NewID(), + Spec: api.NetworkSpec{ + Ingress: true, + Annotations: api.Annotations{ + Name: "ingress", + }, + DriverConfig: &api.Driver{}, + IPAM: &api.IPAMOptions{ + Driver: &api.Driver{}, + Configs: []*api.IPAMConfig{ + { + Subnet: "10.255.0.0/16", + }, + }, + }, + }, + } +} diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/nodeset.go b/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/nodeset.go index 6e0b18e49d..7f899d8b26 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/nodeset.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/nodeset.go @@ -111,6 +111,10 @@ func (ns *nodeSet) tree(serviceID string, preferences []*api.PlacementPreference tree = next } + if node.ActiveTasksCountByService != nil { + tree.tasks += node.ActiveTasksCountByService[serviceID] + } + if tree.nodeHeap.lessFunc == nil { tree.nodeHeap.lessFunc = nodeLess } diff --git a/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/scheduler.go b/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/scheduler.go index 55420a2d42..7935187e93 100644 --- a/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/scheduler.go +++ b/components/engine/vendor/github.com/docker/swarmkit/manager/scheduler/scheduler.go @@ -1,7 +1,6 @@ package scheduler import ( - "container/list" "time" "github.com/docker/swarmkit/api" @@ -30,7 +29,7 @@ type schedulingDecision struct { // Scheduler assigns tasks to nodes. type Scheduler struct { store *store.MemoryStore - unassignedTasks *list.List + unassignedTasks map[string]*api.Task // preassignedTasks already have NodeID, need resource validation preassignedTasks map[string]*api.Task nodeSet nodeSet @@ -47,7 +46,7 @@ type Scheduler struct { func New(store *store.MemoryStore) *Scheduler { return &Scheduler{ store: store, - unassignedTasks: list.New(), + unassignedTasks: make(map[string]*api.Task), preassignedTasks: make(map[string]*api.Task), allTasks: make(map[string]*api.Task), stopChan: make(chan struct{}), @@ -191,7 +190,7 @@ func (s *Scheduler) Stop() { // enqueue queues a task for scheduling. func (s *Scheduler) enqueue(t *api.Task) { - s.unassignedTasks.PushBack(t) + s.unassignedTasks[t.ID] = t } func (s *Scheduler) createTask(ctx context.Context, t *api.Task) int { @@ -333,15 +332,12 @@ func (s *Scheduler) processPreassignedTasks(ctx context.Context) { // tick attempts to schedule the queue. func (s *Scheduler) tick(ctx context.Context) { tasksByCommonSpec := make(map[string]map[string]*api.Task) - schedulingDecisions := make(map[string]schedulingDecision, s.unassignedTasks.Len()) + schedulingDecisions := make(map[string]schedulingDecision, len(s.unassignedTasks)) - var next *list.Element - for e := s.unassignedTasks.Front(); e != nil; e = next { - next = e.Next() - t := s.allTasks[e.Value.(*api.Task).ID] + for taskID, t := range s.unassignedTasks { if t == nil || t.NodeID != "" { // task deleted or already assigned - s.unassignedTasks.Remove(e) + delete(s.unassignedTasks, taskID) continue } @@ -362,8 +358,8 @@ func (s *Scheduler) tick(ctx context.Context) { if tasksByCommonSpec[taskGroupKey] == nil { tasksByCommonSpec[taskGroupKey] = make(map[string]*api.Task) } - tasksByCommonSpec[taskGroupKey][t.ID] = t - s.unassignedTasks.Remove(e) + tasksByCommonSpec[taskGroupKey][taskID] = t + delete(s.unassignedTasks, taskID) } for _, taskGroup := range tasksByCommonSpec { @@ -602,6 +598,12 @@ func (s *Scheduler) scheduleNTasksOnNodes(ctx context.Context, n int, taskGroup nodeIter := 0 nodeCount := len(nodes) for taskID, t := range taskGroup { + // Skip tasks which were already scheduled because they ended + // up in two groups at once. + if _, exists := schedulingDecisions[taskID]; exists { + continue + } + node := &nodes[nodeIter%nodeCount] log.G(ctx).WithField("task.id", t.ID).Debugf("assigning to node %s", node.ID)