Merge pull request #514 from thaJeztah/18.03-bump-libnetwork

[18.03] Update libnetwork
2018-04-18 15:44:33 -07:00
parent 3d5ce4f418 85143bcb4d
commit 4fd6bf8c4a
20 changed files with 407 additions and 201 deletions
@@ -979,9 +979,6 @@ func (daemon *Daemon) releaseNetwork(container *container.Container) {
 		logrus.Warnf("error locating sandbox id %s: %v", sid, err)
 		return
 	}
-	if err := sb.DisableService(); err != nil {
-		logrus.WithFields(logrus.Fields{"container": container.ID, "sandbox": sid}).WithError(err).Error("Error removing service from sandbox")
-	}

 	if err := sb.Delete(); err != nil {
 		logrus.Errorf("Error deleting sandbox id %s for container %s: %v", sid, container.ID, err)
@@ -3,7 +3,7 @@
 # LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When
 # updating the binary version, consider updating github.com/docker/libnetwork
 # in vendor.conf accordingly
-LIBNETWORK_COMMIT=1b91bc94094ecfdae41daa465cc0c8df37dfb3dd
+LIBNETWORK_COMMIT=c15b372ef22125880d378167dde44f4b134e1a77

 install_proxy() {
 	case "$1" in
@@ -33,7 +33,7 @@ github.com/tonistiigi/fsutil dea3a0da73aee887fc02142d995be764106ac5e2
 #get libnetwork packages

 # When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly
-github.com/docker/libnetwork 1b91bc94094ecfdae41daa465cc0c8df37dfb3dd
+github.com/docker/libnetwork c15b372ef22125880d378167dde44f4b134e1a77
 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
@@ -1,6 +1,6 @@
 # libnetwork - networking for containers

-[![Circle CI](https://circleci.com/gh/docker/libnetwork/tree/master.svg?style=svg)](https://circleci.com/gh/docker/libnetwork/tree/master) [![Coverage Status](https://coveralls.io/repos/docker/libnetwork/badge.svg)](https://coveralls.io/r/docker/libnetwork) [![GoDoc](https://godoc.org/github.com/docker/libnetwork?status.svg)](https://godoc.org/github.com/docker/libnetwork)
+[![Circle CI](https://circleci.com/gh/docker/libnetwork/tree/master.svg?style=svg)](https://circleci.com/gh/docker/libnetwork/tree/master) [![Coverage Status](https://coveralls.io/repos/docker/libnetwork/badge.svg)](https://coveralls.io/r/docker/libnetwork) [![GoDoc](https://godoc.org/github.com/docker/libnetwork?status.svg)](https://godoc.org/github.com/docker/libnetwork) [![Go Report Card](https://goreportcard.com/badge/github.com/docker/libnetwork)](https://goreportcard.com/report/github.com/docker/libnetwork)

 Libnetwork provides a native Go implementation for connecting containers

@@ -638,14 +638,15 @@ func (ep *endpoint) addServiceInfoToCluster(sb *sandbox) error {
 	}

 	buf, err := proto.Marshal(&EndpointRecord{
-		Name:         name,
-		ServiceName:  ep.svcName,
-		ServiceID:    ep.svcID,
-		VirtualIP:    ep.virtualIP.String(),
-		IngressPorts: ingressPorts,
-		Aliases:      ep.svcAliases,
-		TaskAliases:  ep.myAliases,
-		EndpointIP:   ep.Iface().Address().IP.String(),
+		Name:            name,
+		ServiceName:     ep.svcName,
+		ServiceID:       ep.svcID,
+		VirtualIP:       ep.virtualIP.String(),
+		IngressPorts:    ingressPorts,
+		Aliases:         ep.svcAliases,
+		TaskAliases:     ep.myAliases,
+		EndpointIP:      ep.Iface().Address().IP.String(),
+		ServiceDisabled: false,
 	})
 	if err != nil {
 		return err
@@ -663,7 +664,7 @@ func (ep *endpoint) addServiceInfoToCluster(sb *sandbox) error {
 	return nil
 }

-func (ep *endpoint) deleteServiceInfoFromCluster(sb *sandbox, method string) error {
+func (ep *endpoint) deleteServiceInfoFromCluster(sb *sandbox, fullRemove bool, method string) error {
 	if ep.isAnonymous() && len(ep.myAliases) == 0 {
 		return nil
 	}
@@ -677,6 +678,15 @@ func (ep *endpoint) deleteServiceInfoFromCluster(sb *sandbox, method string) err
 	defer sb.Service.Unlock()
 	logrus.Debugf("deleteServiceInfoFromCluster from %s START for %s %s", method, ep.svcName, ep.ID())

+	// Avoid a race w/ with a container that aborts preemptively.  This would
+	// get caught in disableServceInNetworkDB, but we check here to make the
+	// nature of the condition more clear.
+	// See comment in addServiceInfoToCluster()
+	if e := sb.getEndpoint(ep.ID()); e == nil {
+		logrus.Warnf("deleteServiceInfoFromCluster suppressing service resolution ep is not anymore in the sandbox %s", ep.ID())
+		return nil
+	}
+
 	c := n.getController()
 	agent := c.getAgent()

@@ -686,9 +696,13 @@ func (ep *endpoint) deleteServiceInfoFromCluster(sb *sandbox, method string) err
 	}

 	if agent != nil {
-		// First delete from networkDB then locally
-		if err := agent.networkDB.DeleteEntry(libnetworkEPTable, n.ID(), ep.ID()); err != nil {
-			logrus.Warnf("deleteServiceInfoFromCluster NetworkDB DeleteEntry failed for %s %s err:%s", ep.id, n.id, err)
+		// First update the networkDB then locally
+		if fullRemove {
+			if err := agent.networkDB.DeleteEntry(libnetworkEPTable, n.ID(), ep.ID()); err != nil {
+				logrus.Warnf("deleteServiceInfoFromCluster NetworkDB DeleteEntry failed for %s %s err:%s", ep.id, n.id, err)
+			}
+		} else {
+			disableServiceInNetworkDB(agent, n, ep)
 		}
 	}

@@ -699,7 +713,7 @@ func (ep *endpoint) deleteServiceInfoFromCluster(sb *sandbox, method string) err
 			if n.ingress {
 				ingressPorts = ep.ingressPorts
 			}
-			if err := c.rmServiceBinding(ep.svcName, ep.svcID, n.ID(), ep.ID(), name, ep.virtualIP, ingressPorts, ep.svcAliases, ep.myAliases, ep.Iface().Address().IP, "deleteServiceInfoFromCluster", true); err != nil {
+			if err := c.rmServiceBinding(ep.svcName, ep.svcID, n.ID(), ep.ID(), name, ep.virtualIP, ingressPorts, ep.svcAliases, ep.myAliases, ep.Iface().Address().IP, "deleteServiceInfoFromCluster", true, fullRemove); err != nil {
 				return err
 			}
 		} else {
@@ -715,6 +729,35 @@ func (ep *endpoint) deleteServiceInfoFromCluster(sb *sandbox, method string) err
 	return nil
 }

+func disableServiceInNetworkDB(a *agent, n *network, ep *endpoint) {
+	var epRec EndpointRecord
+
+	logrus.Debugf("disableServiceInNetworkDB for %s %s", ep.svcName, ep.ID())
+
+	// Update existing record to indicate that the service is disabled
+	inBuf, err := a.networkDB.GetEntry(libnetworkEPTable, n.ID(), ep.ID())
+	if err != nil {
+		logrus.Warnf("disableServiceInNetworkDB GetEntry failed for %s %s err:%s", ep.id, n.id, err)
+		return
+	}
+	// Should never fail
+	if err := proto.Unmarshal(inBuf, &epRec); err != nil {
+		logrus.Errorf("disableServiceInNetworkDB unmarshal failed for %s %s err:%s", ep.id, n.id, err)
+		return
+	}
+	epRec.ServiceDisabled = true
+	// Should never fail
+	outBuf, err := proto.Marshal(&epRec)
+	if err != nil {
+		logrus.Errorf("disableServiceInNetworkDB marshalling failed for %s %s err:%s", ep.id, n.id, err)
+		return
+	}
+	// Send update to the whole cluster
+	if err := a.networkDB.UpdateEntry(libnetworkEPTable, n.ID(), ep.ID(), outBuf); err != nil {
+		logrus.Warnf("disableServiceInNetworkDB UpdateEntry failed for %s %s err:%s", ep.id, n.id, err)
+	}
+}
+
 func (n *network) addDriverWatches() {
 	if !n.isClusterEligible() {
 		return
@@ -844,7 +887,6 @@ func (c *controller) handleEpTableEvent(ev events.Event) {
 		nid   string
 		eid   string
 		value []byte
-		isAdd bool
 		epRec EndpointRecord
 	)

@@ -853,12 +895,15 @@ func (c *controller) handleEpTableEvent(ev events.Event) {
 		nid = event.NetworkID
 		eid = event.Key
 		value = event.Value
-		isAdd = true
 	case networkdb.DeleteEvent:
 		nid = event.NetworkID
 		eid = event.Key
 		value = event.Value
 	case networkdb.UpdateEvent:
+		nid = event.NetworkID
+		eid = event.Key
+		value = event.Value
+	default:
 		logrus.Errorf("Unexpected update service table event = %#v", event)
 		return
 	}
@@ -883,7 +928,8 @@ func (c *controller) handleEpTableEvent(ev events.Event) {
 		return
 	}

-	if isAdd {
+	switch ev.(type) {
+	case networkdb.CreateEvent:
 		logrus.Debugf("handleEpTableEvent ADD %s R:%v", eid, epRec)
 		if svcID != "" {
 			// This is a remote task part of a service
@@ -897,11 +943,12 @@ func (c *controller) handleEpTableEvent(ev events.Event) {
 				logrus.Errorf("failed adding container name resolution for %s epRec:%v err:%v", eid, epRec, err)
 			}
 		}
-	} else {
+
+	case networkdb.DeleteEvent:
 		logrus.Debugf("handleEpTableEvent DEL %s R:%v", eid, epRec)
 		if svcID != "" {
 			// This is a remote task part of a service
-			if err := c.rmServiceBinding(svcName, svcID, nid, eid, containerName, vip, ingressPorts, serviceAliases, taskAliases, ip, "handleEpTableEvent", true); err != nil {
+			if err := c.rmServiceBinding(svcName, svcID, nid, eid, containerName, vip, ingressPorts, serviceAliases, taskAliases, ip, "handleEpTableEvent", true, true); err != nil {
 				logrus.Errorf("failed removing service binding for %s epRec:%v err:%v", eid, epRec, err)
 				return
 			}
@@ -911,5 +958,18 @@ func (c *controller) handleEpTableEvent(ev events.Event) {
 				logrus.Errorf("failed removing container name resolution for %s epRec:%v err:%v", eid, epRec, err)
 			}
 		}
+	case networkdb.UpdateEvent:
+		logrus.Debugf("handleEpTableEvent UPD %s R:%v", eid, epRec)
+		// We currently should only get these to inform us that an endpoint
+		// is disabled.  Report if otherwise.
+		if svcID == "" || !epRec.ServiceDisabled {
+			logrus.Errorf("Unexpected update table event for %s epRec:%v", eid, epRec)
+			return
+		}
+		// This is a remote task that is part of a service that is now disabled
+		if err := c.rmServiceBinding(svcName, svcID, nid, eid, containerName, vip, ingressPorts, serviceAliases, taskAliases, ip, "handleEpTableEvent", true, false); err != nil {
+			logrus.Errorf("failed disabling service binding for %s epRec:%v err:%v", eid, epRec, err)
+			return
+		}
 	}
 }
@@ -77,6 +77,8 @@ type EndpointRecord struct {
 	Aliases []string `protobuf:"bytes,7,rep,name=aliases" json:"aliases,omitempty"`
 	// List of aliases task specific aliases
 	TaskAliases []string `protobuf:"bytes,8,rep,name=task_aliases,json=taskAliases" json:"task_aliases,omitempty"`
+	// Whether this enpoint's service has been disabled
+	ServiceDisabled bool `protobuf:"varint,9,opt,name=service_disabled,json=serviceDisabled,proto3" json:"service_disabled,omitempty"`
 }

 func (m *EndpointRecord) Reset()                    { *m = EndpointRecord{} }
@@ -139,6 +141,13 @@ func (m *EndpointRecord) GetTaskAliases() []string {
 	return nil
 }

+func (m *EndpointRecord) GetServiceDisabled() bool {
+	if m != nil {
+		return m.ServiceDisabled
+	}
+	return false
+}
+
 // PortConfig specifies an exposed port which can be
 // addressed using the given name. This can be later queried
 // using a service discovery api or a DNS SRV query. The node
@@ -202,7 +211,7 @@ func (this *EndpointRecord) GoString() string {
 	if this == nil {
 		return "nil"
 	}
-	s := make([]string, 0, 12)
+	s := make([]string, 0, 13)
 	s = append(s, "&libnetwork.EndpointRecord{")
 	s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n")
 	s = append(s, "ServiceName: "+fmt.Sprintf("%#v", this.ServiceName)+",\n")
@@ -214,6 +223,7 @@ func (this *EndpointRecord) GoString() string {
 	}
 	s = append(s, "Aliases: "+fmt.Sprintf("%#v", this.Aliases)+",\n")
 	s = append(s, "TaskAliases: "+fmt.Sprintf("%#v", this.TaskAliases)+",\n")
+	s = append(s, "ServiceDisabled: "+fmt.Sprintf("%#v", this.ServiceDisabled)+",\n")
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -325,6 +335,16 @@ func (m *EndpointRecord) MarshalTo(dAtA []byte) (int, error) {
 			i += copy(dAtA[i:], s)
 		}
 	}
+	if m.ServiceDisabled {
+		dAtA[i] = 0x48
+		i++
+		if m.ServiceDisabled {
+			dAtA[i] = 1
+		} else {
+			dAtA[i] = 0
+		}
+		i++
+	}
 	return i, nil
 }

@@ -367,24 +387,6 @@ func (m *PortConfig) MarshalTo(dAtA []byte) (int, error) {
 	return i, nil
 }

-func encodeFixed64Agent(dAtA []byte, offset int, v uint64) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	dAtA[offset+4] = uint8(v >> 32)
-	dAtA[offset+5] = uint8(v >> 40)
-	dAtA[offset+6] = uint8(v >> 48)
-	dAtA[offset+7] = uint8(v >> 56)
-	return offset + 8
-}
-func encodeFixed32Agent(dAtA []byte, offset int, v uint32) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	return offset + 4
-}
 func encodeVarintAgent(dAtA []byte, offset int, v uint64) int {
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
@@ -435,6 +437,9 @@ func (m *EndpointRecord) Size() (n int) {
 			n += 1 + l + sovAgent(uint64(l))
 		}
 	}
+	if m.ServiceDisabled {
+		n += 2
+	}
 	return n
 }

@@ -483,6 +488,7 @@ func (this *EndpointRecord) String() string {
 		`IngressPorts:` + strings.Replace(fmt.Sprintf("%v", this.IngressPorts), "PortConfig", "PortConfig", 1) + `,`,
 		`Aliases:` + fmt.Sprintf("%v", this.Aliases) + `,`,
 		`TaskAliases:` + fmt.Sprintf("%v", this.TaskAliases) + `,`,
+		`ServiceDisabled:` + fmt.Sprintf("%v", this.ServiceDisabled) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -771,6 +777,26 @@ func (m *EndpointRecord) Unmarshal(dAtA []byte) error {
 			}
 			m.TaskAliases = append(m.TaskAliases, string(dAtA[iNdEx:postIndex]))
 			iNdEx = postIndex
+		case 9:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ServiceDisabled", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowAgent
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.ServiceDisabled = bool(v != 0)
 		default:
 			iNdEx = preIndex
 			skippy, err := skipAgent(dAtA[iNdEx:])
@@ -1036,33 +1062,34 @@ var (
 func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) }

 var fileDescriptorAgent = []byte{
-	// 437 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0xc1, 0x6e, 0xd3, 0x30,
-	0x18, 0xc7, 0x9b, 0x36, 0x6c, 0xcd, 0x97, 0xb6, 0x54, 0x16, 0x42, 0x51, 0x0e, 0x69, 0xa8, 0x84,
-	0xd4, 0x03, 0xea, 0xa4, 0x71, 0xdc, 0x89, 0xb5, 0x1c, 0x72, 0x41, 0x96, 0xd7, 0x71, 0x0d, 0x69,
-	0x63, 0x82, 0xb5, 0x10, 0x47, 0xb6, 0x37, 0xae, 0xdc, 0x40, 0x7b, 0x87, 0x9d, 0x78, 0x19, 0x4e,
-	0x88, 0x23, 0xa7, 0x89, 0xe5, 0x09, 0x78, 0x04, 0x64, 0x27, 0x5e, 0x35, 0x69, 0x37, 0xfb, 0xf7,
-	0xff, 0xd9, 0xfa, 0xbe, 0x3f, 0xf8, 0x59, 0x41, 0x2b, 0xb5, 0xac, 0x05, 0x57, 0x1c, 0x41, 0xc9,
-	0xb6, 0x15, 0x55, 0x5f, 0xb8, 0xb8, 0x08, 0x9f, 0x15, 0xbc, 0xe0, 0x06, 0x1f, 0xe9, 0x53, 0x6b,
-	0xcc, 0x7f, 0xf5, 0x61, 0xf2, 0xb6, 0xca, 0x6b, 0xce, 0x2a, 0x45, 0xe8, 0x8e, 0x8b, 0x1c, 0x21,
-	0x70, 0xab, 0xec, 0x33, 0x0d, 0x9c, 0xd8, 0x59, 0x78, 0xc4, 0x9c, 0xd1, 0x0b, 0x18, 0x49, 0x2a,
-	0xae, 0xd8, 0x8e, 0xa6, 0x26, 0xeb, 0x9b, 0xcc, 0xef, 0xd8, 0x3b, 0xad, 0xbc, 0x02, 0xb0, 0x0a,
-	0xcb, 0x83, 0x81, 0x16, 0x4e, 0xc7, 0xcd, 0xed, 0xcc, 0x3b, 0x6b, 0x69, 0xb2, 0x26, 0x5e, 0x27,
-	0x24, 0xb9, 0xb6, 0xaf, 0x98, 0x50, 0x97, 0x59, 0x99, 0xb2, 0x3a, 0x70, 0xf7, 0xf6, 0xfb, 0x96,
-	0x26, 0x98, 0x78, 0x9d, 0x90, 0xd4, 0xe8, 0x08, 0x7c, 0xda, 0x0d, 0xa9, 0xf5, 0x27, 0x46, 0x9f,
-	0x34, 0xb7, 0x33, 0xb0, 0xb3, 0x27, 0x98, 0x80, 0x55, 0x92, 0x1a, 0x9d, 0xc0, 0x98, 0x55, 0x85,
-	0xa0, 0x52, 0xa6, 0x35, 0x17, 0x4a, 0x06, 0x07, 0xf1, 0x60, 0xe1, 0x1f, 0x3f, 0x5f, 0xee, 0x0b,
-	0x59, 0x62, 0x2e, 0xd4, 0x8a, 0x57, 0x1f, 0x59, 0x41, 0x46, 0x9d, 0xac, 0x91, 0x44, 0x01, 0x1c,
-	0x66, 0x25, 0xcb, 0x24, 0x95, 0xc1, 0x61, 0x3c, 0x58, 0x78, 0xc4, 0x5e, 0x75, 0x0d, 0x2a, 0x93,
-	0x17, 0xa9, 0x8d, 0x87, 0x26, 0xf6, 0x35, 0x7b, 0xd3, 0xa2, 0xf9, 0xb7, 0x3e, 0xc0, 0xfe, 0xe7,
-	0x47, 0xcb, 0x3c, 0x81, 0xa1, 0x29, 0x7f, 0xc7, 0x4b, 0x53, 0xe4, 0xe4, 0x78, 0xf6, 0xf8, 0x5c,
-	0x4b, 0xdc, 0x69, 0xe4, 0xfe, 0x01, 0x9a, 0x81, 0xaf, 0x32, 0x51, 0x50, 0x65, 0x16, 0x33, 0x3d,
-	0x8f, 0x09, 0xb4, 0x48, 0xbf, 0x44, 0x2f, 0x61, 0x52, 0x5f, 0x6e, 0x4b, 0x26, 0x3f, 0xd1, 0xbc,
-	0x75, 0x5c, 0xe3, 0x8c, 0xef, 0xa9, 0xd6, 0xe6, 0x1f, 0x60, 0x68, 0x7f, 0x47, 0x01, 0x0c, 0x36,
-	0x2b, 0x3c, 0xed, 0x85, 0x4f, 0xaf, 0x6f, 0x62, 0xdf, 0xe2, 0xcd, 0x0a, 0xeb, 0xe4, 0x7c, 0x8d,
-	0xa7, 0xce, 0xc3, 0xe4, 0x7c, 0x8d, 0x51, 0x08, 0xee, 0xd9, 0x6a, 0x83, 0xa7, 0xfd, 0x70, 0x7a,
-	0x7d, 0x13, 0x8f, 0x6c, 0xa4, 0x59, 0xe8, 0x7e, 0xff, 0x11, 0xf5, 0x4e, 0x83, 0x3f, 0x77, 0x51,
-	0xef, 0xdf, 0x5d, 0xe4, 0x7c, 0x6d, 0x22, 0xe7, 0x67, 0x13, 0x39, 0xbf, 0x9b, 0xc8, 0xf9, 0xdb,
-	0x44, 0xce, 0xf6, 0xc0, 0x6c, 0xf3, 0xfa, 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xce, 0x12, 0x15,
-	0x67, 0xac, 0x02, 0x00, 0x00,
+	// 459 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0x31, 0x6f, 0xd3, 0x4c,
+	0x18, 0xc7, 0xe3, 0xc4, 0x6f, 0x1b, 0x3f, 0x4e, 0x52, 0xeb, 0xf4, 0x0a, 0x59, 0x1e, 0x1c, 0x13,
+	0x09, 0x29, 0x48, 0x28, 0x95, 0xca, 0xd8, 0x89, 0x26, 0x0c, 0x5e, 0x90, 0x75, 0x4d, 0x59, 0x83,
+	0x13, 0x1f, 0xe6, 0x54, 0xe3, 0xb3, 0xee, 0xae, 0x65, 0x65, 0x03, 0xf5, 0x3b, 0x74, 0xe2, 0xcb,
+	0x30, 0x32, 0x32, 0x55, 0xd4, 0x9f, 0x80, 0x95, 0x0d, 0xdd, 0xf9, 0xae, 0x11, 0x52, 0xb7, 0xf3,
+	0xef, 0xff, 0x3b, 0xeb, 0xb9, 0xff, 0x03, 0x7e, 0x5e, 0x92, 0x5a, 0x2e, 0x1a, 0xce, 0x24, 0x43,
+	0x50, 0xd1, 0x6d, 0x4d, 0xe4, 0x27, 0xc6, 0x2f, 0xa3, 0xff, 0x4b, 0x56, 0x32, 0x8d, 0x8f, 0xd5,
+	0xa9, 0x33, 0x66, 0x7f, 0xfa, 0x30, 0x79, 0x5d, 0x17, 0x0d, 0xa3, 0xb5, 0xc4, 0x64, 0xc7, 0x78,
+	0x81, 0x10, 0xb8, 0x75, 0xfe, 0x91, 0x84, 0x4e, 0xe2, 0xcc, 0x3d, 0xac, 0xcf, 0xe8, 0x29, 0x8c,
+	0x04, 0xe1, 0xd7, 0x74, 0x47, 0x36, 0x3a, 0xeb, 0xeb, 0xcc, 0x37, 0xec, 0x8d, 0x52, 0x5e, 0x00,
+	0x58, 0x85, 0x16, 0xe1, 0x40, 0x09, 0x67, 0xe3, 0xf6, 0x6e, 0xea, 0x9d, 0x77, 0x34, 0x5d, 0x61,
+	0xcf, 0x08, 0x69, 0xa1, 0xec, 0x6b, 0xca, 0xe5, 0x55, 0x5e, 0x6d, 0x68, 0x13, 0xba, 0x7b, 0xfb,
+	0x6d, 0x47, 0xd3, 0x0c, 0x7b, 0x46, 0x48, 0x1b, 0x74, 0x0c, 0x3e, 0x31, 0x43, 0x2a, 0xfd, 0x3f,
+	0xad, 0x4f, 0xda, 0xbb, 0x29, 0xd8, 0xd9, 0xd3, 0x0c, 0x83, 0x55, 0xd2, 0x06, 0x9d, 0xc2, 0x98,
+	0xd6, 0x25, 0x27, 0x42, 0x6c, 0x1a, 0xc6, 0xa5, 0x08, 0x0f, 0x92, 0xc1, 0xdc, 0x3f, 0x79, 0xb2,
+	0xd8, 0x17, 0xb2, 0xc8, 0x18, 0x97, 0x4b, 0x56, 0xbf, 0xa7, 0x25, 0x1e, 0x19, 0x59, 0x21, 0x81,
+	0x42, 0x38, 0xcc, 0x2b, 0x9a, 0x0b, 0x22, 0xc2, 0xc3, 0x64, 0x30, 0xf7, 0xb0, 0xfd, 0x54, 0x35,
+	0xc8, 0x5c, 0x5c, 0x6e, 0x6c, 0x3c, 0xd4, 0xb1, 0xaf, 0xd8, 0x2b, 0xa3, 0x3c, 0x87, 0xc0, 0xd6,
+	0x50, 0x50, 0x91, 0x6f, 0x2b, 0x52, 0x84, 0x5e, 0xe2, 0xcc, 0x87, 0xf8, 0xc8, 0xf0, 0x95, 0xc1,
+	0xb3, 0x2f, 0x7d, 0x80, 0xfd, 0x10, 0x8f, 0xf6, 0x7e, 0x0a, 0x43, 0xbd, 0xa7, 0x1d, 0xab, 0x74,
+	0xe7, 0x93, 0x93, 0xe9, 0xe3, 0x4f, 0x58, 0x64, 0x46, 0xc3, 0x0f, 0x17, 0xd0, 0x14, 0x7c, 0x99,
+	0xf3, 0x92, 0x48, 0xdd, 0x81, 0x5e, 0xc9, 0x18, 0x43, 0x87, 0xd4, 0x4d, 0xf4, 0x0c, 0x26, 0xcd,
+	0xd5, 0xb6, 0xa2, 0xe2, 0x03, 0x29, 0x3a, 0xc7, 0xd5, 0xce, 0xf8, 0x81, 0x2a, 0x6d, 0xf6, 0x0e,
+	0x86, 0xf6, 0xef, 0x28, 0x84, 0xc1, 0x7a, 0x99, 0x05, 0xbd, 0xe8, 0xe8, 0xe6, 0x36, 0xf1, 0x2d,
+	0x5e, 0x2f, 0x33, 0x95, 0x5c, 0xac, 0xb2, 0xc0, 0xf9, 0x37, 0xb9, 0x58, 0x65, 0x28, 0x02, 0xf7,
+	0x7c, 0xb9, 0xce, 0x82, 0x7e, 0x14, 0xdc, 0xdc, 0x26, 0x23, 0x1b, 0x29, 0x16, 0xb9, 0x5f, 0xbf,
+	0xc5, 0xbd, 0xb3, 0xf0, 0xe7, 0x7d, 0xdc, 0xfb, 0x7d, 0x1f, 0x3b, 0x9f, 0xdb, 0xd8, 0xf9, 0xde,
+	0xc6, 0xce, 0x8f, 0x36, 0x76, 0x7e, 0xb5, 0xb1, 0xb3, 0x3d, 0xd0, 0xaf, 0x79, 0xf9, 0x37, 0x00,
+	0x00, 0xff, 0xff, 0x55, 0x29, 0x75, 0x5c, 0xd7, 0x02, 0x00, 0x00,
 }
@@ -37,6 +37,9 @@ message EndpointRecord {

 	// List of aliases task specific aliases
 	repeated string task_aliases = 8;
+
+	// Whether this enpoint's service has been disabled
+	bool service_disabled = 9;
 }

 // PortConfig specifies an exposed port which can be
@@ -137,15 +137,16 @@ type bridgeNetwork struct {
 }

 type driver struct {
-	config         *configuration
-	network        *bridgeNetwork
-	natChain       *iptables.ChainInfo
-	filterChain    *iptables.ChainInfo
-	isolationChain *iptables.ChainInfo
-	networks       map[string]*bridgeNetwork
-	store          datastore.DataStore
-	nlh            *netlink.Handle
-	configNetwork  sync.Mutex
+	config          *configuration
+	network         *bridgeNetwork
+	natChain        *iptables.ChainInfo
+	filterChain     *iptables.ChainInfo
+	isolationChain1 *iptables.ChainInfo
+	isolationChain2 *iptables.ChainInfo
+	networks        map[string]*bridgeNetwork
+	store           datastore.DataStore
+	nlh             *netlink.Handle
+	configNetwork   sync.Mutex
 	sync.Mutex
 }

@@ -266,15 +267,15 @@ func (n *bridgeNetwork) registerIptCleanFunc(clean iptableCleanFunc) {
 	n.iptCleanFuncs = append(n.iptCleanFuncs, clean)
 }

-func (n *bridgeNetwork) getDriverChains() (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
+func (n *bridgeNetwork) getDriverChains() (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
 	n.Lock()
 	defer n.Unlock()

 	if n.driver == nil {
-		return nil, nil, nil, types.BadRequestErrorf("no driver found")
+		return nil, nil, nil, nil, types.BadRequestErrorf("no driver found")
 	}

-	return n.driver.natChain, n.driver.filterChain, n.driver.isolationChain, nil
+	return n.driver.natChain, n.driver.filterChain, n.driver.isolationChain1, n.driver.isolationChain2, nil
 }

 func (n *bridgeNetwork) getNetworkBridgeName() string {
@@ -311,33 +312,18 @@ func (n *bridgeNetwork) isolateNetwork(others []*bridgeNetwork, enable bool) err
 		return nil
 	}

-	// Install the rules to isolate this networks against each of the other networks
-	for _, o := range others {
-		o.Lock()
-		otherConfig := o.config
-		o.Unlock()
-
-		if otherConfig.Internal {
-			continue
-		}
-
-		if thisConfig.BridgeName != otherConfig.BridgeName {
-			if err := setINC(thisConfig.BridgeName, otherConfig.BridgeName, enable); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
+	// Install the rules to isolate this network against each of the other networks
+	return setINC(thisConfig.BridgeName, enable)
 }

 func (d *driver) configure(option map[string]interface{}) error {
 	var (
-		config         *configuration
-		err            error
-		natChain       *iptables.ChainInfo
-		filterChain    *iptables.ChainInfo
-		isolationChain *iptables.ChainInfo
+		config          *configuration
+		err             error
+		natChain        *iptables.ChainInfo
+		filterChain     *iptables.ChainInfo
+		isolationChain1 *iptables.ChainInfo
+		isolationChain2 *iptables.ChainInfo
 	)

 	genericData, ok := option[netlabel.GenericData]
@@ -365,7 +351,7 @@ func (d *driver) configure(option map[string]interface{}) error {
 			}
 		}
 		removeIPChains()
-		natChain, filterChain, isolationChain, err = setupIPChains(config)
+		natChain, filterChain, isolationChain1, isolationChain2, err = setupIPChains(config)
 		if err != nil {
 			return err
 		}
@@ -384,7 +370,8 @@ func (d *driver) configure(option map[string]interface{}) error {
 	d.Lock()
 	d.natChain = natChain
 	d.filterChain = filterChain
-	d.isolationChain = isolationChain
+	d.isolationChain1 = isolationChain1
+	d.isolationChain2 = isolationChain2
 	d.config = config
 	d.Unlock()

@@ -12,52 +12,85 @@ import (

 // DockerChain: DOCKER iptable chain name
 const (
-	DockerChain    = "DOCKER"
-	IsolationChain = "DOCKER-ISOLATION"
+	DockerChain = "DOCKER"
+	// Isolation between bridge networks is achieved in two stages by means
+	// of the following two chains in the filter table. The first chain matches
+	// on the source interface being a bridge network's bridge and the
+	// destination being a different interface. A positive match leads to the
+	// second isolation chain. No match returns to the parent chain. The second
+	// isolation chain matches on destination interface being a bridge network's
+	// bridge. A positive match identifies a packet originated from one bridge
+	// network's bridge destined to another bridge network's bridge and will
+	// result in the packet being dropped. No match returns to the parent chain.
+	IsolationChain1 = "DOCKER-ISOLATION-STAGE-1"
+	IsolationChain2 = "DOCKER-ISOLATION-STAGE-2"
 )

-func setupIPChains(config *configuration) (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
+func setupIPChains(config *configuration) (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
 	// Sanity check.
 	if config.EnableIPTables == false {
-		return nil, nil, nil, errors.New("cannot create new chains, EnableIPTable is disabled")
+		return nil, nil, nil, nil, errors.New("cannot create new chains, EnableIPTable is disabled")
 	}

 	hairpinMode := !config.EnableUserlandProxy

 	natChain, err := iptables.NewChain(DockerChain, iptables.Nat, hairpinMode)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to create NAT chain: %v", err)
+		return nil, nil, nil, nil, fmt.Errorf("failed to create NAT chain %s: %v", DockerChain, err)
 	}
 	defer func() {
 		if err != nil {
 			if err := iptables.RemoveExistingChain(DockerChain, iptables.Nat); err != nil {
-				logrus.Warnf("failed on removing iptables NAT chain on cleanup: %v", err)
+				logrus.Warnf("failed on removing iptables NAT chain %s on cleanup: %v", DockerChain, err)
 			}
 		}
 	}()

 	filterChain, err := iptables.NewChain(DockerChain, iptables.Filter, false)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to create FILTER chain: %v", err)
+		return nil, nil, nil, nil, fmt.Errorf("failed to create FILTER chain %s: %v", DockerChain, err)
 	}
 	defer func() {
 		if err != nil {
 			if err := iptables.RemoveExistingChain(DockerChain, iptables.Filter); err != nil {
-				logrus.Warnf("failed on removing iptables FILTER chain on cleanup: %v", err)
+				logrus.Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", DockerChain, err)
 			}
 		}
 	}()

-	isolationChain, err := iptables.NewChain(IsolationChain, iptables.Filter, false)
+	isolationChain1, err := iptables.NewChain(IsolationChain1, iptables.Filter, false)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to create FILTER isolation chain: %v", err)
+		return nil, nil, nil, nil, fmt.Errorf("failed to create FILTER isolation chain: %v", err)
+	}
+	defer func() {
+		if err != nil {
+			if err := iptables.RemoveExistingChain(IsolationChain1, iptables.Filter); err != nil {
+				logrus.Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", IsolationChain1, err)
+			}
+		}
+	}()
+
+	isolationChain2, err := iptables.NewChain(IsolationChain2, iptables.Filter, false)
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("failed to create FILTER isolation chain: %v", err)
+	}
+	defer func() {
+		if err != nil {
+			if err := iptables.RemoveExistingChain(IsolationChain2, iptables.Filter); err != nil {
+				logrus.Warnf("failed on removing iptables FILTER chain %s on cleanup: %v", IsolationChain2, err)
+			}
+		}
+	}()
+
+	if err := iptables.AddReturnRule(IsolationChain1); err != nil {
+		return nil, nil, nil, nil, err
 	}

-	if err := iptables.AddReturnRule(IsolationChain); err != nil {
-		return nil, nil, nil, err
+	if err := iptables.AddReturnRule(IsolationChain2); err != nil {
+		return nil, nil, nil, nil, err
 	}

-	return natChain, filterChain, isolationChain, nil
+	return natChain, filterChain, isolationChain1, isolationChain2, nil
 }

 func (n *bridgeNetwork) setupIPTables(config *networkConfiguration, i *bridgeInterface) error {
@@ -94,7 +127,7 @@ func (n *bridgeNetwork) setupIPTables(config *networkConfiguration, i *bridgeInt
 		n.registerIptCleanFunc(func() error {
 			return setupIPTablesInternal(config.BridgeName, maskedAddrv4, config.EnableICC, config.EnableIPMasquerade, hairpinMode, false)
 		})
-		natChain, filterChain, _, err := n.getDriverChains()
+		natChain, filterChain, _, _, err := n.getDriverChains()
 		if err != nil {
 			return fmt.Errorf("Failed to setup IP tables, cannot acquire chain info %s", err.Error())
 		}
@@ -117,7 +150,7 @@ func (n *bridgeNetwork) setupIPTables(config *networkConfiguration, i *bridgeInt
 	}

 	d.Lock()
-	err = iptables.EnsureJumpRule("FORWARD", IsolationChain)
+	err = iptables.EnsureJumpRule("FORWARD", IsolationChain1)
 	d.Unlock()
 	if err != nil {
 		return err
@@ -245,42 +278,56 @@ func setIcc(bridgeIface string, iccEnable, insert bool) error {
 	return nil
 }

-// Control Inter Network Communication. Install/remove only if it is not/is present.
-func setINC(iface1, iface2 string, enable bool) error {
+// Control Inter Network Communication. Install[Remove] only if it is [not] present.
+func setINC(iface string, enable bool) error {
 	var (
-		table = iptables.Filter
-		chain = IsolationChain
-		args  = [2][]string{{"-i", iface1, "-o", iface2, "-j", "DROP"}, {"-i", iface2, "-o", iface1, "-j", "DROP"}}
+		action    = iptables.Insert
+		actionMsg = "add"
+		chains    = []string{IsolationChain1, IsolationChain2}
+		rules     = [][]string{
+			{"-i", iface, "!", "-o", iface, "-j", IsolationChain2},
+			{"-o", iface, "-j", "DROP"},
+		}
 	)

-	if enable {
-		for i := 0; i < 2; i++ {
-			if iptables.Exists(table, chain, args[i]...) {
-				continue
-			}
-			if err := iptables.RawCombinedOutput(append([]string{"-I", chain}, args[i]...)...); err != nil {
-				return fmt.Errorf("unable to add inter-network communication rule: %v", err)
-			}
-		}
-	} else {
-		for i := 0; i < 2; i++ {
-			if !iptables.Exists(table, chain, args[i]...) {
-				continue
-			}
-			if err := iptables.RawCombinedOutput(append([]string{"-D", chain}, args[i]...)...); err != nil {
-				return fmt.Errorf("unable to remove inter-network communication rule: %v", err)
+	if !enable {
+		action = iptables.Delete
+		actionMsg = "remove"
+	}
+
+	for i, chain := range chains {
+		if err := iptables.ProgramRule(iptables.Filter, chain, action, rules[i]); err != nil {
+			msg := fmt.Sprintf("unable to %s inter-network communication rule: %v", actionMsg, err)
+			if enable {
+				if i == 1 {
+					// Rollback the rule installed on first chain
+					if err2 := iptables.ProgramRule(iptables.Filter, chains[0], iptables.Delete, rules[0]); err2 != nil {
+						logrus.Warn("Failed to rollback iptables rule after failure (%v): %v", err, err2)
+					}
+				}
+				return fmt.Errorf(msg)
 			}
+			logrus.Warn(msg)
 		}
 	}

 	return nil
 }

+// Obsolete chain from previous docker versions
+const oldIsolationChain = "DOCKER-ISOLATION"
+
 func removeIPChains() {
+	// Remove obsolete rules from default chains
+	iptables.ProgramRule(iptables.Filter, "FORWARD", iptables.Delete, []string{"-j", oldIsolationChain})
+
+	// Remove chains
 	for _, chainInfo := range []iptables.ChainInfo{
 		{Name: DockerChain, Table: iptables.Nat},
 		{Name: DockerChain, Table: iptables.Filter},
-		{Name: IsolationChain, Table: iptables.Filter},
+		{Name: IsolationChain1, Table: iptables.Filter},
+		{Name: IsolationChain2, Table: iptables.Filter},
+		{Name: oldIsolationChain, Table: iptables.Filter},
 	} {
 		if err := chainInfo.Remove(); err != nil {
 			logrus.Warnf("Failed to remove existing iptables entries in table %s chain %s : %v", chainInfo.Table, chainInfo.Name, err)
@@ -290,8 +337,8 @@ func removeIPChains() {

 func setupInternalNetworkRules(bridgeIface string, addr net.Addr, icc, insert bool) error {
 	var (
-		inDropRule  = iptRule{table: iptables.Filter, chain: IsolationChain, args: []string{"-i", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}}
-		outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain, args: []string{"-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}}
+		inDropRule  = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{"-i", bridgeIface, "!", "-d", addr.String(), "-j", "DROP"}}
+		outDropRule = iptRule{table: iptables.Filter, chain: IsolationChain1, args: []string{"-o", bridgeIface, "!", "-s", addr.String(), "-j", "DROP"}}
 	)
 	if err := programChainRule(inDropRule, "DROP INCOMING", insert); err != nil {
 		return err
@@ -3,15 +3,36 @@ package overlay
 import (
 	"io/ioutil"
 	"path"
+	"strconv"
 	"strings"

 	"github.com/sirupsen/logrus"
 )

-var sysctlConf = map[string]string{
-	"net.ipv4.neigh.default.gc_thresh1": "8192",
-	"net.ipv4.neigh.default.gc_thresh2": "49152",
-	"net.ipv4.neigh.default.gc_thresh3": "65536",
+type conditionalCheck func(val1, val2 string) bool
+
+type osValue struct {
+	value   string
+	checkFn conditionalCheck
+}
+
+var osConfig = map[string]osValue{
+	"net.ipv4.neigh.default.gc_thresh1": {"8192", checkHigher},
+	"net.ipv4.neigh.default.gc_thresh2": {"49152", checkHigher},
+	"net.ipv4.neigh.default.gc_thresh3": {"65536", checkHigher},
+}
+
+func propertyIsValid(val1, val2 string, check conditionalCheck) bool {
+	if check == nil || check(val1, val2) {
+		return true
+	}
+	return false
+}
+
+func checkHigher(val1, val2 string) bool {
+	val1Int, _ := strconv.ParseInt(val1, 10, 32)
+	val2Int, _ := strconv.ParseInt(val2, 10, 32)
+	return val1Int < val2Int
 }

 // writeSystemProperty writes the value to a path under /proc/sys as determined from the key.
@@ -21,10 +42,31 @@ func writeSystemProperty(key, value string) error {
 	return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644)
 }

+func readSystemProperty(key string) (string, error) {
+	keyPath := strings.Replace(key, ".", "/", -1)
+	value, err := ioutil.ReadFile(path.Join("/proc/sys", keyPath))
+	if err != nil {
+		return "", err
+	}
+	return string(value), nil
+}
+
 func applyOStweaks() {
-	for k, v := range sysctlConf {
-		if err := writeSystemProperty(k, v); err != nil {
-			logrus.Errorf("error setting the kernel parameter %s = %s, err: %s", k, v, err)
+	for k, v := range osConfig {
+		// read the existing property from disk
+		oldv, err := readSystemProperty(k)
+		if err != nil {
+			logrus.Errorf("error reading the kernel parameter %s, error: %s", k, err)
+			continue
+		}
+
+		if propertyIsValid(oldv, v.value, v.checkFn) {
+			// write new prop value to disk
+			if err := writeSystemProperty(k, v.value); err != nil {
+				logrus.Errorf("error setting the kernel parameter %s = %s, (leaving as %s) error: %s", k, v.value, oldv, err)
+				continue
+			}
+			logrus.Debugf("updated kernel parameter %s = %s (was %s)", k, v.value, oldv)
 		}
 	}
 }
@@ -620,7 +620,7 @@ func (ep *endpoint) rename(name string) error {
 	}

 	if c.isAgent() {
-		if err = ep.deleteServiceInfoFromCluster(sb, "rename"); err != nil {
+		if err = ep.deleteServiceInfoFromCluster(sb, true, "rename"); err != nil {
 			return types.InternalErrorf("Could not delete service state for endpoint %s from cluster on rename: %v", ep.Name(), err)
 		}
 	} else {
@@ -644,7 +644,7 @@ func (ep *endpoint) rename(name string) error {
 		}
 		defer func() {
 			if err != nil {
-				ep.deleteServiceInfoFromCluster(sb, "rename")
+				ep.deleteServiceInfoFromCluster(sb, true, "rename")
 				ep.name = oldName
 				ep.anonymous = oldAnonymous
 				ep.addServiceInfoToCluster(sb)
@@ -755,8 +755,14 @@ func (ep *endpoint) sbLeave(sb *sandbox, force bool, options ...EndpointOption)
 		}
 	}

+	if ep.svcID != "" {
+		if err := ep.deleteServiceInfoFromCluster(sb, true, "sbLeave"); err != nil {
+			logrus.Warnf("Failed to clean up service info on container %s disconnect: %v", ep.name, err)
+		}
+	}
+
 	if err := sb.clearNetworkResources(ep); err != nil {
-		logrus.Warnf("Could not cleanup network resources on container %s disconnect: %v", ep.name, err)
+		logrus.Warnf("Failed to clean up network resources on container %s disconnect: %v", ep.name, err)
 	}

 	// Update the store about the sandbox detach only after we
@@ -769,7 +775,7 @@ func (ep *endpoint) sbLeave(sb *sandbox, force bool, options ...EndpointOption)
 	}

 	if e := ep.deleteDriverInfoFromCluster(); e != nil {
-		logrus.Errorf("Could not delete endpoint state for endpoint %s from cluster: %v", ep.Name(), e)
+		logrus.Errorf("Failed to delete endpoint state for endpoint %s from cluster: %v", ep.Name(), e)
 	}

 	sb.deleteHostsEntries(n.getSvcRecords(ep))
@@ -21,8 +21,8 @@ func (nDB *NetworkDB) handleNodeEvent(nEvent *NodeEvent) bool {
 	// time.
 	nDB.networkClock.Witness(nEvent.LTime)

-	nDB.RLock()
-	defer nDB.RUnlock()
+	nDB.Lock()
+	defer nDB.Unlock()

 	// check if the node exists
 	n, _, _ := nDB.findNode(nEvent.NodeName)
@@ -313,7 +313,7 @@ func (nDB *NetworkDB) Peers(nid string) []PeerInfo {
 		} else {
 			// Added for testing purposes, this condition should never happen else mean that the network list
 			// is out of sync with the node list
-			peers = append(peers, PeerInfo{})
+			peers = append(peers, PeerInfo{Name: nodeName, IP: "unknown"})
 		}
 	}
 	return peers
@@ -84,7 +84,11 @@ func dbPeers(ctx interface{}, w http.ResponseWriter, r *http.Request) {
 		peers := nDB.Peers(r.Form["nid"][0])
 		rsp := &diagnostic.TableObj{Length: len(peers)}
 		for i, peerInfo := range peers {
-			rsp.Elements = append(rsp.Elements, &diagnostic.PeerEntryObj{Index: i, Name: peerInfo.Name, IP: peerInfo.IP})
+			if peerInfo.IP == "unknown" {
+				rsp.Elements = append(rsp.Elements, &diagnostic.PeerEntryObj{Index: i, Name: "orphan-" + peerInfo.Name, IP: peerInfo.IP})
+			} else {
+				rsp.Elements = append(rsp.Elements, &diagnostic.PeerEntryObj{Index: i, Name: peerInfo.Name, IP: peerInfo.IP})
+			}
 		}
 		log.WithField("response", fmt.Sprintf("%+v", rsp)).Info("network peers done")
 		diagnostic.HTTPReply(w, diagnostic.CommandSucceed(rsp), json)
@@ -452,7 +452,6 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {
 				logrus.Warnf("[resolver] connect failed: %s", err)
 				continue
 			}
-
 			queryType := dns.TypeToString[query.Question[0].Qtype]
 			logrus.Debugf("[resolver] query %s (%s) from %s, forwarding to %s:%s", name, queryType,
 				extConn.LocalAddr().String(), proto, extDNS.IPStr)
@@ -492,6 +491,11 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {
 			}
 			r.forwardQueryEnd()
 			if resp != nil {
+				if resp.Rcode == dns.RcodeServerFailure {
+					// for Server Failure response, continue to the next external DNS server
+					logrus.Debugf("[resolver] external DNS %s:%s responded with ServFail for %q", proto, extDNS.IPStr, name)
+					continue
+				}
 				answers := 0
 				for _, rr := range resp.Answer {
 					h := rr.Header()
@@ -511,10 +515,10 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {
 				if resp.Answer == nil || answers == 0 {
 					logrus.Debugf("[resolver] external DNS %s:%s did not return any %s records for %q", proto, extDNS.IPStr, queryType, name)
 				}
+				resp.Compress = true
 			} else {
 				logrus.Debugf("[resolver] external DNS %s:%s returned empty response for %q", proto, extDNS.IPStr, name)
 			}
-			resp.Compress = true
 			break
 		}
 		if resp == nil {
@@ -703,7 +703,7 @@ func (sb *sandbox) DisableService() (err error) {
 	}()
 	for _, ep := range sb.getConnectedEndpoints() {
 		if ep.isServiceEnabled() {
-			if err := ep.deleteServiceInfoFromCluster(sb, "DisableService"); err != nil {
+			if err := ep.deleteServiceInfoFromCluster(sb, false, "DisableService"); err != nil {
 				failedEps = append(failedEps, ep.Name())
 				logrus.Warnf("failed update state for endpoint %s into cluster: %v", ep.Name(), err)
 			}
@@ -79,13 +79,18 @@ func (s *service) printIPToEndpoint(ip string) (string, bool) {
 	return s.ipToEndpoint.String(ip)
 }

+type lbBackend struct {
+	ip       net.IP
+	disabled bool
+}
+
 type loadBalancer struct {
 	vip    net.IP
 	fwMark uint32

 	// Map of backend IPs backing this loadbalancer on this
 	// network. It is keyed with endpoint ID.
-	backEnds map[string]net.IP
+	backEnds map[string]*lbBackend

 	// Back pointer to service to which the loadbalancer belongs.
 	service *service
@@ -198,23 +198,8 @@ func (c *controller) cleanupServiceBindings(cleanupNID string) {
 			if cleanupNID != "" && nid != cleanupNID {
 				continue
 			}
-
-			for eid, ip := range lb.backEnds {
-				epID := eid
-				epIP := ip
-				service := s
-				loadBalancer := lb
-				networkID := nid
-				cleanupFuncs = append(cleanupFuncs, func() {
-					// ContainerName and taskAliases are not available here, this is still fine because the Service discovery
-					// cleanup already happened before. The only thing that rmServiceBinding is still doing here a part from the Load
-					// Balancer bookeeping, is to keep consistent the mapping of endpoint to IP.
-					if err := c.rmServiceBinding(service.name, service.id, networkID, epID, "", loadBalancer.vip,
-						service.ingressPorts, service.aliases, []string{}, epIP, "cleanupServiceBindings", false); err != nil {
-						logrus.Errorf("Failed to remove service bindings for service %s network %s endpoint %s while cleanup: %v",
-							service.id, networkID, epID, err)
-					}
-				})
+			for eid, be := range lb.backEnds {
+				cleanupFuncs = append(cleanupFuncs, makeServiceCleanupFunc(c, s, nid, eid, lb.vip, be.ip))
 			}
 		}
 		s.Unlock()
@@ -226,6 +211,17 @@ func (c *controller) cleanupServiceBindings(cleanupNID string) {

 }

+func makeServiceCleanupFunc(c *controller, s *service, nID, eID string, vip net.IP, ip net.IP) func() {
+	// ContainerName and taskAliases are not available here, this is still fine because the Service discovery
+	// cleanup already happened before. The only thing that rmServiceBinding is still doing here a part from the Load
+	// Balancer bookeeping, is to keep consistent the mapping of endpoint to IP.
+	return func() {
+		if err := c.rmServiceBinding(s.name, s.id, nID, eID, "", vip, s.ingressPorts, s.aliases, []string{}, ip, "cleanupServiceBindings", false, true); err != nil {
+			logrus.Errorf("Failed to remove service bindings for service %s network %s endpoint %s while cleanup: %v", s.id, nID, eID, err)
+		}
+	}
+}
+
 func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName string, vip net.IP, ingressPorts []*PortConfig, serviceAliases, taskAliases []string, ip net.IP, method string) error {
 	var addService bool

@@ -271,7 +267,7 @@ func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName s
 		lb = &loadBalancer{
 			vip:      vip,
 			fwMark:   fwMarkCtr,
-			backEnds: make(map[string]net.IP),
+			backEnds: make(map[string]*lbBackend),
 			service:  s,
 		}

@@ -282,7 +278,7 @@ func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName s
 		addService = true
 	}

-	lb.backEnds[eID] = ip
+	lb.backEnds[eID] = &lbBackend{ip, false}

 	ok, entries := s.assignIPToEndpoint(ip.String(), eID)
 	if !ok || entries > 1 {
@@ -307,7 +303,7 @@ func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName s
 	return nil
 }

-func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName string, vip net.IP, ingressPorts []*PortConfig, serviceAliases []string, taskAliases []string, ip net.IP, method string, deleteSvcRecords bool) error {
+func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName string, vip net.IP, ingressPorts []*PortConfig, serviceAliases []string, taskAliases []string, ip net.IP, method string, deleteSvcRecords bool, fullRemove bool) error {

 	var rmService bool

@@ -338,13 +334,19 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st
 		return nil
 	}

-	_, ok = lb.backEnds[eID]
+	be, ok := lb.backEnds[eID]
 	if !ok {
-		logrus.Warnf("rmServiceBinding %s %s %s aborted lb.backEnds[eid] !ok", method, svcName, eID)
+		logrus.Warnf("rmServiceBinding %s %s %s aborted lb.backEnds[eid] && lb.disabled[eid] !ok", method, svcName, eID)
 		return nil
 	}

-	delete(lb.backEnds, eID)
+	if fullRemove {
+		// delete regardless
+		delete(lb.backEnds, eID)
+	} else {
+		be.disabled = true
+	}
+
 	if len(lb.backEnds) == 0 {
 		// All the backends for this service have been
 		// removed. Time to remove the load balancer and also
@@ -367,7 +369,7 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st
 	// Remove loadbalancer service(if needed) and backend in all
 	// sandboxes in the network only if the vip is valid.
 	if len(vip) != 0 && entries == 0 {
-		n.(*network).rmLBBackend(ip, vip, lb, ingressPorts, rmService)
+		n.(*network).rmLBBackend(ip, vip, lb, ingressPorts, rmService, fullRemove)
 	}

 	// Delete the name resolutions
@@ -103,8 +103,10 @@ func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
 		}

 		lb.service.Lock()
-		for _, ip := range lb.backEnds {
-			sb.addLBBackend(ip, lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, gwIP, n.ingress)
+		for _, be := range lb.backEnds {
+			if !be.disabled {
+				sb.addLBBackend(be.ip, lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, gwIP, n.ingress)
+			}
 		}
 		lb.service.Unlock()
 	}
@@ -135,7 +137,7 @@ func (n *network) addLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []
 // Remove loadbalancer backend from all sandboxes which has a
 // connection to this network. If needed remove the service entry as
 // well, as specified by the rmService bool.
-func (n *network) rmLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig, rmService bool) {
+func (n *network) rmLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig, rmService bool, fullRemove bool) {
 	n.WalkEndpoints(func(e Endpoint) bool {
 		ep := e.(*endpoint)
 		if sb, ok := ep.getSandbox(); ok {
@@ -148,7 +150,7 @@ func (n *network) rmLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*
 				gwIP = ep.Iface().Address().IP
 			}

-			sb.rmLBBackend(ip, vip, lb.fwMark, ingressPorts, ep.Iface().Address(), gwIP, rmService, n.ingress)
+			sb.rmLBBackend(ip, vip, lb.fwMark, ingressPorts, ep.Iface().Address(), gwIP, rmService, fullRemove, n.ingress)
 		}

 		return false
@@ -215,7 +217,7 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P
 }

 // Remove loadbalancer backend from one connected sandbox.
-func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, gwIP net.IP, rmService bool, isIngressNetwork bool) {
+func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, gwIP net.IP, rmService bool, fullRemove bool, isIngressNetwork bool) {
 	if sb.osSbox == nil {
 		return
 	}
@@ -242,8 +244,15 @@ func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*Po
 		Weight:        1,
 	}

-	if err := i.DelDestination(s, d); err != nil && err != syscall.ENOENT {
-		logrus.Errorf("Failed to delete real server %s for vip %s fwmark %d in sbox %s (%s): %v", ip, vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
+	if fullRemove {
+		if err := i.DelDestination(s, d); err != nil && err != syscall.ENOENT {
+			logrus.Errorf("Failed to delete real server %s for vip %s fwmark %d in sbox %s (%s): %v", ip, vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
+		}
+	} else {
+		d.Weight = 0
+		if err := i.UpdateDestination(s, d); err != nil && err != syscall.ENOENT {
+			logrus.Errorf("Failed to set LB weight of real server %s to 0 for vip %s fwmark %d in sbox %s (%s): %v", ip, vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
+		}
 	}

 	if rmService {
@@ -44,7 +44,10 @@ func (n *network) addLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []

 		var endpoints []hcsshim.HNSEndpoint

-		for eid := range lb.backEnds {
+		for eid, be := range lb.backEnds {
+			if be.disabled {
+				continue
+			}
 			//Call HNS to get back ID (GUID) corresponding to the endpoint.
 			hnsEndpoint, err := hcsshim.GetHNSEndpointByName(eid)
 			if err != nil {
@@ -114,9 +117,9 @@ func (n *network) addLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []
 	}
 }

-func (n *network) rmLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig, rmService bool) {
+func (n *network) rmLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig, rmService bool, fullRemove bool) {
 	if system.GetOSVersion().Build > 16236 {
-		if len(lb.backEnds) > 0 {
+		if numEnabledBackends(lb) > 0 {
 			//Reprogram HNS (actually VFP) with the existing backends.
 			n.addLBBackend(ip, vip, lb, ingressPorts)
 		} else {
@@ -143,6 +146,16 @@ func (n *network) rmLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*
 	}
 }

+func numEnabledBackends(lb *loadBalancer) int {
+	nEnabled := 0
+	for _, be := range lb.backEnds {
+		if !be.disabled {
+			nEnabled++
+		}
+	}
+	return nEnabled
+}
+
 func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
 }