From ea569af511fd54fe8aba20aba497c00dcea41333 Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Mon, 19 Jun 2017 19:42:24 -0700 Subject: [PATCH] Libnetwork vendoring Fixes in the Service Discovery areas Signed-off-by: Flavio Crisciani Upstream-commit: e1953f1ca717c2609a0ee801c8af5b874d9fc3d2 Component: engine --- components/engine/vendor.conf | 2 +- .../github.com/docker/libnetwork/agent.go | 27 +-- .../docker/libnetwork/common/setmatrix.go | 32 ++-- .../docker/libnetwork/controller.go | 5 +- .../libnetwork/drivers/overlay/ov_network.go | 9 +- .../drivers/solaris/overlay/ov_network.go | 9 +- .../libnetwork/drivers/windows/windows.go | 9 + .../drivers/windows/windows_store.go | 14 +- .../docker/libnetwork/ipam/allocator.go | 2 +- .../docker/libnetwork/ipams/remote/api/api.go | 2 +- .../github.com/docker/libnetwork/network.go | 164 ++++++++++-------- .../libnetwork/osl/namespace_unsupported.go | 1 + .../github.com/docker/libnetwork/sandbox.go | 4 + .../github.com/docker/libnetwork/service.go | 8 +- .../docker/libnetwork/service_common.go | 114 +++++++----- .../docker/libnetwork/service_linux.go | 6 +- 16 files changed, 237 insertions(+), 171 deletions(-) diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index e524c518ab..e23fe0bf14 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -26,7 +26,7 @@ github.com/imdario/mergo 0.2.1 golang.org/x/sync de49d9dcd27d4f764488181bea099dfe6179bcf0 #get libnetwork packages -github.com/docker/libnetwork f4a15a0890383619ad797b3bd2481cc6f46a978d +github.com/docker/libnetwork 6426d1e66f33c0b0c8bb135b7ee547447f54d043 github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec diff --git a/components/engine/vendor/github.com/docker/libnetwork/agent.go b/components/engine/vendor/github.com/docker/libnetwork/agent.go index 3fa5341481..a45a569500 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/agent.go +++ b/components/engine/vendor/github.com/docker/libnetwork/agent.go @@ -165,13 +165,13 @@ func (c *controller) handleKeyChange(keys []*types.EncryptionKey) error { a.networkDB.SetKey(added) } - key, tag, err := c.getPrimaryKeyTag(subsysGossip) + key, _, err := c.getPrimaryKeyTag(subsysGossip) if err != nil { return err } a.networkDB.SetPrimaryKey(key) - key, tag, err = c.getPrimaryKeyTag(subsysIPSec) + key, tag, err := c.getPrimaryKeyTag(subsysIPSec) if err != nil { return err } @@ -648,13 +648,13 @@ func (ep *endpoint) addServiceInfoToCluster(sb *sandbox) error { TaskAliases: ep.myAliases, EndpointIP: ep.Iface().Address().IP.String(), }) - if err != nil { return err } if agent != nil { if err := agent.networkDB.CreateEntry(libnetworkEPTable, n.ID(), ep.ID(), buf); err != nil { + logrus.Warnf("addServiceInfoToCluster NetworkDB CreateEntry failed for %s %s err:%s", ep.id, n.id, err) return err } } @@ -686,6 +686,13 @@ func (ep *endpoint) deleteServiceInfoFromCluster(sb *sandbox, method string) err name = ep.MyAliases()[0] } + if agent != nil { + // First delete from networkDB then locally + if err := agent.networkDB.DeleteEntry(libnetworkEPTable, n.ID(), ep.ID()); err != nil { + logrus.Warnf("deleteServiceInfoFromCluster NetworkDB DeleteEntry failed for %s %s err:%s", ep.id, n.id, err) + } + } + if ep.Iface().Address() != nil { if ep.svcID != "" { // This is a task part of a service @@ -693,7 +700,7 @@ func (ep *endpoint) deleteServiceInfoFromCluster(sb *sandbox, method string) err if n.ingress { ingressPorts = ep.ingressPorts } - if err := c.rmServiceBinding(ep.svcName, ep.svcID, n.ID(), ep.ID(), name, ep.virtualIP, ingressPorts, ep.svcAliases, ep.myAliases, ep.Iface().Address().IP, "deleteServiceInfoFromCluster"); err != nil { + if err := c.rmServiceBinding(ep.svcName, ep.svcID, n.ID(), ep.ID(), name, ep.virtualIP, ingressPorts, ep.svcAliases, ep.myAliases, ep.Iface().Address().IP, "deleteServiceInfoFromCluster", true); err != nil { return err } } else { @@ -704,12 +711,6 @@ func (ep *endpoint) deleteServiceInfoFromCluster(sb *sandbox, method string) err } } - if agent != nil { - if err := agent.networkDB.DeleteEntry(libnetworkEPTable, n.ID(), ep.ID()); err != nil { - return err - } - } - logrus.Debugf("deleteServiceInfoFromCluster from %s END for %s %s", method, ep.svcName, ep.ID()) return nil @@ -883,7 +884,7 @@ func (c *controller) handleEpTableEvent(ev events.Event) { } if isAdd { - logrus.Debugf("handleEpTableEvent ADD %s R:%v", isAdd, eid, epRec) + logrus.Debugf("handleEpTableEvent ADD %s R:%v", eid, epRec) if svcID != "" { // This is a remote task part of a service if err := c.addServiceBinding(svcName, svcID, nid, eid, containerName, vip, ingressPorts, serviceAliases, taskAliases, ip, "handleEpTableEvent"); err != nil { @@ -897,10 +898,10 @@ func (c *controller) handleEpTableEvent(ev events.Event) { } } } else { - logrus.Debugf("handleEpTableEvent DEL %s R:%v", isAdd, eid, epRec) + logrus.Debugf("handleEpTableEvent DEL %s R:%v", eid, epRec) if svcID != "" { // This is a remote task part of a service - if err := c.rmServiceBinding(svcName, svcID, nid, eid, containerName, vip, ingressPorts, serviceAliases, taskAliases, ip, "handleEpTableEvent"); err != nil { + if err := c.rmServiceBinding(svcName, svcID, nid, eid, containerName, vip, ingressPorts, serviceAliases, taskAliases, ip, "handleEpTableEvent", true); err != nil { logrus.Errorf("failed removing service binding for %s epRec:%v err:%s", eid, epRec, err) return } diff --git a/components/engine/vendor/github.com/docker/libnetwork/common/setmatrix.go b/components/engine/vendor/github.com/docker/libnetwork/common/setmatrix.go index 0fdb542be4..72be5bbbfc 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/common/setmatrix.go +++ b/components/engine/vendor/github.com/docker/libnetwork/common/setmatrix.go @@ -10,24 +10,26 @@ import ( type SetMatrix interface { // Get returns the members of the set for a specific key as a slice. Get(key string) ([]interface{}, bool) - // Contains is used to verify is an element is in a set for a specific key + // Contains is used to verify if an element is in a set for a specific key // returns true if the element is in the set // returns true if there is a set for the key Contains(key string, value interface{}) (bool, bool) - // Insert inserts the mapping between the IP and the endpoint identifier - // returns true if the mapping was not present, false otherwise - // returns also the number of endpoints associated to the IP + // Insert inserts the value in the set of a key + // returns true if the value is inserted (was not already in the set), false otherwise + // returns also the length of the set for the key Insert(key string, value interface{}) (bool, int) - // Remove removes the mapping between the IP and the endpoint identifier - // returns true if the mapping was deleted, false otherwise - // returns also the number of endpoints associated to the IP + // Remove removes the value in the set for a specific key + // returns true if the value is deleted, false otherwise + // returns also the length of the set for the key Remove(key string, value interface{}) (bool, int) - // Cardinality returns the number of elements in the set of a specfic key - // returns false if the key is not in the map + // Cardinality returns the number of elements in the set for a key + // returns false if the set is not present Cardinality(key string) (int, bool) // String returns the string version of the set, empty otherwise - // returns false if the key is not in the map + // returns false if the set is not present String(key string) (string, bool) + // Returns all the keys in the map + Keys() []string } type setMatrix struct { @@ -121,3 +123,13 @@ func (s *setMatrix) String(key string) (string, bool) { } return set.String(), ok } + +func (s *setMatrix) Keys() []string { + s.Lock() + defer s.Unlock() + keys := make([]string, 0, len(s.matrix)) + for k := range s.matrix { + keys = append(keys, k) + } + return keys +} diff --git a/components/engine/vendor/github.com/docker/libnetwork/controller.go b/components/engine/vendor/github.com/docker/libnetwork/controller.go index ae7dac0b82..1696e07067 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/controller.go +++ b/components/engine/vendor/github.com/docker/libnetwork/controller.go @@ -1014,7 +1014,7 @@ func (c *controller) NetworkByID(id string) (Network, error) { } // NewSandbox creates a new sandbox for the passed container id -func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (sBox Sandbox, err error) { +func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (Sandbox, error) { if containerID == "" { return nil, types.BadRequestErrorf("invalid container ID") } @@ -1054,7 +1054,6 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s extDNS: []extDNSEntry{}, } } - sBox = sb heap.Init(&sb.endpoints) @@ -1073,6 +1072,8 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s sb.id = "ingress_sbox" } c.Unlock() + + var err error defer func() { if err != nil { c.Lock() diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go index 6be88d9179..3a4ea41bfc 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go @@ -670,7 +670,7 @@ func (n *network) initSandbox(restore bool) error { // In the restore case network sandbox already exist; but we don't know // what epoch number it was created with. It has to be retrieved by // searching the net namespaces. - key := "" + var key string if restore { key = osl.GenerateKey("-" + n.id) } else { @@ -872,15 +872,10 @@ func (n *network) Value() []byte { netJSON = append(netJSON, sj) } - b, err := json.Marshal(netJSON) - if err != nil { - return []byte{} - } - m["secure"] = n.secure m["subnets"] = netJSON m["mtu"] = n.mtu - b, err = json.Marshal(m) + b, err := json.Marshal(m) if err != nil { return []byte{} } diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_network.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_network.go index 5e3dd5abe1..b0567261c7 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_network.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/solaris/overlay/ov_network.go @@ -457,7 +457,7 @@ func (n *network) initSandbox(restore bool) error { // In the restore case network sandbox already exist; but we don't know // what epoch number it was created with. It has to be retrieved by // searching the net namespaces. - key := "" + var key string if restore { key = osl.GenerateKey("-" + n.id) } else { @@ -570,15 +570,10 @@ func (n *network) Value() []byte { netJSON = append(netJSON, sj) } - b, err := json.Marshal(netJSON) - if err != nil { - return []byte{} - } - m["secure"] = n.secure m["subnets"] = netJSON m["mtu"] = n.mtu - b, err = json.Marshal(m) + b, err := json.Marshal(m) if err != nil { return []byte{} } diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows.go index c1c69f2991..eb1522a74a 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows.go @@ -324,6 +324,9 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d } n, err := d.getNetwork(id) + if err != nil { + return err + } n.created = true return d.storeUpdate(config) } @@ -530,7 +533,13 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, } epOption, err := parseEndpointOptions(epOptions) + if err != nil { + return err + } epConnectivity, err := parseEndpointConnectivity(epOptions) + if err != nil { + return err + } macAddress := ifInfo.MacAddress() // Use the macaddress if it was provided diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows_store.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows_store.go index 3c5cb04e21..bbed5aeaa2 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows_store.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/windows/windows_store.go @@ -232,9 +232,12 @@ func (ep *hnsEndpoint) MarshalJSON() ([]byte, error) { epMap["Type"] = ep.Type epMap["profileID"] = ep.profileID epMap["MacAddress"] = ep.macAddress.String() - epMap["Addr"] = ep.addr.String() - epMap["gateway"] = ep.gateway.String() - + if ep.addr.IP != nil { + epMap["Addr"] = ep.addr.String() + } + if ep.gateway != nil { + epMap["gateway"] = ep.gateway.String() + } epMap["epOption"] = ep.epOption epMap["epConnectivity"] = ep.epConnectivity epMap["PortMapping"] = ep.portMapping @@ -251,7 +254,6 @@ func (ep *hnsEndpoint) UnmarshalJSON(b []byte) error { if err = json.Unmarshal(b, &epMap); err != nil { return fmt.Errorf("Failed to unmarshal to endpoint: %v", err) } - if v, ok := epMap["MacAddress"]; ok { if ep.macAddress, err = net.ParseMAC(v.(string)); err != nil { return types.InternalErrorf("failed to decode endpoint MAC address (%s) after json unmarshal: %v", v.(string), err) @@ -262,7 +264,9 @@ func (ep *hnsEndpoint) UnmarshalJSON(b []byte) error { return types.InternalErrorf("failed to decode endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err) } } - + if v, ok := epMap["gateway"]; ok { + ep.gateway = net.ParseIP(v.(string)) + } ep.id = epMap["id"].(string) ep.Type = epMap["Type"].(string) ep.nid = epMap["nid"].(string) diff --git a/components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go b/components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go index 4243d57a74..b3876ffded 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ipam/allocator.go @@ -448,7 +448,7 @@ func (a *Allocator) RequestAddress(poolID string, prefAddress net.IP, opts map[s c := p for c.Range != nil { k = c.ParentKey - c, ok = aSpace.subnets[k] + c = aSpace.subnets[k] } aSpace.Unlock() diff --git a/components/engine/vendor/github.com/docker/libnetwork/ipams/remote/api/api.go b/components/engine/vendor/github.com/docker/libnetwork/ipams/remote/api/api.go index 23f3eda7d1..543c99bb00 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/ipams/remote/api/api.go +++ b/components/engine/vendor/github.com/docker/libnetwork/ipams/remote/api/api.go @@ -9,7 +9,7 @@ type Response struct { Error string } -// IsSuccess returns wheter the plugin response is successful +// IsSuccess returns whether the plugin response is successful func (r *Response) IsSuccess() bool { return r.Error == "" } diff --git a/components/engine/vendor/github.com/docker/libnetwork/network.go b/components/engine/vendor/github.com/docker/libnetwork/network.go index ed9a61b456..9f99064e11 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/network.go +++ b/components/engine/vendor/github.com/docker/libnetwork/network.go @@ -92,12 +92,20 @@ type EndpointWalker func(ep Endpoint) bool // Its an indication to defer PTR queries also to that external server. type ipInfo struct { name string + serviceID string extResolver bool } +// svcMapEntry is the body of the element into the svcMap +// The ip is a string because the SetMatrix does not accept non hashable values +type svcMapEntry struct { + ip string + serviceID string +} + type svcInfo struct { - svcMap map[string][]net.IP - svcIPv6Map map[string][]net.IP + svcMap common.SetMatrix + svcIPv6Map common.SetMatrix ipMap common.SetMatrix service map[string][]servicePorts } @@ -933,6 +941,9 @@ func (n *network) delete(force bool) error { id := n.id n.Unlock() + c.networkLocker.Lock(id) + defer c.networkLocker.Unlock(id) + n, err := c.getNetworkFromStore(id) if err != nil { return &UnknownNetworkError{name: name, id: id} @@ -991,12 +1002,6 @@ func (n *network) delete(force bool) error { c.cleanupServiceBindings(n.ID()) - // The network had been left, the service discovery can be cleaned up - c.Lock() - logrus.Debugf("network %s delete, clean svcRecords", n.id) - delete(c.svcRecords, n.id) - c.Unlock() - removeFromStore: // deleteFromStore performs an atomic delete operation and the // network.epCnt will help prevent any possible @@ -1070,6 +1075,9 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi ep := &endpoint{name: name, generic: make(map[string]interface{}), iface: &endpointInterface{}} ep.id = stringid.GenerateRandomID() + n.ctrlr.networkLocker.Lock(n.id) + defer n.ctrlr.networkLocker.Unlock(n.id) + // Initialize ep.network with a possibly stale copy of n. We need this to get network from // store. But once we get it from store we will have the most uptodate copy possibly. ep.network = n @@ -1228,75 +1236,77 @@ func (n *network) updateSvcRecord(ep *endpoint, localEps []*endpoint, isAdd bool ipv6 = iface.AddressIPv6().IP } + serviceID := ep.svcID + if serviceID == "" { + serviceID = ep.ID() + } if isAdd { // If anonymous endpoint has an alias use the first alias // for ip->name mapping. Not having the reverse mapping // breaks some apps if ep.isAnonymous() { if len(myAliases) > 0 { - n.addSvcRecords(ep.ID(), myAliases[0], iface.Address().IP, ipv6, true, "updateSvcRecord") + n.addSvcRecords(ep.ID(), myAliases[0], serviceID, iface.Address().IP, ipv6, true, "updateSvcRecord") } } else { - n.addSvcRecords(ep.ID(), epName, iface.Address().IP, ipv6, true, "updateSvcRecord") + n.addSvcRecords(ep.ID(), epName, serviceID, iface.Address().IP, ipv6, true, "updateSvcRecord") } for _, alias := range myAliases { - n.addSvcRecords(ep.ID(), alias, iface.Address().IP, ipv6, false, "updateSvcRecord") + n.addSvcRecords(ep.ID(), alias, serviceID, iface.Address().IP, ipv6, false, "updateSvcRecord") } } else { if ep.isAnonymous() { if len(myAliases) > 0 { - n.deleteSvcRecords(ep.ID(), myAliases[0], iface.Address().IP, ipv6, true, "updateSvcRecord") + n.deleteSvcRecords(ep.ID(), myAliases[0], serviceID, iface.Address().IP, ipv6, true, "updateSvcRecord") } } else { - n.deleteSvcRecords(ep.ID(), epName, iface.Address().IP, ipv6, true, "updateSvcRecord") + n.deleteSvcRecords(ep.ID(), epName, serviceID, iface.Address().IP, ipv6, true, "updateSvcRecord") } for _, alias := range myAliases { - n.deleteSvcRecords(ep.ID(), alias, iface.Address().IP, ipv6, false, "updateSvcRecord") + n.deleteSvcRecords(ep.ID(), alias, serviceID, iface.Address().IP, ipv6, false, "updateSvcRecord") } } } } -func addIPToName(ipMap common.SetMatrix, name string, ip net.IP) { +func addIPToName(ipMap common.SetMatrix, name, serviceID string, ip net.IP) { reverseIP := netutils.ReverseIP(ip.String()) ipMap.Insert(reverseIP, ipInfo{ - name: name, + name: name, + serviceID: serviceID, }) } -func addNameToIP(svcMap map[string][]net.IP, name string, epIP net.IP) { - ipList := svcMap[name] - for _, ip := range ipList { - if ip.Equal(epIP) { - return - } - } - svcMap[name] = append(svcMap[name], epIP) +func delIPToName(ipMap common.SetMatrix, name, serviceID string, ip net.IP) { + reverseIP := netutils.ReverseIP(ip.String()) + ipMap.Remove(reverseIP, ipInfo{ + name: name, + serviceID: serviceID, + }) } -func delNameToIP(svcMap map[string][]net.IP, name string, epIP net.IP) { - ipList := svcMap[name] - for i, ip := range ipList { - if ip.Equal(epIP) { - ipList = append(ipList[:i], ipList[i+1:]...) - break - } - } - svcMap[name] = ipList - - if len(ipList) == 0 { - delete(svcMap, name) - } +func addNameToIP(svcMap common.SetMatrix, name, serviceID string, epIP net.IP) { + svcMap.Insert(name, svcMapEntry{ + ip: epIP.String(), + serviceID: serviceID, + }) } -func (n *network) addSvcRecords(eID, name string, epIP net.IP, epIPv6 net.IP, ipMapUpdate bool, method string) { +func delNameToIP(svcMap common.SetMatrix, name, serviceID string, epIP net.IP) { + svcMap.Remove(name, svcMapEntry{ + ip: epIP.String(), + serviceID: serviceID, + }) +} + +func (n *network) addSvcRecords(eID, name, serviceID string, epIP, epIPv6 net.IP, ipMapUpdate bool, method string) { // Do not add service names for ingress network as this is a // routing only network if n.ingress { return } - logrus.Debugf("%s (%s).addSvcRecords(%s, %s, %s, %t) %s", eID, n.ID()[0:7], name, epIP, epIPv6, ipMapUpdate, method) + logrus.Debugf("%s (%s).addSvcRecords(%s, %s, %s, %t) %s sid:%s", eID, n.ID()[0:7], name, epIP, epIPv6, ipMapUpdate, method, serviceID) c := n.getController() c.Lock() @@ -1305,34 +1315,34 @@ func (n *network) addSvcRecords(eID, name string, epIP net.IP, epIPv6 net.IP, ip sr, ok := c.svcRecords[n.ID()] if !ok { sr = svcInfo{ - svcMap: make(map[string][]net.IP), - svcIPv6Map: make(map[string][]net.IP), + svcMap: common.NewSetMatrix(), + svcIPv6Map: common.NewSetMatrix(), ipMap: common.NewSetMatrix(), } c.svcRecords[n.ID()] = sr } if ipMapUpdate { - addIPToName(sr.ipMap, name, epIP) + addIPToName(sr.ipMap, name, serviceID, epIP) if epIPv6 != nil { - addIPToName(sr.ipMap, name, epIPv6) + addIPToName(sr.ipMap, name, serviceID, epIPv6) } } - addNameToIP(sr.svcMap, name, epIP) + addNameToIP(sr.svcMap, name, serviceID, epIP) if epIPv6 != nil { - addNameToIP(sr.svcIPv6Map, name, epIPv6) + addNameToIP(sr.svcIPv6Map, name, serviceID, epIPv6) } } -func (n *network) deleteSvcRecords(eID, name string, epIP net.IP, epIPv6 net.IP, ipMapUpdate bool, method string) { +func (n *network) deleteSvcRecords(eID, name, serviceID string, epIP net.IP, epIPv6 net.IP, ipMapUpdate bool, method string) { // Do not delete service names from ingress network as this is a // routing only network if n.ingress { return } - logrus.Debugf("%s (%s).deleteSvcRecords(%s, %s, %s, %t) %s", eID, n.ID()[0:7], name, epIP, epIPv6, ipMapUpdate, method) + logrus.Debugf("%s (%s).deleteSvcRecords(%s, %s, %s, %t) %s sid:%s ", eID, n.ID()[0:7], name, epIP, epIPv6, ipMapUpdate, method, serviceID) c := n.getController() c.Lock() @@ -1344,21 +1354,17 @@ func (n *network) deleteSvcRecords(eID, name string, epIP net.IP, epIPv6 net.IP, } if ipMapUpdate { - sr.ipMap.Remove(netutils.ReverseIP(epIP.String()), ipInfo{ - name: name, - }) + delIPToName(sr.ipMap, name, serviceID, epIP) if epIPv6 != nil { - sr.ipMap.Remove(netutils.ReverseIP(epIPv6.String()), ipInfo{ - name: name, - }) + delIPToName(sr.ipMap, name, serviceID, epIPv6) } } - delNameToIP(sr.svcMap, name, epIP) + delNameToIP(sr.svcMap, name, serviceID, epIP) if epIPv6 != nil { - delNameToIP(sr.svcIPv6Map, name, epIPv6) + delNameToIP(sr.svcIPv6Map, name, serviceID, epIPv6) } } @@ -1376,19 +1382,31 @@ func (n *network) getSvcRecords(ep *endpoint) []etchosts.Record { n.ctrlr.Lock() defer n.ctrlr.Unlock() - sr, _ := n.ctrlr.svcRecords[n.id] + sr, ok := n.ctrlr.svcRecords[n.id] + if !ok || sr.svcMap == nil { + return nil + } - for h, ip := range sr.svcMap { - if strings.Split(h, ".")[0] == epName { + svcMapKeys := sr.svcMap.Keys() + // Loop on service names on this network + for _, k := range svcMapKeys { + if strings.Split(k, ".")[0] == epName { continue } - if len(ip) == 0 { - logrus.Warnf("Found empty list of IP addresses for service %s on network %s (%s)", h, n.name, n.id) + // Get all the IPs associated to this service + mapEntryList, ok := sr.svcMap.Get(k) + if !ok { + // The key got deleted continue } + if len(mapEntryList) == 0 { + logrus.Warnf("Found empty list of IP addresses for service %s on network %s (%s)", k, n.name, n.id) + continue + } + recs = append(recs, etchosts.Record{ - Hosts: h, - IP: ip[0].String(), + Hosts: k, + IP: mapEntryList[0].(svcMapEntry).ip, }) } @@ -1845,8 +1863,7 @@ func (n *network) ResolveName(req string, ipType int) ([]net.IP, bool) { } req = strings.TrimSuffix(req, ".") - var ip []net.IP - ip, ok = sr.svcMap[req] + ipSet, ok := sr.svcMap.Get(req) if ipType == types.IPv6 { // If the name resolved to v4 address then its a valid name in @@ -1856,13 +1873,20 @@ func (n *network) ResolveName(req string, ipType int) ([]net.IP, bool) { if ok && n.enableIPv6 == false { ipv6Miss = true } - ip = sr.svcIPv6Map[req] + ipSet, ok = sr.svcIPv6Map.Get(req) } - if ip != nil { - ipLocal := make([]net.IP, len(ip)) - copy(ipLocal, ip) - return ipLocal, false + if ok && len(ipSet) > 0 { + // this map is to avoid IP duplicates, this can happen during a transition period where 2 services are using the same IP + noDup := make(map[string]bool) + var ipLocal []net.IP + for _, ip := range ipSet { + if _, dup := noDup[ip.(svcMapEntry).ip]; !dup { + noDup[ip.(svcMapEntry).ip] = true + ipLocal = append(ipLocal, net.ParseIP(ip.(svcMapEntry).ip)) + } + } + return ipLocal, ok } return nil, ipv6Miss @@ -1904,7 +1928,7 @@ func (n *network) ResolveIP(ip string) string { return "" } // NOTE it is possible to have more than one element in the Set, this will happen - // because of interleave of diffent events from differnt sources (local container create vs + // because of interleave of different events from different sources (local container create vs // network db notifications) // In such cases the resolution will be based on the first element of the set, and can vary // during the system stabilitation diff --git a/components/engine/vendor/github.com/docker/libnetwork/osl/namespace_unsupported.go b/components/engine/vendor/github.com/docker/libnetwork/osl/namespace_unsupported.go index e385046121..74372e2492 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/osl/namespace_unsupported.go +++ b/components/engine/vendor/github.com/docker/libnetwork/osl/namespace_unsupported.go @@ -7,6 +7,7 @@ package osl func GC() { } +// GetSandboxForExternalKey returns sandbox object for the supplied path func GetSandboxForExternalKey(path string, key string) (Sandbox, error) { return nil, nil } diff --git a/components/engine/vendor/github.com/docker/libnetwork/sandbox.go b/components/engine/vendor/github.com/docker/libnetwork/sandbox.go index 472dbeafe7..9454c5c286 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/sandbox.go +++ b/components/engine/vendor/github.com/docker/libnetwork/sandbox.go @@ -629,6 +629,10 @@ func (sb *sandbox) SetKey(basePath string) error { } sb.Lock() + if sb.inDelete { + sb.Unlock() + return types.ForbiddenErrorf("failed to SetKey: sandbox %q delete in progress", sb.id) + } oldosSbox := sb.osSbox sb.Unlock() diff --git a/components/engine/vendor/github.com/docker/libnetwork/service.go b/components/engine/vendor/github.com/docker/libnetwork/service.go index c890e01a39..5a0d7e0057 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/service.go +++ b/components/engine/vendor/github.com/docker/libnetwork/service.go @@ -85,14 +85,8 @@ type loadBalancer struct { // Map of backend IPs backing this loadbalancer on this // network. It is keyed with endpoint ID. - backEnds map[string]loadBalancerBackend + backEnds map[string]net.IP // Back pointer to service to which the loadbalancer belongs. service *service } - -type loadBalancerBackend struct { - ip net.IP - containerName string - taskAliases []string -} diff --git a/components/engine/vendor/github.com/docker/libnetwork/service_common.go b/components/engine/vendor/github.com/docker/libnetwork/service_common.go index e57ce1498b..7e3367c71c 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/service_common.go +++ b/components/engine/vendor/github.com/docker/libnetwork/service_common.go @@ -15,29 +15,35 @@ func (c *controller) addEndpointNameResolution(svcName, svcID, nID, eID, contain return err } - logrus.Debugf("addEndpointNameResolution %s %s add_service:%t", eID, svcName, addService) + logrus.Debugf("addEndpointNameResolution %s %s add_service:%t sAliases:%v tAliases:%v", eID, svcName, addService, serviceAliases, taskAliases) // Add container resolution mappings c.addContainerNameResolution(nID, eID, containerName, taskAliases, ip, method) + serviceID := svcID + if serviceID == "" { + // This is the case of a normal container not part of a service + serviceID = eID + } + // Add endpoint IP to special "tasks.svc_name" so that the applications have access to DNS RR. - n.(*network).addSvcRecords(eID, "tasks."+svcName, ip, nil, false, method) + n.(*network).addSvcRecords(eID, "tasks."+svcName, serviceID, ip, nil, false, method) for _, alias := range serviceAliases { - n.(*network).addSvcRecords(eID, "tasks."+alias, ip, nil, false, method) + n.(*network).addSvcRecords(eID, "tasks."+alias, serviceID, ip, nil, false, method) } // Add service name to vip in DNS, if vip is valid. Otherwise resort to DNS RR if len(vip) == 0 { - n.(*network).addSvcRecords(eID, svcName, ip, nil, false, method) + n.(*network).addSvcRecords(eID, svcName, serviceID, ip, nil, false, method) for _, alias := range serviceAliases { - n.(*network).addSvcRecords(eID, alias, ip, nil, false, method) + n.(*network).addSvcRecords(eID, alias, serviceID, ip, nil, false, method) } } if addService && len(vip) != 0 { - n.(*network).addSvcRecords(eID, svcName, vip, nil, false, method) + n.(*network).addSvcRecords(eID, svcName, serviceID, vip, nil, false, method) for _, alias := range serviceAliases { - n.(*network).addSvcRecords(eID, alias, vip, nil, false, method) + n.(*network).addSvcRecords(eID, alias, serviceID, vip, nil, false, method) } } @@ -52,11 +58,11 @@ func (c *controller) addContainerNameResolution(nID, eID, containerName string, logrus.Debugf("addContainerNameResolution %s %s", eID, containerName) // Add resolution for container name - n.(*network).addSvcRecords(eID, containerName, ip, nil, true, method) + n.(*network).addSvcRecords(eID, containerName, eID, ip, nil, true, method) // Add resolution for taskaliases for _, alias := range taskAliases { - n.(*network).addSvcRecords(eID, alias, ip, nil, true, method) + n.(*network).addSvcRecords(eID, alias, eID, ip, nil, true, method) } return nil @@ -68,32 +74,38 @@ func (c *controller) deleteEndpointNameResolution(svcName, svcID, nID, eID, cont return err } - logrus.Debugf("deleteEndpointNameResolution %s %s rm_service:%t suppress:%t", eID, svcName, rmService, multipleEntries) + logrus.Debugf("deleteEndpointNameResolution %s %s rm_service:%t suppress:%t sAliases:%v tAliases:%v", eID, svcName, rmService, multipleEntries, serviceAliases, taskAliases) // Delete container resolution mappings c.delContainerNameResolution(nID, eID, containerName, taskAliases, ip, method) + serviceID := svcID + if serviceID == "" { + // This is the case of a normal container not part of a service + serviceID = eID + } + // Delete the special "tasks.svc_name" backend record. if !multipleEntries { - n.(*network).deleteSvcRecords(eID, "tasks."+svcName, ip, nil, false, method) + n.(*network).deleteSvcRecords(eID, "tasks."+svcName, serviceID, ip, nil, false, method) for _, alias := range serviceAliases { - n.(*network).deleteSvcRecords(eID, "tasks."+alias, ip, nil, false, method) + n.(*network).deleteSvcRecords(eID, "tasks."+alias, serviceID, ip, nil, false, method) } } // If we are doing DNS RR delete the endpoint IP from DNS record right away. if !multipleEntries && len(vip) == 0 { - n.(*network).deleteSvcRecords(eID, svcName, ip, nil, false, method) + n.(*network).deleteSvcRecords(eID, svcName, serviceID, ip, nil, false, method) for _, alias := range serviceAliases { - n.(*network).deleteSvcRecords(eID, alias, ip, nil, false, method) + n.(*network).deleteSvcRecords(eID, alias, serviceID, ip, nil, false, method) } } // Remove the DNS record for VIP only if we are removing the service if rmService && len(vip) != 0 && !multipleEntries { - n.(*network).deleteSvcRecords(eID, svcName, vip, nil, false, method) + n.(*network).deleteSvcRecords(eID, svcName, serviceID, vip, nil, false, method) for _, alias := range serviceAliases { - n.(*network).deleteSvcRecords(eID, alias, vip, nil, false, method) + n.(*network).deleteSvcRecords(eID, alias, serviceID, vip, nil, false, method) } } @@ -108,11 +120,11 @@ func (c *controller) delContainerNameResolution(nID, eID, containerName string, logrus.Debugf("delContainerNameResolution %s %s", eID, containerName) // Delete resolution for container name - n.(*network).deleteSvcRecords(eID, containerName, ip, nil, true, method) + n.(*network).deleteSvcRecords(eID, containerName, eID, ip, nil, true, method) // Delete resolution for taskaliases for _, alias := range taskAliases { - n.(*network).deleteSvcRecords(eID, alias, ip, nil, true, method) + n.(*network).deleteSvcRecords(eID, alias, eID, ip, nil, true, method) } return nil @@ -152,6 +164,7 @@ func (c *controller) getLBIndex(sid, nid string, ingressPorts []*PortConfig) int func (c *controller) cleanupServiceBindings(cleanupNID string) { var cleanupFuncs []func() + logrus.Debugf("cleanupServiceBindings for %s", cleanupNID) c.Lock() services := make([]*service, 0, len(c.serviceBindings)) for _, s := range c.serviceBindings { @@ -171,16 +184,27 @@ func (c *controller) cleanupServiceBindings(cleanupNID string) { continue } - for eid, be := range lb.backEnds { + // The network is being deleted, erase all the associated service discovery records + // TODO(fcrisciani) separate the Load Balancer from the Service discovery, this operation + // can be done safely here, but the rmServiceBinding is still keeping consistency in the + // data structures that are tracking the endpoint to IP mapping. + c.Lock() + logrus.Debugf("cleanupServiceBindings erasing the svcRecords for %s", nid) + delete(c.svcRecords, nid) + c.Unlock() + + for eid, ip := range lb.backEnds { + epID := eid + epIP := ip service := s loadBalancer := lb networkID := nid - epID := eid - epIP := be.ip - cleanupFuncs = append(cleanupFuncs, func() { - if err := c.rmServiceBinding(service.name, service.id, networkID, epID, be.containerName, loadBalancer.vip, - service.ingressPorts, service.aliases, be.taskAliases, epIP, "cleanupServiceBindings"); err != nil { + // ContainerName and taskAliases are not available here, this is still fine because the Service discovery + // cleanup already happened before. The only thing that rmServiceBinding is still doing here a part from the Load + // Balancer bookeeping, is to keep consistent the mapping of endpoint to IP. + if err := c.rmServiceBinding(service.name, service.id, networkID, epID, "", loadBalancer.vip, + service.ingressPorts, service.aliases, []string{}, epIP, "cleanupServiceBindings", false); err != nil { logrus.Errorf("Failed to remove service bindings for service %s network %s endpoint %s while cleanup: %v", service.id, networkID, epID, err) } @@ -228,8 +252,7 @@ func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName s } s.Unlock() } - logrus.Debugf("addServiceBinding from %s START for %s %s", method, svcName, eID) - + logrus.Debugf("addServiceBinding from %s START for %s %s p:%p nid:%s skey:%v", method, svcName, eID, s, nID, skey) defer s.Unlock() lb, ok := s.loadBalancers[nID] @@ -242,7 +265,7 @@ func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName s lb = &loadBalancer{ vip: vip, fwMark: fwMarkCtr, - backEnds: make(map[string]loadBalancerBackend), + backEnds: make(map[string]net.IP), service: s, } @@ -253,9 +276,7 @@ func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName s addService = true } - lb.backEnds[eID] = loadBalancerBackend{ip: ip, - containerName: containerName, - taskAliases: taskAliases} + lb.backEnds[eID] = ip ok, entries := s.assignIPToEndpoint(ip.String(), eID) if !ok || entries > 1 { @@ -277,7 +298,7 @@ func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName s return nil } -func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName string, vip net.IP, ingressPorts []*PortConfig, serviceAliases []string, taskAliases []string, ip net.IP, method string) error { +func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName string, vip net.IP, ingressPorts []*PortConfig, serviceAliases []string, taskAliases []string, ip net.IP, method string, deleteSvcRecords bool) error { var rmService bool @@ -294,7 +315,6 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st c.Lock() s, ok := c.serviceBindings[skey] c.Unlock() - logrus.Debugf("rmServiceBinding from %s START for %s %s", method, svcName, eID) if !ok { logrus.Warnf("rmServiceBinding %s %s %s aborted c.serviceBindings[skey] !ok", method, svcName, eID) return nil @@ -302,6 +322,7 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st s.Lock() defer s.Unlock() + logrus.Debugf("rmServiceBinding from %s START for %s %s p:%p nid:%s sKey:%v deleteSvc:%t", method, svcName, eID, s, nID, skey, deleteSvcRecords) lb, ok := s.loadBalancers[nID] if !ok { logrus.Warnf("rmServiceBinding %s %s %s aborted s.loadBalancers[nid] !ok", method, svcName, eID) @@ -322,17 +343,7 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st rmService = true delete(s.loadBalancers, nID) - } - - if len(s.loadBalancers) == 0 { - // All loadbalancers for the service removed. Time to - // remove the service itself. - c.Lock() - - // Mark the object as deleted so that the add won't use it wrongly - s.deleted = true - delete(c.serviceBindings, skey) - c.Unlock() + logrus.Debugf("rmServiceBinding %s delete %s, p:%p in loadbalancers len:%d", eID, nID, lb, len(s.loadBalancers)) } ok, entries := s.removeIPToEndpoint(ip.String(), eID) @@ -348,7 +359,22 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st } // Delete the name resolutions - c.deleteEndpointNameResolution(svcName, svcID, nID, eID, containerName, vip, serviceAliases, taskAliases, ip, rmService, entries > 0, "rmServiceBinding") + if deleteSvcRecords { + c.deleteEndpointNameResolution(svcName, svcID, nID, eID, containerName, vip, serviceAliases, taskAliases, ip, rmService, entries > 0, "rmServiceBinding") + } + + if len(s.loadBalancers) == 0 { + // All loadbalancers for the service removed. Time to + // remove the service itself. + c.Lock() + + // Mark the object as deleted so that the add won't use it wrongly + s.deleted = true + // NOTE The delete from the serviceBindings map has to be the last operation else we are allowing a race between this service + // that is getting deleted and a new service that will be created if the entry is not anymore there + delete(c.serviceBindings, skey) + c.Unlock() + } logrus.Debugf("rmServiceBinding from %s END for %s %s", method, svcName, eID) return nil diff --git a/components/engine/vendor/github.com/docker/libnetwork/service_linux.go b/components/engine/vendor/github.com/docker/libnetwork/service_linux.go index 70af7e33bc..1cf7ee91aa 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/service_linux.go +++ b/components/engine/vendor/github.com/docker/libnetwork/service_linux.go @@ -102,8 +102,8 @@ func (sb *sandbox) populateLoadbalancers(ep *endpoint) { } lb.service.Lock() - for _, l := range lb.backEnds { - sb.addLBBackend(l.ip, lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, gwIP, n.ingress) + for _, ip := range lb.backEnds { + sb.addLBBackend(ip, lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, gwIP, n.ingress) } lb.service.Unlock() } @@ -520,7 +520,7 @@ func writePortsToFile(ports []*PortConfig) (string, error) { } defer f.Close() - buf, err := proto.Marshal(&EndpointRecord{ + buf, _ := proto.Marshal(&EndpointRecord{ IngressPorts: ports, })