Merge pull request #19889 from Microsoft/jjh/reliabilitytake2

Windows CI: TP4 reliability hack
Upstream-commit: a39ad952ec1efc3b7536d3d8a93e96a22cd1818b
Component: engine

components/engine/daemon/execdriver/windows/run.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/execdriver"
@@ -223,10 +224,32 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd
 
 	configuration := string(configurationb)
 
-	err = hcsshim.CreateComputeSystem(c.ID, configuration)
-	if err != nil {
-		logrus.Debugln("Failed to create temporary container ", err)
-		return execdriver.ExitStatus{ExitCode: -1}, err
+	// TODO Windows TP5 timeframe. Remove when TP4 is no longer supported.
+	// The following a workaround for Windows TP4 which has a networking
+	// bug which fairly frequently returns an error. Back off and retry.
+	maxAttempts := 1
+	if TP4RetryHack {
+		maxAttempts = 5
+	}
+	i := 0
+	for i < maxAttempts {
+		i++
+		err = hcsshim.CreateComputeSystem(c.ID, configuration)
+		if err != nil {
+			if TP4RetryHack {
+				if !strings.Contains(err.Error(), `Win32 API call returned error r1=2147746291`) && // Invalid class string
+					!strings.Contains(err.Error(), `Win32 API call returned error r1=2147943568`) && // Element not found
+					!strings.Contains(err.Error(), `Win32 API call returned error r1=2147942402`) && // The system cannot find the file specified
+					!strings.Contains(err.Error(), `Win32 API call returned error r1=2147943622`) { // The network is not present or not started
+					logrus.Debugln("Failed to create temporary container ", err)
+					return execdriver.ExitStatus{ExitCode: -1}, err
+				}
+				logrus.Warnf("Invoking Windows TP4 retry hack (%d of %d)", i, maxAttempts-1)
+				time.Sleep(50 * time.Millisecond)
+			}
+		} else {
+			break
+		}
 	}
 
 	// Start the container

components/engine/daemon/execdriver/windows/windows.go

@@ -4,6 +4,7 @@ package windows
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -12,8 +13,13 @@ import (
 	"github.com/docker/docker/dockerversion"
 	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/engine-api/types/container"
+	"golang.org/x/sys/windows/registry"
 )
 
+// TP4RetryHack is a hack to retry CreateComputeSystem if it fails with
+// known return codes from Windows due to bugs in TP4.
+var TP4RetryHack bool
+
 // This is a daemon development variable only and should not be
 // used for running production containers on Windows.
 var dummyMode bool
@@ -89,6 +95,37 @@ func NewDriver(root string, options []string) (*Driver, error) {
 		}
 	}
 
+	// TODO Windows TP5 timeframe. Remove this next block of code once TP4
+	// is no longer supported. Also remove the workaround in run.go.
+	//
+	// Hack for TP4 - determine the version of Windows from the registry.
+	// This overcomes an issue on TP4 which causes CreateComputeSystem to
+	// intermittently fail. It's predominantly here to make Windows to Windows
+	// CI more reliable.
+	TP4RetryHack = false
+	k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE)
+	if err != nil {
+		return &Driver{}, err
+	}
+	defer k.Close()
+
+	s, _, err := k.GetStringValue("BuildLab")
+	if err != nil {
+		return &Driver{}, err
+	}
+	parts := strings.Split(s, ".")
+	if len(parts) < 1 {
+		return &Driver{}, err
+	}
+	var val int
+	if val, err = strconv.Atoi(parts[0]); err != nil {
+		return &Driver{}, err
+	}
+	if val < 14250 {
+		TP4RetryHack = true
+	}
+	// End of Windows TP4 hack
+
 	return &Driver{
 		root:             root,
 		activeContainers: make(map[string]*activeContainer),