diff --git a/components/engine/Makefile b/components/engine/Makefile index fc5b7d9964..403c6792ee 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -57,6 +57,7 @@ DOCKER_ENVS := \ -e no_proxy \ -e VERSION \ -e PLATFORM \ + -e DEFAULT_PRODUCT_LICENSE \ -e PRODUCT # note: we _cannot_ add "-e DOCKER_BUILDTAGS" here because even if it's unset in the shell, that would shadow the "ENV DOCKER_BUILDTAGS" set in our Dockerfile, which is very important for our official builds diff --git a/components/engine/api/swagger.yaml b/components/engine/api/swagger.yaml index 204d5b860f..30fbf189de 100644 --- a/components/engine/api/swagger.yaml +++ b/components/engine/api/swagger.yaml @@ -3896,6 +3896,14 @@ definitions: - "name=seccomp,profile=default" - "name=selinux" - "name=userns" + ProductLicense: + description: | + Reports a summary of the product license on the daemon. + + If a commercial license has been applied to the daemon, information + such as number of nodes, and expiration are included. + type: "string" + example: "Community Engine" # PluginsInfo is a temp struct holding Plugins name diff --git a/components/engine/api/types/types.go b/components/engine/api/types/types.go index 06c0ca3a69..ca1b3753a5 100644 --- a/components/engine/api/types/types.go +++ b/components/engine/api/types/types.go @@ -204,6 +204,7 @@ type Info struct { RuncCommit Commit InitCommit Commit SecurityOptions []string + ProductLicense string `json:",omitempty"` } // KeyValue holds a key/value pair diff --git a/components/engine/builder/remotecontext/detect.go b/components/engine/builder/remotecontext/detect.go index 49b196ed7b..144eb570ab 100644 --- a/components/engine/builder/remotecontext/detect.go +++ b/components/engine/builder/remotecontext/detect.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "runtime" "strings" "github.com/containerd/continuity/driver" @@ -173,6 +174,9 @@ func StatAt(remote builder.Source, path string) (os.FileInfo, error) { func FullPath(remote builder.Source, path string) (string, error) { fullPath, err := remote.Root().ResolveScopedPath(path, true) if err != nil { + if runtime.GOOS == "windows" { + return "", fmt.Errorf("failed to resolve scoped path %s (%s): %s. Possible cause is a forbidden path outside the build context", path, fullPath, err) + } return "", fmt.Errorf("Forbidden path outside the build context: %s (%s)", path, fullPath) // backwards compat with old error } return fullPath, nil diff --git a/components/engine/daemon/graphdriver/lcow/lcow.go b/components/engine/daemon/graphdriver/lcow/lcow.go index 27a7e044a9..8a89cc12d0 100644 --- a/components/engine/daemon/graphdriver/lcow/lcow.go +++ b/components/engine/daemon/graphdriver/lcow/lcow.go @@ -212,6 +212,17 @@ func (d *Driver) getVMID(id string) string { return id } +// remapLongToShortContainerPath does the mapping of a long container path for a +// SCSI attached disk, to a short container path where it's actually mounted. +func remapLongToShortContainerPath(longContainerPath string, attachCounter uint64, svmName string) string { + shortContainerPath := longContainerPath + if shortContainerPath != "" && shortContainerPath != toolsScratchPath { + shortContainerPath = fmt.Sprintf("/tmp/d%d", attachCounter) + logrus.Debugf("lcowdriver: UVM %s: remapping %s --> %s", svmName, longContainerPath, shortContainerPath) + } + return shortContainerPath +} + // startServiceVMIfNotRunning starts a service utility VM if it is not currently running. // It can optionally be started with a mapped virtual disk. Returns a opengcs config structure // representing the VM. @@ -239,6 +250,8 @@ func (d *Driver) startServiceVMIfNotRunning(id string, mvdToAdd []hcsshim.Mapped if exists { // Service VM is already up and running. In this case, just hot add the vhds. + // Note that hotAddVHDs will remap long to short container paths, so no need + // for us to that here. logrus.Debugf("%s: service vm already exists. Just hot adding: %+v", title, mvdToAdd) if err := svm.hotAddVHDs(mvdToAdd...); err != nil { logrus.Debugf("%s: failed to hot add vhds on service vm creation: %s", title, err) @@ -302,10 +315,23 @@ func (d *Driver) startServiceVMIfNotRunning(id string, mvdToAdd []hcsshim.Mapped logrus.Debugf("%s: releasing cachedScratchMutex", title) d.cachedScratchMutex.Unlock() - // If requested to start it with a mapped virtual disk, add it now. - svm.config.MappedVirtualDisks = append(svm.config.MappedVirtualDisks, mvdToAdd...) - for _, mvd := range svm.config.MappedVirtualDisks { - svm.attachedVHDs[mvd.HostPath] = 1 + // Add mapped virtual disks. First those that are already in the configuration. Generally, + // the only one that will be here is the service VMs scratch. The exception is when invoked + // via the graphdrivers DiffGetter implementation. + for i, mvd := range svm.config.MappedVirtualDisks { + svm.attachCounter++ + svm.attachedVHDs[mvd.HostPath] = &attachedVHD{refCount: 1, attachCounter: svm.attachCounter} + + // No-op for the service VMs scratch disk. Only applicable in the DiffGetter interface invocation. + svm.config.MappedVirtualDisks[i].ContainerPath = remapLongToShortContainerPath(mvd.ContainerPath, svm.attachCounter, svm.config.Name) + } + + // Then the remaining ones to add, and adding them to the startup configuration. + for _, mvd := range mvdToAdd { + svm.attachCounter++ + svm.attachedVHDs[mvd.HostPath] = &attachedVHD{refCount: 1, attachCounter: svm.attachCounter} + mvd.ContainerPath = remapLongToShortContainerPath(mvd.ContainerPath, svm.attachCounter, svm.config.Name) + svm.config.MappedVirtualDisks = append(svm.config.MappedVirtualDisks, mvd) } // Start it. @@ -351,6 +377,7 @@ func (d *Driver) startServiceVMIfNotRunning(id string, mvdToAdd []hcsshim.Mapped return nil, fmt.Errorf("failed to hot-add %s failed: %s", scratchTargetFile, err) } svm.scratchAttached = true + // Don't need to ref-count here as it will be done via hotAddVHDsAtStart() call above. } logrus.Debugf("%s: (%s) success", title, context) @@ -787,8 +814,13 @@ func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) { } // Obtain the tar stream for it - logrus.Debugf("%s: %s %s, size %d, ReadOnly %t", title, ld.filename, mvd.ContainerPath, ld.size, ld.isSandbox) - tarReadCloser, err := svm.config.VhdToTar(mvd.HostPath, mvd.ContainerPath, ld.isSandbox, ld.size) + // The actual container path will have be remapped to a short name, so use that. + actualContainerPath := svm.getShortContainerPath(&mvd) + if actualContainerPath == "" { + return nil, fmt.Errorf("failed to get short container path for %+v in SVM %s", mvd, svm.config.Name) + } + logrus.Debugf("%s: %s %s, size %d, ReadOnly %t", title, ld.filename, actualContainerPath, ld.size, ld.isSandbox) + tarReadCloser, err := svm.config.VhdToTar(mvd.HostPath, actualContainerPath, ld.isSandbox, ld.size) if err != nil { svm.hotRemoveVHDs(mvd) d.terminateServiceVM(id, fmt.Sprintf("diff %s", id), false) @@ -960,6 +992,17 @@ func (d *Driver) getAllMounts(id string) ([]hcsshim.MappedVirtualDisk, error) { } func hostToGuest(hostpath string) string { + // This is the "long" container path. At the point of which we are + // calculating this, we don't know which service VM we're going to be + // using, so we can't translate this to a short path yet, instead + // deferring until the point of which it's added to an SVM. We don't + // use long container paths in SVMs for SCSI disks, otherwise it can cause + // command line operations that we invoke to fail due to being over ~4200 + // characters when there are ~47 layers involved. An example of this is + // the mount call to create the overlay across multiple SCSI-attached disks. + // It doesn't affect VPMem attached layers during container creation as + // these get mapped by openGCS to /tmp/N/M where N is a container instance + // number, and M is a layer number. return fmt.Sprintf("/tmp/%s", filepath.Base(filepath.Dir(hostpath))) } @@ -1002,7 +1045,12 @@ func (fgc *fileGetCloserFromSVM) Close() error { func (fgc *fileGetCloserFromSVM) Get(filename string) (io.ReadCloser, error) { errOut := &bytes.Buffer{} outOut := &bytes.Buffer{} - file := path.Join(fgc.mvd.ContainerPath, filename) + // Must map to the actual "short" container path where the SCSI disk was mounted + actualContainerPath := fgc.svm.getShortContainerPath(fgc.mvd) + if actualContainerPath == "" { + return nil, fmt.Errorf("inconsistency detected: couldn't get short container path for %+v in utility VM %s", fgc.mvd, fgc.svm.config.Name) + } + file := path.Join(actualContainerPath, filename) if err := fgc.svm.runProcess(fmt.Sprintf("cat %s", file), nil, outOut, errOut); err != nil { logrus.Debugf("cat %s failed: %s", file, errOut.String()) return nil, err diff --git a/components/engine/daemon/graphdriver/lcow/lcow_svm.go b/components/engine/daemon/graphdriver/lcow/lcow_svm.go index 9a27ac9496..fdb0553dee 100644 --- a/components/engine/daemon/graphdriver/lcow/lcow_svm.go +++ b/components/engine/daemon/graphdriver/lcow/lcow_svm.go @@ -3,6 +3,7 @@ package lcow // import "github.com/docker/docker/daemon/graphdriver/lcow" import ( + "bytes" "errors" "fmt" "io" @@ -34,6 +35,13 @@ type serviceVMMapItem struct { refCount int // refcount for VM } +// attachedVHD is for reference counting SCSI disks attached to a service VM, +// and for a counter used to generate a short path name for the container path. +type attachedVHD struct { + refCount int + attachCounter uint64 +} + type serviceVM struct { sync.Mutex // Serialises operations being performed in this service VM. scratchAttached bool // Has a scratch been attached? @@ -47,8 +55,9 @@ type serviceVM struct { stopStatus chan interface{} stopError error - attachedVHDs map[string]int // Map ref counting all the VHDS we've hot-added/hot-removed. - unionMounts map[string]int // Map ref counting all the union filesystems we mounted. + attachCounter uint64 // Increasing counter for each add + attachedVHDs map[string]*attachedVHD // Map ref counting all the VHDS we've hot-added/hot-removed. + unionMounts map[string]int // Map ref counting all the union filesystems we mounted. } // add will add an id to the service vm map. There are three cases: @@ -73,7 +82,7 @@ func (svmMap *serviceVMMap) add(id string) (svm *serviceVM, alreadyExists bool, newSVM := &serviceVM{ startStatus: make(chan interface{}), stopStatus: make(chan interface{}), - attachedVHDs: make(map[string]int), + attachedVHDs: make(map[string]*attachedVHD), unionMounts: make(map[string]int), config: &client.Config{}, } @@ -203,15 +212,18 @@ func (svm *serviceVM) hotAddVHDsAtStart(mvds ...hcsshim.MappedVirtualDisk) error defer svm.Unlock() for i, mvd := range mvds { if _, ok := svm.attachedVHDs[mvd.HostPath]; ok { - svm.attachedVHDs[mvd.HostPath]++ + svm.attachedVHDs[mvd.HostPath].refCount++ + logrus.Debugf("lcowdriver: UVM %s: %s already present, refCount now %d", svm.config.Name, mvd.HostPath, svm.attachedVHDs[mvd.HostPath].refCount) continue } - if err := svm.config.HotAddVhd(mvd.HostPath, mvd.ContainerPath, mvd.ReadOnly, !mvd.AttachOnly); err != nil { + svm.attachCounter++ + shortContainerPath := remapLongToShortContainerPath(mvd.ContainerPath, svm.attachCounter, svm.config.Name) + if err := svm.config.HotAddVhd(mvd.HostPath, shortContainerPath, mvd.ReadOnly, !mvd.AttachOnly); err != nil { svm.hotRemoveVHDsNoLock(mvds[:i]...) return err } - svm.attachedVHDs[mvd.HostPath] = 1 + svm.attachedVHDs[mvd.HostPath] = &attachedVHD{refCount: 1, attachCounter: svm.attachCounter} } return nil } @@ -238,15 +250,17 @@ func (svm *serviceVM) hotRemoveVHDsNoLock(mvds ...hcsshim.MappedVirtualDisk) err // defers the VM start to the first operation, it's possible that nothing have been hot-added // when Put() is called. To avoid Put returning an error in that case, we simply continue if we // don't find the vhd attached. + logrus.Debugf("lcowdriver: UVM %s: %s is not attached, not doing anything", svm.config.Name, mvd.HostPath) continue } - if svm.attachedVHDs[mvd.HostPath] > 1 { - svm.attachedVHDs[mvd.HostPath]-- + if svm.attachedVHDs[mvd.HostPath].refCount > 1 { + svm.attachedVHDs[mvd.HostPath].refCount-- + logrus.Debugf("lcowdriver: UVM %s: %s refCount dropped to %d. not removing from UVM", svm.config.Name, mvd.HostPath, svm.attachedVHDs[mvd.HostPath].refCount) continue } - // last VHD, so remove from VM and map + // last reference to VHD, so remove from VM and map if err := svm.config.HotRemoveVhd(mvd.HostPath); err == nil { delete(svm.attachedVHDs, mvd.HostPath) } else { @@ -270,6 +284,19 @@ func (svm *serviceVM) createExt4VHDX(destFile string, sizeGB uint32, cacheFile s return svm.config.CreateExt4Vhdx(destFile, sizeGB, cacheFile) } +// getShortContainerPath looks up where a SCSI disk was actually mounted +// in a service VM when we remapped a long path name to a short name. +func (svm *serviceVM) getShortContainerPath(mvd *hcsshim.MappedVirtualDisk) string { + if mvd.ContainerPath == "" { + return "" + } + avhd, ok := svm.attachedVHDs[mvd.HostPath] + if !ok { + return "" + } + return fmt.Sprintf("/tmp/d%d", avhd.attachCounter) +} + func (svm *serviceVM) createUnionMount(mountName string, mvds ...hcsshim.MappedVirtualDisk) (err error) { if len(mvds) == 0 { return fmt.Errorf("createUnionMount: error must have at least 1 layer") @@ -288,11 +315,11 @@ func (svm *serviceVM) createUnionMount(mountName string, mvds ...hcsshim.MappedV var lowerLayers []string if mvds[0].ReadOnly { - lowerLayers = append(lowerLayers, mvds[0].ContainerPath) + lowerLayers = append(lowerLayers, svm.getShortContainerPath(&mvds[0])) } for i := 1; i < len(mvds); i++ { - lowerLayers = append(lowerLayers, mvds[i].ContainerPath) + lowerLayers = append(lowerLayers, svm.getShortContainerPath(&mvds[i])) } logrus.Debugf("Doing the overlay mount with union directory=%s", mountName) @@ -303,15 +330,15 @@ func (svm *serviceVM) createUnionMount(mountName string, mvds ...hcsshim.MappedV var cmd string if len(mvds) == 1 { // `FROM SCRATCH` case and the only layer. No overlay required. - cmd = fmt.Sprintf("mount %s %s", mvds[0].ContainerPath, mountName) + cmd = fmt.Sprintf("mount %s %s", svm.getShortContainerPath(&mvds[0]), mountName) } else if mvds[0].ReadOnly { // Readonly overlay cmd = fmt.Sprintf("mount -t overlay overlay -olowerdir=%s %s", strings.Join(lowerLayers, ","), mountName) } else { - upper := fmt.Sprintf("%s/upper", mvds[0].ContainerPath) - work := fmt.Sprintf("%s/work", mvds[0].ContainerPath) + upper := fmt.Sprintf("%s/upper", svm.getShortContainerPath(&mvds[0])) + work := fmt.Sprintf("%s/work", svm.getShortContainerPath(&mvds[0])) if err = svm.runProcess(fmt.Sprintf("mkdir -p %s %s", upper, work), nil, nil, nil); err != nil { return err @@ -359,7 +386,15 @@ func (svm *serviceVM) deleteUnionMount(mountName string, disks ...hcsshim.Mapped } func (svm *serviceVM) runProcess(command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { - process, err := svm.config.RunProcess(command, stdin, stdout, stderr) + var process hcsshim.Process + var err error + errOut := &bytes.Buffer{} + + if stderr != nil { + process, err = svm.config.RunProcess(command, stdin, stdout, stderr) + } else { + process, err = svm.config.RunProcess(command, stdin, stdout, errOut) + } if err != nil { return err } @@ -372,7 +407,12 @@ func (svm *serviceVM) runProcess(command string, stdin io.Reader, stdout io.Writ } if exitCode != 0 { - return fmt.Errorf("svm.runProcess: command %s failed with exit code %d", command, exitCode) + // If the caller isn't explicitly capturing stderr output, then capture it here instead. + e := fmt.Sprintf("svm.runProcess: command %s failed with exit code %d", command, exitCode) + if stderr == nil { + e = fmt.Sprintf("%s. (%s)", e, errOut.String()) + } + return fmt.Errorf(e) } return nil } diff --git a/components/engine/daemon/info.go b/components/engine/daemon/info.go index 5bcc5d72fa..3575fdda38 100644 --- a/components/engine/daemon/info.go +++ b/components/engine/daemon/info.go @@ -73,6 +73,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { daemon.fillDriverInfo(v) daemon.fillPluginsInfo(v) daemon.fillSecurityOptions(v, sysInfo) + daemon.fillLicense(v) return v, nil } diff --git a/components/engine/daemon/licensing.go b/components/engine/daemon/licensing.go new file mode 100644 index 0000000000..3e9fcdbd3d --- /dev/null +++ b/components/engine/daemon/licensing.go @@ -0,0 +1,10 @@ +package daemon // import "github.com/docker/docker/daemon" + +import ( + "github.com/docker/docker/api/types" + "github.com/docker/docker/dockerversion" +) + +func (daemon *Daemon) fillLicense(v *types.Info) { + v.ProductLicense = dockerversion.DefaultProductLicense +} diff --git a/components/engine/daemon/licensing_test.go b/components/engine/daemon/licensing_test.go new file mode 100644 index 0000000000..5a97ce8fa2 --- /dev/null +++ b/components/engine/daemon/licensing_test.go @@ -0,0 +1,18 @@ +package daemon // import "github.com/docker/docker/daemon" + +import ( + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/dockerversion" + "gotest.tools/assert" +) + +func TestfillLicense(t *testing.T) { + v := &types.Info{} + d := &Daemon{ + root: "/var/lib/docker/", + } + d.fillLicense(v) + assert.Assert(t, v.ProductLicense == dockerversion.DefaultProductLicense) +} diff --git a/components/engine/dockerversion/version_lib.go b/components/engine/dockerversion/version_lib.go index ff1816503b..77b87891be 100644 --- a/components/engine/dockerversion/version_lib.go +++ b/components/engine/dockerversion/version_lib.go @@ -6,13 +6,14 @@ package dockerversion // import "github.com/docker/docker/dockerversion" // Default build-time variable for library-import. // This file is overridden on build with build-time informations. const ( - GitCommit = "library-import" - Version = "library-import" - BuildTime = "library-import" - IAmStatic = "library-import" - ContainerdCommitID = "library-import" - RuncCommitID = "library-import" - InitCommitID = "library-import" - PlatformName = "" - ProductName = "" + GitCommit = "library-import" + Version = "library-import" + BuildTime = "library-import" + IAmStatic = "library-import" + ContainerdCommitID = "library-import" + RuncCommitID = "library-import" + InitCommitID = "library-import" + PlatformName = "" + ProductName = "" + DefaultProductLicense = "" ) diff --git a/components/engine/docs/api/version-history.md b/components/engine/docs/api/version-history.md index 416187f292..b5b81034d4 100644 --- a/components/engine/docs/api/version-history.md +++ b/components/engine/docs/api/version-history.md @@ -19,6 +19,8 @@ keywords: "API, Docker, rcli, REST, documentation" * `GET /info` now returns an empty string, instead of `` for `KernelVersion` and `OperatingSystem` if the daemon was unable to obtain this information. +* `GET /info` now returns information about the product license, if a license + has been applied to the daemon. ## V1.38 API changes diff --git a/components/engine/hack/make/.go-autogen b/components/engine/hack/make/.go-autogen index 342f5ec95b..5d1aab930b 100644 --- a/components/engine/hack/make/.go-autogen +++ b/components/engine/hack/make/.go-autogen @@ -15,13 +15,14 @@ package dockerversion // Default build-time variable for library-import. // This file is overridden on build with build-time informations. const ( - GitCommit string = "$GITCOMMIT" - Version string = "$VERSION" - BuildTime string = "$BUILDTIME" - IAmStatic string = "${IAMSTATIC:-true}" - ContainerdCommitID string = "${CONTAINERD_COMMIT}" - PlatformName string = "${PLATFORM}" - ProductName string = "${PRODUCT}" + GitCommit string = "$GITCOMMIT" + Version string = "$VERSION" + BuildTime string = "$BUILDTIME" + IAmStatic string = "${IAMSTATIC:-true}" + ContainerdCommitID string = "${CONTAINERD_COMMIT}" + PlatformName string = "${PLATFORM}" + ProductName string = "${PRODUCT}" + DefaultProductLicense string = "${DEFAULT_PRODUCT_LICENSE}" ) // AUTOGENERATED FILE; see /go/src/github.com/docker/docker/hack/make/.go-autogen diff --git a/components/engine/hack/make/.go-autogen.ps1 b/components/engine/hack/make/.go-autogen.ps1 index 98686e1362..cd4d87c557 100644 --- a/components/engine/hack/make/.go-autogen.ps1 +++ b/components/engine/hack/make/.go-autogen.ps1 @@ -16,7 +16,8 @@ param( [Parameter(Mandatory=$true)][string]$CommitString, [Parameter(Mandatory=$true)][string]$DockerVersion, [Parameter(Mandatory=$false)][string]$Platform, - [Parameter(Mandatory=$false)][string]$Product + [Parameter(Mandatory=$false)][string]$Product, + [Parameter(Mandatory=$false)][string]$DefaultProductLicense ) $ErrorActionPreference = "Stop" @@ -42,11 +43,12 @@ package dockerversion // Default build-time variable for library-import. // This file is overridden on build with build-time informations. const ( - GitCommit string = "'+$CommitString+'" - Version string = "'+$DockerVersion+'" - BuildTime string = "'+$buildDateTime+'" - PlatformName string = "'+$Platform+'" - ProductName string = "'+$Product+'" + GitCommit string = "'+$CommitString+'" + Version string = "'+$DockerVersion+'" + BuildTime string = "'+$buildDateTime+'" + PlatformName string = "'+$Platform+'" + ProductName string = "'+$Product+'" + DefaultProductLicense string = "'+$DefaultProductLicense+'" ) // AUTOGENERATED FILE; see hack\make\.go-autogen.ps1