Merge pull request #22866 from jstarks/foreign_layers

Support layers from external URLs
Upstream-commit: 7d08f3a5ad16446952b9b461feb0895a5c73ea58
Component: engine

components/engine/layer/filestore.go

@@ -2,6 +2,7 @@ package layer
 
 import (
 	"compress/gzip"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -13,6 +14,7 @@ import (
 	"strings"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution"
 	"github.com/docker/distribution/digest"
 	"github.com/docker/docker/pkg/ioutils"
 )
@@ -24,6 +26,9 @@ var (
 		// digest.SHA384, // Currently not used
 		// digest.SHA512, // Currently not used
 	}
+
+	// ErrNoForeignSource is returned when no foreign source is set for a layer.
+	ErrNoForeignSource = errors.New("layer does not have a foreign source")
 )
 
 type fileMetadataStore struct {
@@ -98,6 +103,14 @@ func (fm *fileMetadataTransaction) SetCacheID(cacheID string) error {
 	return ioutil.WriteFile(filepath.Join(fm.root, "cache-id"), []byte(cacheID), 0644)
 }
 
+func (fm *fileMetadataTransaction) SetForeignSource(ref distribution.Descriptor) error {
+	jsonRef, err := json.Marshal(ref)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(filepath.Join(fm.root, "descriptor.json"), jsonRef, 0644)
+}
+
 func (fm *fileMetadataTransaction) TarSplitWriter(compressInput bool) (io.WriteCloser, error) {
 	f, err := os.OpenFile(filepath.Join(fm.root, "tar-split.json.gz"), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
 	if err != nil {
@@ -191,6 +204,23 @@ func (fms *fileMetadataStore) GetCacheID(layer ChainID) (string, error) {
 	return content, nil
 }
 
+func (fms *fileMetadataStore) GetForeignSource(layer ChainID) (distribution.Descriptor, error) {
+	content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "descriptor.json"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return distribution.Descriptor{}, ErrNoForeignSource
+		}
+		return distribution.Descriptor{}, err
+	}
+
+	var ref distribution.Descriptor
+	err = json.Unmarshal(content, &ref)
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+	return ref, err
+}
+
 func (fms *fileMetadataStore) TarSplitReader(layer ChainID) (io.ReadCloser, error) {
 	fz, err := os.Open(fms.getLayerFilename(layer, "tar-split.json.gz"))
 	if err != nil {

components/engine/layer/layer.go

@@ -14,6 +14,7 @@ import (
 	"io"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution"
 	"github.com/docker/distribution/digest"
 	"github.com/docker/docker/pkg/archive"
 )
@@ -107,6 +108,14 @@ type Layer interface {
 	Metadata() (map[string]string, error)
 }
 
+// ForeignSourcer is an interface used to describe the source of layers
+// and objects representing layers, when the source is a foreign URL.
+type ForeignSourcer interface {
+	// ForeignSource returns the descriptor for this layer if it is
+	// a foreign layer, or nil for ordinary layers.
+	ForeignSource() *distribution.Descriptor
+}
+
 // RWLayer represents a layer which is
 // read and writable
 type RWLayer interface {
@@ -168,6 +177,7 @@ type MountInit func(root string) error
 // read-only and read-write layers.
 type Store interface {
 	Register(io.Reader, ChainID) (Layer, error)
+	RegisterForeign(io.Reader, ChainID, *distribution.Descriptor) (Layer, error)
 	Get(ChainID) (Layer, error)
 	Release(Layer) ([]Metadata, error)
 
@@ -188,6 +198,7 @@ type MetadataTransaction interface {
 	SetParent(parent ChainID) error
 	SetDiffID(DiffID) error
 	SetCacheID(string) error
+	SetForeignSource(distribution.Descriptor) error
 	TarSplitWriter(compressInput bool) (io.WriteCloser, error)
 
 	Commit(ChainID) error
@@ -207,6 +218,7 @@ type MetadataStore interface {
 	GetParent(ChainID) (ChainID, error)
 	GetDiffID(ChainID) (DiffID, error)
 	GetCacheID(ChainID) (string, error)
+	GetForeignSource(ChainID) (distribution.Descriptor, error)
 	TarSplitReader(ChainID) (io.ReadCloser, error)
 
 	SetMountID(string, string) error

components/engine/layer/layer_store.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution"
 	"github.com/docker/distribution/digest"
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
@@ -137,6 +138,13 @@ func (ls *layerStore) loadLayer(layer ChainID) (*roLayer, error) {
 		references: map[Layer]struct{}{},
 	}
 
+	foreignSrc, err := ls.store.GetForeignSource(layer)
+	if err == nil {
+		cl.foreignSrc = &foreignSrc
+	} else if err != ErrNoForeignSource {
+		return nil, fmt.Errorf("failed to get foreign reference for %s: %s", layer, err)
+	}
+
 	if parent != "" {
 		p, err := ls.loadLayer(parent)
 		if err != nil {
@@ -228,6 +236,10 @@ func (ls *layerStore) applyTar(tx MetadataTransaction, ts io.Reader, parent stri
 }
 
 func (ls *layerStore) Register(ts io.Reader, parent ChainID) (Layer, error) {
+	return ls.RegisterForeign(ts, parent, nil)
+}
+
+func (ls *layerStore) RegisterForeign(ts io.Reader, parent ChainID, foreignSrc *distribution.Descriptor) (Layer, error) {
 	// err is used to hold the error which will always trigger
 	// cleanup of creates sources but may not be an error returned
 	// to the caller (already exists).
@@ -258,6 +270,7 @@ func (ls *layerStore) Register(ts io.Reader, parent ChainID) (Layer, error) {
 	layer := &roLayer{
 		parent:         p,
 		cacheID:        stringid.GenerateRandomID(),
+		foreignSrc:     foreignSrc,
 		referenceCount: 1,
 		layerStore:     ls,
 		references:     map[Layer]struct{}{},

components/engine/layer/layer_unix.go

@@ -7,3 +7,9 @@ import "github.com/docker/docker/pkg/stringid"
 func (ls *layerStore) mountID(name string) string {
 	return stringid.GenerateRandomID()
 }
+
+// ForeignSourceSupported returns whether layers downloaded from foreign sources are
+// supported in this daemon.
+func ForeignSourceSupported() bool {
+	return false
+}

components/engine/layer/layer_windows.go

@@ -96,3 +96,9 @@ func (ls *layerStore) mountID(name string) string {
 func (ls *layerStore) GraphDriver() graphdriver.Driver {
 	return ls.driver
 }
+
+// ForeignSourceSupported returns whether layers downloaded from foreign sources are
+// supported in this daemon.
+func ForeignSourceSupported() bool {
+	return true
+}

components/engine/layer/ro_layer.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/docker/distribution"
 	"github.com/docker/distribution/digest"
 )
 
@@ -14,6 +15,7 @@ type roLayer struct {
 	cacheID    string
 	size       int64
 	layerStore *layerStore
+	foreignSrc *distribution.Descriptor
 
 	referenceCount int
 	references     map[Layer]struct{}
@@ -123,6 +125,11 @@ func storeLayer(tx MetadataTransaction, layer *roLayer) error {
 			return err
 		}
 	}
+	if layer.foreignSrc != nil {
+		if err := tx.SetForeignSource(*layer.foreignSrc); err != nil {
+			return err
+		}
+	}
 
 	return nil
 }

components/engine/layer/ro_layer_windows.go

@@ -0,0 +1,9 @@
+package layer
+
+import "github.com/docker/distribution"
+
+var _ ForeignSourcer = &roLayer{}
+
+func (rl *roLayer) ForeignSource() *distribution.Descriptor {
+	return rl.foreignSrc
+}