Move graph and tags to graph sub pkg

Docker-DCO-1.1-Signed-off-by: Michael Crosby <michael@crosbymichael.com> (github: crosbymichael)
Upstream-commit: 01b6b2be73a6f40e0179e0217385eea6b41100a5
Component: engine

components/engine/graph/graph.go

@@ -0,0 +1,408 @@
+package graph
+
+import (
+	"fmt"
+	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/dockerversion"
+	"github.com/dotcloud/docker/graphdriver"
+	"github.com/dotcloud/docker/image"
+	"github.com/dotcloud/docker/runconfig"
+	"github.com/dotcloud/docker/utils"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"syscall"
+	"time"
+)
+
+// A Graph is a store for versioned filesystem images and the relationship between them.
+type Graph struct {
+	Root    string
+	idIndex *utils.TruncIndex
+	driver  graphdriver.Driver
+}
+
+// NewGraph instantiates a new graph at the given root path in the filesystem.
+// `root` will be created if it doesn't exist.
+func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) {
+	abspath, err := filepath.Abs(root)
+	if err != nil {
+		return nil, err
+	}
+	// Create the root directory if it doesn't exists
+	if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) {
+		return nil, err
+	}
+
+	graph := &Graph{
+		Root:    abspath,
+		idIndex: utils.NewTruncIndex(),
+		driver:  driver,
+	}
+	if err := graph.restore(); err != nil {
+		return nil, err
+	}
+	return graph, nil
+}
+
+func (graph *Graph) restore() error {
+	dir, err := ioutil.ReadDir(graph.Root)
+	if err != nil {
+		return err
+	}
+	for _, v := range dir {
+		id := v.Name()
+		if graph.driver.Exists(id) {
+			graph.idIndex.Add(id)
+		}
+	}
+	utils.Debugf("Restored %d elements", len(dir))
+	return nil
+}
+
+// FIXME: Implement error subclass instead of looking at the error text
+// Note: This is the way golang implements os.IsNotExists on Plan9
+func (graph *Graph) IsNotExist(err error) bool {
+	return err != nil && (strings.Contains(err.Error(), "does not exist") || strings.Contains(err.Error(), "No such"))
+}
+
+// Exists returns true if an image is registered at the given id.
+// If the image doesn't exist or if an error is encountered, false is returned.
+func (graph *Graph) Exists(id string) bool {
+	if _, err := graph.Get(id); err != nil {
+		return false
+	}
+	return true
+}
+
+// Get returns the image with the given id, or an error if the image doesn't exist.
+func (graph *Graph) Get(name string) (*image.Image, error) {
+	id, err := graph.idIndex.Get(name)
+	if err != nil {
+		return nil, err
+	}
+	// FIXME: return nil when the image doesn't exist, instead of an error
+	img, err := image.LoadImage(graph.ImageRoot(id))
+	if err != nil {
+		return nil, err
+	}
+	if img.ID != id {
+		return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID)
+	}
+	img.SetGraph(graph)
+
+	if img.Size < 0 {
+		rootfs, err := graph.driver.Get(img.ID)
+		if err != nil {
+			return nil, fmt.Errorf("Driver %s failed to get image rootfs %s: %s", graph.driver, img.ID, err)
+		}
+		defer graph.driver.Put(img.ID)
+
+		var size int64
+		if img.Parent == "" {
+			if size, err = utils.TreeSize(rootfs); err != nil {
+				return nil, err
+			}
+		} else {
+			parentFs, err := graph.driver.Get(img.Parent)
+			if err != nil {
+				return nil, err
+			}
+			changes, err := archive.ChangesDirs(rootfs, parentFs)
+			if err != nil {
+				return nil, err
+			}
+			size = archive.ChangesSize(rootfs, changes)
+		}
+
+		img.Size = size
+		if err := img.SaveSize(graph.ImageRoot(id)); err != nil {
+			return nil, err
+		}
+	}
+	return img, nil
+}
+
+// Create creates a new image and registers it in the graph.
+func (graph *Graph) Create(layerData archive.ArchiveReader, containerID, containerImage, comment, author string, containerConfig, config *runconfig.Config) (*image.Image, error) {
+	img := &image.Image{
+		ID:            utils.GenerateRandomID(),
+		Comment:       comment,
+		Created:       time.Now().UTC(),
+		DockerVersion: dockerversion.VERSION,
+		Author:        author,
+		Config:        config,
+		Architecture:  runtime.GOARCH,
+		OS:            runtime.GOOS,
+	}
+	if containerID != "" {
+		img.Parent = containerImage
+		img.Container = containerID
+		img.ContainerConfig = *containerConfig
+	}
+	if err := graph.Register(nil, layerData, img); err != nil {
+		return nil, err
+	}
+	return img, nil
+}
+
+// Register imports a pre-existing image into the graph.
+// FIXME: pass img as first argument
+func (graph *Graph) Register(jsonData []byte, layerData archive.ArchiveReader, img *image.Image) (err error) {
+	defer func() {
+		// If any error occurs, remove the new dir from the driver.
+		// Don't check for errors since the dir might not have been created.
+		// FIXME: this leaves a possible race condition.
+		if err != nil {
+			graph.driver.Remove(img.ID)
+		}
+	}()
+	if err := utils.ValidateID(img.ID); err != nil {
+		return err
+	}
+	// (This is a convenience to save time. Race conditions are taken care of by os.Rename)
+	if graph.Exists(img.ID) {
+		return fmt.Errorf("Image %s already exists", img.ID)
+	}
+
+	// Ensure that the image root does not exist on the filesystem
+	// when it is not registered in the graph.
+	// This is common when you switch from one graph driver to another
+	if err := os.RemoveAll(graph.ImageRoot(img.ID)); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	// If the driver has this ID but the graph doesn't, remove it from the driver to start fresh.
+	// (the graph is the source of truth).
+	// Ignore errors, since we don't know if the driver correctly returns ErrNotExist.
+	// (FIXME: make that mandatory for drivers).
+	graph.driver.Remove(img.ID)
+
+	tmp, err := graph.Mktemp("")
+	defer os.RemoveAll(tmp)
+	if err != nil {
+		return fmt.Errorf("Mktemp failed: %s", err)
+	}
+
+	// Create root filesystem in the driver
+	if err := graph.driver.Create(img.ID, img.Parent); err != nil {
+		return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err)
+	}
+	// Mount the root filesystem so we can apply the diff/layer
+	rootfs, err := graph.driver.Get(img.ID)
+	if err != nil {
+		return fmt.Errorf("Driver %s failed to get image rootfs %s: %s", graph.driver, img.ID, err)
+	}
+	defer graph.driver.Put(img.ID)
+	img.SetGraph(graph)
+	if err := image.StoreImage(img, jsonData, layerData, tmp, rootfs); err != nil {
+		return err
+	}
+	// Commit
+	if err := os.Rename(tmp, graph.ImageRoot(img.ID)); err != nil {
+		return err
+	}
+	graph.idIndex.Add(img.ID)
+	return nil
+}
+
+// TempLayerArchive creates a temporary archive of the given image's filesystem layer.
+//   The archive is stored on disk and will be automatically deleted as soon as has been read.
+//   If output is not nil, a human-readable progress bar will be written to it.
+//   FIXME: does this belong in Graph? How about MktempFile, let the caller use it for archives?
+func (graph *Graph) TempLayerArchive(id string, compression archive.Compression, sf *utils.StreamFormatter, output io.Writer) (*archive.TempArchive, error) {
+	image, err := graph.Get(id)
+	if err != nil {
+		return nil, err
+	}
+	tmp, err := graph.Mktemp("")
+	if err != nil {
+		return nil, err
+	}
+	a, err := image.TarLayer()
+	if err != nil {
+		return nil, err
+	}
+	progress := utils.ProgressReader(a, 0, output, sf, false, utils.TruncateID(id), "Buffering to disk")
+	defer progress.Close()
+	return archive.NewTempArchive(progress, tmp)
+}
+
+// Mktemp creates a temporary sub-directory inside the graph's filesystem.
+func (graph *Graph) Mktemp(id string) (string, error) {
+	dir := path.Join(graph.Root, "_tmp", utils.GenerateRandomID())
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return "", err
+	}
+	return dir, nil
+}
+
+// setupInitLayer populates a directory with mountpoints suitable
+// for bind-mounting dockerinit into the container. The mountpoint is simply an
+// empty file at /.dockerinit
+//
+// This extra layer is used by all containers as the top-most ro layer. It protects
+// the container from unwanted side-effects on the rw layer.
+func SetupInitLayer(initLayer string) error {
+	for pth, typ := range map[string]string{
+		"/dev/pts":         "dir",
+		"/dev/shm":         "dir",
+		"/proc":            "dir",
+		"/sys":             "dir",
+		"/.dockerinit":     "file",
+		"/.dockerenv":      "file",
+		"/etc/resolv.conf": "file",
+		"/etc/hosts":       "file",
+		"/etc/hostname":    "file",
+		"/dev/console":     "file",
+		// "var/run": "dir",
+		// "var/lock": "dir",
+	} {
+		parts := strings.Split(pth, "/")
+		prev := "/"
+		for _, p := range parts[1:] {
+			prev = path.Join(prev, p)
+			syscall.Unlink(path.Join(initLayer, prev))
+		}
+
+		if _, err := os.Stat(path.Join(initLayer, pth)); err != nil {
+			if os.IsNotExist(err) {
+				switch typ {
+				case "dir":
+					if err := os.MkdirAll(path.Join(initLayer, pth), 0755); err != nil {
+						return err
+					}
+				case "file":
+					if err := os.MkdirAll(path.Join(initLayer, path.Dir(pth)), 0755); err != nil {
+						return err
+					}
+					f, err := os.OpenFile(path.Join(initLayer, pth), os.O_CREATE, 0755)
+					if err != nil {
+						return err
+					}
+					f.Close()
+				}
+			} else {
+				return err
+			}
+		}
+	}
+
+	// Layer is ready to use, if it wasn't before.
+	return nil
+}
+
+// Check if given error is "not empty".
+// Note: this is the way golang does it internally with os.IsNotExists.
+func isNotEmpty(err error) bool {
+	switch pe := err.(type) {
+	case nil:
+		return false
+	case *os.PathError:
+		err = pe.Err
+	case *os.LinkError:
+		err = pe.Err
+	}
+	return strings.Contains(err.Error(), " not empty")
+}
+
+// Delete atomically removes an image from the graph.
+func (graph *Graph) Delete(name string) error {
+	id, err := graph.idIndex.Get(name)
+	if err != nil {
+		return err
+	}
+	tmp, err := graph.Mktemp("")
+	if err != nil {
+		return err
+	}
+	graph.idIndex.Delete(id)
+	err = os.Rename(graph.ImageRoot(id), tmp)
+	if err != nil {
+		return err
+	}
+	// Remove rootfs data from the driver
+	graph.driver.Remove(id)
+	// Remove the trashed image directory
+	return os.RemoveAll(tmp)
+}
+
+// Map returns a list of all images in the graph, addressable by ID.
+func (graph *Graph) Map() (map[string]*image.Image, error) {
+	images := make(map[string]*image.Image)
+	err := graph.walkAll(func(image *image.Image) {
+		images[image.ID] = image
+	})
+	if err != nil {
+		return nil, err
+	}
+	return images, nil
+}
+
+// walkAll iterates over each image in the graph, and passes it to a handler.
+// The walking order is undetermined.
+func (graph *Graph) walkAll(handler func(*image.Image)) error {
+	files, err := ioutil.ReadDir(graph.Root)
+	if err != nil {
+		return err
+	}
+	for _, st := range files {
+		if img, err := graph.Get(st.Name()); err != nil {
+			// Skip image
+			continue
+		} else if handler != nil {
+			handler(img)
+		}
+	}
+	return nil
+}
+
+// ByParent returns a lookup table of images by their parent.
+// If an image of id ID has 3 children images, then the value for key ID
+// will be a list of 3 images.
+// If an image has no children, it will not have an entry in the table.
+func (graph *Graph) ByParent() (map[string][]*image.Image, error) {
+	byParent := make(map[string][]*image.Image)
+	err := graph.walkAll(func(img *image.Image) {
+		parent, err := graph.Get(img.Parent)
+		if err != nil {
+			return
+		}
+		if children, exists := byParent[parent.ID]; exists {
+			byParent[parent.ID] = append(children, img)
+		} else {
+			byParent[parent.ID] = []*image.Image{img}
+		}
+	})
+	return byParent, err
+}
+
+// Heads returns all heads in the graph, keyed by id.
+// A head is an image which is not the parent of another image in the graph.
+func (graph *Graph) Heads() (map[string]*image.Image, error) {
+	heads := make(map[string]*image.Image)
+	byParent, err := graph.ByParent()
+	if err != nil {
+		return nil, err
+	}
+	err = graph.walkAll(func(image *image.Image) {
+		// If it's not in the byParent lookup table, then
+		// it's not a parent -> so it's a head!
+		if _, exists := byParent[image.ID]; !exists {
+			heads[image.ID] = image
+		}
+	})
+	return heads, err
+}
+
+func (graph *Graph) ImageRoot(id string) string {
+	return path.Join(graph.Root, id)
+}
+
+func (graph *Graph) Driver() graphdriver.Driver {
+	return graph.driver
+}

components/engine/graph/tags.go

@@ -0,0 +1,235 @@
+package graph
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/dotcloud/docker/image"
+	"github.com/dotcloud/docker/utils"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+const DEFAULTTAG = "latest"
+
+type TagStore struct {
+	path         string
+	graph        *Graph
+	Repositories map[string]Repository
+}
+
+type Repository map[string]string
+
+func NewTagStore(path string, graph *Graph) (*TagStore, error) {
+	abspath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, err
+	}
+	store := &TagStore{
+		path:         abspath,
+		graph:        graph,
+		Repositories: make(map[string]Repository),
+	}
+	// Load the json file if it exists, otherwise create it.
+	if err := store.Reload(); os.IsNotExist(err) {
+		if err := store.Save(); err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+	return store, nil
+}
+
+func (store *TagStore) Save() error {
+	// Store the json ball
+	jsonData, err := json.Marshal(store)
+	if err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(store.path, jsonData, 0600); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (store *TagStore) Reload() error {
+	jsonData, err := ioutil.ReadFile(store.path)
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(jsonData, store); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (store *TagStore) LookupImage(name string) (*image.Image, error) {
+	// FIXME: standardize on returning nil when the image doesn't exist, and err for everything else
+	// (so we can pass all errors here)
+	repos, tag := utils.ParseRepositoryTag(name)
+	if tag == "" {
+		tag = DEFAULTTAG
+	}
+	img, err := store.GetImage(repos, tag)
+	if err != nil {
+		return nil, err
+	} else if img == nil {
+		if img, err = store.graph.Get(name); err != nil {
+			return nil, err
+		}
+	}
+	return img, nil
+}
+
+// Return a reverse-lookup table of all the names which refer to each image
+// Eg. {"43b5f19b10584": {"base:latest", "base:v1"}}
+func (store *TagStore) ByID() map[string][]string {
+	byID := make(map[string][]string)
+	for repoName, repository := range store.Repositories {
+		for tag, id := range repository {
+			name := repoName + ":" + tag
+			if _, exists := byID[id]; !exists {
+				byID[id] = []string{name}
+			} else {
+				byID[id] = append(byID[id], name)
+				sort.Strings(byID[id])
+			}
+		}
+	}
+	return byID
+}
+
+func (store *TagStore) ImageName(id string) string {
+	if names, exists := store.ByID()[id]; exists && len(names) > 0 {
+		return names[0]
+	}
+	return utils.TruncateID(id)
+}
+
+func (store *TagStore) DeleteAll(id string) error {
+	names, exists := store.ByID()[id]
+	if !exists || len(names) == 0 {
+		return nil
+	}
+	for _, name := range names {
+		if strings.Contains(name, ":") {
+			nameParts := strings.Split(name, ":")
+			if _, err := store.Delete(nameParts[0], nameParts[1]); err != nil {
+				return err
+			}
+		} else {
+			if _, err := store.Delete(name, ""); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (store *TagStore) Delete(repoName, tag string) (bool, error) {
+	deleted := false
+	if err := store.Reload(); err != nil {
+		return false, err
+	}
+	if r, exists := store.Repositories[repoName]; exists {
+		if tag != "" {
+			if _, exists2 := r[tag]; exists2 {
+				delete(r, tag)
+				if len(r) == 0 {
+					delete(store.Repositories, repoName)
+				}
+				deleted = true
+			} else {
+				return false, fmt.Errorf("No such tag: %s:%s", repoName, tag)
+			}
+		} else {
+			delete(store.Repositories, repoName)
+			deleted = true
+		}
+	} else {
+		fmt.Errorf("No such repository: %s", repoName)
+	}
+	return deleted, store.Save()
+}
+
+func (store *TagStore) Set(repoName, tag, imageName string, force bool) error {
+	img, err := store.LookupImage(imageName)
+	if err != nil {
+		return err
+	}
+	if tag == "" {
+		tag = DEFAULTTAG
+	}
+	if err := validateRepoName(repoName); err != nil {
+		return err
+	}
+	if err := validateTagName(tag); err != nil {
+		return err
+	}
+	if err := store.Reload(); err != nil {
+		return err
+	}
+	var repo Repository
+	if r, exists := store.Repositories[repoName]; exists {
+		repo = r
+	} else {
+		repo = make(map[string]string)
+		if old, exists := store.Repositories[repoName]; exists && !force {
+			return fmt.Errorf("Conflict: Tag %s:%s is already set to %s", repoName, tag, old)
+		}
+		store.Repositories[repoName] = repo
+	}
+	repo[tag] = img.ID
+	return store.Save()
+}
+
+func (store *TagStore) Get(repoName string) (Repository, error) {
+	if err := store.Reload(); err != nil {
+		return nil, err
+	}
+	if r, exists := store.Repositories[repoName]; exists {
+		return r, nil
+	}
+	return nil, nil
+}
+
+func (store *TagStore) GetImage(repoName, tagOrID string) (*image.Image, error) {
+	repo, err := store.Get(repoName)
+	if err != nil {
+		return nil, err
+	} else if repo == nil {
+		return nil, nil
+	}
+	if revision, exists := repo[tagOrID]; exists {
+		return store.graph.Get(revision)
+	}
+	// If no matching tag is found, search through images for a matching image id
+	for _, revision := range repo {
+		if strings.HasPrefix(revision, tagOrID) {
+			return store.graph.Get(revision)
+		}
+	}
+	return nil, nil
+}
+
+// Validate the name of a repository
+func validateRepoName(name string) error {
+	if name == "" {
+		return fmt.Errorf("Repository name can't be empty")
+	}
+	return nil
+}
+
+// Validate the name of a tag
+func validateTagName(name string) error {
+	if name == "" {
+		return fmt.Errorf("Tag name can't be empty")
+	}
+	if strings.Contains(name, "/") || strings.Contains(name, ":") {
+		return fmt.Errorf("Illegal tag name: %s", name)
+	}
+	return nil
+}

components/engine/graph/tags_unit_test.go

@@ -0,0 +1,110 @@
+package graph
+
+import (
+	"bytes"
+	"github.com/dotcloud/docker/graphdriver"
+	_ "github.com/dotcloud/docker/graphdriver/vfs" // import the vfs driver so it is used in the tests
+	"github.com/dotcloud/docker/image"
+	"github.com/dotcloud/docker/utils"
+	"github.com/dotcloud/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
+	"io"
+	"os"
+	"path"
+	"testing"
+)
+
+const (
+	testImageName = "myapp"
+	testImageID   = "foo"
+)
+
+func fakeTar() (io.Reader, error) {
+	content := []byte("Hello world!\n")
+	buf := new(bytes.Buffer)
+	tw := tar.NewWriter(buf)
+	for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} {
+		hdr := new(tar.Header)
+		hdr.Size = int64(len(content))
+		hdr.Name = name
+		if err := tw.WriteHeader(hdr); err != nil {
+			return nil, err
+		}
+		tw.Write([]byte(content))
+	}
+	tw.Close()
+	return buf, nil
+}
+
+func mkTestTagStore(root string, t *testing.T) *TagStore {
+	driver, err := graphdriver.New(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	graph, err := NewGraph(root, driver)
+	if err != nil {
+		t.Fatal(err)
+	}
+	store, err := NewTagStore(path.Join(root, "tags"), graph)
+	if err != nil {
+		t.Fatal(err)
+	}
+	archive, err := fakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
+	img := &image.Image{ID: testImageID}
+	// FIXME: this fails on Darwin with:
+	// tags_unit_test.go:36: mkdir /var/folders/7g/b3ydb5gx4t94ndr_cljffbt80000gq/T/docker-test569b-tRunner-075013689/vfs/dir/foo/etc/postgres: permission denied
+	if err := graph.Register(nil, archive, img); err != nil {
+		t.Fatal(err)
+	}
+	if err := store.Set(testImageName, "", testImageID, false); err != nil {
+		t.Fatal(err)
+	}
+	return store
+}
+
+func TestLookupImage(t *testing.T) {
+	tmp, err := utils.TestDirectory("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	store := mkTestTagStore(tmp, t)
+	defer store.graph.driver.Cleanup()
+
+	if img, err := store.LookupImage(testImageName); err != nil {
+		t.Fatal(err)
+	} else if img == nil {
+		t.Errorf("Expected 1 image, none found")
+	}
+	if img, err := store.LookupImage(testImageName + ":" + DEFAULTTAG); err != nil {
+		t.Fatal(err)
+	} else if img == nil {
+		t.Errorf("Expected 1 image, none found")
+	}
+
+	if img, err := store.LookupImage(testImageName + ":" + "fail"); err == nil {
+		t.Errorf("Expected error, none found")
+	} else if img != nil {
+		t.Errorf("Expected 0 image, 1 found")
+	}
+
+	if img, err := store.LookupImage("fail:fail"); err == nil {
+		t.Errorf("Expected error, none found")
+	} else if img != nil {
+		t.Errorf("Expected 0 image, 1 found")
+	}
+
+	if img, err := store.LookupImage(testImageID); err != nil {
+		t.Fatal(err)
+	} else if img == nil {
+		t.Errorf("Expected 1 image, none found")
+	}
+
+	if img, err := store.LookupImage(testImageName + ":" + testImageID); err != nil {
+		t.Fatal(err)
+	} else if img == nil {
+		t.Errorf("Expected 1 image, none found")
+	}
+}