toolshed/docker-cli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge component 'engine' from git@github.com:docker/engine master

Browse Source
This commit is contained in:
GordonTheTurtle
2018-06-13 17:07:06 +00:00
parent ccbd518d6d 2d48a7f46a
commit 39057b0c96
498 changed files with 39734 additions and 1281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
components/engine/Makefile
View File

@ -31,6 +31,7 @@ DOCKER_ENVS := \
-e DOCKER_BUILD_ARGS \
-e DOCKER_BUILD_GOGC \
-e DOCKER_BUILD_PKGS \
-e DOCKER_BUILDKIT \
-e DOCKER_BASH_COMPLETION_PATH \
-e DOCKER_CLI_PATH \
-e DOCKER_DEBUG \

4
components/engine/TESTING.md
View File

@ -8,11 +8,11 @@ questions you may have as an aspiring Moby contributor.
Moby has two test suites (and one legacy test suite):
* Unit tests - use standard `go test` and
[gotestyourself/assert](https://godoc.org/github.com/gotestyourself/gotestyourself/assert) assertions. They are located in
[gotest.tools/assert](https://godoc.org/gotest.tools/assert) assertions. They are located in
the package they test. Unit tests should be fast and test only their own
package.
* API integration tests - use standard `go test` and
[gotestyourself/assert](https://godoc.org/github.com/gotestyourself/gotestyourself/assert) assertions. They are located in
[gotest.tools/assert](https://godoc.org/gotest.tools/assert) assertions. They are located in
`./integration/<component>` directories, where `component` is: container,
image, volume, etc. These tests perform HTTP requests to an API endpoint and
check the HTTP response and daemon state after the call.

70
components/engine/api/server/backend/build/backend.go
View File

@ -8,10 +8,12 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/builder"
buildkit "github.com/docker/docker/builder/builder-next"
"github.com/docker/docker/builder/fscache"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/stringid"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
// ImageComponent provides an interface for working with images
@ -30,24 +32,39 @@ type Backend struct {
builder Builder
fsCache *fscache.FSCache
imageComponent ImageComponent
buildkit *buildkit.Builder
}
// NewBackend creates a new build backend from components
func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSCache) (*Backend, error) {
return &Backend{imageComponent: components, builder: builder, fsCache: fsCache}, nil
func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSCache, buildkit *buildkit.Builder) (*Backend, error) {
return &Backend{imageComponent: components, builder: builder, fsCache: fsCache, buildkit: buildkit}, nil
}
// Build builds an image from a Source
func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string, error) {
options := config.Options
useBuildKit := options.Version == types.BuilderBuildKit
tagger, err := NewTagger(b.imageComponent, config.ProgressWriter.StdoutFormatter, options.Tags)
if err != nil {
return "", err
}
build, err := b.builder.Build(ctx, config)
if err != nil {
return "", err
var build *builder.Result
if useBuildKit {
build, err = b.buildkit.Build(ctx, config)
if err != nil {
return "", err
}
} else {
build, err = b.builder.Build(ctx, config)
if err != nil {
return "", err
}
}
if build == nil {
return "", nil
}
var imageID = build.ImageID
@ -62,19 +79,48 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
}
}
stdout := config.ProgressWriter.StdoutFormatter
fmt.Fprintf(stdout, "Successfully built %s\n", stringid.TruncateID(imageID))
err = tagger.TagImages(image.ID(imageID))
if !useBuildKit {
stdout := config.ProgressWriter.StdoutFormatter
fmt.Fprintf(stdout, "Successfully built %s\n", stringid.TruncateID(imageID))
err = tagger.TagImages(image.ID(imageID))
}
return imageID, err
}
// PruneCache removes all cached build sources
func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
size, err := b.fsCache.Prune(ctx)
if err != nil {
return nil, errors.Wrap(err, "failed to prune build cache")
eg, ctx := errgroup.WithContext(ctx)
var fsCacheSize uint64
eg.Go(func() error {
var err error
fsCacheSize, err = b.fsCache.Prune(ctx)
if err != nil {
return errors.Wrap(err, "failed to prune fscache")
}
return nil
})
var buildCacheSize int64
eg.Go(func() error {
var err error
buildCacheSize, err = b.buildkit.Prune(ctx)
if err != nil {
return errors.Wrap(err, "failed to prune build cache")
}
return nil
})
if err := eg.Wait(); err != nil {
return nil, err
}
return &types.BuildCachePruneReport{SpaceReclaimed: size}, nil
return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize)}, nil
}
// Cancel cancels the build by ID
func (b *Backend) Cancel(ctx context.Context, id string) error {
return b.buildkit.Cancel(ctx, id)
}
func squashBuild(build *builder.Result, imageComponent ImageComponent) (string, error) {

4
components/engine/api/server/middleware/debug_test.go
View File

@ -3,8 +3,8 @@ package middleware // import "github.com/docker/docker/api/server/middleware"
import (
"testing"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestMaskSecretKeys(t *testing.T) {

4
components/engine/api/server/middleware/version_test.go
View File

@ -8,8 +8,8 @@ import (
"testing"
"github.com/docker/docker/api/server/httputils"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestVersionMiddlewareVersion(t *testing.T) {

2
components/engine/api/server/router/build/backend.go
View File

@ -15,6 +15,8 @@ type Backend interface {
// Prune build cache
PruneCache(context.Context) (*types.BuildCachePruneReport, error)
Cancel(context.Context, string) error
}
type experimentalProvider interface {

1
components/engine/api/server/router/build/build.go
View File

@ -25,5 +25,6 @@ func (r *buildRouter) initRoutes() {
r.routes = []router.Route{
router.NewPostRoute("/build", r.postBuild, router.WithCancel),
router.NewPostRoute("/build/prune", r.postPrune, router.WithCancel),
router.NewPostRoute("/build/cancel", r.postCancel),
}
}

153
components/engine/api/server/router/build/build_routes.go
View File

@ -1,6 +1,7 @@
package build // import "github.com/docker/docker/api/server/router/build"
import (
"bufio"
"bytes"
"context"
"encoding/base64"
@ -145,10 +146,26 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
options.CacheFrom = cacheFrom
}
options.SessionID = r.FormValue("session")
options.BuildID = r.FormValue("buildid")
builderVersion, err := parseVersion(r.FormValue("version"))
if err != nil {
return nil, err
}
options.Version = builderVersion
return options, nil
}
func parseVersion(s string) (types.BuilderVersion, error) {
if s == "" || s == string(types.BuilderV1) {
return types.BuilderV1, nil
}
if s == string(types.BuilderBuildKit) {
return types.BuilderBuildKit, nil
}
return "", errors.Errorf("invalid version %s", s)
}
func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
report, err := br.backend.PruneCache(ctx)
if err != nil {
@ -157,6 +174,17 @@ func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *
return httputils.WriteJSON(w, http.StatusOK, report)
}
func (br *buildRouter) postCancel(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
w.Header().Set("Content-Type", "application/json")
id := r.FormValue("id")
if id == "" {
return errors.Errorf("build ID not provided")
}
return br.backend.Cancel(ctx, id)
}
func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
var (
notVerboseBuffer = bytes.NewBuffer(nil)
@ -165,18 +193,34 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
w.Header().Set("Content-Type", "application/json")
output := ioutils.NewWriteFlusher(w)
body := r.Body
var ww io.Writer = w
if body != nil {
// there is a possibility that output is written before request body
// has been fully read so we need to protect against it.
// this can be removed when
// https://github.com/golang/go/issues/15527
// https://github.com/golang/go/issues/22209
// has been fixed
body, ww = wrapOutputBufferedUntilRequestRead(body, ww)
}
output := ioutils.NewWriteFlusher(ww)
defer output.Close()
errf := func(err error) error {
if httputils.BoolValue(r, "q") && notVerboseBuffer.Len() > 0 {
output.Write(notVerboseBuffer.Bytes())
}
logrus.Debugf("isflushed %v", output.Flushed())
// Do not write the error in the http output if it's still empty.
// This prevents from writing a 200(OK) when there is an internal error.
if !output.Flushed() {
return err
}
_, err = w.Write(streamformatter.FormatError(err))
_, err = output.Write(streamformatter.FormatError(err))
if err != nil {
logrus.Warnf("could not write error response: %v", err)
}
@ -205,10 +249,14 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
return progress.NewProgressReader(in, progressOutput, r.ContentLength, "Downloading context", buildOptions.RemoteContext)
}
if buildOptions.Version == types.BuilderBuildKit && !br.daemon.HasExperimental() {
return errdefs.InvalidParameter(errors.New("buildkit is only supported with experimental mode"))
}
wantAux := versions.GreaterThanOrEqualTo(version, "1.30")
imgID, err := br.backend.Build(ctx, backend.BuildConfig{
Source: r.Body,
Source: body,
Options: buildOptions,
ProgressWriter: buildProgressWriter(out, wantAux, createProgressReader),
})
@ -267,3 +315,102 @@ func buildProgressWriter(out io.Writer, wantAux bool, createProgressReader func(
ProgressReaderFunc: createProgressReader,
}
}
type flusher interface {
Flush()
}
func wrapOutputBufferedUntilRequestRead(rc io.ReadCloser, out io.Writer) (io.ReadCloser, io.Writer) {
var fl flusher = &ioutils.NopFlusher{}
if f, ok := out.(flusher); ok {
fl = f
}
w := &wcf{
buf: bytes.NewBuffer(nil),
Writer: out,
flusher: fl,
}
r := bufio.NewReader(rc)
_, err := r.Peek(1)
if err != nil {
return rc, out
}
rc = &rcNotifier{
Reader: r,
Closer: rc,
notify: w.notify,
}
return rc, w
}
type rcNotifier struct {
io.Reader
io.Closer
notify func()
}
func (r *rcNotifier) Read(b []byte) (int, error) {
n, err := r.Reader.Read(b)
if err != nil {
r.notify()
}
return n, err
}
func (r *rcNotifier) Close() error {
r.notify()
return r.Closer.Close()
}
type wcf struct {
io.Writer
flusher
mu sync.Mutex
ready bool
buf *bytes.Buffer
flushed bool
}
func (w *wcf) Flush() {
w.mu.Lock()
w.flushed = true
if !w.ready {
w.mu.Unlock()
return
}
w.mu.Unlock()
w.flusher.Flush()
}
func (w *wcf) Flushed() bool {
w.mu.Lock()
b := w.flushed
w.mu.Unlock()
return b
}
func (w *wcf) Write(b []byte) (int, error) {
w.mu.Lock()
if !w.ready {
n, err := w.buf.Write(b)
w.mu.Unlock()
return n, err
}
w.mu.Unlock()
return w.Writer.Write(b)
}
func (w *wcf) notify() {
w.mu.Lock()
if !w.ready {
if w.buf.Len() > 0 {
io.Copy(w.Writer, w.buf)
}
if w.flushed {
w.flusher.Flush()
}
w.ready = true
}
w.mu.Unlock()
}

9
components/engine/api/server/router/system/system.go
View File

@ -2,6 +2,7 @@ package system // import "github.com/docker/docker/api/server/router/system"
import (
"github.com/docker/docker/api/server/router"
buildkit "github.com/docker/docker/builder/builder-next"
"github.com/docker/docker/builder/fscache"
)
@ -11,15 +12,17 @@ type systemRouter struct {
backend Backend
cluster ClusterBackend
routes []router.Route
builder *fscache.FSCache
fscache *fscache.FSCache // legacy
builder *buildkit.Builder
}
// NewRouter initializes a new system router
func NewRouter(b Backend, c ClusterBackend, fscache *fscache.FSCache) router.Router {
func NewRouter(b Backend, c ClusterBackend, fscache *fscache.FSCache, builder *buildkit.Builder) router.Router {
r := &systemRouter{
backend: b,
cluster: c,
builder: fscache,
fscache: fscache,
builder: builder,
}
r.routes = []router.Route{

41
components/engine/api/server/router/system/system_routes.go
View File

@ -17,6 +17,7 @@ import (
"github.com/docker/docker/pkg/ioutils"
pkgerrors "github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
)
func optionsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
@ -69,15 +70,45 @@ func (s *systemRouter) getVersion(ctx context.Context, w http.ResponseWriter, r
}
func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
du, err := s.backend.SystemDiskUsage(ctx)
if err != nil {
eg, ctx := errgroup.WithContext(ctx)
var du *types.DiskUsage
eg.Go(func() error {
var err error
du, err = s.backend.SystemDiskUsage(ctx)
return err
})
var builderSize int64 // legacy
eg.Go(func() error {
var err error
builderSize, err = s.fscache.DiskUsage(ctx)
if err != nil {
return pkgerrors.Wrap(err, "error getting fscache build cache usage")
}
return nil
})
var buildCache []*types.BuildCache
eg.Go(func() error {
var err error
buildCache, err = s.builder.DiskUsage(ctx)
if err != nil {
return pkgerrors.Wrap(err, "error getting build cache usage")
}
return nil
})
if err := eg.Wait(); err != nil {
return err
}
builderSize, err := s.builder.DiskUsage(ctx)
if err != nil {
return pkgerrors.Wrap(err, "error getting build cache usage")
for _, b := range buildCache {
builderSize += b.Size
}
du.BuilderSize = builderSize
du.BuildCache = buildCache
return httputils.WriteJSON(w, http.StatusOK, du)
}

16
components/engine/api/types/client.go
View File

@ -181,8 +181,24 @@ type ImageBuildOptions struct {
Target string
SessionID string
Platform string
// Version specifies the version of the unerlying builder to use
Version BuilderVersion
// BuildID is an optional identifier that can be passed together with the
// build request. The same identifier can be used to gracefully cancel the
// build with the cancel request.
BuildID string
}
// BuilderVersion sets the version of underlying builder to use
type BuilderVersion string
const (
// BuilderV1 is the first generation builder in docker daemon
BuilderV1 BuilderVersion = "1"
// BuilderBuildKit is builder based on moby/buildkit project
BuilderBuildKit = "2"
)
// ImageBuildResponse holds information
// returned by a server after building
// an image.

4
components/engine/api/types/filters/parse_test.go
View File

@ -4,8 +4,8 @@ import (
"errors"
"testing"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestParseArgs(t *testing.T) {

17
components/engine/api/types/types.go
View File

@ -512,7 +512,8 @@ type DiskUsage struct {
Images []*ImageSummary
Containers []*Container
Volumes []*Volume
BuilderSize int64
BuildCache []*BuildCache
BuilderSize int64 // deprecated
}
// ContainersPruneReport contains the response for Engine API:
@ -585,3 +586,17 @@ type PushResult struct {
type BuildResult struct {
ID string
}
// BuildCache contains information about a build cache record
type BuildCache struct {
ID string
Mutable bool
InUse bool
Size int64
CreatedAt time.Time
LastUsedAt *time.Time
UsageCount int
Parent string
Description string
}

724
components/engine/builder/builder-next/adapters/containerimage/pull.go Normal file
View File

@ -0,0 +1,724 @@
package containerimage
import (
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"runtime"
"sync"
"time"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
ctdreference "github.com/containerd/containerd/reference"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/containerd/remotes/docker/schema1"
distreference "github.com/docker/distribution/reference"
"github.com/docker/docker/distribution"
"github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/distribution/xfer"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
pkgprogress "github.com/docker/docker/pkg/progress"
"github.com/docker/docker/reference"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/auth"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/util/flightcontrol"
"github.com/moby/buildkit/util/imageutil"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/tracing"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/identity"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/time/rate"
)
const preferLocal = true // FIXME: make this optional from the op
// SourceOpt is options for creating the image source
type SourceOpt struct {
SessionManager *session.Manager
ContentStore content.Store
CacheAccessor cache.Accessor
ReferenceStore reference.Store
DownloadManager distribution.RootFSDownloadManager
MetadataStore metadata.V2MetadataService
ImageStore image.Store
}
type imageSource struct {
SourceOpt
g flightcontrol.Group
}
// NewSource creates a new image source
func NewSource(opt SourceOpt) (source.Source, error) {
is := &imageSource{
SourceOpt: opt,
}
return is, nil
}
func (is *imageSource) ID() string {
return source.DockerImageScheme
}
func (is *imageSource) getResolver(ctx context.Context) remotes.Resolver {
return docker.NewResolver(docker.ResolverOptions{
Client: tracing.DefaultClient,
Credentials: is.getCredentialsFromSession(ctx),
})
}
func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(string) (string, string, error) {
id := session.FromContext(ctx)
if id == "" {
return nil
}
return func(host string) (string, string, error) {
timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
caller, err := is.SessionManager.Get(timeoutCtx, id)
if err != nil {
return "", "", err
}
return auth.CredentialsFunc(tracing.ContextWithSpanFromContext(context.TODO(), ctx), caller)(host)
}
}
func (is *imageSource) resolveLocal(refStr string) ([]byte, error) {
ref, err := distreference.ParseNormalizedNamed(refStr)
if err != nil {
return nil, err
}
dgst, err := is.ReferenceStore.Get(ref)
if err != nil {
return nil, err
}
img, err := is.ImageStore.Get(image.ID(dgst))
if err != nil {
return nil, err
}
return img.RawJSON(), nil
}
func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error) {
if preferLocal {
dt, err := is.resolveLocal(ref)
if err == nil {
return "", dt, nil
}
}
type t struct {
dgst digest.Digest
dt []byte
}
res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx), is.ContentStore, "")
if err != nil {
return nil, err
}
return &t{dgst: dgst, dt: dt}, nil
})
if err != nil {
return "", nil, err
}
typed := res.(*t)
return typed.dgst, typed.dt, nil
}
func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (source.SourceInstance, error) {
imageIdentifier, ok := id.(*source.ImageIdentifier)
if !ok {
return nil, errors.Errorf("invalid image identifier %v", id)
}
p := &puller{
src: imageIdentifier,
is: is,
resolver: is.getResolver(ctx),
}
return p, nil
}
type puller struct {
is *imageSource
resolveOnce sync.Once
resolveLocalOnce sync.Once
src *source.ImageIdentifier
desc ocispec.Descriptor
ref string
resolveErr error
resolver remotes.Resolver
config []byte
}
func (p *puller) mainManifestKey(dgst digest.Digest) (digest.Digest, error) {
dt, err := json.Marshal(struct {
Digest digest.Digest
OS string
Arch string
}{
Digest: p.desc.Digest,
OS: runtime.GOOS,
Arch: runtime.GOARCH,
})
if err != nil {
return "", err
}
return digest.FromBytes(dt), nil
}
func (p *puller) resolveLocal() {
p.resolveLocalOnce.Do(func() {
dgst := p.src.Reference.Digest()
if dgst != "" {
info, err := p.is.ContentStore.Info(context.TODO(), dgst)
if err == nil {
p.ref = p.src.Reference.String()
desc := ocispec.Descriptor{
Size: info.Size,
Digest: dgst,
}
ra, err := p.is.ContentStore.ReaderAt(context.TODO(), desc)
if err == nil {
mt, err := imageutil.DetectManifestMediaType(ra)
if err == nil {
desc.MediaType = mt
p.desc = desc
}
}
}
}
if preferLocal {
dt, err := p.is.resolveLocal(p.src.Reference.String())
if err == nil {
p.config = dt
}
}
})
}
func (p *puller) resolve(ctx context.Context) error {
p.resolveOnce.Do(func() {
resolveProgressDone := oneOffProgress(ctx, "resolve "+p.src.Reference.String())
ref, err := distreference.ParseNormalizedNamed(p.src.Reference.String())
if err != nil {
p.resolveErr = err
resolveProgressDone(err)
return
}
if p.desc.Digest == "" && p.config == nil {
origRef, desc, err := p.resolver.Resolve(ctx, ref.String())
if err != nil {
p.resolveErr = err
resolveProgressDone(err)
return
}
p.desc = desc
p.ref = origRef
}
// Schema 1 manifests cannot be resolved to an image config
// since the conversion must take place after all the content
// has been read.
// It may be possible to have a mapping between schema 1 manifests
// and the schema 2 manifests they are converted to.
if p.config == nil && p.desc.MediaType != images.MediaTypeDockerSchema1Manifest {
ref, err := distreference.WithDigest(ref, p.desc.Digest)
if err != nil {
p.resolveErr = err
resolveProgressDone(err)
return
}
_, dt, err := p.is.ResolveImageConfig(ctx, ref.String())
if err != nil {
p.resolveErr = err
resolveProgressDone(err)
return
}
p.config = dt
}
resolveProgressDone(nil)
})
return p.resolveErr
}
func (p *puller) CacheKey(ctx context.Context, index int) (string, bool, error) {
p.resolveLocal()
if p.desc.Digest != "" && index == 0 {
dgst, err := p.mainManifestKey(p.desc.Digest)
if err != nil {
return "", false, err
}
return dgst.String(), false, nil
}
if p.config != nil {
return cacheKeyFromConfig(p.config).String(), true, nil
}
if err := p.resolve(ctx); err != nil {
return "", false, err
}
if p.desc.Digest != "" && index == 0 {
dgst, err := p.mainManifestKey(p.desc.Digest)
if err != nil {
return "", false, err
}
return dgst.String(), false, nil
}
return cacheKeyFromConfig(p.config).String(), true, nil
}
func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
p.resolveLocal()
if err := p.resolve(ctx); err != nil {
return nil, err
}
if p.config != nil {
img, err := p.is.ImageStore.Get(image.ID(digest.FromBytes(p.config)))
if err == nil {
if len(img.RootFS.DiffIDs) == 0 {
return nil, nil
}
ref, err := p.is.CacheAccessor.GetFromSnapshotter(ctx, string(img.RootFS.ChainID()), cache.WithDescription(fmt.Sprintf("from local %s", p.ref)))
if err != nil {
return nil, err
}
return ref, nil
}
}
ongoing := newJobs(p.ref)
pctx, stopProgress := context.WithCancel(ctx)
pw, _, ctx := progress.FromContext(ctx)
defer pw.Close()
progressDone := make(chan struct{})
go func() {
showProgress(pctx, ongoing, p.is.ContentStore, pw)
close(progressDone)
}()
defer func() {
<-progressDone
}()
fetcher, err := p.resolver.Fetcher(ctx, p.ref)
if err != nil {
stopProgress()
return nil, err
}
var (
schema1Converter *schema1.Converter
handlers []images.Handler
)
if p.desc.MediaType == images.MediaTypeDockerSchema1Manifest {
schema1Converter = schema1.NewConverter(p.is.ContentStore, fetcher)
handlers = append(handlers, schema1Converter)
// TODO: Optimize to do dispatch and integrate pulling with download manager,
// leverage existing blob mapping and layer storage
} else {
// TODO: need a wrapper snapshot interface that combines content
// and snapshots as 1) buildkit shouldn't have a dependency on contentstore
// or 2) cachemanager should manage the contentstore
handlers = append(handlers, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
switch desc.MediaType {
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest,
images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex,
images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
default:
return nil, images.ErrSkipDesc
}
ongoing.add(desc)
return nil, nil
}))
// Get all the children for a descriptor
childrenHandler := images.ChildrenHandler(p.is.ContentStore)
// Set any children labels for that content
childrenHandler = images.SetChildrenLabels(p.is.ContentStore, childrenHandler)
// Filter the childen by the platform
childrenHandler = images.FilterPlatforms(childrenHandler, platforms.Default())
handlers = append(handlers,
remotes.FetchHandler(p.is.ContentStore, fetcher),
childrenHandler,
)
}
if err := images.Dispatch(ctx, images.Handlers(handlers...), p.desc); err != nil {
stopProgress()
return nil, err
}
defer stopProgress()
if schema1Converter != nil {
p.desc, err = schema1Converter.Convert(ctx)
if err != nil {
return nil, err
}
}
mfst, err := images.Manifest(ctx, p.is.ContentStore, p.desc, platforms.Default())
if err != nil {
return nil, err
}
config, err := images.Config(ctx, p.is.ContentStore, p.desc, platforms.Default())
if err != nil {
return nil, err
}
dt, err := content.ReadBlob(ctx, p.is.ContentStore, config)
if err != nil {
return nil, err
}
var img ocispec.Image
if err := json.Unmarshal(dt, &img); err != nil {
return nil, err
}
if len(mfst.Layers) != len(img.RootFS.DiffIDs) {
return nil, errors.Errorf("invalid config for manifest")
}
pchan := make(chan pkgprogress.Progress, 10)
defer close(pchan)
go func() {
m := map[string]struct {
st time.Time
limiter *rate.Limiter
}{}
for p := range pchan {
if p.Action == "Extracting" {
st, ok := m[p.ID]
if !ok {
st.st = time.Now()
st.limiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 1)
m[p.ID] = st
}
var end *time.Time
if p.LastUpdate || st.limiter.Allow() {
if p.LastUpdate {
tm := time.Now()
end = &tm
}
pw.Write("extracting "+p.ID, progress.Status{
Action: "extract",
Started: &st.st,
Completed: end,
})
}
}
}
}()
if len(mfst.Layers) == 0 {
return nil, nil
}
layers := make([]xfer.DownloadDescriptor, 0, len(mfst.Layers))
for i, desc := range mfst.Layers {
ongoing.add(desc)
layers = append(layers, &layerDescriptor{
desc: desc,
diffID: layer.DiffID(img.RootFS.DiffIDs[i]),
fetcher: fetcher,
ref: p.src.Reference,
is: p.is,
})
}
defer func() {
<-progressDone
for _, desc := range mfst.Layers {
p.is.ContentStore.Delete(context.TODO(), desc.Digest)
}
}()
r := image.NewRootFS()
rootFS, release, err := p.is.DownloadManager.Download(ctx, *r, runtime.GOOS, layers, pkgprogress.ChanOutput(pchan))
if err != nil {
return nil, err
}
stopProgress()
ref, err := p.is.CacheAccessor.GetFromSnapshotter(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("pulled from %s", p.ref)))
release()
if err != nil {
return nil, err
}
return ref, nil
}
// Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error)
type layerDescriptor struct {
is *imageSource
fetcher remotes.Fetcher
desc ocispec.Descriptor
diffID layer.DiffID
ref ctdreference.Spec
}
func (ld *layerDescriptor) Key() string {
return "v2:" + ld.desc.Digest.String()
}
func (ld *layerDescriptor) ID() string {
return ld.desc.Digest.String()
}
func (ld *layerDescriptor) DiffID() (layer.DiffID, error) {
return ld.diffID, nil
}
func (ld *layerDescriptor) Download(ctx context.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) {
rc, err := ld.fetcher.Fetch(ctx, ld.desc)
if err != nil {
return nil, 0, err
}
defer rc.Close()
refKey := remotes.MakeRefKey(ctx, ld.desc)
ld.is.ContentStore.Abort(ctx, refKey)
if err := content.WriteBlob(ctx, ld.is.ContentStore, refKey, rc, ld.desc); err != nil {
ld.is.ContentStore.Abort(ctx, refKey)
return nil, 0, err
}
ra, err := ld.is.ContentStore.ReaderAt(ctx, ld.desc)
if err != nil {
return nil, 0, err
}
return ioutil.NopCloser(content.NewReader(ra)), ld.desc.Size, nil
}
func (ld *layerDescriptor) Close() {
// ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest))
}
func (ld *layerDescriptor) Registered(diffID layer.DiffID) {
// Cache mapping from this layer's DiffID to the blobsum
ld.is.MetadataStore.Add(diffID, metadata.V2Metadata{Digest: ld.desc.Digest, SourceRepository: ld.ref.Locator})
}
func showProgress(ctx context.Context, ongoing *jobs, cs content.Store, pw progress.Writer) {
var (
ticker = time.NewTicker(100 * time.Millisecond)
statuses = map[string]statusInfo{}
done bool
)
defer ticker.Stop()
for {
select {
case <-ticker.C:
case <-ctx.Done():
done = true
}
resolved := "resolved"
if !ongoing.isResolved() {
resolved = "resolving"
}
statuses[ongoing.name] = statusInfo{
Ref: ongoing.name,
Status: resolved,
}
actives := make(map[string]statusInfo)
if !done {
active, err := cs.ListStatuses(ctx)
if err != nil {
// log.G(ctx).WithError(err).Error("active check failed")
continue
}
// update status of active entries!
for _, active := range active {
actives[active.Ref] = statusInfo{
Ref: active.Ref,
Status: "downloading",
Offset: active.Offset,
Total: active.Total,
StartedAt: active.StartedAt,
UpdatedAt: active.UpdatedAt,
}
}
}
// now, update the items in jobs that are not in active
for _, j := range ongoing.jobs() {
refKey := remotes.MakeRefKey(ctx, j.Descriptor)
if a, ok := actives[refKey]; ok {
started := j.started
pw.Write(j.Digest.String(), progress.Status{
Action: a.Status,
Total: int(a.Total),
Current: int(a.Offset),
Started: &started,
})
continue
}
if !j.done {
info, err := cs.Info(context.TODO(), j.Digest)
if err != nil {
if errdefs.IsNotFound(err) {
// pw.Write(j.Digest.String(), progress.Status{
// Action: "waiting",
// })
continue
}
} else {
j.done = true
}
if done || j.done {
started := j.started
createdAt := info.CreatedAt
pw.Write(j.Digest.String(), progress.Status{
Action: "done",
Current: int(info.Size),
Total: int(info.Size),
Completed: &createdAt,
Started: &started,
})
}
}
}
if done {
return
}
}
}
// jobs provides a way of identifying the download keys for a particular task
// encountering during the pull walk.
//
// This is very minimal and will probably be replaced with something more
// featured.
type jobs struct {
name string
added map[digest.Digest]job
mu sync.Mutex
resolved bool
}
type job struct {
ocispec.Descriptor
done bool
started time.Time
}
func newJobs(name string) *jobs {
return &jobs{
name: name,
added: make(map[digest.Digest]job),
}
}
func (j *jobs) add(desc ocispec.Descriptor) {
j.mu.Lock()
defer j.mu.Unlock()
if _, ok := j.added[desc.Digest]; ok {
return
}
j.added[desc.Digest] = job{
Descriptor: desc,
started: time.Now(),
}
}
func (j *jobs) jobs() []job {
j.mu.Lock()
defer j.mu.Unlock()
descs := make([]job, 0, len(j.added))
for _, j := range j.added {
descs = append(descs, j)
}
return descs
}
func (j *jobs) isResolved() bool {
j.mu.Lock()
defer j.mu.Unlock()
return j.resolved
}
type statusInfo struct {
Ref string
Status string
Offset int64
Total int64
StartedAt time.Time
UpdatedAt time.Time
}
func oneOffProgress(ctx context.Context, id string) func(err error) error {
pw, _, _ := progress.FromContext(ctx)
now := time.Now()
st := progress.Status{
Started: &now,
}
pw.Write(id, st)
return func(err error) error {
// TODO: set error on status
now := time.Now()
st.Completed = &now
pw.Write(id, st)
pw.Close()
return err
}
}
// cacheKeyFromConfig returns a stable digest from image config. If image config
// is a known oci image we will use chainID of layers.
func cacheKeyFromConfig(dt []byte) digest.Digest {
var img ocispec.Image
err := json.Unmarshal(dt, &img)
if err != nil {
return digest.FromBytes(dt)
}
if img.RootFS.Type != "layers" {
return digest.FromBytes(dt)
}
return identity.ChainID(img.RootFS.DiffIDs)
}

113
components/engine/builder/builder-next/adapters/snapshot/layer.go Normal file
View File

@ -0,0 +1,113 @@
package snapshot
import (
"context"
"os"
"path/filepath"
"github.com/boltdb/bolt"
"github.com/docker/docker/layer"
"github.com/docker/docker/pkg/ioutils"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
func (s *snapshotter) EnsureLayer(ctx context.Context, key string) ([]layer.DiffID, error) {
if l, err := s.getLayer(key, true); err != nil {
return nil, err
} else if l != nil {
return getDiffChain(l), nil
}
id, committed := s.getGraphDriverID(key)
if !committed {
return nil, errors.Errorf("can not convert active %s to layer", key)
}
info, err := s.Stat(ctx, key)
if err != nil {
return nil, err
}
eg, gctx := errgroup.WithContext(ctx)
// TODO: add flightcontrol
var parentChainID layer.ChainID
if info.Parent != "" {
eg.Go(func() error {
diffIDs, err := s.EnsureLayer(gctx, info.Parent)
if err != nil {
return err
}
parentChainID = layer.CreateChainID(diffIDs)
return nil
})
}
tmpDir, err := ioutils.TempDir("", "docker-tarsplit")
if err != nil {
return nil, err
}
defer os.RemoveAll(tmpDir)
tarSplitPath := filepath.Join(tmpDir, "tar-split")
var diffID layer.DiffID
var size int64
eg.Go(func() error {
parent := ""
if p := info.Parent; p != "" {
if l, err := s.getLayer(p, true); err != nil {
return err
} else if l != nil {
parent, err = getGraphID(l)
if err != nil {
return err
}
} else {
parent, _ = s.getGraphDriverID(info.Parent)
}
}
diffID, size, err = s.reg.ChecksumForGraphID(id, parent, "", tarSplitPath)
return err
})
if err := eg.Wait(); err != nil {
return nil, err
}
l, err := s.reg.RegisterByGraphID(id, parentChainID, diffID, tarSplitPath, size)
if err != nil {
return nil, err
}
if err := s.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(key))
b.Put(keyChainID, []byte(l.ChainID()))
return nil
}); err != nil {
return nil, err
}
s.mu.Lock()
s.refs[key] = l
s.mu.Unlock()
return getDiffChain(l), nil
}
func getDiffChain(l layer.Layer) []layer.DiffID {
if p := l.Parent(); p != nil {
return append(getDiffChain(p), l.DiffID())
}
return []layer.DiffID{l.DiffID()}
}
func getGraphID(l layer.Layer) (string, error) {
if l, ok := l.(interface {
CacheID() string
}); ok {
return l.CacheID(), nil
}
return "", errors.Errorf("couldn't access cacheID for %s", l.ChainID())
}

445
components/engine/builder/builder-next/adapters/snapshot/snapshot.go Normal file
View File

@ -0,0 +1,445 @@
package snapshot
import (
"context"
"path/filepath"
"strconv"
"strings"
"sync"
"github.com/boltdb/bolt"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/snapshots"
"github.com/docker/docker/daemon/graphdriver"
"github.com/docker/docker/layer"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/snapshot"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
var keyParent = []byte("parent")
var keyCommitted = []byte("committed")
var keyChainID = []byte("chainid")
var keySize = []byte("size")
// Opt defines options for creating the snapshotter
type Opt struct {
GraphDriver graphdriver.Driver
LayerStore layer.Store
Root string
}
type graphIDRegistrar interface {
RegisterByGraphID(string, layer.ChainID, layer.DiffID, string, int64) (layer.Layer, error)
Release(layer.Layer) ([]layer.Metadata, error)
checksumCalculator
}
type checksumCalculator interface {
ChecksumForGraphID(id, parent, oldTarDataPath, newTarDataPath string) (diffID layer.DiffID, size int64, err error)
}
type snapshotter struct {
opt Opt
refs map[string]layer.Layer
db *bolt.DB
mu sync.Mutex
reg graphIDRegistrar
}
var _ snapshot.SnapshotterBase = &snapshotter{}
// NewSnapshotter creates a new snapshotter
func NewSnapshotter(opt Opt) (snapshot.SnapshotterBase, error) {
dbPath := filepath.Join(opt.Root, "snapshots.db")
db, err := bolt.Open(dbPath, 0600, nil)
if err != nil {
return nil, errors.Wrapf(err, "failed to open database file %s", dbPath)
}
reg, ok := opt.LayerStore.(graphIDRegistrar)
if !ok {
return nil, errors.Errorf("layerstore doesn't support graphID registration")
}
s := &snapshotter{
opt: opt,
db: db,
refs: map[string]layer.Layer{},
reg: reg,
}
return s, nil
}
func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error {
origParent := parent
if parent != "" {
if l, err := s.getLayer(parent, false); err != nil {
return err
} else if l != nil {
parent, err = getGraphID(l)
if err != nil {
return err
}
} else {
parent, _ = s.getGraphDriverID(parent)
}
}
if err := s.opt.GraphDriver.Create(key, parent, nil); err != nil {
return err
}
if err := s.db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(key))
if err != nil {
return err
}
if err := b.Put(keyParent, []byte(origParent)); err != nil {
return err
}
return nil
}); err != nil {
return err
}
return nil
}
func (s *snapshotter) chainID(key string) (layer.ChainID, bool) {
if strings.HasPrefix(key, "sha256:") {
dgst, err := digest.Parse(key)
if err != nil {
return "", false
}
return layer.ChainID(dgst), true
}
return "", false
}
func (s *snapshotter) getLayer(key string, withCommitted bool) (layer.Layer, error) {
s.mu.Lock()
l, ok := s.refs[key]
if !ok {
id, ok := s.chainID(key)
if !ok {
if !withCommitted {
s.mu.Unlock()
return nil, nil
}
if err := s.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(key))
if b == nil {
return nil
}
v := b.Get(keyChainID)
if v != nil {
id = layer.ChainID(v)
}
return nil
}); err != nil {
s.mu.Unlock()
return nil, err
}
if id == "" {
s.mu.Unlock()
return nil, nil
}
}
var err error
l, err = s.opt.LayerStore.Get(id)
if err != nil {
s.mu.Unlock()
return nil, err
}
s.refs[key] = l
if err := s.db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(key))
return err
}); err != nil {
s.mu.Unlock()
return nil, err
}
}
s.mu.Unlock()
return l, nil
}
func (s *snapshotter) getGraphDriverID(key string) (string, bool) {
var gdID string
if err := s.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(key))
if b == nil {
return errors.Errorf("not found") // TODO: typed
}
v := b.Get(keyCommitted)
if v != nil {
gdID = string(v)
}
return nil
}); err != nil || gdID == "" {
return key, false
}
return gdID, true
}
func (s *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) {
inf := snapshots.Info{
Kind: snapshots.KindActive,
}
l, err := s.getLayer(key, false)
if err != nil {
return snapshots.Info{}, err
}
if l != nil {
if p := l.Parent(); p != nil {
inf.Parent = p.ChainID().String()
}
inf.Kind = snapshots.KindCommitted
inf.Name = key
return inf, nil
}
l, err = s.getLayer(key, true)
if err != nil {
return snapshots.Info{}, err
}
id, committed := s.getGraphDriverID(key)
if committed {
inf.Kind = snapshots.KindCommitted
}
if err := s.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(id))
if b == nil && l == nil {
return errors.Errorf("snapshot %s not found", id) // TODO: typed
}
inf.Name = key
if b != nil {
v := b.Get(keyParent)
if v != nil {
inf.Parent = string(v)
return nil
}
}
if l != nil {
if p := l.Parent(); p != nil {
inf.Parent = p.ChainID().String()
}
inf.Kind = snapshots.KindCommitted
}
return nil
}); err != nil {
return snapshots.Info{}, err
}
return inf, nil
}
func (s *snapshotter) Mounts(ctx context.Context, key string) (snapshot.Mountable, error) {
l, err := s.getLayer(key, true)
if err != nil {
return nil, err
}
if l != nil {
id := identity.NewID()
rwlayer, err := s.opt.LayerStore.CreateRWLayer(id, l.ChainID(), nil)
if err != nil {
return nil, err
}
rootfs, err := rwlayer.Mount("")
if err != nil {
return nil, err
}
mnt := []mount.Mount{{
Source: rootfs.Path(),
Type: "bind",
Options: []string{"rbind"},
}}
return &constMountable{
mounts: mnt,
release: func() error {
_, err := s.opt.LayerStore.ReleaseRWLayer(rwlayer)
return err
},
}, nil
}
id, _ := s.getGraphDriverID(key)
rootfs, err := s.opt.GraphDriver.Get(id, "")
if err != nil {
return nil, err
}
mnt := []mount.Mount{{
Source: rootfs.Path(),
Type: "bind",
Options: []string{"rbind"},
}}
return &constMountable{
mounts: mnt,
release: func() error {
return s.opt.GraphDriver.Put(id)
},
}, nil
}
func (s *snapshotter) Remove(ctx context.Context, key string) error {
l, err := s.getLayer(key, true)
if err != nil {
return err
}
id, _ := s.getGraphDriverID(key)
var found bool
if err := s.db.Update(func(tx *bolt.Tx) error {
found = tx.Bucket([]byte(key)) != nil
if found {
tx.DeleteBucket([]byte(key))
if id != key {
tx.DeleteBucket([]byte(id))
}
}
return nil
}); err != nil {
return err
}
if l != nil {
s.mu.Lock()
delete(s.refs, key)
s.mu.Unlock()
_, err := s.opt.LayerStore.Release(l)
return err
}
if !found { // this happens when removing views
return nil
}
return s.opt.GraphDriver.Remove(id)
}
func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error {
return s.db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(name))
if err != nil {
return err
}
if err := b.Put(keyCommitted, []byte(key)); err != nil {
return err
}
return nil
})
}
func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) (snapshot.Mountable, error) {
return s.Mounts(ctx, parent)
}
func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error {
return errors.Errorf("not-implemented")
}
func (s *snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) {
// not implemented
return s.Stat(ctx, info.Name)
}
func (s *snapshotter) Usage(ctx context.Context, key string) (us snapshots.Usage, retErr error) {
usage := snapshots.Usage{}
if l, err := s.getLayer(key, true); err != nil {
return usage, err
} else if l != nil {
s, err := l.DiffSize()
if err != nil {
return usage, err
}
usage.Size = s
return usage, nil
}
size := int64(-1)
if err := s.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(key))
if b == nil {
return nil
}
v := b.Get(keySize)
if v != nil {
s, err := strconv.Atoi(string(v))
if err != nil {
return err
}
size = int64(s)
}
return nil
}); err != nil {
return usage, err
}
if size != -1 {
usage.Size = size
return usage, nil
}
id, _ := s.getGraphDriverID(key)
info, err := s.Stat(ctx, key)
if err != nil {
return usage, err
}
var parent string
if info.Parent != "" {
if l, err := s.getLayer(info.Parent, false); err != nil {
return usage, err
} else if l != nil {
parent, err = getGraphID(l)
if err != nil {
return usage, err
}
} else {
parent, _ = s.getGraphDriverID(info.Parent)
}
}
diffSize, err := s.opt.GraphDriver.DiffSize(id, parent)
if err != nil {
return usage, err
}
if err := s.db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(key))
if err != nil {
return err
}
return b.Put(keySize, []byte(strconv.Itoa(int(diffSize))))
}); err != nil {
return usage, err
}
usage.Size = diffSize
return usage, nil
}
func (s *snapshotter) Close() error {
return s.db.Close()
}
type constMountable struct {
mounts []mount.Mount
release func() error
}
func (m *constMountable) Mount() ([]mount.Mount, error) {
return m.mounts, nil
}
func (m *constMountable) Release() error {
if m.release == nil {
return nil
}
return m.release()
}

419
components/engine/builder/builder-next/builder.go Normal file
View File

@ -0,0 +1,419 @@
package buildkit
import (
"context"
"encoding/json"
"io"
"strings"
"sync"
"time"
"github.com/containerd/containerd/content"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/builder"
"github.com/docker/docker/daemon/images"
"github.com/docker/docker/pkg/jsonmessage"
controlapi "github.com/moby/buildkit/api/services/control"
"github.com/moby/buildkit/control"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/util/tracing"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
grpcmetadata "google.golang.org/grpc/metadata"
)
// Opt is option struct required for creating the builder
type Opt struct {
SessionManager *session.Manager
Root string
Dist images.DistributionServices
}
// Builder can build using BuildKit backend
type Builder struct {
controller *control.Controller
reqBodyHandler *reqBodyHandler
mu sync.Mutex
jobs map[string]*buildJob
}
// New creates a new builder
func New(opt Opt) (*Builder, error) {
reqHandler := newReqBodyHandler(tracing.DefaultTransport)
c, err := newController(reqHandler, opt)
if err != nil {
return nil, err
}
b := &Builder{
controller: c,
reqBodyHandler: reqHandler,
jobs: map[string]*buildJob{},
}
return b, nil
}
// Cancel cancels a build using ID
func (b *Builder) Cancel(ctx context.Context, id string) error {
b.mu.Lock()
if j, ok := b.jobs[id]; ok && j.cancel != nil {
j.cancel()
}
b.mu.Unlock()
return nil
}
// DiskUsage returns a report about space used by build cache
func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) {
duResp, err := b.controller.DiskUsage(ctx, &controlapi.DiskUsageRequest{})
if err != nil {
return nil, err
}
var items []*types.BuildCache
for _, r := range duResp.Record {
items = append(items, &types.BuildCache{
ID: r.ID,
Mutable: r.Mutable,
InUse: r.InUse,
Size: r.Size_,
CreatedAt: r.CreatedAt,
LastUsedAt: r.LastUsedAt,
UsageCount: int(r.UsageCount),
Parent: r.Parent,
Description: r.Description,
})
}
return items, nil
}
// Prune clears all reclaimable build cache
func (b *Builder) Prune(ctx context.Context) (int64, error) {
ch := make(chan *controlapi.UsageRecord)
eg, ctx := errgroup.WithContext(ctx)
eg.Go(func() error {
defer close(ch)
return b.controller.Prune(&controlapi.PruneRequest{}, &pruneProxy{
streamProxy: streamProxy{ctx: ctx},
ch: ch,
})
})
var size int64
eg.Go(func() error {
for r := range ch {
size += r.Size_
}
return nil
})
if err := eg.Wait(); err != nil {
return 0, err
}
return size, nil
}
// Build executes a build request
func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.Result, error) {
var rc = opt.Source
if buildID := opt.Options.BuildID; buildID != "" {
b.mu.Lock()
upload := false
if strings.HasPrefix(buildID, "upload-request:") {
upload = true
buildID = strings.TrimPrefix(buildID, "upload-request:")
}
if _, ok := b.jobs[buildID]; !ok {
b.jobs[buildID] = newBuildJob()
}
j := b.jobs[buildID]
var cancel func()
ctx, cancel = context.WithCancel(ctx)
j.cancel = cancel
b.mu.Unlock()
if upload {
ctx2, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
err := j.SetUpload(ctx2, rc)
return nil, err
}
if remoteContext := opt.Options.RemoteContext; remoteContext == "upload-request" {
ctx2, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
var err error
rc, err = j.WaitUpload(ctx2)
if err != nil {
return nil, err
}
opt.Options.RemoteContext = ""
}
defer func() {
delete(b.jobs, buildID)
}()
}
var out builder.Result
id := identity.NewID()
frontendAttrs := map[string]string{}
if opt.Options.Target != "" {
frontendAttrs["target"] = opt.Options.Target
}
if opt.Options.Dockerfile != "" && opt.Options.Dockerfile != "." {
frontendAttrs["filename"] = opt.Options.Dockerfile
}
if opt.Options.RemoteContext != "" {
if opt.Options.RemoteContext != "client-session" {
frontendAttrs["context"] = opt.Options.RemoteContext
}
} else {
url, cancel := b.reqBodyHandler.newRequest(rc)
defer cancel()
frontendAttrs["context"] = url
}
cacheFrom := append([]string{}, opt.Options.CacheFrom...)
frontendAttrs["cache-from"] = strings.Join(cacheFrom, ",")
for k, v := range opt.Options.BuildArgs {
if v == nil {
continue
}
frontendAttrs["build-arg:"+k] = *v
}
for k, v := range opt.Options.Labels {
frontendAttrs["label:"+k] = v
}
if opt.Options.NoCache {
frontendAttrs["no-cache"] = ""
}
exporterAttrs := map[string]string{}
if len(opt.Options.Tags) > 0 {
exporterAttrs["name"] = strings.Join(opt.Options.Tags, ",")
}
req := &controlapi.SolveRequest{
Ref: id,
Exporter: "moby",
ExporterAttrs: exporterAttrs,
Frontend: "dockerfile.v0",
FrontendAttrs: frontendAttrs,
Session: opt.Options.SessionID,
}
eg, ctx := errgroup.WithContext(ctx)
eg.Go(func() error {
resp, err := b.controller.Solve(ctx, req)
if err != nil {
return err
}
id, ok := resp.ExporterResponse["containerimage.digest"]
if !ok {
return errors.Errorf("missing image id")
}
out.ImageID = id
return nil
})
ch := make(chan *controlapi.StatusResponse)
eg.Go(func() error {
defer close(ch)
return b.controller.Status(&controlapi.StatusRequest{
Ref: id,
}, &statusProxy{streamProxy: streamProxy{ctx: ctx}, ch: ch})
})
eg.Go(func() error {
for sr := range ch {
dt, err := sr.Marshal()
if err != nil {
return err
}
auxJSONBytes, err := json.Marshal(dt)
if err != nil {
return err
}
auxJSON := new(json.RawMessage)
*auxJSON = auxJSONBytes
msgJSON, err := json.Marshal(&jsonmessage.JSONMessage{ID: "moby.buildkit.trace", Aux: auxJSON})
if err != nil {
return err
}
msgJSON = append(msgJSON, []byte("\r\n")...)
n, err := opt.ProgressWriter.Output.Write(msgJSON)
if err != nil {
return err
}
if n != len(msgJSON) {
return io.ErrShortWrite
}
}
return nil
})
if err := eg.Wait(); err != nil {
return nil, err
}
return &out, nil
}
type streamProxy struct {
ctx context.Context
}
func (sp *streamProxy) SetHeader(_ grpcmetadata.MD) error {
return nil
}
func (sp *streamProxy) SendHeader(_ grpcmetadata.MD) error {
return nil
}
func (sp *streamProxy) SetTrailer(_ grpcmetadata.MD) {
}
func (sp *streamProxy) Context() context.Context {
return sp.ctx
}
func (sp *streamProxy) RecvMsg(m interface{}) error {
return io.EOF
}
type statusProxy struct {
streamProxy
ch chan *controlapi.StatusResponse
}
func (sp *statusProxy) Send(resp *controlapi.StatusResponse) error {
return sp.SendMsg(resp)
}
func (sp *statusProxy) SendMsg(m interface{}) error {
if sr, ok := m.(*controlapi.StatusResponse); ok {
sp.ch <- sr
}
return nil
}
type pruneProxy struct {
streamProxy
ch chan *controlapi.UsageRecord
}
func (sp *pruneProxy) Send(resp *controlapi.UsageRecord) error {
return sp.SendMsg(resp)
}
func (sp *pruneProxy) SendMsg(m interface{}) error {
if sr, ok := m.(*controlapi.UsageRecord); ok {
sp.ch <- sr
}
return nil
}
type contentStoreNoLabels struct {
content.Store
}
func (c *contentStoreNoLabels) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) {
return content.Info{}, nil
}
type wrapRC struct {
io.ReadCloser
once sync.Once
err error
waitCh chan struct{}
}
func (w *wrapRC) Read(b []byte) (int, error) {
n, err := w.ReadCloser.Read(b)
if err != nil {
e := err
if e == io.EOF {
e = nil
}
w.close(e)
}
return n, err
}
func (w *wrapRC) Close() error {
err := w.ReadCloser.Close()
w.close(err)
return err
}
func (w *wrapRC) close(err error) {
w.once.Do(func() {
w.err = err
close(w.waitCh)
})
}
func (w *wrapRC) wait() error {
<-w.waitCh
return w.err
}
type buildJob struct {
cancel func()
waitCh chan func(io.ReadCloser) error
}
func newBuildJob() *buildJob {
return &buildJob{waitCh: make(chan func(io.ReadCloser) error)}
}
func (j *buildJob) WaitUpload(ctx context.Context) (io.ReadCloser, error) {
done := make(chan struct{})
var upload io.ReadCloser
fn := func(rc io.ReadCloser) error {
w := &wrapRC{ReadCloser: rc, waitCh: make(chan struct{})}
upload = w
close(done)
return w.wait()
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case j.waitCh <- fn:
<-done
return upload, nil
}
}
func (j *buildJob) SetUpload(ctx context.Context, rc io.ReadCloser) error {
select {
case <-ctx.Done():
return ctx.Err()
case fn := <-j.waitCh:
return fn(rc)
}
}

157
components/engine/builder/builder-next/controller.go Normal file
View File

@ -0,0 +1,157 @@
package buildkit
import (
"net/http"
"os"
"path/filepath"
"github.com/containerd/containerd/content/local"
"github.com/docker/docker/builder/builder-next/adapters/containerimage"
"github.com/docker/docker/builder/builder-next/adapters/snapshot"
containerimageexp "github.com/docker/docker/builder/builder-next/exporter"
mobyworker "github.com/docker/docker/builder/builder-next/worker"
"github.com/docker/docker/daemon/graphdriver"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/cache/remotecache"
"github.com/moby/buildkit/control"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/frontend"
"github.com/moby/buildkit/frontend/dockerfile"
"github.com/moby/buildkit/frontend/gateway"
"github.com/moby/buildkit/snapshot/blobmapping"
"github.com/moby/buildkit/solver/boltdbcachestorage"
"github.com/moby/buildkit/worker"
"github.com/pkg/errors"
)
func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
if err := os.MkdirAll(opt.Root, 0700); err != nil {
return nil, err
}
dist := opt.Dist
root := opt.Root
var driver graphdriver.Driver
if ls, ok := dist.LayerStore.(interface {
Driver() graphdriver.Driver
}); ok {
driver = ls.Driver()
} else {
return nil, errors.Errorf("could not access graphdriver")
}
sbase, err := snapshot.NewSnapshotter(snapshot.Opt{
GraphDriver: driver,
LayerStore: dist.LayerStore,
Root: root,
})
if err != nil {
return nil, err
}
store, err := local.NewStore(filepath.Join(root, "content"))
if err != nil {
return nil, err
}
store = &contentStoreNoLabels{store}
md, err := metadata.NewStore(filepath.Join(root, "metadata.db"))
if err != nil {
return nil, err
}
snapshotter := blobmapping.NewSnapshotter(blobmapping.Opt{
Content: store,
Snapshotter: sbase,
MetadataStore: md,
})
cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: snapshotter,
MetadataStore: md,
})
if err != nil {
return nil, err
}
src, err := containerimage.NewSource(containerimage.SourceOpt{
SessionManager: opt.SessionManager,
CacheAccessor: cm,
ContentStore: store,
DownloadManager: dist.DownloadManager,
MetadataStore: dist.V2MetadataService,
ImageStore: dist.ImageStore,
ReferenceStore: dist.ReferenceStore,
})
if err != nil {
return nil, err
}
exec, err := newExecutor(root)
if err != nil {
return nil, err
}
differ, ok := sbase.(containerimageexp.Differ)
if !ok {
return nil, errors.Errorf("snapshotter doesn't support differ")
}
exp, err := containerimageexp.New(containerimageexp.Opt{
ImageStore: dist.ImageStore,
ReferenceStore: dist.ReferenceStore,
Differ: differ,
})
if err != nil {
return nil, err
}
cacheStorage, err := boltdbcachestorage.NewStore(filepath.Join(opt.Root, "cache.db"))
if err != nil {
return nil, err
}
frontends := map[string]frontend.Frontend{}
frontends["dockerfile.v0"] = dockerfile.NewDockerfileFrontend()
frontends["gateway.v0"] = gateway.NewGatewayFrontend()
wopt := mobyworker.Opt{
ID: "moby",
SessionManager: opt.SessionManager,
MetadataStore: md,
ContentStore: store,
CacheManager: cm,
Snapshotter: snapshotter,
Executor: exec,
ImageSource: src,
DownloadManager: dist.DownloadManager,
V2MetadataService: dist.V2MetadataService,
Exporters: map[string]exporter.Exporter{
"moby": exp,
},
Transport: rt,
}
wc := &worker.Controller{}
w, err := mobyworker.NewWorker(wopt)
if err != nil {
return nil, err
}
wc.Add(w)
ci := remotecache.NewCacheImporter(remotecache.ImportOpt{
Worker: w,
SessionManager: opt.SessionManager,
})
return control.NewController(control.Opt{
SessionManager: opt.SessionManager,
WorkerController: wc,
Frontends: frontends,
CacheKeyStorage: cacheStorage,
// CacheExporter: ce,
CacheImporter: ci,
})
}

17
components/engine/builder/builder-next/executor_unix.go Normal file
View File

@ -0,0 +1,17 @@
// +build !windows
package buildkit
import (
"path/filepath"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/executor/runcexecutor"
)
func newExecutor(root string) (executor.Executor, error) {
return runcexecutor.New(runcexecutor.Opt{
Root: filepath.Join(root, "executor"),
CommandCandidates: []string{"docker-runc", "runc"},
})
}

21
components/engine/builder/builder-next/executor_windows.go Normal file
View File

@ -0,0 +1,21 @@
package buildkit
import (
"context"
"errors"
"io"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/executor"
)
func newExecutor(_ string) (executor.Executor, error) {
return &winExecutor{}, nil
}
type winExecutor struct {
}
func (e *winExecutor) Exec(ctx context.Context, meta executor.Meta, rootfs cache.Mountable, mounts []executor.Mount, stdin io.ReadCloser, stdout, stderr io.WriteCloser) error {
return errors.New("buildkit executor not implemented for windows")
}

146
components/engine/builder/builder-next/exporter/export.go Normal file
View File

@ -0,0 +1,146 @@
package containerimage
import (
"context"
"fmt"
"strings"
distref "github.com/docker/distribution/reference"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/docker/reference"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/exporter"
digest "github.com/opencontainers/go-digest"
"github.com/sirupsen/logrus"
)
const (
keyImageName = "name"
exporterImageConfig = "containerimage.config"
)
// Differ can make a moby layer from a snapshot
type Differ interface {
EnsureLayer(ctx context.Context, key string) ([]layer.DiffID, error)
}
// Opt defines a struct for creating new exporter
type Opt struct {
ImageStore image.Store
ReferenceStore reference.Store
Differ Differ
}
type imageExporter struct {
opt Opt
}
// New creates a new moby imagestore exporter
func New(opt Opt) (exporter.Exporter, error) {
im := &imageExporter{opt: opt}
return im, nil
}
func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) {
i := &imageExporterInstance{imageExporter: e}
for k, v := range opt {
switch k {
case keyImageName:
for _, v := range strings.Split(v, ",") {
ref, err := distref.ParseNormalizedNamed(v)
if err != nil {
return nil, err
}
i.targetNames = append(i.targetNames, ref)
}
case exporterImageConfig:
i.config = []byte(v)
default:
logrus.Warnf("image exporter: unknown option %s", k)
}
}
return i, nil
}
type imageExporterInstance struct {
*imageExporter
targetNames []distref.Named
config []byte
}
func (e *imageExporterInstance) Name() string {
return "exporting to image"
}
func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableRef, opt map[string][]byte) (map[string]string, error) {
if config, ok := opt[exporterImageConfig]; ok {
e.config = config
}
config := e.config
var diffs []digest.Digest
if ref != nil {
layersDone := oneOffProgress(ctx, "exporting layers")
if err := ref.Finalize(ctx); err != nil {
return nil, err
}
diffIDs, err := e.opt.Differ.EnsureLayer(ctx, ref.ID())
if err != nil {
return nil, err
}
diffs = make([]digest.Digest, len(diffIDs))
for i := range diffIDs {
diffs[i] = digest.Digest(diffIDs[i])
}
layersDone(nil)
}
if len(config) == 0 {
var err error
config, err = emptyImageConfig()
if err != nil {
return nil, err
}
}
history, err := parseHistoryFromConfig(config)
if err != nil {
return nil, err
}
diffs, history = normalizeLayersAndHistory(diffs, history, ref)
config, err = patchImageConfig(config, diffs, history)
if err != nil {
return nil, err
}
configDigest := digest.FromBytes(config)
configDone := oneOffProgress(ctx, fmt.Sprintf("writing image %s", configDigest))
id, err := e.opt.ImageStore.Create(config)
if err != nil {
return nil, configDone(err)
}
configDone(nil)
if e.opt.ReferenceStore != nil {
for _, targetName := range e.targetNames {
tagDone := oneOffProgress(ctx, "naming to "+targetName.String())
if err := e.opt.ReferenceStore.AddTag(targetName, digest.Digest(id), true); err != nil {
return nil, tagDone(err)
}
tagDone(nil)
}
}
return map[string]string{
"containerimage.digest": id.String(),
}, nil
}

177
components/engine/builder/builder-next/exporter/writer.go Normal file
View File

@ -0,0 +1,177 @@
package containerimage
import (
"context"
"encoding/json"
"runtime"
"time"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/system"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
// const (
// emptyGZLayer = digest.Digest("sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1")
// )
func emptyImageConfig() ([]byte, error) {
img := ocispec.Image{
Architecture: runtime.GOARCH,
OS: runtime.GOOS,
}
img.RootFS.Type = "layers"
img.Config.WorkingDir = "/"
img.Config.Env = []string{"PATH=" + system.DefaultPathEnv}
dt, err := json.Marshal(img)
return dt, errors.Wrap(err, "failed to create empty image config")
}
func parseHistoryFromConfig(dt []byte) ([]ocispec.History, error) {
var config struct {
History []ocispec.History
}
if err := json.Unmarshal(dt, &config); err != nil {
return nil, errors.Wrap(err, "failed to unmarshal history from config")
}
return config.History, nil
}
func patchImageConfig(dt []byte, dps []digest.Digest, history []ocispec.History) ([]byte, error) {
m := map[string]json.RawMessage{}
if err := json.Unmarshal(dt, &m); err != nil {
return nil, errors.Wrap(err, "failed to parse image config for patch")
}
var rootFS ocispec.RootFS
rootFS.Type = "layers"
rootFS.DiffIDs = append(rootFS.DiffIDs, dps...)
dt, err := json.Marshal(rootFS)
if err != nil {
return nil, errors.Wrap(err, "failed to marshal rootfs")
}
m["rootfs"] = dt
dt, err = json.Marshal(history)
if err != nil {
return nil, errors.Wrap(err, "failed to marshal history")
}
m["history"] = dt
if _, ok := m["created"]; !ok {
var tm *time.Time
for _, h := range history {
if h.Created != nil {
tm = h.Created
}
}
dt, err = json.Marshal(&tm)
if err != nil {
return nil, errors.Wrap(err, "failed to marshal creation time")
}
m["created"] = dt
}
dt, err = json.Marshal(m)
return dt, errors.Wrap(err, "failed to marshal config after patch")
}
func normalizeLayersAndHistory(diffs []digest.Digest, history []ocispec.History, ref cache.ImmutableRef) ([]digest.Digest, []ocispec.History) {
refMeta := getRefMetadata(ref, len(diffs))
var historyLayers int
for _, h := range history {
if !h.EmptyLayer {
historyLayers++
}
}
if historyLayers > len(diffs) {
// this case shouldn't happen but if it does force set history layers empty
// from the bottom
logrus.Warn("invalid image config with unaccounted layers")
historyCopy := make([]ocispec.History, 0, len(history))
var l int
for _, h := range history {
if l >= len(diffs) {
h.EmptyLayer = true
}
if !h.EmptyLayer {
l++
}
historyCopy = append(historyCopy, h)
}
history = historyCopy
}
if len(diffs) > historyLayers {
// some history items are missing. add them based on the ref metadata
for _, md := range refMeta[historyLayers:] {
history = append(history, ocispec.History{
Created: &md.createdAt,
CreatedBy: md.description,
Comment: "buildkit.exporter.image.v0",
})
}
}
var layerIndex int
for i, h := range history {
if !h.EmptyLayer {
if h.Created == nil {
h.Created = &refMeta[layerIndex].createdAt
}
layerIndex++
}
history[i] = h
}
return diffs, history
}
type refMetadata struct {
description string
createdAt time.Time
}
func getRefMetadata(ref cache.ImmutableRef, limit int) []refMetadata {
if limit <= 0 {
return nil
}
meta := refMetadata{
description: "created by buildkit", // shouldn't be shown but don't fail build
createdAt: time.Now(),
}
if ref == nil {
return append(getRefMetadata(nil, limit-1), meta)
}
if descr := cache.GetDescription(ref.Metadata()); descr != "" {
meta.description = descr
}
meta.createdAt = cache.GetCreatedAt(ref.Metadata())
p := ref.Parent()
if p != nil {
defer p.Release(context.TODO())
}
return append(getRefMetadata(p, limit-1), meta)
}
func oneOffProgress(ctx context.Context, id string) func(err error) error {
pw, _, _ := progress.FromContext(ctx)
now := time.Now()
st := progress.Status{
Started: &now,
}
pw.Write(id, st)
return func(err error) error {
// TODO: set error on status
now := time.Now()
st.Completed = &now
pw.Write(id, st)
pw.Close()
return err
}
}

67
components/engine/builder/builder-next/reqbodyhandler.go Normal file
View File

@ -0,0 +1,67 @@
package buildkit
import (
"io"
"net/http"
"strings"
"sync"
"github.com/moby/buildkit/identity"
"github.com/pkg/errors"
)
const urlPrefix = "build-context-"
type reqBodyHandler struct {
mu sync.Mutex
rt http.RoundTripper
requests map[string]io.ReadCloser
}
func newReqBodyHandler(rt http.RoundTripper) *reqBodyHandler {
return &reqBodyHandler{
rt: rt,
requests: map[string]io.ReadCloser{},
}
}
func (h *reqBodyHandler) newRequest(rc io.ReadCloser) (string, func()) {
id := identity.NewID()
h.mu.Lock()
h.requests[id] = rc
h.mu.Unlock()
return "http://" + urlPrefix + id, func() {
h.mu.Lock()
delete(h.requests, id)
h.mu.Unlock()
}
}
func (h *reqBodyHandler) RoundTrip(req *http.Request) (*http.Response, error) {
host := req.URL.Host
if strings.HasPrefix(host, urlPrefix) {
if req.Method != "GET" {
return nil, errors.Errorf("invalid request")
}
id := strings.TrimPrefix(host, urlPrefix)
h.mu.Lock()
rc, ok := h.requests[id]
delete(h.requests, id)
h.mu.Unlock()
if !ok {
return nil, errors.Errorf("context not found")
}
resp := &http.Response{
Status: "200 OK",
StatusCode: 200,
Body: rc,
ContentLength: -1,
}
return resp, nil
}
return h.rt.RoundTrip(req)
}

321
components/engine/builder/builder-next/worker/worker.go Normal file
View File

@ -0,0 +1,321 @@
package worker
import (
"context"
"fmt"
"io"
"io/ioutil"
nethttp "net/http"
"runtime"
"time"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/rootfs"
"github.com/docker/docker/distribution"
distmetadata "github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/distribution/xfer"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
pkgprogress "github.com/docker/docker/pkg/progress"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/frontend"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/solver/llbsolver/ops"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/source/git"
"github.com/moby/buildkit/source/http"
"github.com/moby/buildkit/source/local"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/progress"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
// Opt defines a structure for creating a worker.
type Opt struct {
ID string
Labels map[string]string
SessionManager *session.Manager
MetadataStore *metadata.Store
Executor executor.Executor
Snapshotter snapshot.Snapshotter
ContentStore content.Store
CacheManager cache.Manager
ImageSource source.Source
Exporters map[string]exporter.Exporter
DownloadManager distribution.RootFSDownloadManager
V2MetadataService distmetadata.V2MetadataService
Transport nethttp.RoundTripper
}
// Worker is a local worker instance with dedicated snapshotter, cache, and so on.
// TODO: s/Worker/OpWorker/g ?
type Worker struct {
Opt
SourceManager *source.Manager
}
// NewWorker instantiates a local worker
func NewWorker(opt Opt) (*Worker, error) {
sm, err := source.NewManager()
if err != nil {
return nil, err
}
cm := opt.CacheManager
sm.Register(opt.ImageSource)
gs, err := git.NewSource(git.Opt{
CacheAccessor: cm,
MetadataStore: opt.MetadataStore,
})
if err != nil {
return nil, err
}
sm.Register(gs)
hs, err := http.NewSource(http.Opt{
CacheAccessor: cm,
MetadataStore: opt.MetadataStore,
Transport: opt.Transport,
})
if err != nil {
return nil, err
}
sm.Register(hs)
ss, err := local.NewSource(local.Opt{
SessionManager: opt.SessionManager,
CacheAccessor: cm,
MetadataStore: opt.MetadataStore,
})
if err != nil {
return nil, err
}
sm.Register(ss)
return &Worker{
Opt: opt,
SourceManager: sm,
}, nil
}
// ID returns worker ID
func (w *Worker) ID() string {
return w.Opt.ID
}
// Labels returns map of all worker labels
func (w *Worker) Labels() map[string]string {
return w.Opt.Labels
}
// LoadRef loads a reference by ID
func (w *Worker) LoadRef(id string) (cache.ImmutableRef, error) {
return w.CacheManager.Get(context.TODO(), id)
}
// ResolveOp converts a LLB vertex into a LLB operation
func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge) (solver.Op, error) {
switch op := v.Sys().(type) {
case *pb.Op_Source:
return ops.NewSourceOp(v, op, w.SourceManager, w)
case *pb.Op_Exec:
return ops.NewExecOp(v, op, w.CacheManager, w.MetadataStore, w.Executor, w)
case *pb.Op_Build:
return ops.NewBuildOp(v, op, s, w)
default:
return nil, errors.Errorf("could not resolve %v", v)
}
}
// ResolveImageConfig returns image config for an image
func (w *Worker) ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error) {
// ImageSource is typically source/containerimage
resolveImageConfig, ok := w.ImageSource.(resolveImageConfig)
if !ok {
return "", nil, errors.Errorf("worker %q does not implement ResolveImageConfig", w.ID())
}
return resolveImageConfig.ResolveImageConfig(ctx, ref)
}
// Exec executes a process directly on a worker
func (w *Worker) Exec(ctx context.Context, meta executor.Meta, rootFS cache.ImmutableRef, stdin io.ReadCloser, stdout, stderr io.WriteCloser) error {
active, err := w.CacheManager.New(ctx, rootFS)
if err != nil {
return err
}
defer active.Release(context.TODO())
return w.Executor.Exec(ctx, meta, active, nil, stdin, stdout, stderr)
}
// DiskUsage returns disk usage report
func (w *Worker) DiskUsage(ctx context.Context, opt client.DiskUsageInfo) ([]*client.UsageInfo, error) {
return w.CacheManager.DiskUsage(ctx, opt)
}
// Prune deletes reclaimable build cache
func (w *Worker) Prune(ctx context.Context, ch chan client.UsageInfo) error {
return w.CacheManager.Prune(ctx, ch)
}
// Exporter returns exporter by name
func (w *Worker) Exporter(name string) (exporter.Exporter, error) {
exp, ok := w.Exporters[name]
if !ok {
return nil, errors.Errorf("exporter %q could not be found", name)
}
return exp, nil
}
// GetRemote returns a remote snapshot reference for a local one
func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef, createIfNeeded bool) (*solver.Remote, error) {
return nil, errors.Errorf("getremote not implemented")
}
// FromRemote converts a remote snapshot reference to a local one
func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.ImmutableRef, error) {
rootfs, err := getLayers(ctx, remote.Descriptors)
if err != nil {
return nil, err
}
layers := make([]xfer.DownloadDescriptor, 0, len(rootfs))
for _, l := range rootfs {
// ongoing.add(desc)
layers = append(layers, &layerDescriptor{
desc: l.Blob,
diffID: layer.DiffID(l.Diff.Digest),
provider: remote.Provider,
w: w,
pctx: ctx,
})
}
defer func() {
for _, l := range rootfs {
w.ContentStore.Delete(context.TODO(), l.Blob.Digest)
}
}()
r := image.NewRootFS()
rootFS, release, err := w.DownloadManager.Download(ctx, *r, runtime.GOOS, layers, &discardProgress{})
if err != nil {
return nil, err
}
defer release()
ref, err := w.CacheManager.GetFromSnapshotter(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("imported %s", remote.Descriptors[len(remote.Descriptors)-1].Digest)))
if err != nil {
return nil, err
}
return ref, nil
}
type discardProgress struct{}
func (*discardProgress) WriteProgress(_ pkgprogress.Progress) error {
return nil
}
// Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error)
type layerDescriptor struct {
provider content.Provider
desc ocispec.Descriptor
diffID layer.DiffID
// ref ctdreference.Spec
w *Worker
pctx context.Context
}
func (ld *layerDescriptor) Key() string {
return "v2:" + ld.desc.Digest.String()
}
func (ld *layerDescriptor) ID() string {
return ld.desc.Digest.String()
}
func (ld *layerDescriptor) DiffID() (layer.DiffID, error) {
return ld.diffID, nil
}
func (ld *layerDescriptor) Download(ctx context.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) {
done := oneOffProgress(ld.pctx, fmt.Sprintf("pulling %s", ld.desc.Digest))
if err := contentutil.Copy(ctx, ld.w.ContentStore, ld.provider, ld.desc); err != nil {
return nil, 0, done(err)
}
done(nil)
ra, err := ld.w.ContentStore.ReaderAt(ctx, ld.desc)
if err != nil {
return nil, 0, err
}
return ioutil.NopCloser(content.NewReader(ra)), ld.desc.Size, nil
}
func (ld *layerDescriptor) Close() {
// ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest)
}
func (ld *layerDescriptor) Registered(diffID layer.DiffID) {
// Cache mapping from this layer's DiffID to the blobsum
ld.w.V2MetadataService.Add(diffID, distmetadata.V2Metadata{Digest: ld.desc.Digest})
}
func getLayers(ctx context.Context, descs []ocispec.Descriptor) ([]rootfs.Layer, error) {
layers := make([]rootfs.Layer, len(descs))
for i, desc := range descs {
diffIDStr := desc.Annotations["containerd.io/uncompressed"]
if diffIDStr == "" {
return nil, errors.Errorf("%s missing uncompressed digest", desc.Digest)
}
diffID, err := digest.Parse(diffIDStr)
if err != nil {
return nil, err
}
layers[i].Diff = ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageLayer,
Digest: diffID,
}
layers[i].Blob = ocispec.Descriptor{
MediaType: desc.MediaType,
Digest: desc.Digest,
Size: desc.Size,
}
}
return layers, nil
}
func oneOffProgress(ctx context.Context, id string) func(err error) error {
pw, _, _ := progress.FromContext(ctx)
now := time.Now()
st := progress.Status{
Started: &now,
}
pw.Write(id, st)
return func(err error) error {
// TODO: set error on status
now := time.Now()
st.Completed = &now
pw.Write(id, st)
pw.Close()
return err
}
}
type resolveImageConfig interface {
ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error)
}

4
components/engine/builder/dockerfile/buildargs_test.go
View File

@ -5,8 +5,8 @@ import (
"strings"
"testing"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func strPtr(source string) *string {

6
components/engine/builder/dockerfile/copy_test.go
View File

@ -5,9 +5,9 @@ import (
"testing"
"github.com/docker/docker/pkg/containerfs"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/gotestyourself/gotestyourself/fs"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
"gotest.tools/fs"
)
func TestIsExistingDirectory(t *testing.T) {

4
components/engine/builder/dockerfile/dispatchers_test.go
View File

@ -14,10 +14,10 @@ import (
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/system"
"github.com/docker/go-connections/nat"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/moby/buildkit/frontend/dockerfile/shell"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func newBuilderWithMockBackend() *Builder {

8
components/engine/builder/dockerfile/evaluator_test.go
View File

@ -7,10 +7,10 @@ import (
"github.com/docker/docker/builder/remotecontext"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/reexec"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/gotestyourself/gotestyourself/skip"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
"gotest.tools/skip"
)
type dispatchTestCase struct {
@ -97,7 +97,7 @@ func initDispatchTestCases() []dispatchTestCase {
}
func TestDispatch(t *testing.T) {
skip.IfCondition(t, os.Getuid() != 0, "skipping test that requires root")
skip.If(t, os.Getuid() != 0, "skipping test that requires root")
testCases := initDispatchTestCases()
for _, testCase := range testCases {

4
components/engine/builder/dockerfile/internals_linux_test.go
View File

@ -6,8 +6,8 @@ import (
"testing"
"github.com/docker/docker/pkg/idtools"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestChownFlagParsing(t *testing.T) {

8
components/engine/builder/dockerfile/internals_test.go
View File

@ -13,9 +13,9 @@ import (
"github.com/docker/docker/builder/remotecontext"
"github.com/docker/docker/pkg/archive"
"github.com/docker/go-connections/nat"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/gotestyourself/gotestyourself/skip"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
"gotest.tools/skip"
)
func TestEmptyDockerfile(t *testing.T) {
@ -61,7 +61,7 @@ func TestNonExistingDockerfile(t *testing.T) {
}
func readAndCheckDockerfile(t *testing.T, testName, contextDir, dockerfilePath, expectedError string) {
skip.IfCondition(t, os.Getuid() != 0, "skipping test that requires root")
skip.If(t, os.Getuid() != 0, "skipping test that requires root")
tarStream, err := archive.Tar(contextDir, archive.Uncompressed)
assert.NilError(t, err)

4
components/engine/builder/dockerfile/internals_windows_test.go
View File

@ -6,8 +6,8 @@ import (
"fmt"
"testing"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestNormalizeDest(t *testing.T) {

4
components/engine/builder/fscache/fscache_test.go
View File

@ -8,9 +8,9 @@ import (
"testing"
"time"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/moby/buildkit/session/filesync"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestFSCache(t *testing.T) {

4
components/engine/builder/remotecontext/git/gitutils_test.go
View File

@ -14,8 +14,8 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestParseRemoteURL(t *testing.T) {

4
components/engine/builder/remotecontext/mimetype_test.go
View File

@ -3,8 +3,8 @@ package remotecontext // import "github.com/docker/docker/builder/remotecontext"
import (
"testing"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestDetectContentType(t *testing.T) {

6
components/engine/builder/remotecontext/remote_test.go
View File

@ -10,9 +10,9 @@ import (
"testing"
"github.com/docker/docker/builder"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/gotestyourself/gotestyourself/fs"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
"gotest.tools/fs"
)
var binaryContext = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00} //xz magic

4
components/engine/builder/remotecontext/tarsum_test.go
View File

@ -9,8 +9,8 @@ import (
"github.com/docker/docker/builder"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/reexec"
"github.com/gotestyourself/gotestyourself/skip"
"github.com/pkg/errors"
"gotest.tools/skip"
)
const (
@ -137,7 +137,7 @@ func TestRemoveDirectory(t *testing.T) {
}
func makeTestArchiveContext(t *testing.T, dir string) builder.Source {
skip.IfCondition(t, os.Getuid() != 0, "skipping test that requires root")
skip.If(t, os.Getuid() != 0, "skipping test that requires root")
tarStream, err := archive.Tar(dir, archive.Uncompressed)
if err != nil {
t.Fatalf("error: %s", err)

21
components/engine/client/build_cancel.go Normal file
View File

@ -0,0 +1,21 @@
package client // import "github.com/docker/docker/client"
import (
"net/url"
"golang.org/x/net/context"
)
// BuildCancel requests the daemon to cancel ongoing build request
func (cli *Client) BuildCancel(ctx context.Context, id string) error {
query := url.Values{}
query.Set("id", id)
serverResp, err := cli.post(ctx, "/build/cancel", query, nil, nil)
if err != nil {
return err
}
defer ensureReaderClosed(serverResp)
return nil
}

8
components/engine/client/client_test.go
View File

@ -10,10 +10,10 @@ import (
"github.com/docker/docker/api"
"github.com/docker/docker/api/types"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/gotestyourself/gotestyourself/env"
"github.com/gotestyourself/gotestyourself/skip"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
"gotest.tools/env"
"gotest.tools/skip"
)
func TestNewEnvClient(t *testing.T) {

4
components/engine/client/config_create_test.go
View File

@ -12,8 +12,8 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/swarm"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestConfigCreateUnsupported(t *testing.T) {

4
components/engine/client/config_inspect_test.go
View File

@ -11,9 +11,9 @@ import (
"testing"
"github.com/docker/docker/api/types/swarm"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/pkg/errors"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestConfigInspectNotFound(t *testing.T) {

4
components/engine/client/config_list_test.go
View File

@ -13,8 +13,8 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/swarm"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestConfigListUnsupported(t *testing.T) {

4
components/engine/client/config_remove_test.go
View File

@ -9,8 +9,8 @@ import (
"strings"
"testing"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestConfigRemoveUnsupported(t *testing.T) {

4
components/engine/client/config_update_test.go
View File

@ -10,8 +10,8 @@ import (
"testing"
"github.com/docker/docker/api/types/swarm"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestConfigUpdateUnsupported(t *testing.T) {

4
components/engine/client/container_logs_test.go
View File

@ -14,8 +14,8 @@ import (
"time"
"github.com/docker/docker/api/types"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestContainerLogsNotFoundError(t *testing.T) {

4
components/engine/client/container_prune_test.go
View File

@ -12,8 +12,8 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestContainersPruneError(t *testing.T) {

4
components/engine/client/container_remove_test.go
View File

@ -10,8 +10,8 @@ import (
"testing"
"github.com/docker/docker/api/types"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestContainerRemoveError(t *testing.T) {

4
components/engine/client/distribution_inspect_test.go
View File

@ -5,9 +5,9 @@ import (
"net/http"
"testing"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/pkg/errors"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestDistributionInspectUnsupported(t *testing.T) {

2
components/engine/client/hijack_test.go
View File

@ -11,9 +11,9 @@ import (
"github.com/docker/docker/api/server/httputils"
"github.com/docker/docker/api/types"
"github.com/gotestyourself/gotestyourself/assert"
"github.com/pkg/errors"
"golang.org/x/net/context"
"gotest.tools/assert"
)
func TestTLSCloseWriter(t *testing.T) {

4
components/engine/client/image_build.go
View File

@ -133,5 +133,9 @@ func (cli *Client) imageBuildOptionsToQuery(options types.ImageBuildOptions) (ur
if options.Platform != "" {
query.Set("platform", strings.ToLower(options.Platform))
}
if options.BuildID != "" {
query.Set("buildid", options.BuildID)
}
query.Set("version", string(options.Version))
return query, nil
}

4
components/engine/client/image_prune_test.go
View File

@ -12,8 +12,8 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestImagesPruneError(t *testing.T) {

4
components/engine/client/image_remove_test.go
View File

@ -11,8 +11,8 @@ import (
"testing"
"github.com/docker/docker/api/types"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestImageRemoveError(t *testing.T) {

1
components/engine/client/interface.go
View File

@ -86,6 +86,7 @@ type DistributionAPIClient interface {
type ImageAPIClient interface {
ImageBuild(ctx context.Context, context io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error)
BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error)
BuildCancel(ctx context.Context, id string) error
ImageCreate(ctx context.Context, parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error)
ImageHistory(ctx context.Context, image string) ([]image.HistoryResponseItem, error)
ImageImport(ctx context.Context, source types.ImageImportSource, ref string, options types.ImageImportOptions) (io.ReadCloser, error)

4
components/engine/client/network_inspect_test.go
View File

@ -12,9 +12,9 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/network"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/pkg/errors"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestNetworkInspectError(t *testing.T) {

4
components/engine/client/network_prune_test.go
View File

@ -12,8 +12,8 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestNetworksPruneError(t *testing.T) {

4
components/engine/client/ping_test.go
View File

@ -8,8 +8,8 @@ import (
"strings"
"testing"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
// TestPingFail tests that when a server sends a non-successful response that we

2
components/engine/client/request_test.go
View File

@ -10,7 +10,7 @@ import (
"testing"
"github.com/docker/docker/api/types"
"github.com/gotestyourself/gotestyourself/assert"
"gotest.tools/assert"
)
// TestSetHostHeader should set fake host for local communications, set real host

4
components/engine/client/secret_create_test.go
View File

@ -12,8 +12,8 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/swarm"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestSecretCreateUnsupported(t *testing.T) {

4
components/engine/client/secret_inspect_test.go
View File

@ -11,9 +11,9 @@ import (
"testing"
"github.com/docker/docker/api/types/swarm"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/pkg/errors"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestSecretInspectUnsupported(t *testing.T) {

4
components/engine/client/secret_list_test.go
View File

@ -13,8 +13,8 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/swarm"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestSecretListUnsupported(t *testing.T) {

4
components/engine/client/secret_remove_test.go
View File

@ -9,8 +9,8 @@ import (
"strings"
"testing"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestSecretRemoveUnsupported(t *testing.T) {

4
components/engine/client/secret_update_test.go
View File

@ -10,8 +10,8 @@ import (
"testing"
"github.com/docker/docker/api/types/swarm"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestSecretUpdateUnsupported(t *testing.T) {

4
components/engine/client/service_create_test.go
View File

@ -13,10 +13,10 @@ import (
"github.com/docker/docker/api/types"
registrytypes "github.com/docker/docker/api/types/registry"
"github.com/docker/docker/api/types/swarm"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go/v1"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestServiceCreateError(t *testing.T) {

4
components/engine/client/service_logs_test.go
View File

@ -14,8 +14,8 @@ import (
"time"
"github.com/docker/docker/api/types"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestServiceLogsError(t *testing.T) {

4
components/engine/client/service_remove_test.go
View File

@ -9,8 +9,8 @@ import (
"strings"
"testing"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestServiceRemoveError(t *testing.T) {

4
components/engine/client/swarm_get_unlock_key_test.go
View File

@ -11,8 +11,8 @@ import (
"testing"
"github.com/docker/docker/api/types"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestSwarmGetUnlockKeyError(t *testing.T) {

4
components/engine/client/volume_inspect_test.go
View File

@ -11,9 +11,9 @@ import (
"testing"
"github.com/docker/docker/api/types"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/pkg/errors"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestVolumeInspectError(t *testing.T) {

4
components/engine/cmd/dockerd/config_unix_test.go
View File

@ -6,9 +6,9 @@ import (
"testing"
"github.com/docker/docker/daemon/config"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/spf13/pflag"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestDaemonParseShmSize(t *testing.T) {

18
components/engine/cmd/dockerd/daemon.go
View File

@ -27,6 +27,7 @@ import (
swarmrouter "github.com/docker/docker/api/server/router/swarm"
systemrouter "github.com/docker/docker/api/server/router/system"
"github.com/docker/docker/api/server/router/volume"
buildkit "github.com/docker/docker/builder/builder-next"
"github.com/docker/docker/builder/dockerfile"
"github.com/docker/docker/builder/fscache"
"github.com/docker/docker/cli/debug"
@ -238,7 +239,8 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
type routerOptions struct {
sessionManager *session.Manager
buildBackend *buildbackend.Backend
buildCache *fscache.FSCache
buildCache *fscache.FSCache // legacy
buildkit *buildkit.Builder
daemon *daemon.Daemon
api *apiserver.Server
cluster *cluster.Cluster
@ -270,7 +272,16 @@ func newRouterOptions(config *config.Config, daemon *daemon.Daemon) (routerOptio
return opts, err
}
bb, err := buildbackend.NewBackend(daemon.ImageService(), manager, buildCache)
buildkit, err := buildkit.New(buildkit.Opt{
SessionManager: sm,
Root: filepath.Join(config.Root, "buildkit"),
Dist: daemon.DistributionServices(),
})
if err != nil {
return opts, err
}
bb, err := buildbackend.NewBackend(daemon.ImageService(), manager, buildCache, buildkit)
if err != nil {
return opts, errors.Wrap(err, "failed to create buildmanager")
}
@ -279,6 +290,7 @@ func newRouterOptions(config *config.Config, daemon *daemon.Daemon) (routerOptio
sessionManager: sm,
buildBackend: bb,
buildCache: buildCache,
buildkit: buildkit,
daemon: daemon,
}, nil
}
@ -452,7 +464,7 @@ func initRouter(opts routerOptions) {
checkpointrouter.NewRouter(opts.daemon, decoder),
container.NewRouter(opts.daemon, decoder),
image.NewRouter(opts.daemon.ImageService()),
systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache),
systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache, opts.buildkit),
volume.NewRouter(opts.daemon.VolumesService()),
build.NewRouter(opts.buildBackend, opts.daemon),
sessionrouter.NewRouter(opts.sessionManager),

6
components/engine/cmd/dockerd/daemon_test.go
View File

@ -4,11 +4,11 @@ import (
"testing"
"github.com/docker/docker/daemon/config"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/gotestyourself/gotestyourself/fs"
"github.com/sirupsen/logrus"
"github.com/spf13/pflag"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
"gotest.tools/fs"
)
func defaultOptions(configFile string) *daemonOptions {

6
components/engine/cmd/dockerd/daemon_unix_test.go
View File

@ -6,9 +6,9 @@ import (
"testing"
"github.com/docker/docker/daemon/config"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/gotestyourself/gotestyourself/fs"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
"gotest.tools/fs"
)
func TestLoadDaemonCliConfigWithDaemonFlags(t *testing.T) {

4
components/engine/cmd/dockerd/options_test.go
View File

@ -6,9 +6,9 @@ import (
cliconfig "github.com/docker/docker/cli/config"
"github.com/docker/docker/daemon/config"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/spf13/pflag"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestCommonOptionsInstallFlags(t *testing.T) {

2
components/engine/container/container_unit_test.go
View File

@ -11,7 +11,7 @@ import (
swarmtypes "github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/daemon/logger/jsonfilelog"
"github.com/docker/docker/pkg/signal"
"github.com/gotestyourself/gotestyourself/assert"
"gotest.tools/assert"
)
func TestContainerStopSignal(t *testing.T) {

4
components/engine/container/view_test.go
View File

@ -8,9 +8,9 @@ import (
"github.com/docker/docker/api/types"
containertypes "github.com/docker/docker/api/types/container"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/pborman/uuid"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
var root string

2
components/engine/daemon/cluster/convert/service_test.go
View File

@ -8,7 +8,7 @@ import (
"github.com/docker/docker/api/types/swarm/runtime"
swarmapi "github.com/docker/swarmkit/api"
google_protobuf3 "github.com/gogo/protobuf/types"
"github.com/gotestyourself/gotestyourself/assert"
"gotest.tools/assert"
)
func TestServiceConvertFromGRPCRuntimeContainer(t *testing.T) {

2
components/engine/daemon/cluster/executor/container/container_test.go
View File

@ -5,7 +5,7 @@ import (
"github.com/docker/docker/api/types/container"
swarmapi "github.com/docker/swarmkit/api"
"github.com/gotestyourself/gotestyourself/assert"
"gotest.tools/assert"
)
func TestIsolationConversion(t *testing.T) {

10
components/engine/daemon/config/config_test.go
View File

@ -8,11 +8,11 @@ import (
"github.com/docker/docker/daemon/discovery"
"github.com/docker/docker/opts"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/gotestyourself/gotestyourself/fs"
"github.com/gotestyourself/gotestyourself/skip"
"github.com/spf13/pflag"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
"gotest.tools/fs"
"gotest.tools/skip"
)
func TestDaemonConfigurationNotFound(t *testing.T) {
@ -461,7 +461,7 @@ func TestReloadSetConfigFileNotExist(t *testing.T) {
// TestReloadDefaultConfigNotExist tests that if the default configuration file
// doesn't exist the daemon still will be reloaded.
func TestReloadDefaultConfigNotExist(t *testing.T) {
skip.IfCondition(t, os.Getuid() != 0, "skipping test that requires root")
skip.If(t, os.Getuid() != 0, "skipping test that requires root")
reloaded := false
configFile := "/etc/docker/daemon.json"
flags := pflag.NewFlagSet("test", pflag.ContinueOnError)

6
components/engine/daemon/config/config_unix_test.go
View File

@ -7,10 +7,10 @@ import (
"github.com/docker/docker/opts"
"github.com/docker/go-units"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/gotestyourself/gotestyourself/fs"
"github.com/spf13/pflag"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
"gotest.tools/fs"
)
func TestGetConflictFreeConfiguration(t *testing.T) {

4
components/engine/daemon/config/config_windows_test.go
View File

@ -7,9 +7,9 @@ import (
"testing"
"github.com/docker/docker/opts"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/spf13/pflag"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestDaemonConfigurationMerge(t *testing.T) {

2
components/engine/daemon/container_unix_test.go
View File

@ -9,7 +9,7 @@ import (
containertypes "github.com/docker/docker/api/types/container"
"github.com/docker/docker/daemon/config"
"github.com/docker/go-connections/nat"
"github.com/gotestyourself/gotestyourself/assert"
"gotest.tools/assert"
)
// TestContainerWarningHostAndPublishPorts that a warning is returned when setting network mode to host and specifying published ports.

2
components/engine/daemon/create_test.go
View File

@ -5,7 +5,7 @@ import (
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/errdefs"
"github.com/gotestyourself/gotestyourself/assert"
"gotest.tools/assert"
)
// Test case for 35752

5
components/engine/daemon/daemon.go
View File

@ -922,6 +922,11 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
return d, nil
}
// DistributionServices returns services controlling daemon storage
func (daemon *Daemon) DistributionServices() images.DistributionServices {
return daemon.imageService.DistributionServices()
}
func (daemon *Daemon) waitForStartupDone() {
<-daemon.startupDone
}

4
components/engine/daemon/daemon_linux_test.go
View File

@ -15,8 +15,8 @@ import (
"github.com/docker/docker/oci"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/mount"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
const mountsFixture = `142 78 0:38 / / rw,relatime - aufs none rw,si=573b861da0b3a05b,dio

4
components/engine/daemon/daemon_test.go
View File

@ -16,9 +16,9 @@ import (
volumesservice "github.com/docker/docker/volume/service"
"github.com/docker/go-connections/nat"
"github.com/docker/libnetwork"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/pkg/errors"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
//

4
components/engine/daemon/delete_test.go
View File

@ -9,8 +9,8 @@ import (
"github.com/docker/docker/api/types"
containertypes "github.com/docker/docker/api/types/container"
"github.com/docker/docker/container"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func newDaemonWithTmpRoot(t *testing.T) (*Daemon, func()) {

4
components/engine/daemon/discovery/discovery_test.go
View File

@ -5,8 +5,8 @@ import (
"testing"
"time"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestDiscoveryOptsErrors(t *testing.T) {

2
components/engine/daemon/exec_linux_test.go
View File

@ -8,9 +8,9 @@ import (
containertypes "github.com/docker/docker/api/types/container"
"github.com/docker/docker/container"
"github.com/docker/docker/daemon/exec"
"github.com/gotestyourself/gotestyourself/assert"
"github.com/opencontainers/runc/libcontainer/apparmor"
"github.com/opencontainers/runtime-spec/specs-go"
"gotest.tools/assert"
)
func TestExecSetPlatformOpt(t *testing.T) {

4
components/engine/daemon/graphdriver/aufs/aufs_test.go
View File

@ -17,8 +17,8 @@ import (
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/reexec"
"github.com/docker/docker/pkg/stringid"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
var (

4
components/engine/daemon/graphdriver/copy/copy_test.go
View File

@ -13,9 +13,9 @@ import (
"time"
"github.com/docker/docker/pkg/system"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"golang.org/x/sys/unix"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestCopy(t *testing.T) {

2
components/engine/daemon/graphdriver/driver_test.go
View File

@ -6,7 +6,7 @@ import (
"path/filepath"
"testing"
"github.com/gotestyourself/gotestyourself/assert"
"gotest.tools/assert"
)
func TestIsEmptyDir(t *testing.T) {

2
components/engine/daemon/graphdriver/graphtest/graphbench_unix.go
View File

@ -9,7 +9,7 @@ import (
contdriver "github.com/containerd/continuity/driver"
"github.com/docker/docker/pkg/stringid"
"github.com/gotestyourself/gotestyourself/assert"
"gotest.tools/assert"
)
// DriverBenchExists benchmarks calls to exist

4
components/engine/daemon/graphdriver/graphtest/graphtest_unix.go
View File

@ -16,9 +16,9 @@ import (
"github.com/docker/docker/daemon/graphdriver/quota"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/go-units"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"golang.org/x/sys/unix"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
var (

4
components/engine/daemon/graphdriver/graphtest/testutil_unix.go
View File

@ -9,9 +9,9 @@ import (
contdriver "github.com/containerd/continuity/driver"
"github.com/docker/docker/daemon/graphdriver"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"golang.org/x/sys/unix"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func verifyFile(t testing.TB, path string, mode os.FileMode, uid, gid uint32) {

6
components/engine/daemon/graphdriver/quota/projectquota_test.go
View File

@ -10,10 +10,10 @@ import (
"path/filepath"
"testing"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/gotestyourself/gotestyourself/fs"
"golang.org/x/sys/unix"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
"gotest.tools/fs"
)
// 10MB

22
components/engine/daemon/images/service.go
View File

@ -3,9 +3,11 @@ package images // import "github.com/docker/docker/daemon/images"
import (
"context"
"os"
"runtime"
"github.com/docker/docker/container"
daemonevents "github.com/docker/docker/daemon/events"
"github.com/docker/docker/distribution"
"github.com/docker/docker/distribution/metadata"
"github.com/docker/docker/distribution/xfer"
"github.com/docker/docker/image"
@ -74,6 +76,26 @@ type ImageService struct {
uploadManager *xfer.LayerUploadManager
}
// DistributionServices provides daemon image storage services
type DistributionServices struct {
DownloadManager distribution.RootFSDownloadManager
V2MetadataService metadata.V2MetadataService
LayerStore layer.Store // TODO: lcow
ImageStore image.Store
ReferenceStore dockerreference.Store
}
// DistributionServices return services controlling daemon image storage
func (i *ImageService) DistributionServices() DistributionServices {
return DistributionServices{
DownloadManager: i.downloadManager,
V2MetadataService: metadata.NewV2MetadataService(i.distributionMetadataStore),
LayerStore: i.layerStores[runtime.GOOS],
ImageStore: i.imageStore,
ReferenceStore: i.referenceStore,
}
}
// CountImages returns the number of images stored by ImageService
// called from info.go
func (i *ImageService) CountImages() int {

4
components/engine/daemon/info_unix_test.go
View File

@ -7,8 +7,8 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/dockerversion"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestParseInitVersion(t *testing.T) {

4
components/engine/daemon/inspect_test.go
View File

@ -7,8 +7,8 @@ import (
"github.com/docker/docker/container"
"github.com/docker/docker/daemon/config"
"github.com/docker/docker/daemon/exec"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestGetInspectData(t *testing.T) {

4
components/engine/daemon/list_test.go
View File

@ -6,8 +6,8 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/container"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
func TestListInvalidFilter(t *testing.T) {

4
components/engine/daemon/logger/adapter_test.go
View File

@ -9,8 +9,8 @@ import (
"github.com/docker/docker/api/types/plugins/logdriver"
protoio "github.com/gogo/protobuf/io"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
// mockLoggingPlugin implements the loggingPlugin interface for testing purposes

4
components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go
View File

@ -21,8 +21,8 @@ import (
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/daemon/logger/loggerutils"
"github.com/docker/docker/dockerversion"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
)
const (

6
components/engine/daemon/logger/jsonfilelog/jsonfilelog_test.go
View File

@ -14,9 +14,9 @@ import (
"github.com/docker/docker/daemon/logger"
"github.com/docker/docker/daemon/logger/jsonfilelog/jsonlog"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/gotestyourself/gotestyourself/fs"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
"gotest.tools/fs"
)
func TestJSONFileLogger(t *testing.T) {

Some files were not shown because too many files have changed in this diff Show More