add manifest command

Enable inspection (aka "shallow pull") of images' manifest info, and
also the creation of manifest lists (aka "fat manifests").

The workflow for creating a manifest list will be:

`docker manifest create new-list-ref-name image-ref [image-ref...]`
`docker manifest annotate new-list-ref-name image-ref --os linux --arch
arm`
`docker manifest push new-list-ref-name`

The annotate step is optional. Most architectures are fine by default.

There is also a `manifest inspect` command to allow for a "shallow pull"
of an image's manifest: `docker manifest inspect
manifest-or-manifest_list`.

To be more in line with the existing external manifest tool, there is
also a `-v` option for inspect that will show information depending on
what the reference maps to (list or single manifest).

Signed-off-by: Christy Perez <christy@linux.vnet.ibm.com>
Signed-off-by: Daniel Nephin <dnephin@docker.com>
This commit is contained in:
Christy Perez
2017-06-15 13:41:54 -05:00
committed by Christy Norman Perez
parent 17886d7547
commit 02719bdbb5
19 changed files with 1948 additions and 12 deletions

View File

@ -0,0 +1,183 @@
package client
import (
"fmt"
"net/http"
"strings"
manifesttypes "github.com/docker/cli/cli/manifest/types"
"github.com/docker/distribution"
"github.com/docker/distribution/reference"
distributionclient "github.com/docker/distribution/registry/client"
"github.com/docker/docker/api/types"
registrytypes "github.com/docker/docker/api/types/registry"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
)
// RegistryClient is a client used to communicate with a Docker distribution
// registry
type RegistryClient interface {
GetManifest(ctx context.Context, ref reference.Named) (manifesttypes.ImageManifest, error)
GetManifestList(ctx context.Context, ref reference.Named) ([]manifesttypes.ImageManifest, error)
MountBlob(ctx context.Context, source reference.Canonical, target reference.Named) error
PutManifest(ctx context.Context, ref reference.Named, manifest distribution.Manifest) (digest.Digest, error)
}
// NewRegistryClient returns a new RegistryClient with a resolver
func NewRegistryClient(resolver AuthConfigResolver, userAgent string, insecure bool) RegistryClient {
return &client{
authConfigResolver: resolver,
insecureRegistry: insecure,
userAgent: userAgent,
}
}
// AuthConfigResolver returns Auth Configuration for an index
type AuthConfigResolver func(ctx context.Context, index *registrytypes.IndexInfo) types.AuthConfig
// PutManifestOptions is the data sent to push a manifest
type PutManifestOptions struct {
MediaType string
Payload []byte
}
type client struct {
authConfigResolver AuthConfigResolver
insecureRegistry bool
userAgent string
}
// ErrBlobCreated returned when a blob mount request was created
type ErrBlobCreated struct {
From reference.Named
Target reference.Named
}
func (err ErrBlobCreated) Error() string {
return fmt.Sprintf("blob mounted from: %v to: %v",
err.From, err.Target)
}
// ErrHTTPProto returned if attempting to use TLS with a non-TLS registry
type ErrHTTPProto struct {
OrigErr string
}
func (err ErrHTTPProto) Error() string {
return err.OrigErr
}
var _ RegistryClient = &client{}
// MountBlob into the registry, so it can be referenced by a manifest
func (c *client) MountBlob(ctx context.Context, sourceRef reference.Canonical, targetRef reference.Named) error {
repoEndpoint, err := newDefaultRepositoryEndpoint(targetRef, c.insecureRegistry)
if err != nil {
return err
}
repo, err := c.getRepositoryForReference(ctx, targetRef, repoEndpoint)
if err != nil {
return err
}
lu, err := repo.Blobs(ctx).Create(ctx, distributionclient.WithMountFrom(sourceRef))
switch err.(type) {
case distribution.ErrBlobMounted:
logrus.Debugf("mount of blob %s succeeded", sourceRef)
return nil
case nil:
default:
return errors.Wrapf(err, "failed to mount blob %s to %s", sourceRef, targetRef)
}
lu.Cancel(ctx)
logrus.Debugf("mount of blob %s created", sourceRef)
return ErrBlobCreated{From: sourceRef, Target: targetRef}
}
// PutManifest sends the manifest to a registry and returns the new digest
func (c *client) PutManifest(ctx context.Context, ref reference.Named, manifest distribution.Manifest) (digest.Digest, error) {
repoEndpoint, err := newDefaultRepositoryEndpoint(ref, c.insecureRegistry)
if err != nil {
return digest.Digest(""), err
}
repo, err := c.getRepositoryForReference(ctx, ref, repoEndpoint)
if err != nil {
return digest.Digest(""), err
}
manifestService, err := repo.Manifests(ctx)
if err != nil {
return digest.Digest(""), err
}
_, opts, err := getManifestOptionsFromReference(ref)
if err != nil {
return digest.Digest(""), err
}
dgst, err := manifestService.Put(ctx, manifest, opts...)
return dgst, errors.Wrapf(err, "failed to put manifest %s", ref)
}
func (c *client) getRepositoryForReference(ctx context.Context, ref reference.Named, repoEndpoint repositoryEndpoint) (distribution.Repository, error) {
httpTransport, err := c.getHTTPTransportForRepoEndpoint(ctx, repoEndpoint)
if err != nil {
if strings.Contains(err.Error(), "server gave HTTP response to HTTPS client") {
return nil, ErrHTTPProto{OrigErr: err.Error()}
}
}
repoName, err := reference.WithName(repoEndpoint.Name())
if err != nil {
return nil, errors.Wrapf(err, "failed to parse repo name from %s", ref)
}
return distributionclient.NewRepository(ctx, repoName, repoEndpoint.BaseURL(), httpTransport)
}
func (c *client) getHTTPTransportForRepoEndpoint(ctx context.Context, repoEndpoint repositoryEndpoint) (http.RoundTripper, error) {
httpTransport, err := getHTTPTransport(
c.authConfigResolver(ctx, repoEndpoint.info.Index),
repoEndpoint.endpoint,
repoEndpoint.Name(),
c.userAgent)
return httpTransport, errors.Wrap(err, "failed to configure transport")
}
// GetManifest returns an ImageManifest for the reference
func (c *client) GetManifest(ctx context.Context, ref reference.Named) (manifesttypes.ImageManifest, error) {
var result manifesttypes.ImageManifest
fetch := func(ctx context.Context, repo distribution.Repository, ref reference.Named) (bool, error) {
var err error
result, err = fetchManifest(ctx, repo, ref)
return result.Ref != nil, err
}
err := c.iterateEndpoints(ctx, ref, fetch)
return result, err
}
// GetManifestList returns a list of ImageManifest for the reference
func (c *client) GetManifestList(ctx context.Context, ref reference.Named) ([]manifesttypes.ImageManifest, error) {
result := []manifesttypes.ImageManifest{}
fetch := func(ctx context.Context, repo distribution.Repository, ref reference.Named) (bool, error) {
var err error
result, err = fetchList(ctx, repo, ref)
return len(result) > 0, err
}
err := c.iterateEndpoints(ctx, ref, fetch)
return result, err
}
func getManifestOptionsFromReference(ref reference.Named) (digest.Digest, []distribution.ManifestServiceOption, error) {
if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
tag := tagged.Tag()
return "", []distribution.ManifestServiceOption{distribution.WithTag(tag)}, nil
}
if digested, isDigested := ref.(reference.Canonical); isDigested {
return digested.Digest(), []distribution.ManifestServiceOption{}, nil
}
return "", nil, errors.Errorf("%s no tag or digest", ref)
}

View File

@ -0,0 +1,133 @@
package client
import (
"fmt"
"net"
"net/http"
"time"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/client/auth"
"github.com/docker/distribution/registry/client/transport"
authtypes "github.com/docker/docker/api/types"
"github.com/docker/docker/registry"
"github.com/pkg/errors"
)
type repositoryEndpoint struct {
info *registry.RepositoryInfo
endpoint registry.APIEndpoint
}
// Name returns the repository name
func (r repositoryEndpoint) Name() string {
repoName := r.info.Name.Name()
// If endpoint does not support CanonicalName, use the RemoteName instead
if r.endpoint.TrimHostname {
repoName = reference.Path(r.info.Name)
}
return repoName
}
// BaseURL returns the endpoint url
func (r repositoryEndpoint) BaseURL() string {
return r.endpoint.URL.String()
}
func newDefaultRepositoryEndpoint(ref reference.Named, insecure bool) (repositoryEndpoint, error) {
repoInfo, err := registry.ParseRepositoryInfo(ref)
if err != nil {
return repositoryEndpoint{}, err
}
endpoint, err := getDefaultEndpointFromRepoInfo(repoInfo)
if err != nil {
return repositoryEndpoint{}, err
}
if insecure {
endpoint.TLSConfig.InsecureSkipVerify = true
}
return repositoryEndpoint{info: repoInfo, endpoint: endpoint}, nil
}
func getDefaultEndpointFromRepoInfo(repoInfo *registry.RepositoryInfo) (registry.APIEndpoint, error) {
var err error
options := registry.ServiceOptions{}
registryService, err := registry.NewService(options)
if err != nil {
return registry.APIEndpoint{}, err
}
endpoints, err := registryService.LookupPushEndpoints(reference.Domain(repoInfo.Name))
if err != nil {
return registry.APIEndpoint{}, err
}
// Default to the highest priority endpoint to return
endpoint := endpoints[0]
if !repoInfo.Index.Secure {
for _, ep := range endpoints {
if ep.URL.Scheme == "http" {
endpoint = ep
}
}
}
return endpoint, nil
}
// getHTTPTransport builds a transport for use in communicating with a registry
func getHTTPTransport(authConfig authtypes.AuthConfig, endpoint registry.APIEndpoint, repoName string, userAgent string) (http.RoundTripper, error) {
// get the http transport, this will be used in a client to upload manifest
base := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: endpoint.TLSConfig,
DisableKeepAlives: true,
}
modifiers := registry.Headers(userAgent, http.Header{})
authTransport := transport.NewTransport(base, modifiers...)
challengeManager, confirmedV2, err := registry.PingV2Registry(endpoint.URL, authTransport)
if err != nil {
return nil, errors.Wrap(err, "error pinging v2 registry")
}
if !confirmedV2 {
return nil, fmt.Errorf("unsupported registry version")
}
if authConfig.RegistryToken != "" {
passThruTokenHandler := &existingTokenHandler{token: authConfig.RegistryToken}
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, passThruTokenHandler))
} else {
creds := registry.NewStaticCredentialStore(&authConfig)
tokenHandler := auth.NewTokenHandler(authTransport, creds, repoName, "*")
basicHandler := auth.NewBasicHandler(creds)
modifiers = append(modifiers, auth.NewAuthorizer(challengeManager, tokenHandler, basicHandler))
}
return transport.NewTransport(base, modifiers...), nil
}
// RepoNameForReference returns the repository name from a reference
func RepoNameForReference(ref reference.Named) (string, error) {
// insecure is fine since this only returns the name
repo, err := newDefaultRepositoryEndpoint(ref, false)
if err != nil {
return "", err
}
return repo.Name(), nil
}
type existingTokenHandler struct {
token string
}
func (th *existingTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", th.token))
return nil
}
func (th *existingTokenHandler) Scheme() string {
return "bearer"
}

View File

@ -0,0 +1,295 @@
package client
import (
"fmt"
"github.com/docker/cli/cli/manifest/types"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema2"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/api/errcode"
"github.com/docker/distribution/registry/api/v2"
distclient "github.com/docker/distribution/registry/client"
"github.com/docker/docker/registry"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/net/context"
)
// fetchManifest pulls a manifest from a registry and returns it. An error
// is returned if no manifest is found matching namedRef.
func fetchManifest(ctx context.Context, repo distribution.Repository, ref reference.Named) (types.ImageManifest, error) {
manifest, err := getManifest(ctx, repo, ref)
if err != nil {
return types.ImageManifest{}, err
}
switch v := manifest.(type) {
// Removed Schema 1 support
case *schema2.DeserializedManifest:
imageManifest, err := pullManifestSchemaV2(ctx, ref, repo, *v)
if err != nil {
return types.ImageManifest{}, err
}
return imageManifest, nil
case *manifestlist.DeserializedManifestList:
return types.ImageManifest{}, errors.Errorf("%s is a manifest list", ref)
}
return types.ImageManifest{}, errors.Errorf("%s is not a manifest", ref)
}
func fetchList(ctx context.Context, repo distribution.Repository, ref reference.Named) ([]types.ImageManifest, error) {
manifest, err := getManifest(ctx, repo, ref)
if err != nil {
return nil, err
}
switch v := manifest.(type) {
case *manifestlist.DeserializedManifestList:
imageManifests, err := pullManifestList(ctx, ref, repo, *v)
if err != nil {
return nil, err
}
return imageManifests, nil
default:
return nil, errors.Errorf("unsupported manifest format: %v", v)
}
}
func getManifest(ctx context.Context, repo distribution.Repository, ref reference.Named) (distribution.Manifest, error) {
manSvc, err := repo.Manifests(ctx)
if err != nil {
return nil, err
}
dgst, opts, err := getManifestOptionsFromReference(ref)
if err != nil {
return nil, errors.Errorf("image manifest for %q does not exist", ref)
}
return manSvc.Get(ctx, dgst, opts...)
}
func pullManifestSchemaV2(ctx context.Context, ref reference.Named, repo distribution.Repository, mfst schema2.DeserializedManifest) (types.ImageManifest, error) {
manifestDigest, err := validateManifestDigest(ref, mfst)
if err != nil {
return types.ImageManifest{}, err
}
configJSON, err := pullManifestSchemaV2ImageConfig(ctx, mfst.Target().Digest, repo)
if err != nil {
return types.ImageManifest{}, err
}
img, err := types.NewImageFromJSON(configJSON)
if err != nil {
return types.ImageManifest{}, err
}
return types.NewImageManifest(ref, manifestDigest, *img, &mfst), nil
}
func pullManifestSchemaV2ImageConfig(ctx context.Context, dgst digest.Digest, repo distribution.Repository) ([]byte, error) {
blobs := repo.Blobs(ctx)
configJSON, err := blobs.Get(ctx, dgst)
if err != nil {
return nil, err
}
verifier := dgst.Verifier()
if err != nil {
return nil, err
}
if _, err := verifier.Write(configJSON); err != nil {
return nil, err
}
if !verifier.Verified() {
return nil, errors.Errorf("image config verification failed for digest %s", dgst)
}
return configJSON, nil
}
// validateManifestDigest computes the manifest digest, and, if pulling by
// digest, ensures that it matches the requested digest.
func validateManifestDigest(ref reference.Named, mfst distribution.Manifest) (digest.Digest, error) {
_, canonical, err := mfst.Payload()
if err != nil {
return "", err
}
// If pull by digest, then verify the manifest digest.
if digested, isDigested := ref.(reference.Canonical); isDigested {
verifier := digested.Digest().Verifier()
if err != nil {
return "", err
}
if _, err := verifier.Write(canonical); err != nil {
return "", err
}
if !verifier.Verified() {
err := fmt.Errorf("manifest verification failed for digest %s", digested.Digest())
return "", err
}
return digested.Digest(), nil
}
return digest.FromBytes(canonical), nil
}
// pullManifestList handles "manifest lists" which point to various
// platform-specific manifests.
func pullManifestList(ctx context.Context, ref reference.Named, repo distribution.Repository, mfstList manifestlist.DeserializedManifestList) ([]types.ImageManifest, error) {
infos := []types.ImageManifest{}
if _, err := validateManifestDigest(ref, mfstList); err != nil {
return nil, err
}
for _, manifestDescriptor := range mfstList.Manifests {
manSvc, err := repo.Manifests(ctx)
if err != nil {
return nil, err
}
manifest, err := manSvc.Get(ctx, manifestDescriptor.Digest)
if err != nil {
return nil, err
}
v, ok := manifest.(*schema2.DeserializedManifest)
if !ok {
return nil, fmt.Errorf("unsupported manifest format: %s", v)
}
manifestRef, err := reference.WithDigest(ref, manifestDescriptor.Digest)
if err != nil {
return nil, err
}
imageManifest, err := pullManifestSchemaV2(ctx, manifestRef, repo, *v)
if err != nil {
return nil, err
}
imageManifest.Platform = manifestDescriptor.Platform
infos = append(infos, imageManifest)
}
return infos, nil
}
func continueOnError(err error) bool {
switch v := err.(type) {
case errcode.Errors:
if len(v) == 0 {
return true
}
return continueOnError(v[0])
case errcode.Error:
e := err.(errcode.Error)
switch e.Code {
case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown, v2.ErrorCodeNameUnknown:
return true
}
return false
case *distclient.UnexpectedHTTPResponseError:
return true
}
return false
}
func (c *client) iterateEndpoints(ctx context.Context, namedRef reference.Named, each func(context.Context, distribution.Repository, reference.Named) (bool, error)) error {
endpoints, err := allEndpoints(namedRef)
if err != nil {
return err
}
repoInfo, err := registry.ParseRepositoryInfo(namedRef)
if err != nil {
return err
}
confirmedTLSRegistries := make(map[string]bool)
for _, endpoint := range endpoints {
if endpoint.Version == registry.APIVersion1 {
logrus.Debugf("skipping v1 endpoint %s", endpoint.URL)
continue
}
if endpoint.URL.Scheme != "https" {
if _, confirmedTLS := confirmedTLSRegistries[endpoint.URL.Host]; confirmedTLS {
logrus.Debugf("skipping non-TLS endpoint %s for host/port that appears to use TLS", endpoint.URL)
continue
}
}
if c.insecureRegistry {
endpoint.TLSConfig.InsecureSkipVerify = true
}
repoEndpoint := repositoryEndpoint{endpoint: endpoint, info: repoInfo}
repo, err := c.getRepositoryForReference(ctx, namedRef, repoEndpoint)
if err != nil {
logrus.Debugf("error with repo endpoint %s: %s", repoEndpoint, err)
if _, ok := err.(ErrHTTPProto); ok {
continue
}
return err
}
if endpoint.URL.Scheme == "http" && !c.insecureRegistry {
logrus.Debugf("skipping non-tls registry endpoint: %s", endpoint.URL)
continue
}
done, err := each(ctx, repo, namedRef)
if err != nil {
if continueOnError(err) {
if endpoint.URL.Scheme == "https" {
confirmedTLSRegistries[endpoint.URL.Host] = true
}
logrus.Debugf("continuing on error (%T) %s", err, err)
continue
}
logrus.Debugf("not continuing on error (%T) %s", err, err)
return err
}
if done {
return nil
}
}
return newNotFoundError(namedRef.String())
}
// allEndpoints returns a list of endpoints ordered by priority (v2, https, v1).
func allEndpoints(namedRef reference.Named) ([]registry.APIEndpoint, error) {
repoInfo, err := registry.ParseRepositoryInfo(namedRef)
if err != nil {
return nil, err
}
registryService, err := registry.NewService(registry.ServiceOptions{})
if err != nil {
return []registry.APIEndpoint{}, err
}
endpoints, err := registryService.LookupPullEndpoints(reference.Domain(repoInfo.Name))
logrus.Debugf("endpoints for %s: %v", namedRef, endpoints)
return endpoints, err
}
type notFoundError struct {
object string
}
func newNotFoundError(ref string) *notFoundError {
return &notFoundError{object: ref}
}
func (n *notFoundError) Error() string {
return fmt.Sprintf("no such manifest: %s", n.object)
}
// NotFound interface
func (n *notFoundError) NotFound() {}
// IsNotFound returns true if the error is a not found error
func IsNotFound(err error) bool {
_, ok := err.(notFound)
return ok
}
type notFound interface {
NotFound()
}