fix: handle all container registries

See coop-cloud/organising#258

This fixes also how we read the digest of the image. I think it was
wrong before. Some registries restrict reading this info and we now just
default to "unknown" for that case.

This also appears to bring a wave of new dependencies due to the generic
handling logic of containers/... package. The abra binary is now 1mb
larger.

The catalogue generation is now slower unfortunately. But it is more
robust.

The generic logic looks in ~/.docker/config.json for log in details, so
you don't have to pass those in manually on the CLI anymore. We just
read those defaults. You can "docker login" to get credentials setup in
that file. Since most folks won't generate the catalogue, this seems
fine for now.

cli/catalogue/catalogue.go

@@ -66,8 +66,6 @@ var catalogueGenerateCommand = cli.Command{
 		internal.PublishFlag,
 		internal.DryFlag,
 		internal.SkipUpdatesFlag,
-		internal.RegistryUsernameFlag,
-		internal.RegistryPasswordFlag,
 	},
 	Before: internal.SubCommandBefore,
 	Description: `
@@ -132,11 +130,7 @@ keys configured on your account.
 				continue
 			}
 
-			versions, err := recipe.GetRecipeVersions(
+			versions, err := recipe.GetRecipeVersions(recipeMeta.Name)
-				recipeMeta.Name,
-				internal.RegistryUsername,
-				internal.RegistryPassword,
-			)
 			if err != nil {
 				logrus.Warn(err)
 			}

cli/internal/cli.go

@@ -336,24 +336,6 @@ var SkipUpdatesFlag = &cli.BoolFlag{
 	Destination: &SkipUpdates,
 }
 
-var RegistryUsername string
-var RegistryUsernameFlag = &cli.StringFlag{
-	Name:        "username, user",
-	Value:       "",
-	Usage:       "Registry username",
-	EnvVar:      "REGISTRY_USERNAME",
-	Destination: &RegistryUsername,
-}
-
-var RegistryPassword string
-var RegistryPasswordFlag = &cli.StringFlag{
-	Name:        "password, pass",
-	Value:       "",
-	Usage:       "Registry password",
-	EnvVar:      "REGISTRY_PASSWORD",
-	Destination: &RegistryUsername,
-}
-
 var AllTags bool
 var AllTagsFlag = &cli.BoolFlag{
 	Name:        "all-tags, a",

cli/recipe/upgrade.go

@@ -113,13 +113,13 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				logrus.Fatal(err)
 			}
 
-			image := reference.Path(img)
+			regVersions, err := client.GetRegistryTags(img)
-			regVersions, err := client.GetRegistryTags(image)
 			if err != nil {
 				logrus.Fatal(err)
 			}
-			logrus.Debugf("retrieved %s from remote registry for %s", regVersions, image)
 
+			image := reference.Path(img)
+			logrus.Debugf("retrieved %s from remote registry for %s", regVersions, image)
 			image = formatter.StripTagMeta(image)
 
 			switch img.(type) {
@@ -142,7 +142,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 
 			var compatible []tagcmp.Tag
 			for _, regVersion := range regVersions {
-				other, err := tagcmp.Parse(regVersion.Name)
+				other, err := tagcmp.Parse(regVersion)
 				if err != nil {
 					continue // skip tags that cannot be parsed
 				}
@@ -232,7 +232,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
 						msg = fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
 						compatibleStrings = []string{"skip"}
 						for _, regVersion := range regVersions {
-							compatibleStrings = append(compatibleStrings, regVersion.Name)
+							compatibleStrings = append(compatibleStrings, regVersion)
 						}
 					}
 

pkg/client/registry.go

@@ -1,193 +1,57 @@
 package client
 
 import (
-	"encoding/base64"
+	"context"
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"net/http"
 	"strings"
 
-	"coopcloud.tech/abra/pkg/web"
+	"github.com/containers/image/docker"
+	"github.com/containers/image/types"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/client"
-	"github.com/hashicorp/go-retryablehttp"
 	"github.com/sirupsen/logrus"
 )
 
-type RawTag struct {
+// GetRegistryTags retrieves all tags of an image from a container registry.
-	Layer string
+func GetRegistryTags(img reference.Named) ([]string, error) {
-	Name  string
+	var tags []string
-}
 
-type RawTags []RawTag
+	ref, err := docker.ParseReference(fmt.Sprintf("//%s", img))
+	if err != nil {
+		return tags, fmt.Errorf("failed to parse image %s, saw: %s", img, err.Error())
+	}
 
-var registryURL = "https://registry.hub.docker.com/v1/repositories/%s/tags"
+	ctx := context.Background()
-
+	tags, err = docker.GetRepositoryTags(ctx, &types.SystemContext{}, ref)
-func GetRegistryTags(image string) (RawTags, error) {
+	if err != nil {
-	var tags RawTags
-
-	tagsUrl := fmt.Sprintf(registryURL, image)
-	if err := web.ReadJSON(tagsUrl, &tags); err != nil {
 		return tags, err
 	}
 
 	return tags, nil
 }
 
-func basicAuth(username, password string) string {
+// GetTagDigest retrieves an image digest from a container registry.
-	auth := username + ":" + password
+func GetTagDigest(cl *client.Client, image reference.Named) (string, error) {
-	return base64.StdEncoding.EncodeToString([]byte(auth))
+	target := fmt.Sprintf("//%s", reference.Path(image))
-}
 
-// getRegv2Token retrieves a registry v2 authentication token.
+	ref, err := docker.ParseReference(target)
-func getRegv2Token(cl *client.Client, image reference.Named, registryUsername, registryPassword string) (string, error) {
-	img := reference.Path(image)
-	tokenURL := "https://auth.docker.io/token"
-	values := fmt.Sprintf("service=registry.docker.io&scope=repository:%s:pull", img)
-
-	fullURL := fmt.Sprintf("%s?%s", tokenURL, values)
-	req, err := retryablehttp.NewRequest("GET", fullURL, nil)
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("failed to parse image %s, saw: %s", image, err.Error())
 	}
 
-	if registryUsername != "" && registryPassword != "" {
+	ctx := context.Background()
-		logrus.Debugf("using registry log in credentials for token request")
+	img, err := ref.NewImage(ctx, nil)
-		auth := basicAuth(registryUsername, registryPassword)
-		req.Header.Add("Authorization", fmt.Sprintf("Basic %s", auth))
-	}
-
-	client := web.NewHTTPRetryClient()
-	res, err := client.Do(req)
 	if err != nil {
-		return "", err
+		logrus.Debugf("failed to query remote registry for %s, saw: %s", image, err.Error())
+		return "", fmt.Errorf("unable to read digest for %s", image)
 	}
-	defer res.Body.Close()
+	defer img.Close()
 
-	if res.StatusCode != http.StatusOK {
+	digest := img.ConfigInfo().Digest.String()
-		_, err := ioutil.ReadAll(res.Body)
-		if err != nil {
-			return "", err
-		}
-	}
-
-	body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return "", nil
-	}
-
-	tokenRes := struct {
-		AccessToken string `json:"access_token"`
-		Expiry      int    `json:"expires_in"`
-		Issued      string `json:"issued_at"`
-		Token       string `json:"token"`
-	}{}
-
-	if err := json.Unmarshal(body, &tokenRes); err != nil {
-		return "", err
-	}
-
-	return tokenRes.Token, nil
-}
-
-// GetTagDigest retrieves an image digest from a v2 registry
-func GetTagDigest(cl *client.Client, image reference.Named, registryUsername, registryPassword string) (string, error) {
-	img := reference.Path(image)
-	tag := image.(reference.NamedTagged).Tag()
-	manifestURL := fmt.Sprintf("https://index.docker.io/v2/%s/manifests/%s", img, tag)
-
-	req, err := retryablehttp.NewRequest("GET", manifestURL, nil)
-	if err != nil {
-		return "", err
-	}
-
-	token, err := getRegv2Token(cl, image, registryUsername, registryPassword)
-	if err != nil {
-		return "", err
-	}
-
-	if token == "" {
-		return "", fmt.Errorf("unable to retrieve registry token?")
-	}
-
-	req.Header = http.Header{
-		"Accept": []string{
-			"application/vnd.docker.distribution.manifest.v2+json",
-			"application/vnd.docker.distribution.manifest.list.v2+json",
-		},
-		"Authorization": []string{fmt.Sprintf("Bearer %s", token)},
-	}
-
-	client := web.NewHTTPRetryClient()
-	res, err := client.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer res.Body.Close()
-
-	if res.StatusCode != http.StatusOK {
-		_, err := ioutil.ReadAll(res.Body)
-		if err != nil {
-			return "", err
-		}
-	}
-
-	body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return "", err
-	}
-
-	registryResT1 := struct {
-		SchemaVersion int
-		MediaType     string
-		Manifests     []struct {
-			MediaType string
-			Size      int
-			Digest    string
-			Platform  struct {
-				Architecture string
-				Os           string
-			}
-		}
-	}{}
-
-	registryResT2 := struct {
-		SchemaVersion int
-		MediaType     string
-		Config        struct {
-			MediaType string
-			Size      int
-			Digest    string
-		}
-		Layers []struct {
-			MediaType string
-			Size      int
-			Digest    string
-		}
-	}{}
-
-	if err := json.Unmarshal(body, &registryResT1); err != nil {
-		return "", err
-	}
-
-	var digest string
-	for _, manifest := range registryResT1.Manifests {
-		if string(manifest.Platform.Architecture) == "amd64" {
-			digest = strings.Split(manifest.Digest, ":")[1][:7]
-		}
-	}
 
 	if digest == "" {
-		if err := json.Unmarshal(body, &registryResT2); err != nil {
+		return digest, fmt.Errorf("unable to read digest for %s", image)
-			return "", err
-		}
-		digest = strings.Split(registryResT2.Config.Digest, ":")[1][:7]
 	}
 
-	if digest == "" {
+	return strings.Split(digest, ":")[1][:7], nil
-		return "", fmt.Errorf("Unable to retrieve amd64 digest for %s", image)
-	}
-
-	return digest, nil
 }

pkg/recipe/recipe.go

@@ -232,7 +232,11 @@ func Get(recipeName string) (Recipe, error) {
 
 	meta, err := GetRecipeMeta(recipeName)
 	if err != nil {
-		return Recipe{}, err
+		if strings.Contains(err.Error(), "does not exist") {
+			meta = RecipeMeta{}
+		} else {
+			return Recipe{}, err
+		}
 	}
 
 	return Recipe{
@@ -799,8 +803,7 @@ func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
 
 	recipeMeta, ok := catl[recipeName]
 	if !ok {
-		err := fmt.Errorf("recipe %s does not exist?", recipeName)
+		return RecipeMeta{}, fmt.Errorf("recipe %s does not exist?", recipeName)
-		return RecipeMeta{}, err
 	}
 
 	if err := EnsureExists(recipeName); err != nil {
@@ -925,7 +928,7 @@ func ReadReposMetadata() (RepoCatalogue, error) {
 }
 
 // GetRecipeVersions retrieves all recipe versions.
-func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (RecipeVersions, error) {
+func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 	versions := RecipeVersions{}
 
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)
@@ -969,7 +972,7 @@ func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (R
 			return err
 		}
 
-		cl, err := client.New("default") // only required for docker.io registry calls
+		cl, err := client.New("default") // only required for container registry calls
 		if err != nil {
 			return err
 		}
@@ -999,18 +1002,19 @@ func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (R
 			var exists bool
 			var digest string
 			if digest, exists = queryCache[img]; !exists {
-				logrus.Debugf("looking up image: %s from %s", img, path)
+				logrus.Debugf("cache miss: querying for image: %s, tag: %s", path, tag)
+
 				var err error
-				digest, err = client.GetTagDigest(cl, img, registryUsername, registryPassword)
+				digest, err = client.GetTagDigest(cl, img)
 				if err != nil {
 					logrus.Warn(err)
-					continue
+					digest = "unknown"
 				}
-				logrus.Debugf("queried for image: %s, tag: %s, digest: %s", path, tag, digest)
+
 				queryCache[img] = digest
-				logrus.Debugf("cached image: %s, tag: %s, digest: %s", path, tag, digest)
+				logrus.Debugf("cached insert: %s, tag: %s, digest: %s", path, tag, digest)
 			} else {
-				logrus.Debugf("reading image: %s, tag: %s, digest: %s from cache", path, tag, digest)
+				logrus.Debugf("cache hit: image: %s, tag: %s, digest: %s", path, tag, digest)
 			}
 
 			versionMeta[service.Name] = ServiceMeta{