fix: handle all container registries
See coop-cloud/organising#258 This fixes also how we read the digest of the image. I think it was wrong before. Some registries restrict reading this info and we now just default to "unknown" for that case. This also appears to bring a wave of new dependencies due to the generic handling logic of containers/... package. The abra binary is now 1mb larger. The catalogue generation is now slower unfortunately. But it is more robust. The generic logic looks in ~/.docker/config.json for log in details, so you don't have to pass those in manually on the CLI anymore. We just read those defaults. You can "docker login" to get credentials setup in that file. Since most folks won't generate the catalogue, this seems fine for now.
This commit is contained in:
Gitea
parent
2fbdcfb958
commit
b295958c17
|
|
@ -66,8 +66,6 @@ var catalogueGenerateCommand = cli.Command{
|
|||
internal.PublishFlag,
|
||||
internal.DryFlag,
|
||||
internal.SkipUpdatesFlag,
|
||||
internal.RegistryUsernameFlag,
|
||||
internal.RegistryPasswordFlag,
|
||||
},
|
||||
Before: internal.SubCommandBefore,
|
||||
Description: `
|
||||
|
|
@ -132,11 +130,7 @@ keys configured on your account.
|
|||
continue
|
||||
}
|
||||
|
||||
versions, err := recipe.GetRecipeVersions(
|
||||
recipeMeta.Name,
|
||||
internal.RegistryUsername,
|
||||
internal.RegistryPassword,
|
||||
)
|
||||
versions, err := recipe.GetRecipeVersions(recipeMeta.Name)
|
||||
if err != nil {
|
||||
logrus.Warn(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -336,24 +336,6 @@ var SkipUpdatesFlag = &cli.BoolFlag{
|
|||
Destination: &SkipUpdates,
|
||||
}
|
||||
|
||||
var RegistryUsername string
|
||||
var RegistryUsernameFlag = &cli.StringFlag{
|
||||
Name: "username, user",
|
||||
Value: "",
|
||||
Usage: "Registry username",
|
||||
EnvVar: "REGISTRY_USERNAME",
|
||||
Destination: &RegistryUsername,
|
||||
}
|
||||
|
||||
var RegistryPassword string
|
||||
var RegistryPasswordFlag = &cli.StringFlag{
|
||||
Name: "password, pass",
|
||||
Value: "",
|
||||
Usage: "Registry password",
|
||||
EnvVar: "REGISTRY_PASSWORD",
|
||||
Destination: &RegistryUsername,
|
||||
}
|
||||
|
||||
var AllTags bool
|
||||
var AllTagsFlag = &cli.BoolFlag{
|
||||
Name: "all-tags, a",
|
||||
|
|
|
|||
|
|
@ -113,13 +113,13 @@ You may invoke this command in "wizard" mode and be prompted for input:
|
|||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
image := reference.Path(img)
|
||||
regVersions, err := client.GetRegistryTags(image)
|
||||
regVersions, err := client.GetRegistryTags(img)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
logrus.Debugf("retrieved %s from remote registry for %s", regVersions, image)
|
||||
|
||||
image := reference.Path(img)
|
||||
logrus.Debugf("retrieved %s from remote registry for %s", regVersions, image)
|
||||
image = formatter.StripTagMeta(image)
|
||||
|
||||
switch img.(type) {
|
||||
|
|
@ -142,7 +142,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
|
|||
|
||||
var compatible []tagcmp.Tag
|
||||
for _, regVersion := range regVersions {
|
||||
other, err := tagcmp.Parse(regVersion.Name)
|
||||
other, err := tagcmp.Parse(regVersion)
|
||||
if err != nil {
|
||||
continue // skip tags that cannot be parsed
|
||||
}
|
||||
|
|
@ -232,7 +232,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
|
|||
msg = fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
|
||||
compatibleStrings = []string{"skip"}
|
||||
for _, regVersion := range regVersions {
|
||||
compatibleStrings = append(compatibleStrings, regVersion.Name)
|
||||
compatibleStrings = append(compatibleStrings, regVersion)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,193 +1,57 @@
|
|||
package client
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"coopcloud.tech/abra/pkg/web"
|
||||
"github.com/containers/image/docker"
|
||||
"github.com/containers/image/types"
|
||||
"github.com/docker/distribution/reference"
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/hashicorp/go-retryablehttp"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type RawTag struct {
|
||||
Layer string
|
||||
Name string
|
||||
}
|
||||
// GetRegistryTags retrieves all tags of an image from a container registry.
|
||||
func GetRegistryTags(img reference.Named) ([]string, error) {
|
||||
var tags []string
|
||||
|
||||
type RawTags []RawTag
|
||||
ref, err := docker.ParseReference(fmt.Sprintf("//%s", img))
|
||||
if err != nil {
|
||||
return tags, fmt.Errorf("failed to parse image %s, saw: %s", img, err.Error())
|
||||
}
|
||||
|
||||
var registryURL = "https://registry.hub.docker.com/v1/repositories/%s/tags"
|
||||
|
||||
func GetRegistryTags(image string) (RawTags, error) {
|
||||
var tags RawTags
|
||||
|
||||
tagsUrl := fmt.Sprintf(registryURL, image)
|
||||
if err := web.ReadJSON(tagsUrl, &tags); err != nil {
|
||||
ctx := context.Background()
|
||||
tags, err = docker.GetRepositoryTags(ctx, &types.SystemContext{}, ref)
|
||||
if err != nil {
|
||||
return tags, err
|
||||
}
|
||||
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
func basicAuth(username, password string) string {
|
||||
auth := username + ":" + password
|
||||
return base64.StdEncoding.EncodeToString([]byte(auth))
|
||||
}
|
||||
// GetTagDigest retrieves an image digest from a container registry.
|
||||
func GetTagDigest(cl *client.Client, image reference.Named) (string, error) {
|
||||
target := fmt.Sprintf("//%s", reference.Path(image))
|
||||
|
||||
// getRegv2Token retrieves a registry v2 authentication token.
|
||||
func getRegv2Token(cl *client.Client, image reference.Named, registryUsername, registryPassword string) (string, error) {
|
||||
img := reference.Path(image)
|
||||
tokenURL := "https://auth.docker.io/token"
|
||||
values := fmt.Sprintf("service=registry.docker.io&scope=repository:%s:pull", img)
|
||||
|
||||
fullURL := fmt.Sprintf("%s?%s", tokenURL, values)
|
||||
req, err := retryablehttp.NewRequest("GET", fullURL, nil)
|
||||
ref, err := docker.ParseReference(target)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", fmt.Errorf("failed to parse image %s, saw: %s", image, err.Error())
|
||||
}
|
||||
|
||||
if registryUsername != "" && registryPassword != "" {
|
||||
logrus.Debugf("using registry log in credentials for token request")
|
||||
auth := basicAuth(registryUsername, registryPassword)
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", auth))
|
||||
}
|
||||
|
||||
client := web.NewHTTPRetryClient()
|
||||
res, err := client.Do(req)
|
||||
ctx := context.Background()
|
||||
img, err := ref.NewImage(ctx, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
logrus.Debugf("failed to query remote registry for %s, saw: %s", image, err.Error())
|
||||
return "", fmt.Errorf("unable to read digest for %s", image)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
defer img.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
_, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
tokenRes := struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
Expiry int `json:"expires_in"`
|
||||
Issued string `json:"issued_at"`
|
||||
Token string `json:"token"`
|
||||
}{}
|
||||
|
||||
if err := json.Unmarshal(body, &tokenRes); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return tokenRes.Token, nil
|
||||
}
|
||||
|
||||
// GetTagDigest retrieves an image digest from a v2 registry
|
||||
func GetTagDigest(cl *client.Client, image reference.Named, registryUsername, registryPassword string) (string, error) {
|
||||
img := reference.Path(image)
|
||||
tag := image.(reference.NamedTagged).Tag()
|
||||
manifestURL := fmt.Sprintf("https://index.docker.io/v2/%s/manifests/%s", img, tag)
|
||||
|
||||
req, err := retryablehttp.NewRequest("GET", manifestURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
token, err := getRegv2Token(cl, image, registryUsername, registryPassword)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if token == "" {
|
||||
return "", fmt.Errorf("unable to retrieve registry token?")
|
||||
}
|
||||
|
||||
req.Header = http.Header{
|
||||
"Accept": []string{
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
},
|
||||
"Authorization": []string{fmt.Sprintf("Bearer %s", token)},
|
||||
}
|
||||
|
||||
client := web.NewHTTPRetryClient()
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
_, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
registryResT1 := struct {
|
||||
SchemaVersion int
|
||||
MediaType string
|
||||
Manifests []struct {
|
||||
MediaType string
|
||||
Size int
|
||||
Digest string
|
||||
Platform struct {
|
||||
Architecture string
|
||||
Os string
|
||||
}
|
||||
}
|
||||
}{}
|
||||
|
||||
registryResT2 := struct {
|
||||
SchemaVersion int
|
||||
MediaType string
|
||||
Config struct {
|
||||
MediaType string
|
||||
Size int
|
||||
Digest string
|
||||
}
|
||||
Layers []struct {
|
||||
MediaType string
|
||||
Size int
|
||||
Digest string
|
||||
}
|
||||
}{}
|
||||
|
||||
if err := json.Unmarshal(body, ®istryResT1); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var digest string
|
||||
for _, manifest := range registryResT1.Manifests {
|
||||
if string(manifest.Platform.Architecture) == "amd64" {
|
||||
digest = strings.Split(manifest.Digest, ":")[1][:7]
|
||||
}
|
||||
}
|
||||
digest := img.ConfigInfo().Digest.String()
|
||||
|
||||
if digest == "" {
|
||||
if err := json.Unmarshal(body, ®istryResT2); err != nil {
|
||||
return "", err
|
||||
}
|
||||
digest = strings.Split(registryResT2.Config.Digest, ":")[1][:7]
|
||||
return digest, fmt.Errorf("unable to read digest for %s", image)
|
||||
}
|
||||
|
||||
if digest == "" {
|
||||
return "", fmt.Errorf("Unable to retrieve amd64 digest for %s", image)
|
||||
}
|
||||
|
||||
return digest, nil
|
||||
return strings.Split(digest, ":")[1][:7], nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -232,7 +232,11 @@ func Get(recipeName string) (Recipe, error) {
|
|||
|
||||
meta, err := GetRecipeMeta(recipeName)
|
||||
if err != nil {
|
||||
return Recipe{}, err
|
||||
if strings.Contains(err.Error(), "does not exist") {
|
||||
meta = RecipeMeta{}
|
||||
} else {
|
||||
return Recipe{}, err
|
||||
}
|
||||
}
|
||||
|
||||
return Recipe{
|
||||
|
|
@ -799,8 +803,7 @@ func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
|
|||
|
||||
recipeMeta, ok := catl[recipeName]
|
||||
if !ok {
|
||||
err := fmt.Errorf("recipe %s does not exist?", recipeName)
|
||||
return RecipeMeta{}, err
|
||||
return RecipeMeta{}, fmt.Errorf("recipe %s does not exist?", recipeName)
|
||||
}
|
||||
|
||||
if err := EnsureExists(recipeName); err != nil {
|
||||
|
|
@ -925,7 +928,7 @@ func ReadReposMetadata() (RepoCatalogue, error) {
|
|||
}
|
||||
|
||||
// GetRecipeVersions retrieves all recipe versions.
|
||||
func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (RecipeVersions, error) {
|
||||
func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
|
||||
versions := RecipeVersions{}
|
||||
|
||||
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
||||
|
|
@ -969,7 +972,7 @@ func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (R
|
|||
return err
|
||||
}
|
||||
|
||||
cl, err := client.New("default") // only required for docker.io registry calls
|
||||
cl, err := client.New("default") // only required for container registry calls
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -999,18 +1002,19 @@ func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (R
|
|||
var exists bool
|
||||
var digest string
|
||||
if digest, exists = queryCache[img]; !exists {
|
||||
logrus.Debugf("looking up image: %s from %s", img, path)
|
||||
logrus.Debugf("cache miss: querying for image: %s, tag: %s", path, tag)
|
||||
|
||||
var err error
|
||||
digest, err = client.GetTagDigest(cl, img, registryUsername, registryPassword)
|
||||
digest, err = client.GetTagDigest(cl, img)
|
||||
if err != nil {
|
||||
logrus.Warn(err)
|
||||
continue
|
||||
digest = "unknown"
|
||||
}
|
||||
logrus.Debugf("queried for image: %s, tag: %s, digest: %s", path, tag, digest)
|
||||
|
||||
queryCache[img] = digest
|
||||
logrus.Debugf("cached image: %s, tag: %s, digest: %s", path, tag, digest)
|
||||
logrus.Debugf("cached insert: %s, tag: %s, digest: %s", path, tag, digest)
|
||||
} else {
|
||||
logrus.Debugf("reading image: %s, tag: %s, digest: %s from cache", path, tag, digest)
|
||||
logrus.Debugf("cache hit: image: %s, tag: %s, digest: %s", path, tag, digest)
|
||||
}
|
||||
|
||||
versionMeta[service.Name] = ServiceMeta{
|
||||
|
|
|
|||
Loading…
Reference in New Issue