Support more container registries #159
|
@ -49,6 +49,8 @@ var CatalogueSkipList = map[string]bool{
|
|||
"pyabra": true,
|
||||
"radicle-seed-node": true,
|
||||
"recipes-catalogue-json": true,
|
||||
"recipes-wishlist": true,
|
||||
"recipes.coopcloud.tech": true,
|
||||
"stack-ssh-deploy": true,
|
||||
"swarm-cronjob": true,
|
||||
"tagcmp": true,
|
||||
|
@ -66,8 +68,6 @@ var catalogueGenerateCommand = cli.Command{
|
|||
internal.PublishFlag,
|
||||
internal.DryFlag,
|
||||
internal.SkipUpdatesFlag,
|
||||
internal.RegistryUsernameFlag,
|
||||
internal.RegistryPasswordFlag,
|
||||
},
|
||||
Before: internal.SubCommandBefore,
|
||||
Description: `
|
||||
|
@ -132,11 +132,7 @@ keys configured on your account.
|
|||
continue
|
||||
}
|
||||
|
||||
versions, err := recipe.GetRecipeVersions(
|
||||
recipeMeta.Name,
|
||||
internal.RegistryUsername,
|
||||
internal.RegistryPassword,
|
||||
)
|
||||
versions, err := recipe.GetRecipeVersions(recipeMeta.Name)
|
||||
if err != nil {
|
||||
logrus.Warn(err)
|
||||
}
|
||||
|
|
|
@ -336,24 +336,6 @@ var SkipUpdatesFlag = &cli.BoolFlag{
|
|||
Destination: &SkipUpdates,
|
||||
}
|
||||
|
||||
var RegistryUsername string
|
||||
var RegistryUsernameFlag = &cli.StringFlag{
|
||||
Name: "username, user",
|
||||
Value: "",
|
||||
Usage: "Registry username",
|
||||
EnvVar: "REGISTRY_USERNAME",
|
||||
Destination: &RegistryUsername,
|
||||
}
|
||||
|
||||
var RegistryPassword string
|
||||
var RegistryPasswordFlag = &cli.StringFlag{
|
||||
Name: "password, pass",
|
||||
Value: "",
|
||||
Usage: "Registry password",
|
||||
EnvVar: "REGISTRY_PASSWORD",
|
||||
Destination: &RegistryUsername,
|
||||
}
|
||||
|
||||
var AllTags bool
|
||||
var AllTagsFlag = &cli.BoolFlag{
|
||||
Name: "all-tags, a",
|
||||
|
|
|
@ -113,13 +113,13 @@ You may invoke this command in "wizard" mode and be prompted for input:
|
|||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
image := reference.Path(img)
|
||||
regVersions, err := client.GetRegistryTags(image)
|
||||
regVersions, err := client.GetRegistryTags(img)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
logrus.Debugf("retrieved %s from remote registry for %s", regVersions, image)
|
||||
|
||||
image := reference.Path(img)
|
||||
logrus.Debugf("retrieved %s from remote registry for %s", regVersions, image)
|
||||
image = formatter.StripTagMeta(image)
|
||||
|
||||
switch img.(type) {
|
||||
|
@ -142,7 +142,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
|
|||
|
||||
var compatible []tagcmp.Tag
|
||||
for _, regVersion := range regVersions {
|
||||
other, err := tagcmp.Parse(regVersion.Name)
|
||||
other, err := tagcmp.Parse(regVersion)
|
||||
if err != nil {
|
||||
continue // skip tags that cannot be parsed
|
||||
}
|
||||
|
@ -232,7 +232,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
|
|||
msg = fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
|
||||
compatibleStrings = []string{"skip"}
|
||||
for _, regVersion := range regVersions {
|
||||
compatibleStrings = append(compatibleStrings, regVersion.Name)
|
||||
compatibleStrings = append(compatibleStrings, regVersion)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
14
go.mod
14
go.mod
|
@ -25,9 +25,11 @@ require (
|
|||
|
||||
require (
|
||||
coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e
|
||||
github.com/Microsoft/hcsshim v0.8.21 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3 // indirect
|
||||
github.com/buger/goterm v1.0.4
|
||||
github.com/containerd/containerd v1.5.5 // indirect
|
||||
github.com/containerd/containerd v1.5.9 // indirect
|
||||
github.com/containers/image v3.0.2+incompatible
|
||||
github.com/containers/storage v1.38.2 // indirect
|
||||
github.com/docker/docker-credential-helpers v0.6.4 // indirect
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/fvbommel/sortorder v1.0.2 // indirect
|
||||
|
@ -39,11 +41,13 @@ require (
|
|||
github.com/libdns/gandi v1.0.2
|
||||
github.com/libdns/libdns v0.2.1
|
||||
github.com/moby/sys/mount v0.2.0 // indirect
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/opencontainers/runc v1.0.2 // indirect
|
||||
github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 // indirect
|
||||
github.com/sergi/go-diff v1.2.0 // indirect
|
||||
github.com/spf13/cobra v1.3.0 // indirect
|
||||
github.com/theupdateframework/notary v0.7.0 // indirect
|
||||
github.com/urfave/cli v1.22.5
|
||||
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.1 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
|
||||
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838
|
||||
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27
|
||||
)
|
||||
|
|
|
@ -1,193 +1,57 @@
|
|||
package client
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"coopcloud.tech/abra/pkg/web"
|
||||
"github.com/containers/image/docker"
|
||||
"github.com/containers/image/types"
|
||||
"github.com/docker/distribution/reference"
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/hashicorp/go-retryablehttp"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type RawTag struct {
|
||||
Layer string
|
||||
Name string
|
||||
}
|
||||
// GetRegistryTags retrieves all tags of an image from a container registry.
|
||||
func GetRegistryTags(img reference.Named) ([]string, error) {
|
||||
var tags []string
|
||||
|
||||
type RawTags []RawTag
|
||||
ref, err := docker.ParseReference(fmt.Sprintf("//%s", img))
|
||||
if err != nil {
|
||||
return tags, fmt.Errorf("failed to parse image %s, saw: %s", img, err.Error())
|
||||
}
|
||||
|
||||
var registryURL = "https://registry.hub.docker.com/v1/repositories/%s/tags"
|
||||
|
||||
func GetRegistryTags(image string) (RawTags, error) {
|
||||
var tags RawTags
|
||||
|
||||
tagsUrl := fmt.Sprintf(registryURL, image)
|
||||
if err := web.ReadJSON(tagsUrl, &tags); err != nil {
|
||||
ctx := context.Background()
|
||||
tags, err = docker.GetRepositoryTags(ctx, &types.SystemContext{}, ref)
|
||||
if err != nil {
|
||||
return tags, err
|
||||
}
|
||||
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
func basicAuth(username, password string) string {
|
||||
auth := username + ":" + password
|
||||
return base64.StdEncoding.EncodeToString([]byte(auth))
|
||||
}
|
||||
// GetTagDigest retrieves an image digest from a container registry.
|
||||
func GetTagDigest(cl *client.Client, image reference.Named) (string, error) {
|
||||
target := fmt.Sprintf("//%s", reference.Path(image))
|
||||
|
||||
// getRegv2Token retrieves a registry v2 authentication token.
|
||||
func getRegv2Token(cl *client.Client, image reference.Named, registryUsername, registryPassword string) (string, error) {
|
||||
img := reference.Path(image)
|
||||
tokenURL := "https://auth.docker.io/token"
|
||||
values := fmt.Sprintf("service=registry.docker.io&scope=repository:%s:pull", img)
|
||||
|
||||
fullURL := fmt.Sprintf("%s?%s", tokenURL, values)
|
||||
req, err := retryablehttp.NewRequest("GET", fullURL, nil)
|
||||
ref, err := docker.ParseReference(target)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", fmt.Errorf("failed to parse image %s, saw: %s", image, err.Error())
|
||||
}
|
||||
|
||||
if registryUsername != "" && registryPassword != "" {
|
||||
logrus.Debugf("using registry log in credentials for token request")
|
||||
auth := basicAuth(registryUsername, registryPassword)
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", auth))
|
||||
}
|
||||
|
||||
client := web.NewHTTPRetryClient()
|
||||
res, err := client.Do(req)
|
||||
ctx := context.Background()
|
||||
img, err := ref.NewImage(ctx, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
logrus.Debugf("failed to query remote registry for %s, saw: %s", image, err.Error())
|
||||
return "", fmt.Errorf("unable to read digest for %s", image)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
defer img.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
_, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
tokenRes := struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
Expiry int `json:"expires_in"`
|
||||
Issued string `json:"issued_at"`
|
||||
Token string `json:"token"`
|
||||
}{}
|
||||
|
||||
if err := json.Unmarshal(body, &tokenRes); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return tokenRes.Token, nil
|
||||
}
|
||||
|
||||
// GetTagDigest retrieves an image digest from a v2 registry
|
||||
func GetTagDigest(cl *client.Client, image reference.Named, registryUsername, registryPassword string) (string, error) {
|
||||
img := reference.Path(image)
|
||||
tag := image.(reference.NamedTagged).Tag()
|
||||
manifestURL := fmt.Sprintf("https://index.docker.io/v2/%s/manifests/%s", img, tag)
|
||||
|
||||
req, err := retryablehttp.NewRequest("GET", manifestURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
token, err := getRegv2Token(cl, image, registryUsername, registryPassword)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if token == "" {
|
||||
return "", fmt.Errorf("unable to retrieve registry token?")
|
||||
}
|
||||
|
||||
req.Header = http.Header{
|
||||
"Accept": []string{
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
},
|
||||
"Authorization": []string{fmt.Sprintf("Bearer %s", token)},
|
||||
}
|
||||
|
||||
client := web.NewHTTPRetryClient()
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
_, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
registryResT1 := struct {
|
||||
SchemaVersion int
|
||||
MediaType string
|
||||
Manifests []struct {
|
||||
MediaType string
|
||||
Size int
|
||||
Digest string
|
||||
Platform struct {
|
||||
Architecture string
|
||||
Os string
|
||||
}
|
||||
}
|
||||
}{}
|
||||
|
||||
registryResT2 := struct {
|
||||
SchemaVersion int
|
||||
MediaType string
|
||||
Config struct {
|
||||
MediaType string
|
||||
Size int
|
||||
Digest string
|
||||
}
|
||||
Layers []struct {
|
||||
MediaType string
|
||||
Size int
|
||||
Digest string
|
||||
}
|
||||
}{}
|
||||
|
||||
if err := json.Unmarshal(body, ®istryResT1); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var digest string
|
||||
for _, manifest := range registryResT1.Manifests {
|
||||
if string(manifest.Platform.Architecture) == "amd64" {
|
||||
digest = strings.Split(manifest.Digest, ":")[1][:7]
|
||||
}
|
||||
}
|
||||
digest := img.ConfigInfo().Digest.String()
|
||||
|
||||
if digest == "" {
|
||||
if err := json.Unmarshal(body, ®istryResT2); err != nil {
|
||||
return "", err
|
||||
}
|
||||
digest = strings.Split(registryResT2.Config.Digest, ":")[1][:7]
|
||||
return digest, fmt.Errorf("unable to read digest for %s", image)
|
||||
}
|
||||
|
||||
if digest == "" {
|
||||
return "", fmt.Errorf("Unable to retrieve amd64 digest for %s", image)
|
||||
}
|
||||
|
||||
return digest, nil
|
||||
return strings.Split(digest, ":")[1][:7], nil
|
||||
}
|
||||
|
|
|
@ -232,7 +232,11 @@ func Get(recipeName string) (Recipe, error) {
|
|||
|
||||
meta, err := GetRecipeMeta(recipeName)
|
||||
if err != nil {
|
||||
return Recipe{}, err
|
||||
if strings.Contains(err.Error(), "does not exist") {
|
||||
meta = RecipeMeta{}
|
||||
} else {
|
||||
return Recipe{}, err
|
||||
}
|
||||
}
|
||||
|
||||
return Recipe{
|
||||
|
@ -795,8 +799,7 @@ func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
|
|||
|
||||
recipeMeta, ok := catl[recipeName]
|
||||
if !ok {
|
||||
err := fmt.Errorf("recipe %s does not exist?", recipeName)
|
||||
return RecipeMeta{}, err
|
||||
return RecipeMeta{}, fmt.Errorf("recipe %s does not exist?", recipeName)
|
||||
}
|
||||
|
||||
if err := EnsureExists(recipeName); err != nil {
|
||||
|
@ -921,7 +924,7 @@ func ReadReposMetadata() (RepoCatalogue, error) {
|
|||
}
|
||||
|
||||
// GetRecipeVersions retrieves all recipe versions.
|
||||
func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (RecipeVersions, error) {
|
||||
func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
|
||||
versions := RecipeVersions{}
|
||||
|
||||
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
||||
|
@ -965,7 +968,7 @@ func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (R
|
|||
return err
|
||||
}
|
||||
|
||||
cl, err := client.New("default") // only required for docker.io registry calls
|
||||
cl, err := client.New("default") // only required for container registry calls
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -995,18 +998,19 @@ func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (R
|
|||
var exists bool
|
||||
var digest string
|
||||
if digest, exists = queryCache[img]; !exists {
|
||||
logrus.Debugf("looking up image: %s from %s", img, path)
|
||||
logrus.Debugf("cache miss: querying for image: %s, tag: %s", path, tag)
|
||||
|
||||
var err error
|
||||
digest, err = client.GetTagDigest(cl, img, registryUsername, registryPassword)
|
||||
digest, err = client.GetTagDigest(cl, img)
|
||||
if err != nil {
|
||||
logrus.Warn(err)
|
||||
continue
|
||||
digest = "unknown"
|
||||
}
|
||||
logrus.Debugf("queried for image: %s, tag: %s, digest: %s", path, tag, digest)
|
||||
|
||||
queryCache[img] = digest
|
||||
logrus.Debugf("cached image: %s, tag: %s, digest: %s", path, tag, digest)
|
||||
logrus.Debugf("cached insert: %s, tag: %s, digest: %s", path, tag, digest)
|
||||
} else {
|
||||
logrus.Debugf("reading image: %s, tag: %s, digest: %s from cache", path, tag, digest)
|
||||
logrus.Debugf("cache hit: image: %s, tag: %s, digest: %s", path, tag, digest)
|
||||
}
|
||||
|
||||
versionMeta[service.Name] = ServiceMeta{
|
||||
|
|
Loading…
Reference in New Issue