toolshed/abra
4
3
Fork 18
Code Issues 32 Pull Requests 3 Packages Releases 41 Activity

Support more container registries #159

Merged
decentral1se merged 3 commits from more-registry-support into main 2022-02-21 08:47:27 +00:00
Conversation 0 Commits 3 Files Changed 7 +529 -246
7 changed files with 529 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cli/catalogue/catalogue.go
View File

@ -49,6 +49,8 @@ var CatalogueSkipList = map[string]bool{
"pyabra": true,
"radicle-seed-node": true,
"recipes-catalogue-json": true,
"recipes-wishlist": true,
"recipes.coopcloud.tech": true,
"stack-ssh-deploy": true,
"swarm-cronjob": true,
"tagcmp": true,
@ -66,8 +68,6 @@ var catalogueGenerateCommand = cli.Command{
internal.PublishFlag,
internal.DryFlag,
internal.SkipUpdatesFlag,
internal.RegistryUsernameFlag,
internal.RegistryPasswordFlag,
},
Before: internal.SubCommandBefore,
Description: `
@ -132,11 +132,7 @@ keys configured on your account.
continue
}
versions, err := recipe.GetRecipeVersions(
recipeMeta.Name,
internal.RegistryUsername,
internal.RegistryPassword,
)
versions, err := recipe.GetRecipeVersions(recipeMeta.Name)
if err != nil {
logrus.Warn(err)
}

18
cli/internal/cli.go
View File

@ -336,24 +336,6 @@ var SkipUpdatesFlag = &cli.BoolFlag{
Destination: &SkipUpdates,
}
var RegistryUsername string
var RegistryUsernameFlag = &cli.StringFlag{
Name: "username, user",
Value: "",
Usage: "Registry username",
EnvVar: "REGISTRY_USERNAME",
Destination: &RegistryUsername,
}
var RegistryPassword string
var RegistryPasswordFlag = &cli.StringFlag{
Name: "password, pass",
Value: "",
Usage: "Registry password",
EnvVar: "REGISTRY_PASSWORD",
Destination: &RegistryUsername,
}
var AllTags bool
var AllTagsFlag = &cli.BoolFlag{
Name: "all-tags, a",

10
cli/recipe/upgrade.go
View File

@ -113,13 +113,13 @@ You may invoke this command in "wizard" mode and be prompted for input:
logrus.Fatal(err)
}
image := reference.Path(img)
regVersions, err := client.GetRegistryTags(image)
regVersions, err := client.GetRegistryTags(img)
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("retrieved %s from remote registry for %s", regVersions, image)
image := reference.Path(img)
logrus.Debugf("retrieved %s from remote registry for %s", regVersions, image)
image = formatter.StripTagMeta(image)
switch img.(type) {
@ -142,7 +142,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
var compatible []tagcmp.Tag
for _, regVersion := range regVersions {
other, err := tagcmp.Parse(regVersion.Name)
other, err := tagcmp.Parse(regVersion)
if err != nil {
continue // skip tags that cannot be parsed
}
@ -232,7 +232,7 @@ You may invoke this command in "wizard" mode and be prompted for input:
msg = fmt.Sprintf("upgrade to which tag? (service: %s, tag: %s)", service.Name, tag)
compatibleStrings = []string{"skip"}
for _, regVersion := range regVersions {
compatibleStrings = append(compatibleStrings, regVersion.Name)
compatibleStrings = append(compatibleStrings, regVersion)
}
}

14
go.mod
View File

@ -25,9 +25,11 @@ require (
require (
coopcloud.tech/libcapsul v0.0.0-20211022074848-c35e78fe3f3e
github.com/Microsoft/hcsshim v0.8.21 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3 // indirect
github.com/buger/goterm v1.0.4
github.com/containerd/containerd v1.5.5 // indirect
github.com/containerd/containerd v1.5.9 // indirect
github.com/containers/image v3.0.2+incompatible
github.com/containers/storage v1.38.2 // indirect
github.com/docker/docker-credential-helpers v0.6.4 // indirect
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
github.com/fvbommel/sortorder v1.0.2 // indirect
@ -39,11 +41,13 @@ require (
github.com/libdns/gandi v1.0.2
github.com/libdns/libdns v0.2.1
github.com/moby/sys/mount v0.2.0 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/runc v1.0.2 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
github.com/spf13/cobra v1.3.0 // indirect
github.com/theupdateframework/notary v0.7.0 // indirect
github.com/urfave/cli v1.22.5
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xanzy/ssh-agent v0.3.1 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27
)

509
go.sum
View File

File diff suppressed because it is too large Load Diff

188
pkg/client/registry.go
View File

@ -1,193 +1,57 @@
package client
import (
"encoding/base64"
"encoding/json"
"context"
"fmt"
"io/ioutil"
"net/http"
"strings"
"coopcloud.tech/abra/pkg/web"
"github.com/containers/image/docker"
"github.com/containers/image/types"
"github.com/docker/distribution/reference"
"github.com/docker/docker/client"
"github.com/hashicorp/go-retryablehttp"
"github.com/sirupsen/logrus"
)
type RawTag struct {
Layer string
Name string
}
// GetRegistryTags retrieves all tags of an image from a container registry.
func GetRegistryTags(img reference.Named) ([]string, error) {
var tags []string
type RawTags []RawTag
ref, err := docker.ParseReference(fmt.Sprintf("//%s", img))
if err != nil {
return tags, fmt.Errorf("failed to parse image %s, saw: %s", img, err.Error())
}
var registryURL = "https://registry.hub.docker.com/v1/repositories/%s/tags"
func GetRegistryTags(image string) (RawTags, error) {
var tags RawTags
tagsUrl := fmt.Sprintf(registryURL, image)
if err := web.ReadJSON(tagsUrl, &tags); err != nil {
ctx := context.Background()
tags, err = docker.GetRepositoryTags(ctx, &types.SystemContext{}, ref)
if err != nil {
return tags, err
}
return tags, nil
}
func basicAuth(username, password string) string {
auth := username + ":" + password
return base64.StdEncoding.EncodeToString([]byte(auth))
}
// GetTagDigest retrieves an image digest from a container registry.
func GetTagDigest(cl *client.Client, image reference.Named) (string, error) {
target := fmt.Sprintf("//%s", reference.Path(image))
// getRegv2Token retrieves a registry v2 authentication token.
func getRegv2Token(cl *client.Client, image reference.Named, registryUsername, registryPassword string) (string, error) {
img := reference.Path(image)
tokenURL := "https://auth.docker.io/token"
values := fmt.Sprintf("service=registry.docker.io&scope=repository:%s:pull", img)
fullURL := fmt.Sprintf("%s?%s", tokenURL, values)
req, err := retryablehttp.NewRequest("GET", fullURL, nil)
ref, err := docker.ParseReference(target)
if err != nil {
return "", err
return "", fmt.Errorf("failed to parse image %s, saw: %s", image, err.Error())
}
if registryUsername != "" && registryPassword != "" {
logrus.Debugf("using registry log in credentials for token request")
auth := basicAuth(registryUsername, registryPassword)
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", auth))
}
client := web.NewHTTPRetryClient()
res, err := client.Do(req)
ctx := context.Background()
img, err := ref.NewImage(ctx, nil)
if err != nil {
return "", err
logrus.Debugf("failed to query remote registry for %s, saw: %s", image, err.Error())
return "", fmt.Errorf("unable to read digest for %s", image)
}
defer res.Body.Close()
defer img.Close()
if res.StatusCode != http.StatusOK {
_, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", nil
}
tokenRes := struct {
AccessToken string `json:"access_token"`
Expiry int `json:"expires_in"`
Issued string `json:"issued_at"`
Token string `json:"token"`
}{}
if err := json.Unmarshal(body, &tokenRes); err != nil {
return "", err
}
return tokenRes.Token, nil
}
// GetTagDigest retrieves an image digest from a v2 registry
func GetTagDigest(cl *client.Client, image reference.Named, registryUsername, registryPassword string) (string, error) {
img := reference.Path(image)
tag := image.(reference.NamedTagged).Tag()
manifestURL := fmt.Sprintf("https://index.docker.io/v2/%s/manifests/%s", img, tag)
req, err := retryablehttp.NewRequest("GET", manifestURL, nil)
if err != nil {
return "", err
}
token, err := getRegv2Token(cl, image, registryUsername, registryPassword)
if err != nil {
return "", err
}
if token == "" {
return "", fmt.Errorf("unable to retrieve registry token?")
}
req.Header = http.Header{
"Accept": []string{
"application/vnd.docker.distribution.manifest.v2+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
},
"Authorization": []string{fmt.Sprintf("Bearer %s", token)},
}
client := web.NewHTTPRetryClient()
res, err := client.Do(req)
if err != nil {
return "", err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
_, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
registryResT1 := struct {
SchemaVersion int
MediaType string
Manifests []struct {
MediaType string
Size int
Digest string
Platform struct {
Architecture string
Os string
}
}
}{}
registryResT2 := struct {
SchemaVersion int
MediaType string
Config struct {
MediaType string
Size int
Digest string
}
Layers []struct {
MediaType string
Size int
Digest string
}
}{}
if err := json.Unmarshal(body, &registryResT1); err != nil {
return "", err
}
var digest string
for _, manifest := range registryResT1.Manifests {
if string(manifest.Platform.Architecture) == "amd64" {
digest = strings.Split(manifest.Digest, ":")[1][:7]
}
}
digest := img.ConfigInfo().Digest.String()
if digest == "" {
if err := json.Unmarshal(body, &registryResT2); err != nil {
return "", err
}
digest = strings.Split(registryResT2.Config.Digest, ":")[1][:7]
return digest, fmt.Errorf("unable to read digest for %s", image)
}
if digest == "" {
return "", fmt.Errorf("Unable to retrieve amd64 digest for %s", image)
}
return digest, nil
return strings.Split(digest, ":")[1][:7], nil
}

26
pkg/recipe/recipe.go
View File

@ -232,7 +232,11 @@ func Get(recipeName string) (Recipe, error) {
meta, err := GetRecipeMeta(recipeName)
if err != nil {
return Recipe{}, err
if strings.Contains(err.Error(), "does not exist") {
meta = RecipeMeta{}
} else {
return Recipe{}, err
}
}
return Recipe{
@ -795,8 +799,7 @@ func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
recipeMeta, ok := catl[recipeName]
if !ok {
err := fmt.Errorf("recipe %s does not exist?", recipeName)
return RecipeMeta{}, err
return RecipeMeta{}, fmt.Errorf("recipe %s does not exist?", recipeName)
}
if err := EnsureExists(recipeName); err != nil {
@ -921,7 +924,7 @@ func ReadReposMetadata() (RepoCatalogue, error) {
}
// GetRecipeVersions retrieves all recipe versions.
func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (RecipeVersions, error) {
func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
versions := RecipeVersions{}
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
@ -965,7 +968,7 @@ func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (R
return err
}
cl, err := client.New("default") // only required for docker.io registry calls
cl, err := client.New("default") // only required for container registry calls
if err != nil {
return err
}
@ -995,18 +998,19 @@ func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (R
var exists bool
var digest string
if digest, exists = queryCache[img]; !exists {
logrus.Debugf("looking up image: %s from %s", img, path)
logrus.Debugf("cache miss: querying for image: %s, tag: %s", path, tag)
var err error
digest, err = client.GetTagDigest(cl, img, registryUsername, registryPassword)
digest, err = client.GetTagDigest(cl, img)
if err != nil {
logrus.Warn(err)
continue
digest = "unknown"
}
logrus.Debugf("queried for image: %s, tag: %s, digest: %s", path, tag, digest)
queryCache[img] = digest
logrus.Debugf("cached image: %s, tag: %s, digest: %s", path, tag, digest)
logrus.Debugf("cached insert: %s, tag: %s, digest: %s", path, tag, digest)
} else {
logrus.Debugf("reading image: %s, tag: %s, digest: %s from cache", path, tag, digest)
logrus.Debugf("cache hit: image: %s, tag: %s, digest: %s", path, tag, digest)
}
versionMeta[service.Name] = ServiceMeta{