wip: login to docker to avoid rate limiting

Part of coop-cloud/organising#226.

.envrc.sample

@@ -4,3 +4,5 @@ go env -w GOPRIVATE=coopcloud.tech
 # export HCLOUD_TOKEN=$(pass show logins/hetzner/cicd/api_key)
 # export CAPSUL_TOKEN=...
 # export GITEA_TOKEN=...
+# export DOCKER_USERNAME=...
+# export DOCKER_PASSWORD=...

cli/autocomplete.go

@@ -56,7 +56,7 @@ Example:
 
 Supported shells are as follows:
 
-		fish
+		fizsh
 		zsh
     bash
 `,
@@ -69,16 +69,16 @@ Supported shells are as follows:
 		}
 
 		supportedShells := map[string]bool{
-			"bash": true,
-			"zsh":  true,
-			"fish": true,
+			"bash":  true,
+			"zsh":   true,
+			"fizsh": true,
 		}
 
 		if _, ok := supportedShells[shellType]; !ok {
 			logrus.Fatalf("%s is not a supported shell right now, sorry", shellType)
 		}
 
-		if shellType == "fish" {
+		if shellType == "fizsh" {
 			shellType = "zsh" // handled the same on the autocompletion side
 		}
 

cli/catalogue/generate.go

@@ -40,12 +40,14 @@ var CatalogueSkipList = map[string]bool{
 	"docker-cp-deploy":      true,
 	"docker-dind-bats-kcov": true,
 	"docs.coopcloud.tech":   true,
+	"drone-abra":            true,
 	"example":               true,
 	"gardening":             true,
 	"go-abra":               true,
 	"organising":            true,
 	"pyabra":                true,
 	"radicle-seed-node":     true,
+	"recipes":               true,
 	"stack-ssh-deploy":      true,
 	"swarm-cronjob":         true,
 	"tagcmp":                true,
@@ -176,7 +178,7 @@ A new catalogue copy can be published to the recipes repository by passing the
 
 			features, category, err := catalogue.GetRecipeFeaturesAndCategory(recipeMeta.Name)
 			if err != nil {
-				logrus.Fatal(err)
+				logrus.Warn(err)
 			}
 
 			catl[recipeMeta.Name] = catalogue.RecipeMeta{

pkg/catalogue/catalogue.go

@@ -411,7 +411,7 @@ func GetImageMetadata(imageRowString string) (image, error) {
 	if err != nil {
 		logrus.Fatal(err)
 	}
-	img.Image = imageName
+	img.Image = strings.ReplaceAll(imageName, "`", "")
 
 	imageURL, err := GetStringInBetween(imgString, "(", ")")
 	if err != nil {
@@ -441,7 +441,7 @@ func GetRecipeFeaturesAndCategory(recipeName string) (features, string, error) {
 		"<!-- metadata -->", "<!-- endmetadata -->",
 	)
 	if err != nil {
-		logrus.Fatal(err)
+		return feat, category, err
 	}
 
 	readmeLines := strings.Split( // Array item from lines
@@ -525,7 +525,7 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 
 		checkOutOpts := &git.CheckoutOptions{
 			Create: false,
-			Force:  true,
+			Keep:   true,
 			Branch: plumbing.ReferenceName(ref.Name()),
 		}
 		if err := worktree.Checkout(checkOutOpts); err != nil {
@@ -553,9 +553,22 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 				path = strings.Split(path, "/")[1]
 			}
 
+			var tag string
+			switch img.(type) {
+			case reference.NamedTagged:
+				tag = img.(reference.NamedTagged).Tag()
+			case reference.Named:
+				logrus.Warnf("%s service is missing image tag?", path)
+				continue
+			}
+
+			logrus.Debugf("looking up image: '%s' from '%s'", img, path)
+
 			digest, err := client.GetTagDigest(img)
 			if err != nil {
-				return err
+				// return err
+				logrus.Warn(err)
+				continue
 			}
 
 			versionMeta[service.Name] = ServiceMeta{
@@ -586,7 +599,7 @@ func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
 	refName := fmt.Sprintf("refs/heads/%s", branch)
 	checkOutOpts := &git.CheckoutOptions{
 		Create: false,
-		Force:  true,
+		Keep:   true,
 		Branch: plumbing.ReferenceName(refName),
 	}
 	if err := worktree.Checkout(checkOutOpts); err != nil {

pkg/client/registry.go

@@ -1,15 +1,20 @@
 package client
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"os"
 
 	"coopcloud.tech/abra/pkg/web"
 	"github.com/docker/distribution/reference"
 	"github.com/hashicorp/go-retryablehttp"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/docker/docker/api/types"
+	"github.com/sirupsen/logrus"
 )
 
 type RawTag struct {
@@ -84,6 +89,20 @@ func GetTagDigest(image reference.Named) (string, error) {
 		return "", err
 	}
 
+	dClient := dockerClient.Client{}
+
+	username, username_ok := os.LookupEnv("DOCKER_USERNAME")
+	password, password_ok := os.LookupEnv("DOCKER_PASSWORD")
+	if username_ok && password_ok {
+		logrus.Debugf("docker login: %s, %s", username, password)
+		dClient.RegistryLogin(context.Background(), types.AuthConfig{
+			Username: username,
+			Password: password,
+			ServerAddress: "docker.io",
+		})
+		return "", nil
+	}
+
 	token, err := getRegv2Token(image)
 	if err != nil {
 		return "", err

pkg/git/clone.go

@@ -76,7 +76,7 @@ func EnsureUpToDate(dir string) error {
 	refName := fmt.Sprintf("refs/heads/%s", branch)
 	checkOutOpts := &git.CheckoutOptions{
 		Create: false,
-		Force:  true,
+		Keep:   true,
 		Branch: plumbing.ReferenceName(refName),
 	}
 	if err := worktree.Checkout(checkOutOpts); err != nil {

pkg/git/read.go

@@ -1,11 +1,17 @@
 package git
 
 import (
+	"io/ioutil"
+	"os/user"
 	"path"
+	"path/filepath"
+	"strings"
 
 	"coopcloud.tech/abra/pkg/config"
 	"github.com/go-git/go-git/v5"
+	gitConfigPkg "github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/format/gitignore"
 	"github.com/sirupsen/logrus"
 )
 
@@ -40,6 +46,12 @@ func IsClean(recipeName string) (bool, error) {
 		return false, err
 	}
 
+	patterns, err := GetExcludesFiles()
+	if err != nil {
+		return false, err
+	}
+	worktree.Excludes = append(patterns, worktree.Excludes...)
+
 	status, err := worktree.Status()
 	if err != nil {
 		return false, err
@@ -53,3 +65,95 @@ func IsClean(recipeName string) (bool, error) {
 
 	return status.IsClean(), nil
 }
+
+// GetExcludesFiles reads the exlude files from a global git ignore
+func GetExcludesFiles() ([]gitignore.Pattern, error) {
+	var err error
+	var patterns []gitignore.Pattern
+
+	cfg, err := parseGitConfig()
+	if err != nil {
+		return patterns, err
+	}
+
+	excludesfile := getExcludesFile(cfg)
+	patterns, err = parseExcludesFile(excludesfile)
+	if err != nil {
+		return patterns, err
+	}
+
+	return patterns, nil
+}
+
+func parseGitConfig() (*gitConfigPkg.Config, error) {
+	cfg := gitConfigPkg.NewConfig()
+
+	usr, err := user.Current()
+	if err != nil {
+		return nil, err
+	}
+
+	b, err := ioutil.ReadFile(usr.HomeDir + "/.gitconfig")
+	if err != nil {
+		return nil, err
+	}
+
+	if err := cfg.Unmarshal(b); err != nil {
+		return nil, err
+	}
+
+	return cfg, err
+}
+
+func getExcludesFile(cfg *gitConfigPkg.Config) string {
+	for _, sec := range cfg.Raw.Sections {
+		if sec.Name == "core" {
+			for _, opt := range sec.Options {
+				if opt.Key == "excludesfile" {
+					return opt.Value
+				}
+			}
+		}
+	}
+	return ""
+}
+
+func parseExcludesFile(excludesfile string) ([]gitignore.Pattern, error) {
+	excludesfile, err := expandTilde(excludesfile)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := ioutil.ReadFile(excludesfile)
+	if err != nil {
+		return nil, err
+	}
+
+	var ps []gitignore.Pattern
+	for _, s := range strings.Split(string(data), "\n") {
+		if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 {
+			ps = append(ps, gitignore.ParsePattern(s, nil))
+		}
+	}
+
+	return ps, nil
+}
+
+func expandTilde(path string) (string, error) {
+	if !strings.HasPrefix(path, "~") {
+		return path, nil
+	}
+	var paths []string
+	u, err := user.Current()
+	if err != nil {
+		return "", err
+	}
+	for _, p := range strings.Split(path, string(filepath.Separator)) {
+		if p == "~" {
+			paths = append(paths, u.HomeDir)
+		} else {
+			paths = append(paths, p)
+		}
+	}
+	return "/" + filepath.Join(paths...), nil
+}

pkg/recipe/recipe.go

@@ -168,7 +168,7 @@ func EnsureVersion(recipeName, version string) error {
 	opts := &git.CheckoutOptions{
 		Branch: tagRef,
 		Create: false,
-		Force:  true,
+		Keep:   true,
 	}
 	if err := worktree.Checkout(opts); err != nil {
 		return err
@@ -220,7 +220,7 @@ func EnsureLatest(recipeName string) error {
 	refName := fmt.Sprintf("refs/heads/%s", branch)
 	checkOutOpts := &git.CheckoutOptions{
 		Create: false,
-		Force:  true,
+		Keep:   true,
 		Branch: plumbing.ReferenceName(refName),
 	}