WIP: implement async recipe cloning

See coop-cloud/organising#159.

cli/catalogue/generate.go

@@ -11,26 +11,86 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
+// CatalogueSkipList is all the repos that are not recipes.
+var CatalogueSkipList = map[string]bool{
+	"abra":                  true,
+	"abra-apps":             true,
+	"abra-aur":              true,
+	"abra-capsul":           true,
+	"abra-gandi":            true,
+	"abra-hetzner":          true,
+	"apps":                  true,
+	"aur-abra-git":          true,
+	"auto-apps-json":        true,
+	"auto-mirror":           true,
+	"backup-bot":            true,
+	"coopcloud.tech":        true,
+	"coturn":                true,
+	"docker-cp-deploy":      true,
+	"docker-dind-bats-kcov": true,
+	"docs.coopcloud.tech":   true,
+	"example":               true,
+	"gardening":             true,
+	"go-abra":               true,
+	"organising":            true,
+	"pyabra":                true,
+	"radicle-seed-node":     true,
+	"stack-ssh-deploy":      true,
+	"swarm-cronjob":         true,
+	"tagcmp":                true,
+	"tyop":                  true,
+}
+
 var catalogueGenerateCommand = &cli.Command{
 	Name:         "generate",
 	Aliases:      []string{"g"},
 	Usage:        "Generate a new copy of the catalogue",
+	ArgsUsage:    "[<recipe>]",
 	BashComplete: func(c *cli.Context) {},
 	Action: func(c *cli.Context) error {
-		catl, err := catalogue.ReadRecipeCatalogue()
+		recipes, err := catalogue.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		for recipeName, recipeMeta := range catl {
+		recipeName := c.Args().First()
-			recipeDir := path.Join(config.ABRA_DIR, "apps", strings.ToLower(recipeName))
+		if recipeName != "" {
-			if err := git.Clone(recipeDir, recipeMeta.Repository); err != nil {
+			recipeMeta, exists := recipes[recipeName]
-				logrus.Fatal(err)
+			if !exists {
+				logrus.Fatalf("'%s' does not exist?", recipeName)
 			}
+			recipes = map[string]catalogue.RecipeMeta{recipeName: recipeMeta}
+		}
 
-			if err := git.EnsureUpToDate(recipeDir); err != nil {
+		logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(recipes))
-				logrus.Fatal(err)
+
-			}
+		ch := make(chan string, len(recipes))
+		for recipeName, recipeMeta := range recipes {
+			go func(rn string, rm catalogue.RecipeMeta) {
+				if _, exists := CatalogueSkipList[rn]; exists {
+					ch <- rn
+					return
+				}
+				if rm.Repository == "" {
+					logrus.Warnf("'%s' has no git clone URL, skipping", rn)
+					ch <- rn
+					return
+				}
+
+				recipeDir := path.Join(config.ABRA_DIR, "apps", strings.ToLower(rn))
+				if err := git.Clone(recipeDir, rm.Repository); err != nil {
+					logrus.Fatal(err)
+				}
+
+				if err := git.EnsureUpToDate(recipeDir); err != nil {
+					logrus.Fatal(err)
+				}
+				ch <- rn
+			}(recipeName, recipeMeta)
+		}
+
+		for range recipes {
+			<-ch // wait for everything
 		}
 
 		// for reach app, build the recipemeta from parsing

pkg/git/clone.go

@@ -17,7 +17,7 @@ func Clone(dir, url string) error {
 		logrus.Debugf("'%s' does not exist, attempting to git clone from '%s'", dir, url)
 		_, err := git.PlainClone(dir, false, &git.CloneOptions{URL: url, Tags: git.AllTags})
 		if err != nil {
-			logrus.Debugf("cloning from default branch failed, attempting from main branch")
+			logrus.Debugf("cloning '%s' default branch failed, attempting from main branch", url)
 			_, err := git.PlainClone(dir, false, &git.CloneOptions{
 				URL:           url,
 				Tags:          git.AllTags,
@@ -45,12 +45,13 @@ func EnsureUpToDate(dir string) error {
 	branch := "master"
 	if _, err := repo.Branch("master"); err != nil {
 		if _, err := repo.Branch("main"); err != nil {
+			logrus.Debugf("failed to select branch in '%s'", dir)
 			return err
 		}
 		branch = "main"
 	}
 
-	logrus.Debugf("choosing '%s' as main git branch for in '%s'", branch, dir)
+	logrus.Debugf("choosing '%s' as main git branch in '%s'", branch, dir)
 
 	worktree, err := repo.Worktree()
 	if err != nil {
@@ -65,10 +66,11 @@ func EnsureUpToDate(dir string) error {
 		Branch: plumbing.ReferenceName(refName),
 	}
 	if err := worktree.Checkout(checkOutOpts); err != nil {
+		logrus.Debugf("failed to check out '%s' in '%s'", refName, dir)
 		return err
 	}
 
-	logrus.Debugf("successfully checked out '%s'", branch)
+	logrus.Debugf("successfully checked out '%s' in '%s'", branch, dir)
 
 	remote, err := repo.Remote("origin")
 	if err != nil {