WIP: implement async recipe cloning

See coop-cloud/organising#159.

cli/catalogue/generate.go

@@ -11,26 +11,86 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
+// CatalogueSkipList is all the repos that are not recipes.
+var CatalogueSkipList = map[string]bool{
+	"abra":                  true,
+	"abra-apps":             true,
+	"abra-aur":              true,
+	"abra-capsul":           true,
+	"abra-gandi":            true,
+	"abra-hetzner":          true,
+	"apps":                  true,
+	"aur-abra-git":          true,
+	"auto-apps-json":        true,
+	"auto-mirror":           true,
+	"backup-bot":            true,
+	"coopcloud.tech":        true,
+	"coturn":                true,
+	"docker-cp-deploy":      true,
+	"docker-dind-bats-kcov": true,
+	"docs.coopcloud.tech":   true,
+	"example":               true,
+	"gardening":             true,
+	"go-abra":               true,
+	"organising":            true,
+	"pyabra":                true,
+	"radicle-seed-node":     true,
+	"stack-ssh-deploy":      true,
+	"swarm-cronjob":         true,
+	"tagcmp":                true,
+	"tyop":                  true,
+}
+
 var catalogueGenerateCommand = &cli.Command{
 	Name:         "generate",
 	Aliases:      []string{"g"},
 	Usage:        "Generate a new copy of the catalogue",
+	ArgsUsage:    "[<recipe>]",
 	BashComplete: func(c *cli.Context) {},
 	Action: func(c *cli.Context) error {
-		catl, err := catalogue.ReadRecipeCatalogue()
+		recipes, err := catalogue.ReadRecipeCatalogue()
 		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		for recipeName, recipeMeta := range catl {
-			recipeDir := path.Join(config.ABRA_DIR, "apps", strings.ToLower(recipeName))
-			if err := git.Clone(recipeDir, recipeMeta.Repository); err != nil {
-				logrus.Fatal(err)
+		recipeName := c.Args().First()
+		if recipeName != "" {
+			recipeMeta, exists := recipes[recipeName]
+			if !exists {
+				logrus.Fatalf("'%s' does not exist?", recipeName)
 			}
+			recipes = map[string]catalogue.RecipeMeta{recipeName: recipeMeta}
+		}
 
-			if err := git.EnsureUpToDate(recipeDir); err != nil {
-				logrus.Fatal(err)
-			}
+		logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(recipes))
+
+		ch := make(chan string, len(recipes))
+		for recipeName, recipeMeta := range recipes {
+			go func(rn string, rm catalogue.RecipeMeta) {
+				if _, exists := CatalogueSkipList[rn]; exists {
+					ch <- rn
+					return
+				}
+				if rm.Repository == "" {
+					logrus.Warnf("'%s' has no git clone URL, skipping", rn)
+					ch <- rn
+					return
+				}
+
+				recipeDir := path.Join(config.ABRA_DIR, "apps", strings.ToLower(rn))
+				if err := git.Clone(recipeDir, rm.Repository); err != nil {
+					logrus.Fatal(err)
+				}
+
+				if err := git.EnsureUpToDate(recipeDir); err != nil {
+					logrus.Fatal(err)
+				}
+				ch <- rn
+			}(recipeName, recipeMeta)
+		}
+
+		for range recipes {
+			<-ch // wait for everything
 		}
 
 		// for reach app, build the recipemeta from parsing