fix: not flaky catalogue generate

See toolshed/abra#464

cli/app/new.go

@@ -93,7 +93,7 @@ var AppNewCommand = &cobra.Command{
 		var recipeVersions recipePkg.RecipeVersions
 		if recipeVersion == "" {
 			var err error
-			recipeVersions, err = recipe.GetRecipeVersions()
+			recipeVersions, _, err = recipe.GetRecipeVersions()
 			if err != nil {
 				log.Fatal(err)
 			}

cli/catalogue/catalogue.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"path"
+	"slices"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -61,52 +62,48 @@ keys configured on your account.`,
 			}
 		}
 
-		repos, err := recipe.ReadReposMetadata()
+		repos, err := recipe.ReadReposMetadata(internal.Debug)
 		if err != nil {
 			log.Fatal(err)
 		}
 
-		var barLength int
+		barLength := len(repos)
-		var logMsg string
 		if recipeName != "" {
 			barLength = 1
-			logMsg = fmt.Sprintf("ensuring %v recipe is cloned & up-to-date", barLength)
-		} else {
-			barLength = len(repos)
-			logMsg = fmt.Sprintf("ensuring %v recipes are cloned & up-to-date, this could take some time...", barLength)
 		}
 
 		if !skipUpdates {
-			log.Warn(logMsg)
+			if err := recipe.UpdateRepositories(repos, recipeName, internal.Debug); err != nil {
-			if err := recipe.UpdateRepositories(repos, recipeName); err != nil {
 				log.Fatal(err)
 			}
 		}
 
+		var warnings []string
 		catl := make(recipe.RecipeCatalogue)
-		catlBar := formatter.CreateProgressbar(barLength, "generating catalogue metadata...")
+		catlBar := formatter.CreateProgressbar(barLength, "collecting catalogue metadata")
 		for _, recipeMeta := range repos {
 			if recipeName != "" && recipeName != recipeMeta.Name {
-				catlBar.Add(1)
+				if !internal.Debug {
-				continue
+					catlBar.Add(1)
-			}
+				}
-
-			// NOTE(d1): the "example" recipe is a temporary special case
-			// https://git.coopcloud.tech/toolshed/organising/issues/666
-			if recipeMeta.Name == "example" {
-				catlBar.Add(1)
 				continue
 			}
 
 			r := recipe.Get(recipeMeta.Name)
-			versions, err := r.GetRecipeVersions()
+			versions, warnMsgs, err := r.GetRecipeVersions()
 			if err != nil {
-				log.Warn(err)
+				warnings = append(warnings, err.Error())
+			}
+			if len(warnMsgs) > 0 {
+				warnings = append(warnings, warnMsgs...)
 			}
 
-			features, category, err := recipe.GetRecipeFeaturesAndCategory(r)
+			features, category, warnMsgs, err := recipe.GetRecipeFeaturesAndCategory(r)
 			if err != nil {
-				log.Warn(err)
+				warnings = append(warnings, err.Error())
+			}
+			if len(warnMsgs) > 0 {
+				warnings = append(warnings, warnMsgs...)
 			}
 
 			catl[recipeMeta.Name] = recipe.RecipeMeta{
@@ -122,7 +119,24 @@ keys configured on your account.`,
 				Features:      features,
 			}
 
-			catlBar.Add(1)
+			if !internal.Debug {
+				catlBar.Add(1)
+			}
+		}
+
+		if err := catlBar.Close(); err != nil {
+			log.Fatal(err)
+		}
+
+		var uniqueWarnings []string
+		for _, w := range warnings {
+			if !slices.Contains(uniqueWarnings, w) {
+				uniqueWarnings = append(uniqueWarnings, w)
+			}
+		}
+
+		for _, warnMsg := range uniqueWarnings {
+			log.Warn(warnMsg)
 		}
 
 		recipesJSON, err := json.MarshalIndent(catl, "", "    ")
@@ -152,7 +166,7 @@ keys configured on your account.`,
 			}
 		}
 
-		log.Infof("generated new recipe catalogue in %s", config.RECIPES_JSON)
+		log.Infof("generated recipe catalogue: %s", config.RECIPES_JSON)
 
 		cataloguePath := path.Join(config.ABRA_DIR, "catalogue")
 		if publishChanges {

cli/recipe/version.go

@@ -37,10 +37,13 @@ var RecipeVersionCommand = &cobra.Command{
 		if !ok {
 			warnMessages = append(warnMessages, "retrieved versions from local recipe repository")
 
-			recipeVersions, err := recipe.GetRecipeVersions()
+			recipeVersions, warnMsg, err := recipe.GetRecipeVersions()
 			if err != nil {
 				warnMessages = append(warnMessages, err.Error())
 			}
+			if len(warnMsg) > 0 {
+				warnMessages = append(warnMessages, warnMsg...)
+			}
 
 			recipeMeta = recipePkg.RecipeMeta{Versions: recipeVersions}
 		}

pkg/catalogue/catalogue.go

@@ -16,13 +16,12 @@ import (
 func EnsureCatalogue() error {
 	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
 	if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
-		log.Warnf("local recipe catalogue is missing, retrieving now")
+		log.Debugf("catalogue is missing, retrieving now")
+
 		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
 		if err := gitPkg.Clone(catalogueDir, url); err != nil {
 			return err
 		}
-
-		log.Debugf("cloned catalogue repository to %s", catalogueDir)
 	}
 
 	return nil

pkg/formatter/formatter.go

@@ -217,7 +217,6 @@ func CreateProgressbar(length int, title string) *progressbar.ProgressBar {
 		progressbar.OptionClearOnFinish(),
 		progressbar.OptionSetPredictTime(false),
 		progressbar.OptionShowCount(),
-		progressbar.OptionFullWidth(),
 		progressbar.OptionSetDescription(title),
 	)
 }

pkg/git/clone.go

@@ -1,9 +1,7 @@
 package git
 
 import (
-	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"coopcloud.tech/abra/pkg/log"
@@ -11,10 +9,23 @@ import (
 	"github.com/go-git/go-git/v5/plumbing"
 )
 
+// gitCloneIgnoreErr checks whether we can ignore a git clone error or not.
+func gitCloneIgnoreErr(err error) bool {
+	if strings.Contains(err.Error(), "authentication required") {
+		return true
+	}
+
+	if strings.Contains(err.Error(), "remote repository is empty") {
+		return true
+	}
+
+	return false
+}
+
 // Clone runs a git clone which accounts for different default branches.
 func Clone(dir, url string) error {
 	if _, err := os.Stat(dir); os.IsNotExist(err) {
-		log.Debugf("%s does not exist, attempting git clone of %s", dir, url)
+		log.Debugf("git clone: %s", dir, url)
 
 		_, err := git.PlainClone(dir, false, &git.CloneOptions{
 			URL:           url,
@@ -23,6 +34,11 @@ func Clone(dir, url string) error {
 			SingleBranch:  true,
 		})
 
+		if err != nil && gitCloneIgnoreErr(err) {
+			log.Debugf("git clone: %s cloned successfully", dir)
+			return nil
+		}
+
 		if err != nil {
 			log.Debug("git clone: main branch failed, attempting master branch")
 
@@ -32,12 +48,13 @@ func Clone(dir, url string) error {
 				ReferenceName: plumbing.ReferenceName("refs/heads/master"),
 				SingleBranch:  true,
 			})
-			if err != nil {
-				if strings.Contains(err.Error(), "authentication required") {
-					name := filepath.Base(dir)
-					return fmt.Errorf("unable to clone %s, does %s exist?", name, url)
-				}
 
+			if err != nil && gitCloneIgnoreErr(err) {
+				log.Debugf("git clone: %s cloned successfully", dir)
+				return nil
+			}
+
+			if err != nil {
 				return err
 			}
 		}

pkg/lint/recipe.go

@@ -409,7 +409,7 @@ func LintHasPublishedVersion(recipe recipe.Recipe) (bool, error) {
 }
 
 func LintMetadataFilledIn(r recipe.Recipe) (bool, error) {
-	features, category, err := recipe.GetRecipeFeaturesAndCategory(r)
+	features, category, _, err := recipe.GetRecipeFeaturesAndCategory(r)
 	if err != nil {
 		return false, err
 	}

pkg/recipe/git.go

@@ -3,6 +3,7 @@ package recipe
 import (
 	"fmt"
 	"os"
+	"slices"
 	"strings"
 
 	"coopcloud.tech/abra/pkg/formatter"
@@ -350,23 +351,26 @@ func (r Recipe) Tags() ([]string, error) {
 }
 
 // GetRecipeVersions retrieves all recipe versions.
-func (r Recipe) GetRecipeVersions() (RecipeVersions, error) {
+func (r Recipe) GetRecipeVersions() (RecipeVersions, []string, error) {
+	var warnMsg []string
+
 	versions := RecipeVersions{}
-	log.Debugf("attempting to open git repository in %s", r.Dir)
+
+	log.Debugf("git: opening repository in %s", r.Dir)
 
 	repo, err := git.PlainOpen(r.Dir)
 	if err != nil {
-		return versions, err
+		return versions, warnMsg, nil
 	}
 
 	worktree, err := repo.Worktree()
 	if err != nil {
-		return versions, err
+		return versions, warnMsg, nil
 	}
 
 	gitTags, err := repo.Tags()
 	if err != nil {
-		return versions, err
+		return versions, warnMsg, nil
 	}
 
 	if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
@@ -384,7 +388,7 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, error) {
 			return err
 		}
 
-		log.Debugf("successfully checked out %s in %s", ref.Name(), r.Dir)
+		log.Debugf("git checkout: %s in %s", ref.Name(), r.Dir)
 
 		config, err := r.GetComposeConfig(nil)
 		if err != nil {
@@ -408,7 +412,7 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, error) {
 			case reference.NamedTagged:
 				tag = img.(reference.NamedTagged).Tag()
 			case reference.Named:
-				log.Warnf("%s service is missing image tag?", path)
+				warnMsg = append(warnMsg, fmt.Sprintf("%s service is missing image tag?", path))
 				continue
 			}
 
@@ -422,19 +426,26 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, error) {
 
 		return nil
 	}); err != nil {
-		return versions, err
+		return versions, warnMsg, nil
 	}
 
 	_, err = gitPkg.CheckoutDefaultBranch(repo, r.Dir)
 	if err != nil {
-		return versions, err
+		return versions, warnMsg, nil
 	}
 
 	sortRecipeVersions(versions)
 
 	log.Debugf("collected %s for %s", versions, r.Dir)
 
-	return versions, nil
+	var uniqueWarnings []string
+	for _, w := range warnMsg {
+		if !slices.Contains(uniqueWarnings, w) {
+			uniqueWarnings = append(uniqueWarnings, w)
+		}
+	}
+
+	return versions, uniqueWarnings, nil
 }
 
 // Head retrieves latest HEAD metadata.

pkg/recipe/recipe.go

@@ -233,16 +233,18 @@ func GetRecipesLocal() ([]string, error) {
 	return recipes, nil
 }
 
-func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, error) {
+func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, []string, error) {
-	feat := Features{}
+	var (
+		category string
+		warnMsgs []string
+		feat     = Features{}
+	)
 
-	var category string
+	log.Debugf("%s: attempt recipe metadata parse", r.ReadmePath)
-
-	log.Debugf("attempting to open %s for recipe metadata parsing", r.ReadmePath)
 
 	readmeFS, err := ioutil.ReadFile(r.ReadmePath)
 	if err != nil {
-		return feat, category, err
+		return feat, category, warnMsgs, err
 	}
 
 	readmeMetadata, err := GetStringInBetween( // Find text between delimiters
@@ -251,7 +253,7 @@ func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, error) {
 		"<!-- metadata -->", "<!-- endmetadata -->",
 	)
 	if err != nil {
-		return feat, category, err
+		return feat, category, warnMsgs, err
 	}
 
 	readmeLines := strings.Split( // Array item from lines
@@ -295,20 +297,25 @@ func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, error) {
 			)
 		}
 		if strings.Contains(val, "**Image**") {
-			imageMetadata, err := GetImageMetadata(strings.TrimSpace(
+			imageMetadata, warnings, err := GetImageMetadata(strings.TrimSpace(
 				strings.TrimPrefix(val, "* **Image**:"),
 			), r.Name)
 			if err != nil {
 				continue
 			}
+			if len(warnings) > 0 {
+				warnMsgs = append(warnMsgs, warnings...)
+			}
 			feat.Image = imageMetadata
 		}
 	}
 
-	return feat, category, nil
+	return feat, category, warnMsgs, nil
 }
 
-func GetImageMetadata(imageRowString, recipeName string) (Image, error) {
+func GetImageMetadata(imageRowString, recipeName string) (Image, []string, error) {
+	var warnMsgs []string
+
 	img := Image{}
 
 	imgFields := strings.Split(imageRowString, ",")
@@ -319,11 +326,18 @@ func GetImageMetadata(imageRowString, recipeName string) (Image, error) {
 
 	if len(imgFields) < 3 {
 		if imageRowString != "" {
-			log.Warnf("%s image meta has incorrect format: %s", recipeName, imageRowString)
+			warnMsgs = append(
+				warnMsgs,
+				fmt.Sprintf("%s: image meta has incorrect format: %s", recipeName, imageRowString),
+			)
 		} else {
-			log.Warnf("%s image meta is empty?", recipeName)
+			warnMsgs = append(
+				warnMsgs,
+				fmt.Sprintf("%s: image meta is empty?", recipeName),
+			)
 		}
-		return img, nil
+
+		return img, warnMsgs, nil
 	}
 
 	img.Rating = imgFields[1]
@@ -333,17 +347,17 @@ func GetImageMetadata(imageRowString, recipeName string) (Image, error) {
 
 	imageName, err := GetStringInBetween(recipeName, imgString, "[", "]")
 	if err != nil {
-		log.Fatal(err)
+		return img, warnMsgs, err
 	}
 	img.Image = strings.ReplaceAll(imageName, "`", "")
 
 	imageURL, err := GetStringInBetween(recipeName, imgString, "(", ")")
 	if err != nil {
-		log.Fatal(err)
+		return img, warnMsgs, err
 	}
 	img.URL = imageURL
 
-	return img, nil
+	return img, warnMsgs, nil
 }
 
 // GetStringInBetween returns empty string if no start or end string found
@@ -534,11 +548,11 @@ type InternalTracker struct {
 type RepoCatalogue map[string]RepoMeta
 
 // ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
-func ReadReposMetadata() (RepoCatalogue, error) {
+func ReadReposMetadata(debug bool) (RepoCatalogue, error) {
 	reposMeta := make(RepoCatalogue)
 
 	pageIdx := 1
-	bar := formatter.CreateProgressbar(-1, "retrieving recipe repos list from git.coopcloud.tech...")
+	bar := formatter.CreateProgressbar(-1, "collecting recipe listing")
 	for {
 		var reposList []RepoMeta
 
@@ -551,19 +565,32 @@ func ReadReposMetadata() (RepoCatalogue, error) {
 		}
 
 		if len(reposList) == 0 {
-			bar.Add(1)
+			if !debug {
+				bar.Add(1)
+			}
 			break
 		}
 
 		for idx, repo := range reposList {
+			// NOTE(d1): the "example" recipe is a temporary special case
+			// https://git.coopcloud.tech/toolshed/organising/issues/666
+			if repo.Name == "example" {
+				continue
+			}
+
 			reposMeta[repo.Name] = reposList[idx]
 		}
 
 		pageIdx++
-		bar.Add(1)
+
+		if !debug {
+			bar.Add(1)
+		}
 	}
 
-	fmt.Println() // newline for spinner
+	if err := bar.Close(); err != nil {
+		return reposMeta, err
+	}
 
 	return reposMeta, nil
 }
@@ -625,7 +652,7 @@ func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]stri
 }
 
 // UpdateRepositories clones and updates all recipe repositories locally.
-func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
+func UpdateRepositories(repos RepoCatalogue, recipeName string, debug bool) error {
 	var barLength int
 	if recipeName != "" {
 		barLength = 1
@@ -633,9 +660,9 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 		barLength = len(repos)
 	}
 
-	cloneLimiter := limit.New(10)
+	cloneLimiter := limit.New(3)
 
-	retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...")
+	retrieveBar := formatter.CreateProgressbar(barLength, "retrieving recipes")
 	ch := make(chan string, barLength)
 	for _, repoMeta := range repos {
 		go func(rm RepoMeta) {
@@ -644,7 +671,9 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 
 			if recipeName != "" && recipeName != rm.Name {
 				ch <- rm.Name
-				retrieveBar.Add(1)
+				if !debug {
+					retrieveBar.Add(1)
+				}
 				return
 			}
 
@@ -653,7 +682,9 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 			}
 
 			ch <- rm.Name
-			retrieveBar.Add(1)
+			if !debug {
+				retrieveBar.Add(1)
+			}
 		}(repoMeta)
 	}
 
@@ -661,6 +692,10 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 		<-ch // wait for everything
 	}
 
+	if err := retrieveBar.Close(); err != nil {
+		return err
+	}
+
 	return nil
 }
 

tests/integration/catalogue.bats

@@ -6,9 +6,13 @@ setup(){
 }
 
 # bats test_tags=slow
-@test "generate entire catalogue" {
+@test "generate catalogue" {
   run $ABRA catalogue generate
   assert_success
+
+  for d in $(ls $ABRA_DIR/recipes); do 
+    assert_exists "$ABRA_DIR/recipes/$d/.git"
+  done
 }
 
 @test "error if unstaged changes" {
@@ -41,4 +45,5 @@ setup(){
 @test "generate only specific recipe" {
   run $ABRA catalogue generate gitea
   assert_success
+  assert_exists "$ABRA_DIR/recipes/gitea/.git"
 }