diff --git a/cli/app/new.go b/cli/app/new.go index 28a7caed..773187c1 100644 --- a/cli/app/new.go +++ b/cli/app/new.go @@ -93,7 +93,7 @@ var AppNewCommand = &cobra.Command{ var recipeVersions recipePkg.RecipeVersions if recipeVersion == "" { var err error - recipeVersions, err = recipe.GetRecipeVersions() + recipeVersions, _, err = recipe.GetRecipeVersions() if err != nil { log.Fatal(err) } diff --git a/cli/catalogue/catalogue.go b/cli/catalogue/catalogue.go index b1bd1e5c..5bbed464 100644 --- a/cli/catalogue/catalogue.go +++ b/cli/catalogue/catalogue.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "path" + "slices" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" @@ -61,52 +62,48 @@ keys configured on your account.`, } } - repos, err := recipe.ReadReposMetadata() + repos, err := recipe.ReadReposMetadata(internal.Debug) if err != nil { log.Fatal(err) } - var barLength int - var logMsg string + barLength := len(repos) if recipeName != "" { barLength = 1 - logMsg = fmt.Sprintf("ensuring %v recipe is cloned & up-to-date", barLength) - } else { - barLength = len(repos) - logMsg = fmt.Sprintf("ensuring %v recipes are cloned & up-to-date, this could take some time...", barLength) } if !skipUpdates { - log.Warn(logMsg) - if err := recipe.UpdateRepositories(repos, recipeName); err != nil { + if err := recipe.UpdateRepositories(repos, recipeName, internal.Debug); err != nil { log.Fatal(err) } } + var warnings []string catl := make(recipe.RecipeCatalogue) - catlBar := formatter.CreateProgressbar(barLength, "generating catalogue metadata...") + catlBar := formatter.CreateProgressbar(barLength, "collecting catalogue metadata") for _, recipeMeta := range repos { if recipeName != "" && recipeName != recipeMeta.Name { - catlBar.Add(1) - continue - } - - // NOTE(d1): the "example" recipe is a temporary special case - // https://git.coopcloud.tech/toolshed/organising/issues/666 - if recipeMeta.Name == "example" { - catlBar.Add(1) + if !internal.Debug { + catlBar.Add(1) + } continue } r := recipe.Get(recipeMeta.Name) - versions, err := r.GetRecipeVersions() + versions, warnMsgs, err := r.GetRecipeVersions() if err != nil { - log.Warn(err) + warnings = append(warnings, err.Error()) + } + if len(warnMsgs) > 0 { + warnings = append(warnings, warnMsgs...) } - features, category, err := recipe.GetRecipeFeaturesAndCategory(r) + features, category, warnMsgs, err := recipe.GetRecipeFeaturesAndCategory(r) if err != nil { - log.Warn(err) + warnings = append(warnings, err.Error()) + } + if len(warnMsgs) > 0 { + warnings = append(warnings, warnMsgs...) } catl[recipeMeta.Name] = recipe.RecipeMeta{ @@ -122,7 +119,24 @@ keys configured on your account.`, Features: features, } - catlBar.Add(1) + if !internal.Debug { + catlBar.Add(1) + } + } + + if err := catlBar.Close(); err != nil { + log.Fatal(err) + } + + var uniqueWarnings []string + for _, w := range warnings { + if !slices.Contains(uniqueWarnings, w) { + uniqueWarnings = append(uniqueWarnings, w) + } + } + + for _, warnMsg := range uniqueWarnings { + log.Warn(warnMsg) } recipesJSON, err := json.MarshalIndent(catl, "", " ") @@ -152,7 +166,7 @@ keys configured on your account.`, } } - log.Infof("generated new recipe catalogue in %s", config.RECIPES_JSON) + log.Infof("generated recipe catalogue: %s", config.RECIPES_JSON) cataloguePath := path.Join(config.ABRA_DIR, "catalogue") if publishChanges { diff --git a/cli/recipe/version.go b/cli/recipe/version.go index 3487dfb0..0ae32bd8 100644 --- a/cli/recipe/version.go +++ b/cli/recipe/version.go @@ -37,10 +37,13 @@ var RecipeVersionCommand = &cobra.Command{ if !ok { warnMessages = append(warnMessages, "retrieved versions from local recipe repository") - recipeVersions, err := recipe.GetRecipeVersions() + recipeVersions, warnMsg, err := recipe.GetRecipeVersions() if err != nil { warnMessages = append(warnMessages, err.Error()) } + if len(warnMsg) > 0 { + warnMessages = append(warnMessages, warnMsg...) + } recipeMeta = recipePkg.RecipeMeta{Versions: recipeVersions} } diff --git a/pkg/catalogue/catalogue.go b/pkg/catalogue/catalogue.go index e3fabb37..76859616 100644 --- a/pkg/catalogue/catalogue.go +++ b/pkg/catalogue/catalogue.go @@ -16,13 +16,12 @@ import ( func EnsureCatalogue() error { catalogueDir := path.Join(config.ABRA_DIR, "catalogue") if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) { - log.Warnf("local recipe catalogue is missing, retrieving now") + log.Debugf("catalogue is missing, retrieving now") + url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME) if err := gitPkg.Clone(catalogueDir, url); err != nil { return err } - - log.Debugf("cloned catalogue repository to %s", catalogueDir) } return nil diff --git a/pkg/formatter/formatter.go b/pkg/formatter/formatter.go index 5d11c955..3c4d1400 100644 --- a/pkg/formatter/formatter.go +++ b/pkg/formatter/formatter.go @@ -217,7 +217,6 @@ func CreateProgressbar(length int, title string) *progressbar.ProgressBar { progressbar.OptionClearOnFinish(), progressbar.OptionSetPredictTime(false), progressbar.OptionShowCount(), - progressbar.OptionFullWidth(), progressbar.OptionSetDescription(title), ) } diff --git a/pkg/git/clone.go b/pkg/git/clone.go index 001900c2..750bc1a3 100644 --- a/pkg/git/clone.go +++ b/pkg/git/clone.go @@ -1,9 +1,7 @@ package git import ( - "fmt" "os" - "path/filepath" "strings" "coopcloud.tech/abra/pkg/log" @@ -11,10 +9,23 @@ import ( "github.com/go-git/go-git/v5/plumbing" ) +// gitCloneIgnoreErr checks whether we can ignore a git clone error or not. +func gitCloneIgnoreErr(err error) bool { + if strings.Contains(err.Error(), "authentication required") { + return true + } + + if strings.Contains(err.Error(), "remote repository is empty") { + return true + } + + return false +} + // Clone runs a git clone which accounts for different default branches. func Clone(dir, url string) error { if _, err := os.Stat(dir); os.IsNotExist(err) { - log.Debugf("%s does not exist, attempting git clone of %s", dir, url) + log.Debugf("git clone: %s", dir, url) _, err := git.PlainClone(dir, false, &git.CloneOptions{ URL: url, @@ -23,6 +34,11 @@ func Clone(dir, url string) error { SingleBranch: true, }) + if err != nil && gitCloneIgnoreErr(err) { + log.Debugf("git clone: %s cloned successfully", dir) + return nil + } + if err != nil { log.Debug("git clone: main branch failed, attempting master branch") @@ -32,12 +48,13 @@ func Clone(dir, url string) error { ReferenceName: plumbing.ReferenceName("refs/heads/master"), SingleBranch: true, }) - if err != nil { - if strings.Contains(err.Error(), "authentication required") { - name := filepath.Base(dir) - return fmt.Errorf("unable to clone %s, does %s exist?", name, url) - } + if err != nil && gitCloneIgnoreErr(err) { + log.Debugf("git clone: %s cloned successfully", dir) + return nil + } + + if err != nil { return err } } diff --git a/pkg/lint/recipe.go b/pkg/lint/recipe.go index 32dd268c..1462fa83 100644 --- a/pkg/lint/recipe.go +++ b/pkg/lint/recipe.go @@ -409,7 +409,7 @@ func LintHasPublishedVersion(recipe recipe.Recipe) (bool, error) { } func LintMetadataFilledIn(r recipe.Recipe) (bool, error) { - features, category, err := recipe.GetRecipeFeaturesAndCategory(r) + features, category, _, err := recipe.GetRecipeFeaturesAndCategory(r) if err != nil { return false, err } diff --git a/pkg/recipe/git.go b/pkg/recipe/git.go index addfdd92..e2f0982d 100644 --- a/pkg/recipe/git.go +++ b/pkg/recipe/git.go @@ -3,6 +3,7 @@ package recipe import ( "fmt" "os" + "slices" "strings" "coopcloud.tech/abra/pkg/formatter" @@ -350,23 +351,26 @@ func (r Recipe) Tags() ([]string, error) { } // GetRecipeVersions retrieves all recipe versions. -func (r Recipe) GetRecipeVersions() (RecipeVersions, error) { +func (r Recipe) GetRecipeVersions() (RecipeVersions, []string, error) { + var warnMsg []string + versions := RecipeVersions{} - log.Debugf("attempting to open git repository in %s", r.Dir) + + log.Debugf("git: opening repository in %s", r.Dir) repo, err := git.PlainOpen(r.Dir) if err != nil { - return versions, err + return versions, warnMsg, nil } worktree, err := repo.Worktree() if err != nil { - return versions, err + return versions, warnMsg, nil } gitTags, err := repo.Tags() if err != nil { - return versions, err + return versions, warnMsg, nil } if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) { @@ -384,7 +388,7 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, error) { return err } - log.Debugf("successfully checked out %s in %s", ref.Name(), r.Dir) + log.Debugf("git checkout: %s in %s", ref.Name(), r.Dir) config, err := r.GetComposeConfig(nil) if err != nil { @@ -408,7 +412,7 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, error) { case reference.NamedTagged: tag = img.(reference.NamedTagged).Tag() case reference.Named: - log.Warnf("%s service is missing image tag?", path) + warnMsg = append(warnMsg, fmt.Sprintf("%s service is missing image tag?", path)) continue } @@ -422,19 +426,26 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, error) { return nil }); err != nil { - return versions, err + return versions, warnMsg, nil } _, err = gitPkg.CheckoutDefaultBranch(repo, r.Dir) if err != nil { - return versions, err + return versions, warnMsg, nil } sortRecipeVersions(versions) log.Debugf("collected %s for %s", versions, r.Dir) - return versions, nil + var uniqueWarnings []string + for _, w := range warnMsg { + if !slices.Contains(uniqueWarnings, w) { + uniqueWarnings = append(uniqueWarnings, w) + } + } + + return versions, uniqueWarnings, nil } // Head retrieves latest HEAD metadata. diff --git a/pkg/recipe/recipe.go b/pkg/recipe/recipe.go index 2434f7ed..97d92ae0 100644 --- a/pkg/recipe/recipe.go +++ b/pkg/recipe/recipe.go @@ -233,16 +233,18 @@ func GetRecipesLocal() ([]string, error) { return recipes, nil } -func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, error) { - feat := Features{} +func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, []string, error) { + var ( + category string + warnMsgs []string + feat = Features{} + ) - var category string - - log.Debugf("attempting to open %s for recipe metadata parsing", r.ReadmePath) + log.Debugf("%s: attempt recipe metadata parse", r.ReadmePath) readmeFS, err := ioutil.ReadFile(r.ReadmePath) if err != nil { - return feat, category, err + return feat, category, warnMsgs, err } readmeMetadata, err := GetStringInBetween( // Find text between delimiters @@ -251,7 +253,7 @@ func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, error) { "", "", ) if err != nil { - return feat, category, err + return feat, category, warnMsgs, err } readmeLines := strings.Split( // Array item from lines @@ -295,20 +297,25 @@ func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, error) { ) } if strings.Contains(val, "**Image**") { - imageMetadata, err := GetImageMetadata(strings.TrimSpace( + imageMetadata, warnings, err := GetImageMetadata(strings.TrimSpace( strings.TrimPrefix(val, "* **Image**:"), ), r.Name) if err != nil { continue } + if len(warnings) > 0 { + warnMsgs = append(warnMsgs, warnings...) + } feat.Image = imageMetadata } } - return feat, category, nil + return feat, category, warnMsgs, nil } -func GetImageMetadata(imageRowString, recipeName string) (Image, error) { +func GetImageMetadata(imageRowString, recipeName string) (Image, []string, error) { + var warnMsgs []string + img := Image{} imgFields := strings.Split(imageRowString, ",") @@ -319,11 +326,18 @@ func GetImageMetadata(imageRowString, recipeName string) (Image, error) { if len(imgFields) < 3 { if imageRowString != "" { - log.Warnf("%s image meta has incorrect format: %s", recipeName, imageRowString) + warnMsgs = append( + warnMsgs, + fmt.Sprintf("%s: image meta has incorrect format: %s", recipeName, imageRowString), + ) } else { - log.Warnf("%s image meta is empty?", recipeName) + warnMsgs = append( + warnMsgs, + fmt.Sprintf("%s: image meta is empty?", recipeName), + ) } - return img, nil + + return img, warnMsgs, nil } img.Rating = imgFields[1] @@ -333,17 +347,17 @@ func GetImageMetadata(imageRowString, recipeName string) (Image, error) { imageName, err := GetStringInBetween(recipeName, imgString, "[", "]") if err != nil { - log.Fatal(err) + return img, warnMsgs, err } img.Image = strings.ReplaceAll(imageName, "`", "") imageURL, err := GetStringInBetween(recipeName, imgString, "(", ")") if err != nil { - log.Fatal(err) + return img, warnMsgs, err } img.URL = imageURL - return img, nil + return img, warnMsgs, nil } // GetStringInBetween returns empty string if no start or end string found @@ -534,11 +548,11 @@ type InternalTracker struct { type RepoCatalogue map[string]RepoMeta // ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea. -func ReadReposMetadata() (RepoCatalogue, error) { +func ReadReposMetadata(debug bool) (RepoCatalogue, error) { reposMeta := make(RepoCatalogue) pageIdx := 1 - bar := formatter.CreateProgressbar(-1, "retrieving recipe repos list from git.coopcloud.tech...") + bar := formatter.CreateProgressbar(-1, "collecting recipe listing") for { var reposList []RepoMeta @@ -551,19 +565,32 @@ func ReadReposMetadata() (RepoCatalogue, error) { } if len(reposList) == 0 { - bar.Add(1) + if !debug { + bar.Add(1) + } break } for idx, repo := range reposList { + // NOTE(d1): the "example" recipe is a temporary special case + // https://git.coopcloud.tech/toolshed/organising/issues/666 + if repo.Name == "example" { + continue + } + reposMeta[repo.Name] = reposList[idx] } pageIdx++ - bar.Add(1) + + if !debug { + bar.Add(1) + } } - fmt.Println() // newline for spinner + if err := bar.Close(); err != nil { + return reposMeta, err + } return reposMeta, nil } @@ -625,7 +652,7 @@ func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]stri } // UpdateRepositories clones and updates all recipe repositories locally. -func UpdateRepositories(repos RepoCatalogue, recipeName string) error { +func UpdateRepositories(repos RepoCatalogue, recipeName string, debug bool) error { var barLength int if recipeName != "" { barLength = 1 @@ -633,9 +660,9 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error { barLength = len(repos) } - cloneLimiter := limit.New(10) + cloneLimiter := limit.New(3) - retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...") + retrieveBar := formatter.CreateProgressbar(barLength, "retrieving recipes") ch := make(chan string, barLength) for _, repoMeta := range repos { go func(rm RepoMeta) { @@ -644,7 +671,9 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error { if recipeName != "" && recipeName != rm.Name { ch <- rm.Name - retrieveBar.Add(1) + if !debug { + retrieveBar.Add(1) + } return } @@ -653,7 +682,9 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error { } ch <- rm.Name - retrieveBar.Add(1) + if !debug { + retrieveBar.Add(1) + } }(repoMeta) } @@ -661,6 +692,10 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error { <-ch // wait for everything } + if err := retrieveBar.Close(); err != nil { + return err + } + return nil } diff --git a/tests/integration/catalogue.bats b/tests/integration/catalogue.bats index 369567b4..0d4c25f7 100644 --- a/tests/integration/catalogue.bats +++ b/tests/integration/catalogue.bats @@ -6,9 +6,13 @@ setup(){ } # bats test_tags=slow -@test "generate entire catalogue" { +@test "generate catalogue" { run $ABRA catalogue generate assert_success + + for d in $(ls $ABRA_DIR/recipes); do + assert_exists "$ABRA_DIR/recipes/$d/.git" + done } @test "error if unstaged changes" { @@ -41,4 +45,5 @@ setup(){ @test "generate only specific recipe" { run $ABRA catalogue generate gitea assert_success + assert_exists "$ABRA_DIR/recipes/gitea/.git" }