toolshed/abra
toolshed/abra
4
2
Fork 18
Code Issues 29 Pull Requests 3 Packages Releases 40 Activity

fix: not flaky catalogue generate
All checks were successful
continuous-integration/drone/pr Build is passing
Details
continuous-integration/drone/push Build is passing
Details

Browse Source 
See #464
This commit is contained in:
decentral1se 2025-01-05 11:57:51 +01:00
parent 2bc77de751
commit 4923984e84
Signed by: decentral1se
GPG Key ID: 03789458B3D0C410
10 changed files with 159 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cli/app/new.go
View File

@ -93,7 +93,7 @@ var AppNewCommand = &cobra.Command{
var recipeVersions recipePkg.RecipeVersions
if recipeVersion == "" {
var err error
recipeVersions, err = recipe.GetRecipeVersions()
recipeVersions, _, err = recipe.GetRecipeVersions()
if err != nil {
log.Fatal(err)
}

62
cli/catalogue/catalogue.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"io/ioutil"
"path"
"slices"
"coopcloud.tech/abra/cli/internal"
"coopcloud.tech/abra/pkg/autocomplete"
@ -61,52 +62,48 @@ keys configured on your account.`,
}
}
repos, err := recipe.ReadReposMetadata()
repos, err := recipe.ReadReposMetadata(internal.Debug)
if err != nil {
log.Fatal(err)
}
var barLength int
var logMsg string
barLength := len(repos)
if recipeName != "" {
barLength = 1
logMsg = fmt.Sprintf("ensuring %v recipe is cloned & up-to-date", barLength)
} else {
barLength = len(repos)
logMsg = fmt.Sprintf("ensuring %v recipes are cloned & up-to-date, this could take some time...", barLength)
}
if !skipUpdates {
log.Warn(logMsg)
if err := recipe.UpdateRepositories(repos, recipeName); err != nil {
if err := recipe.UpdateRepositories(repos, recipeName, internal.Debug); err != nil {
log.Fatal(err)
}
}
var warnings []string
catl := make(recipe.RecipeCatalogue)
catlBar := formatter.CreateProgressbar(barLength, "generating catalogue metadata...")
catlBar := formatter.CreateProgressbar(barLength, "collecting catalogue metadata")
for _, recipeMeta := range repos {
if recipeName != "" && recipeName != recipeMeta.Name {
catlBar.Add(1)
continue
}
// NOTE(d1): the "example" recipe is a temporary special case
// https://git.coopcloud.tech/toolshed/organising/issues/666
if recipeMeta.Name == "example" {
catlBar.Add(1)
if !internal.Debug {
catlBar.Add(1)
}
continue
}
r := recipe.Get(recipeMeta.Name)
versions, err := r.GetRecipeVersions()
versions, warnMsgs, err := r.GetRecipeVersions()
if err != nil {
log.Warn(err)
warnings = append(warnings, err.Error())
}
if len(warnMsgs) > 0 {
warnings = append(warnings, warnMsgs...)
}
features, category, err := recipe.GetRecipeFeaturesAndCategory(r)
features, category, warnMsgs, err := recipe.GetRecipeFeaturesAndCategory(r)
if err != nil {
log.Warn(err)
warnings = append(warnings, err.Error())
}
if len(warnMsgs) > 0 {
warnings = append(warnings, warnMsgs...)
}
catl[recipeMeta.Name] = recipe.RecipeMeta{
@ -122,7 +119,24 @@ keys configured on your account.`,
Features: features,
}
catlBar.Add(1)
if !internal.Debug {
catlBar.Add(1)
}
}
if err := catlBar.Close(); err != nil {
log.Fatal(err)
}
var uniqueWarnings []string
for _, w := range warnings {
if !slices.Contains(uniqueWarnings, w) {
uniqueWarnings = append(uniqueWarnings, w)
}
}
for _, warnMsg := range uniqueWarnings {
log.Warn(warnMsg)
}
recipesJSON, err := json.MarshalIndent(catl, "", " ")
@ -152,7 +166,7 @@ keys configured on your account.`,
}
}
log.Infof("generated new recipe catalogue in %s", config.RECIPES_JSON)
log.Infof("generated recipe catalogue: %s", config.RECIPES_JSON)
cataloguePath := path.Join(config.ABRA_DIR, "catalogue")
if publishChanges {

5
cli/recipe/version.go
View File

@ -37,10 +37,13 @@ var RecipeVersionCommand = &cobra.Command{
if !ok {
warnMessages = append(warnMessages, "retrieved versions from local recipe repository")
recipeVersions, err := recipe.GetRecipeVersions()
recipeVersions, warnMsg, err := recipe.GetRecipeVersions()
if err != nil {
warnMessages = append(warnMessages, err.Error())
}
if len(warnMsg) > 0 {
warnMessages = append(warnMessages, warnMsg...)
}
recipeMeta = recipePkg.RecipeMeta{Versions: recipeVersions}
}

5
pkg/catalogue/catalogue.go
View File

@ -16,13 +16,12 @@ import (
func EnsureCatalogue() error {
catalogueDir := path.Join(config.ABRA_DIR, "catalogue")
if _, err := os.Stat(catalogueDir); err != nil && os.IsNotExist(err) {
log.Warnf("local recipe catalogue is missing, retrieving now")
log.Debugf("catalogue is missing, retrieving now")
url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, config.CATALOGUE_JSON_REPO_NAME)
if err := gitPkg.Clone(catalogueDir, url); err != nil {
return err
}
log.Debugf("cloned catalogue repository to %s", catalogueDir)
}
return nil

1
pkg/formatter/formatter.go
View File

@ -217,7 +217,6 @@ func CreateProgressbar(length int, title string) *progressbar.ProgressBar {
progressbar.OptionClearOnFinish(),
progressbar.OptionSetPredictTime(false),
progressbar.OptionShowCount(),
progressbar.OptionFullWidth(),
progressbar.OptionSetDescription(title),
)
}

33
pkg/git/clone.go
View File

@ -1,9 +1,7 @@
package git
import (
"fmt"
"os"
"path/filepath"
"strings"
"coopcloud.tech/abra/pkg/log"
@ -11,10 +9,23 @@ import (
"github.com/go-git/go-git/v5/plumbing"
)
// gitCloneIgnoreErr checks whether we can ignore a git clone error or not.
func gitCloneIgnoreErr(err error) bool {
if strings.Contains(err.Error(), "authentication required") {
return true
}
if strings.Contains(err.Error(), "remote repository is empty") {
return true
}
return false
}
// Clone runs a git clone which accounts for different default branches.
func Clone(dir, url string) error {
if _, err := os.Stat(dir); os.IsNotExist(err) {
log.Debugf("%s does not exist, attempting git clone of %s", dir, url)
log.Debugf("git clone: %s", dir, url)
_, err := git.PlainClone(dir, false, &git.CloneOptions{
URL: url,
@ -23,6 +34,11 @@ func Clone(dir, url string) error {
SingleBranch: true,
})
if err != nil && gitCloneIgnoreErr(err) {
log.Debugf("git clone: %s cloned successfully", dir)
return nil
}
if err != nil {
log.Debug("git clone: main branch failed, attempting master branch")
@ -32,12 +48,13 @@ func Clone(dir, url string) error {
ReferenceName: plumbing.ReferenceName("refs/heads/master"),
SingleBranch: true,
})
if err != nil {
if strings.Contains(err.Error(), "authentication required") {
name := filepath.Base(dir)
return fmt.Errorf("unable to clone %s, does %s exist?", name, url)
}
if err != nil && gitCloneIgnoreErr(err) {
log.Debugf("git clone: %s cloned successfully", dir)
return nil
}
if err != nil {
return err
}
}

2
pkg/lint/recipe.go
View File

@ -409,7 +409,7 @@ func LintHasPublishedVersion(recipe recipe.Recipe) (bool, error) {
}
func LintMetadataFilledIn(r recipe.Recipe) (bool, error) {
features, category, err := recipe.GetRecipeFeaturesAndCategory(r)
features, category, _, err := recipe.GetRecipeFeaturesAndCategory(r)
if err != nil {
return false, err
}

31
pkg/recipe/git.go
View File

@ -3,6 +3,7 @@ package recipe
import (
"fmt"
"os"
"slices"
"strings"
"coopcloud.tech/abra/pkg/formatter"
@ -350,23 +351,26 @@ func (r Recipe) Tags() ([]string, error) {
}
// GetRecipeVersions retrieves all recipe versions.
func (r Recipe) GetRecipeVersions() (RecipeVersions, error) {
func (r Recipe) GetRecipeVersions() (RecipeVersions, []string, error) {
var warnMsg []string
versions := RecipeVersions{}
log.Debugf("attempting to open git repository in %s", r.Dir)
log.Debugf("git: opening repository in %s", r.Dir)
repo, err := git.PlainOpen(r.Dir)
if err != nil {
return versions, err
return versions, warnMsg, nil
}
worktree, err := repo.Worktree()
if err != nil {
return versions, err
return versions, warnMsg, nil
}
gitTags, err := repo.Tags()
if err != nil {
return versions, err
return versions, warnMsg, nil
}
if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
@ -384,7 +388,7 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, error) {
return err
}
log.Debugf("successfully checked out %s in %s", ref.Name(), r.Dir)
log.Debugf("git checkout: %s in %s", ref.Name(), r.Dir)
config, err := r.GetComposeConfig(nil)
if err != nil {
@ -408,7 +412,7 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, error) {
case reference.NamedTagged:
tag = img.(reference.NamedTagged).Tag()
case reference.Named:
log.Warnf("%s service is missing image tag?", path)
warnMsg = append(warnMsg, fmt.Sprintf("%s service is missing image tag?", path))
continue
}
@ -422,19 +426,26 @@ func (r Recipe) GetRecipeVersions() (RecipeVersions, error) {
return nil
}); err != nil {
return versions, err
return versions, warnMsg, nil
}
_, err = gitPkg.CheckoutDefaultBranch(repo, r.Dir)
if err != nil {
return versions, err
return versions, warnMsg, nil
}
sortRecipeVersions(versions)
log.Debugf("collected %s for %s", versions, r.Dir)
return versions, nil
var uniqueWarnings []string
for _, w := range warnMsg {
if !slices.Contains(uniqueWarnings, w) {
uniqueWarnings = append(uniqueWarnings, w)
}
}
return versions, uniqueWarnings, nil
}
// Head retrieves latest HEAD metadata.

87
pkg/recipe/recipe.go
View File

@ -233,16 +233,18 @@ func GetRecipesLocal() ([]string, error) {
return recipes, nil
}
func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, error) {
feat := Features{}
func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, []string, error) {
var (
category string
warnMsgs []string
feat = Features{}
)
var category string
log.Debugf("attempting to open %s for recipe metadata parsing", r.ReadmePath)
log.Debugf("%s: attempt recipe metadata parse", r.ReadmePath)
readmeFS, err := ioutil.ReadFile(r.ReadmePath)
if err != nil {
return feat, category, err
return feat, category, warnMsgs, err
}
readmeMetadata, err := GetStringInBetween( // Find text between delimiters
@ -251,7 +253,7 @@ func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, error) {
"<!-- metadata -->", "<!-- endmetadata -->",
)
if err != nil {
return feat, category, err
return feat, category, warnMsgs, err
}
readmeLines := strings.Split( // Array item from lines
@ -295,20 +297,25 @@ func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, error) {
)
}
if strings.Contains(val, "**Image**") {
imageMetadata, err := GetImageMetadata(strings.TrimSpace(
imageMetadata, warnings, err := GetImageMetadata(strings.TrimSpace(
strings.TrimPrefix(val, "* **Image**:"),
), r.Name)
if err != nil {
continue
}
if len(warnings) > 0 {
warnMsgs = append(warnMsgs, warnings...)
}
feat.Image = imageMetadata
}
}
return feat, category, nil
return feat, category, warnMsgs, nil
}
func GetImageMetadata(imageRowString, recipeName string) (Image, error) {
func GetImageMetadata(imageRowString, recipeName string) (Image, []string, error) {
var warnMsgs []string
img := Image{}
imgFields := strings.Split(imageRowString, ",")
@ -319,11 +326,18 @@ func GetImageMetadata(imageRowString, recipeName string) (Image, error) {
if len(imgFields) < 3 {
if imageRowString != "" {
log.Warnf("%s image meta has incorrect format: %s", recipeName, imageRowString)
warnMsgs = append(
warnMsgs,
fmt.Sprintf("%s: image meta has incorrect format: %s", recipeName, imageRowString),
)
} else {
log.Warnf("%s image meta is empty?", recipeName)
warnMsgs = append(
warnMsgs,
fmt.Sprintf("%s: image meta is empty?", recipeName),
)
}
return img, nil
return img, warnMsgs, nil
}
img.Rating = imgFields[1]
@ -333,17 +347,17 @@ func GetImageMetadata(imageRowString, recipeName string) (Image, error) {
imageName, err := GetStringInBetween(recipeName, imgString, "[", "]")
if err != nil {
log.Fatal(err)
return img, warnMsgs, err
}
img.Image = strings.ReplaceAll(imageName, "`", "")
imageURL, err := GetStringInBetween(recipeName, imgString, "(", ")")
if err != nil {
log.Fatal(err)
return img, warnMsgs, err
}
img.URL = imageURL
return img, nil
return img, warnMsgs, nil
}
// GetStringInBetween returns empty string if no start or end string found
@ -534,11 +548,11 @@ type InternalTracker struct {
type RepoCatalogue map[string]RepoMeta
// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
func ReadReposMetadata() (RepoCatalogue, error) {
func ReadReposMetadata(debug bool) (RepoCatalogue, error) {
reposMeta := make(RepoCatalogue)
pageIdx := 1
bar := formatter.CreateProgressbar(-1, "retrieving recipe repos list from git.coopcloud.tech...")
bar := formatter.CreateProgressbar(-1, "collecting recipe listing")
for {
var reposList []RepoMeta
@ -551,19 +565,32 @@ func ReadReposMetadata() (RepoCatalogue, error) {
}
if len(reposList) == 0 {
bar.Add(1)
if !debug {
bar.Add(1)
}
break
}
for idx, repo := range reposList {
// NOTE(d1): the "example" recipe is a temporary special case
// https://git.coopcloud.tech/toolshed/organising/issues/666
if repo.Name == "example" {
continue
}
reposMeta[repo.Name] = reposList[idx]
}
pageIdx++
bar.Add(1)
if !debug {
bar.Add(1)
}
}
fmt.Println() // newline for spinner
if err := bar.Close(); err != nil {
return reposMeta, err
}
return reposMeta, nil
}
@ -625,7 +652,7 @@ func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]stri
}
// UpdateRepositories clones and updates all recipe repositories locally.
func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
func UpdateRepositories(repos RepoCatalogue, recipeName string, debug bool) error {
var barLength int
if recipeName != "" {
barLength = 1
@ -633,9 +660,9 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
barLength = len(repos)
}
cloneLimiter := limit.New(10)
cloneLimiter := limit.New(3)
retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...")
retrieveBar := formatter.CreateProgressbar(barLength, "retrieving recipes")
ch := make(chan string, barLength)
for _, repoMeta := range repos {
go func(rm RepoMeta) {
@ -644,7 +671,9 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
if recipeName != "" && recipeName != rm.Name {
ch <- rm.Name
retrieveBar.Add(1)
if !debug {
retrieveBar.Add(1)
}
return
}
@ -653,7 +682,9 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
}
ch <- rm.Name
retrieveBar.Add(1)
if !debug {
retrieveBar.Add(1)
}
}(repoMeta)
}
@ -661,6 +692,10 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
<-ch // wait for everything
}
if err := retrieveBar.Close(); err != nil {
return err
}
return nil
}

7
tests/integration/catalogue.bats
View File

@ -6,9 +6,13 @@ setup(){
}
# bats test_tags=slow
@test "generate entire catalogue" {
@test "generate catalogue" {
run $ABRA catalogue generate
assert_success
for d in $(ls $ABRA_DIR/recipes); do
assert_exists "$ABRA_DIR/recipes/$d/.git"
done
}
@test "error if unstaged changes" {
@ -41,4 +45,5 @@ setup(){
@test "generate only specific recipe" {
run $ABRA catalogue generate gitea
assert_success
assert_exists "$ABRA_DIR/recipes/gitea/.git"
}