diff --git a/cli/catalogue/catalogue.go b/cli/catalogue/catalogue.go index 52613c5c..656e4ea8 100644 --- a/cli/catalogue/catalogue.go +++ b/cli/catalogue/catalogue.go @@ -12,52 +12,12 @@ import ( "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/formatter" gitPkg "coopcloud.tech/abra/pkg/git" - "coopcloud.tech/abra/pkg/limit" "coopcloud.tech/abra/pkg/recipe" "github.com/go-git/go-git/v5" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) -// CatalogueSkipList is all the repos that are not recipes. -var CatalogueSkipList = map[string]bool{ - "abra": true, - "abra-apps": true, - "abra-aur": true, - "abra-bash": true, - "abra-capsul": true, - "abra-gandi": true, - "abra-hetzner": true, - "apps": true, - "aur-abra-git": true, - "auto-recipes-catalogue-json": true, - "auto-mirror": true, - "backup-bot": true, - "backup-bot-two": true, - "beta.coopcloud.tech": true, - "comrade-renovate-bot": true, - "coopcloud.tech": true, - "coturn": true, - "docker-cp-deploy": true, - "docker-dind-bats-kcov": true, - "docs.coopcloud.tech": true, - "drone-abra": true, - "example": true, - "gardening": true, - "go-abra": true, - "organising": true, - "pyabra": true, - "radicle-seed-node": true, - "recipes-catalogue-json": true, - "recipes-wishlist": true, - "recipes.coopcloud.tech": true, - "stack-ssh-deploy": true, - "swarm-cronjob": true, - "tagcmp": true, - "traefik-cert-dumper": true, - "tyop": true, -} - var catalogueGenerateCommand = cli.Command{ Name: "generate", Aliases: []string{"g"}, @@ -119,7 +79,7 @@ keys configured on your account. if !internal.SkipUpdates { logrus.Warn(logMsg) - if err := updateRepositories(repos, recipeName); err != nil { + if err := recipe.UpdateRepositories(repos, recipeName); err != nil { logrus.Fatal(err) } } @@ -132,7 +92,7 @@ keys configured on your account. continue } - if _, exists := CatalogueSkipList[recipeMeta.Name]; exists { + if _, exists := catalogue.CatalogueSkipList[recipeMeta.Name]; exists { catlBar.Add(1) continue } @@ -261,62 +221,3 @@ var CatalogueCommand = cli.Command{ catalogueGenerateCommand, }, } - -func updateRepositories(repos recipe.RepoCatalogue, recipeName string) error { - var barLength int - if recipeName != "" { - barLength = 1 - } else { - barLength = len(repos) - } - - cloneLimiter := limit.New(10) - - retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...") - ch := make(chan string, barLength) - for _, repoMeta := range repos { - go func(rm recipe.RepoMeta) { - cloneLimiter.Begin() - defer cloneLimiter.End() - - if recipeName != "" && recipeName != rm.Name { - ch <- rm.Name - retrieveBar.Add(1) - return - } - if _, exists := CatalogueSkipList[rm.Name]; exists { - ch <- rm.Name - retrieveBar.Add(1) - return - } - - recipeDir := path.Join(config.RECIPES_DIR, rm.Name) - - if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil { - logrus.Fatal(err) - } - - isClean, err := gitPkg.IsClean(recipeDir) - if err != nil { - logrus.Fatal(err) - } - - if !isClean { - logrus.Fatalf("%s has locally unstaged changes", rm.Name) - } - - if err := recipe.EnsureUpToDate(rm.Name); err != nil { - logrus.Fatal(err) - } - - ch <- rm.Name - retrieveBar.Add(1) - }(repoMeta) - } - - for range repos { - <-ch // wait for everything - } - - return nil -} diff --git a/cli/recipe/fetch.go b/cli/recipe/fetch.go new file mode 100644 index 00000000..70187911 --- /dev/null +++ b/cli/recipe/fetch.go @@ -0,0 +1,40 @@ +package recipe + +import ( + "coopcloud.tech/abra/cli/internal" + "coopcloud.tech/abra/pkg/autocomplete" + "coopcloud.tech/abra/pkg/recipe" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var recipeFetchCommand = cli.Command{ + Name: "fetch", + Usage: "Fetch recipe local copies", + Aliases: []string{"f"}, + ArgsUsage: "[]", + Description: "Fetchs all recipes without arguments.", + Flags: []cli.Flag{ + internal.DebugFlag, + }, + Before: internal.SubCommandBefore, + BashComplete: autocomplete.RecipeNameComplete, + Action: func(c *cli.Context) error { + recipeName := c.Args().First() + if recipeName != "" { + internal.ValidateRecipe(c, true) + return nil // ValidateRecipe ensures latest checkout + } + + repos, err := recipe.ReadReposMetadata() + if err != nil { + logrus.Fatal(err) + } + + if err := recipe.UpdateRepositories(repos, recipeName); err != nil { + logrus.Fatal(err) + } + + return nil + }, +} diff --git a/cli/recipe/recipe.go b/cli/recipe/recipe.go index 5d2d72ab..8333f0f0 100644 --- a/cli/recipe/recipe.go +++ b/cli/recipe/recipe.go @@ -22,12 +22,13 @@ manner. Abra supports convenient automation for recipe maintainenace, see the "abra recipe upgrade", "abra recipe sync" and "abra recipe release" commands. `, Subcommands: []cli.Command{ - recipeListCommand, - recipeVersionCommand, - recipeReleaseCommand, - recipeNewCommand, - recipeUpgradeCommand, - recipeSyncCommand, + recipeFetchCommand, recipeLintCommand, + recipeListCommand, + recipeNewCommand, + recipeReleaseCommand, + recipeSyncCommand, + recipeUpgradeCommand, + recipeVersionCommand, }, } diff --git a/pkg/catalogue/catalogue.go b/pkg/catalogue/catalogue.go index e7fe816e..3cf6b2c0 100644 --- a/pkg/catalogue/catalogue.go +++ b/pkg/catalogue/catalogue.go @@ -12,6 +12,45 @@ import ( "github.com/sirupsen/logrus" ) +// CatalogueSkipList is all the repos that are not recipes. +var CatalogueSkipList = map[string]bool{ + "abra": true, + "abra-apps": true, + "abra-aur": true, + "abra-bash": true, + "abra-capsul": true, + "abra-gandi": true, + "abra-hetzner": true, + "apps": true, + "aur-abra-git": true, + "auto-recipes-catalogue-json": true, + "auto-mirror": true, + "backup-bot": true, + "backup-bot-two": true, + "beta.coopcloud.tech": true, + "comrade-renovate-bot": true, + "coopcloud.tech": true, + "coturn": true, + "docker-cp-deploy": true, + "docker-dind-bats-kcov": true, + "docs.coopcloud.tech": true, + "drone-abra": true, + "example": true, + "gardening": true, + "go-abra": true, + "organising": true, + "pyabra": true, + "radicle-seed-node": true, + "recipes-catalogue-json": true, + "recipes-wishlist": true, + "recipes.coopcloud.tech": true, + "stack-ssh-deploy": true, + "swarm-cronjob": true, + "tagcmp": true, + "traefik-cert-dumper": true, + "tyop": true, +} + // EnsureCatalogue ensures that the catalogue is cloned locally & present. func EnsureCatalogue() error { catalogueDir := path.Join(config.ABRA_DIR, "catalogue") diff --git a/pkg/git/branch.go b/pkg/git/branch.go index 6edebee7..11de4f42 100644 --- a/pkg/git/branch.go +++ b/pkg/git/branch.go @@ -10,9 +10,10 @@ import ( // Check if a branch exists in a repo. Use this and not repository.Branch(), // because the latter does not actually check for existing branches. See -// https://github.com/go-git/go-git/issues/518 for more. +// https://github.com/gogit/gogit/issues/518 for more. func HasBranch(repository *git.Repository, name string) bool { var exist bool + if iter, err := repository.Branches(); err == nil { iterFunc := func(reference *plumbing.Reference) error { if name == reference.Name().Short() { @@ -23,6 +24,7 @@ func HasBranch(repository *git.Repository, name string) bool { } _ = iter.ForEach(iterFunc) } + return exist } diff --git a/pkg/git/clone.go b/pkg/git/clone.go index dcf626f5..d9ff190d 100644 --- a/pkg/git/clone.go +++ b/pkg/git/clone.go @@ -15,22 +15,32 @@ import ( func Clone(dir, url string) error { if _, err := os.Stat(dir); os.IsNotExist(err) { logrus.Debugf("%s does not exist, attempting to git clone from %s", dir, url) - _, err := git.PlainClone(dir, false, &git.CloneOptions{URL: url, Tags: git.AllTags}) + + _, err := git.PlainClone(dir, false, &git.CloneOptions{ + URL: url, + Tags: git.AllTags, + ReferenceName: plumbing.ReferenceName("refs/heads/master"), + SingleBranch: true, + }) if err != nil { logrus.Debugf("cloning %s default branch failed, attempting from main branch", url) + _, err := git.PlainClone(dir, false, &git.CloneOptions{ URL: url, Tags: git.AllTags, ReferenceName: plumbing.ReferenceName("refs/heads/main"), + SingleBranch: true, }) if err != nil { if strings.Contains(err.Error(), "authentication required") { name := filepath.Base(dir) return fmt.Errorf("unable to clone %s, does %s exist?", name, url) } + return err } } + logrus.Debugf("%s has been git cloned successfully", dir) } else { logrus.Debugf("%s already exists", dir) diff --git a/pkg/recipe/recipe.go b/pkg/recipe/recipe.go index 072d69b7..4e1df32a 100644 --- a/pkg/recipe/recipe.go +++ b/pkg/recipe/recipe.go @@ -17,6 +17,7 @@ import ( "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/formatter" gitPkg "coopcloud.tech/abra/pkg/git" + "coopcloud.tech/abra/pkg/limit" "coopcloud.tech/abra/pkg/upstream/stack" loader "coopcloud.tech/abra/pkg/upstream/stack" "coopcloud.tech/abra/pkg/web" @@ -1011,3 +1012,63 @@ func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]stri return versions, nil } + +// UpdateRepositories clones and updates all recipe repositories locally. +func UpdateRepositories(repos RepoCatalogue, recipeName string) error { + var barLength int + if recipeName != "" { + barLength = 1 + } else { + barLength = len(repos) + } + + cloneLimiter := limit.New(10) + + retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...") + ch := make(chan string, barLength) + for _, repoMeta := range repos { + go func(rm RepoMeta) { + cloneLimiter.Begin() + defer cloneLimiter.End() + + if recipeName != "" && recipeName != rm.Name { + ch <- rm.Name + retrieveBar.Add(1) + return + } + if _, exists := catalogue.CatalogueSkipList[rm.Name]; exists { + ch <- rm.Name + retrieveBar.Add(1) + return + } + + recipeDir := path.Join(config.RECIPES_DIR, rm.Name) + + if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil { + logrus.Fatal(err) + } + + isClean, err := gitPkg.IsClean(recipeDir) + if err != nil { + logrus.Fatal(err) + } + + if !isClean { + logrus.Fatalf("%s has locally unstaged changes", rm.Name) + } + + if err := EnsureUpToDate(rm.Name); err != nil { + logrus.Fatal(err) + } + + ch <- rm.Name + retrieveBar.Add(1) + }(repoMeta) + } + + for range repos { + <-ch // wait for everything + } + + return nil +}