From 8f6e1de1a105fcd986ed32e435fc23f08f984157 Mon Sep 17 00:00:00 2001 From: cellarspoon Date: Sun, 19 Dec 2021 16:26:27 +0100 Subject: [PATCH] refactor: merge catalogue/catalogue, catalogue/generate --- cli/catalogue/catalogue.go | 258 +++++++++++++++++++++++++++++++++++ cli/catalogue/generate.go | 272 ------------------------------------- 2 files changed, 258 insertions(+), 272 deletions(-) delete mode 100644 cli/catalogue/generate.go diff --git a/cli/catalogue/catalogue.go b/cli/catalogue/catalogue.go index c7969668..5ceffb19 100644 --- a/cli/catalogue/catalogue.go +++ b/cli/catalogue/catalogue.go @@ -1,9 +1,267 @@ package catalogue import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path" + + "coopcloud.tech/abra/cli/formatter" + "coopcloud.tech/abra/cli/internal" + "coopcloud.tech/abra/pkg/autocomplete" + "coopcloud.tech/abra/pkg/catalogue" + "coopcloud.tech/abra/pkg/config" + gitPkg "coopcloud.tech/abra/pkg/git" + "coopcloud.tech/abra/pkg/limit" + "github.com/AlecAivazis/survey/v2" + "github.com/go-git/go-git/v5" + "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) +// CatalogueSkipList is all the repos that are not recipes. +var CatalogueSkipList = map[string]bool{ + "abra": true, + "abra-apps": true, + "abra-aur": true, + "abra-bash": true, + "abra-capsul": true, + "abra-gandi": true, + "abra-hetzner": true, + "apps": true, + "aur-abra-git": true, + "auto-apps-json": true, + "auto-mirror": true, + "backup-bot": true, + "backup-bot-two": true, + "comrade-renovate-bot": true, + "coopcloud.tech": true, + "coturn": true, + "docker-cp-deploy": true, + "docker-dind-bats-kcov": true, + "docs.coopcloud.tech": true, + "drone-abra": true, + "example": true, + "gardening": true, + "go-abra": true, + "organising": true, + "pyabra": true, + "radicle-seed-node": true, + "recipes": true, + "stack-ssh-deploy": true, + "swarm-cronjob": true, + "tagcmp": true, + "traefik-cert-dumper": true, + "tyop": true, +} + +var catalogueGenerateCommand = &cli.Command{ + Name: "generate", + Aliases: []string{"g"}, + Usage: "Generate a new copy of the catalogue", + Flags: []cli.Flag{ + internal.PushFlag, + internal.CommitFlag, + internal.CommitMessageFlag, + }, + Description: ` +This command generates a new copy of the recipe catalogue which can be found on: + + https://recipes.coopcloud.tech + +It polls the entire git.coopcloud.tech/coop-cloud/... recipe repository +listing, parses README and tags to produce recipe metadata and produces a +apps.json file which is placed in your ~/.abra/catalogue/recipes.json. + +It is possible to generate new metadata for a single recipe by passing +. The existing local catalogue will be updated, not overwritten. + +A new catalogue copy can be published to the recipes repository by passing the +"--commit" and "--push" flags. The recipes repository is available here: + + https://git.coopcloud.tech/coop-cloud/recipes + +`, + ArgsUsage: "[]", + Action: func(c *cli.Context) error { + recipeName := c.Args().First() + if recipeName != "" { + internal.ValidateRecipe(c) + } + + catalogueDir := path.Join(config.ABRA_DIR, "catalogue") + url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, "recipes") + if err := gitPkg.Clone(catalogueDir, url); err != nil { + return err + } + + repos, err := catalogue.ReadReposMetadata() + if err != nil { + logrus.Fatal(err) + } + + logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(repos)) + + cloneLimiter := limit.New(10) + retrieveBar := formatter.CreateProgressbar(len(repos), "retrieving recipes...") + ch := make(chan string, len(repos)) + for _, repoMeta := range repos { + go func(rm catalogue.RepoMeta) { + cloneLimiter.Begin() + defer cloneLimiter.End() + + if recipeName != "" && recipeName != rm.Name { + ch <- rm.Name + retrieveBar.Add(1) + return + } + if _, exists := CatalogueSkipList[rm.Name]; exists { + ch <- rm.Name + retrieveBar.Add(1) + return + } + + recipeDir := path.Join(config.ABRA_DIR, "apps", rm.Name) + + if err := gitPkg.Clone(recipeDir, rm.SSHURL); err != nil { + logrus.Fatal(err) + } + + isClean, err := gitPkg.IsClean(rm.Name) + if err != nil { + logrus.Fatal(err) + } + + if !isClean { + logrus.Fatalf("'%s' has locally unstaged changes", rm.Name) + } + + if err := gitPkg.EnsureUpToDate(recipeDir); err != nil { + logrus.Fatal(err) + } + + ch <- rm.Name + retrieveBar.Add(1) + }(repoMeta) + } + + for range repos { + <-ch // wait for everything + } + + catl := make(catalogue.RecipeCatalogue) + catlBar := formatter.CreateProgressbar(len(repos), "generating catalogue...") + for _, recipeMeta := range repos { + if recipeName != "" && recipeName != recipeMeta.Name { + catlBar.Add(1) + continue + } + + if _, exists := CatalogueSkipList[recipeMeta.Name]; exists { + catlBar.Add(1) + continue + } + + versions, err := catalogue.GetRecipeVersions(recipeMeta.Name) + if err != nil { + logrus.Fatal(err) + } + + features, category, err := catalogue.GetRecipeFeaturesAndCategory(recipeMeta.Name) + if err != nil { + logrus.Warn(err) + } + + catl[recipeMeta.Name] = catalogue.RecipeMeta{ + Name: recipeMeta.Name, + Repository: recipeMeta.CloneURL, + Icon: recipeMeta.AvatarURL, + DefaultBranch: recipeMeta.DefaultBranch, + Description: recipeMeta.Description, + Website: recipeMeta.Website, + Versions: versions, + Category: category, + Features: features, + } + catlBar.Add(1) + } + + recipesJSON, err := json.MarshalIndent(catl, "", " ") + if err != nil { + logrus.Fatal(err) + } + + if _, err := os.Stat(config.APPS_JSON); err != nil && os.IsNotExist(err) { + if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0644); err != nil { + logrus.Fatal(err) + } + } else { + if recipeName != "" { + catlFS, err := catalogue.ReadRecipeCatalogue() + if err != nil { + logrus.Fatal(err) + } + catlFS[recipeName] = catl[recipeName] + + updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", " ") + if err != nil { + logrus.Fatal(err) + } + if err := ioutil.WriteFile(config.APPS_JSON, updatedRecipesJSON, 0644); err != nil { + logrus.Fatal(err) + } + } + } + + cataloguePath := path.Join(config.ABRA_DIR, "catalogue", "recipes.json") + logrus.Infof("generated new recipe catalogue in %s", cataloguePath) + + if internal.Commit { + repoPath := path.Join(config.ABRA_DIR, "catalogue") + commitRepo, err := git.PlainOpen(repoPath) + if err != nil { + logrus.Fatal(err) + } + + commitWorktree, err := commitRepo.Worktree() + if err != nil { + logrus.Fatal(err) + } + + if internal.CommitMessage == "" { + prompt := &survey.Input{ + Message: "commit message", + Default: "chore: publish new catalogue changes", + } + if err := survey.AskOne(prompt, &internal.CommitMessage); err != nil { + logrus.Fatal(err) + } + } + + err = commitWorktree.AddGlob("**.json") + if err != nil { + logrus.Fatal(err) + } + logrus.Debug("staged **.json for commit") + + _, err = commitWorktree.Commit(internal.CommitMessage, &git.CommitOptions{}) + if err != nil { + logrus.Fatal(err) + } + logrus.Info("changes commited") + + if err := commitRepo.Push(&git.PushOptions{}); err != nil { + logrus.Fatal(err) + } + logrus.Info("changes pushed") + } + + return nil + }, + BashComplete: autocomplete.RecipeNameComplete, +} + // CatalogueCommand defines the `abra catalogue` command and sub-commands. var CatalogueCommand = &cli.Command{ Name: "catalogue", diff --git a/cli/catalogue/generate.go b/cli/catalogue/generate.go deleted file mode 100644 index a7b879b6..00000000 --- a/cli/catalogue/generate.go +++ /dev/null @@ -1,272 +0,0 @@ -package catalogue - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path" - - "coopcloud.tech/abra/cli/formatter" - "coopcloud.tech/abra/cli/internal" - "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/catalogue" - "coopcloud.tech/abra/pkg/config" - gitPkg "coopcloud.tech/abra/pkg/git" - "coopcloud.tech/abra/pkg/limit" - "github.com/AlecAivazis/survey/v2" - "github.com/go-git/go-git/v5" - "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" -) - -// CatalogueSkipList is all the repos that are not recipes. -var CatalogueSkipList = map[string]bool{ - "abra": true, - "abra-apps": true, - "abra-aur": true, - "abra-bash": true, - "abra-capsul": true, - "abra-gandi": true, - "abra-hetzner": true, - "apps": true, - "aur-abra-git": true, - "auto-apps-json": true, - "auto-mirror": true, - "backup-bot": true, - "backup-bot-two": true, - "comrade-renovate-bot": true, - "coopcloud.tech": true, - "coturn": true, - "docker-cp-deploy": true, - "docker-dind-bats-kcov": true, - "docs.coopcloud.tech": true, - "drone-abra": true, - "example": true, - "gardening": true, - "go-abra": true, - "organising": true, - "pyabra": true, - "radicle-seed-node": true, - "recipes": true, - "stack-ssh-deploy": true, - "swarm-cronjob": true, - "tagcmp": true, - "traefik-cert-dumper": true, - "tyop": true, -} - -var commit bool -var commitFlag = &cli.BoolFlag{ - Name: "commit", - Usage: "Commits new generated catalogue changes", - Value: false, - Aliases: []string{"c"}, - Destination: &commit, -} - -var catalogueGenerateCommand = &cli.Command{ - Name: "generate", - Aliases: []string{"g"}, - Usage: "Generate a new copy of the catalogue", - Flags: []cli.Flag{ - internal.PushFlag, - commitFlag, - internal.CommitMessageFlag, - }, - Description: ` -This command generates a new copy of the recipe catalogue which can be found on: - - https://recipes.coopcloud.tech - -It polls the entire git.coopcloud.tech/coop-cloud/... recipe repository -listing, parses README and tags to produce recipe metadata and produces a -apps.json file which is placed in your ~/.abra/catalogue/recipes.json. - -It is possible to generate new metadata for a single recipe by passing -. The existing local catalogue will be updated, not overwritten. - -A new catalogue copy can be published to the recipes repository by passing the -"--commit" and "--push" flags. The recipes repository is available here: - - https://git.coopcloud.tech/coop-cloud/recipes - -`, - ArgsUsage: "[]", - Action: func(c *cli.Context) error { - recipeName := c.Args().First() - if recipeName != "" { - internal.ValidateRecipe(c) - } - - catalogueDir := path.Join(config.ABRA_DIR, "catalogue") - url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, "recipes") - if err := gitPkg.Clone(catalogueDir, url); err != nil { - return err - } - - repos, err := catalogue.ReadReposMetadata() - if err != nil { - logrus.Fatal(err) - } - - logrus.Debugf("ensuring '%v' recipe(s) are locally present and up-to-date", len(repos)) - - cloneLimiter := limit.New(10) - retrieveBar := formatter.CreateProgressbar(len(repos), "retrieving recipes...") - ch := make(chan string, len(repos)) - for _, repoMeta := range repos { - go func(rm catalogue.RepoMeta) { - cloneLimiter.Begin() - defer cloneLimiter.End() - - if recipeName != "" && recipeName != rm.Name { - ch <- rm.Name - retrieveBar.Add(1) - return - } - if _, exists := CatalogueSkipList[rm.Name]; exists { - ch <- rm.Name - retrieveBar.Add(1) - return - } - - recipeDir := path.Join(config.ABRA_DIR, "apps", rm.Name) - - if err := gitPkg.Clone(recipeDir, rm.SSHURL); err != nil { - logrus.Fatal(err) - } - - isClean, err := gitPkg.IsClean(rm.Name) - if err != nil { - logrus.Fatal(err) - } - - if !isClean { - logrus.Fatalf("'%s' has locally unstaged changes", rm.Name) - } - - if err := gitPkg.EnsureUpToDate(recipeDir); err != nil { - logrus.Fatal(err) - } - - ch <- rm.Name - retrieveBar.Add(1) - }(repoMeta) - } - - for range repos { - <-ch // wait for everything - } - - catl := make(catalogue.RecipeCatalogue) - catlBar := formatter.CreateProgressbar(len(repos), "generating catalogue...") - for _, recipeMeta := range repos { - if recipeName != "" && recipeName != recipeMeta.Name { - catlBar.Add(1) - continue - } - - if _, exists := CatalogueSkipList[recipeMeta.Name]; exists { - catlBar.Add(1) - continue - } - - versions, err := catalogue.GetRecipeVersions(recipeMeta.Name) - if err != nil { - logrus.Fatal(err) - } - - features, category, err := catalogue.GetRecipeFeaturesAndCategory(recipeMeta.Name) - if err != nil { - logrus.Warn(err) - } - - catl[recipeMeta.Name] = catalogue.RecipeMeta{ - Name: recipeMeta.Name, - Repository: recipeMeta.CloneURL, - Icon: recipeMeta.AvatarURL, - DefaultBranch: recipeMeta.DefaultBranch, - Description: recipeMeta.Description, - Website: recipeMeta.Website, - Versions: versions, - Category: category, - Features: features, - } - catlBar.Add(1) - } - - recipesJSON, err := json.MarshalIndent(catl, "", " ") - if err != nil { - logrus.Fatal(err) - } - - if _, err := os.Stat(config.APPS_JSON); err != nil && os.IsNotExist(err) { - if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0644); err != nil { - logrus.Fatal(err) - } - } else { - if recipeName != "" { - catlFS, err := catalogue.ReadRecipeCatalogue() - if err != nil { - logrus.Fatal(err) - } - catlFS[recipeName] = catl[recipeName] - - updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", " ") - if err != nil { - logrus.Fatal(err) - } - if err := ioutil.WriteFile(config.APPS_JSON, updatedRecipesJSON, 0644); err != nil { - logrus.Fatal(err) - } - } - } - - cataloguePath := path.Join(config.ABRA_DIR, "catalogue", "recipes.json") - logrus.Infof("generated new recipe catalogue in %s", cataloguePath) - - if commit { - repoPath := path.Join(config.ABRA_DIR, "catalogue") - commitRepo, err := git.PlainOpen(repoPath) - if err != nil { - logrus.Fatal(err) - } - - commitWorktree, err := commitRepo.Worktree() - if err != nil { - logrus.Fatal(err) - } - - if internal.CommitMessage == "" { - prompt := &survey.Input{ - Message: "commit message", - Default: "chore: publish new catalogue changes", - } - if err := survey.AskOne(prompt, &internal.CommitMessage); err != nil { - logrus.Fatal(err) - } - } - - err = commitWorktree.AddGlob("**.json") - if err != nil { - logrus.Fatal(err) - } - logrus.Debug("staged **.json for commit") - - _, err = commitWorktree.Commit(internal.CommitMessage, &git.CommitOptions{}) - if err != nil { - logrus.Fatal(err) - } - logrus.Info("changes commited") - - if err := commitRepo.Push(&git.PushOptions{}); err != nil { - logrus.Fatal(err) - } - logrus.Info("changes pushed") - } - - return nil - }, - BashComplete: autocomplete.RecipeNameComplete, -}