package catalogue import ( "encoding/json" "fmt" "io/ioutil" "path" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/formatter" gitPkg "coopcloud.tech/abra/pkg/git" "coopcloud.tech/abra/pkg/limit" "coopcloud.tech/abra/pkg/recipe" "github.com/go-git/go-git/v5" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) // CatalogueSkipList is all the repos that are not recipes. var CatalogueSkipList = map[string]bool{ "abra": true, "abra-apps": true, "abra-aur": true, "abra-bash": true, "abra-capsul": true, "abra-gandi": true, "abra-hetzner": true, "apps": true, "aur-abra-git": true, "auto-apps-json": true, "auto-mirror": true, "backup-bot": true, "backup-bot-two": true, "beta.coopcloud.tech": true, "comrade-renovate-bot": true, "coopcloud.tech": true, "coturn": true, "docker-cp-deploy": true, "docker-dind-bats-kcov": true, "docs.coopcloud.tech": true, "drone-abra": true, "example": true, "gardening": true, "go-abra": true, "organising": true, "outline-with-patch": true, "pyabra": true, "radicle-seed-node": true, "recipes": true, "stack-ssh-deploy": true, "swarm-cronjob": true, "tagcmp": true, "traefik-cert-dumper": true, "tyop": true, } var catalogueGenerateCommand = cli.Command{ Name: "generate", Aliases: []string{"g"}, Usage: "Generate the recipe catalogue", Flags: []cli.Flag{ internal.DebugFlag, internal.NoInputFlag, internal.PublishFlag, internal.DryFlag, internal.SkipUpdatesFlag, internal.RegistryUsernameFlag, internal.RegistryPasswordFlag, }, Before: internal.SubCommandBefore, Description: ` This command generates a new copy of the recipe catalogue which can be found on: https://recipes.coopcloud.tech It polls the entire git.coopcloud.tech/coop-cloud/... recipe repository listing, parses README.md and git tags of those repositories to produce recipe metadata and produces a recipes JSON file. It is possible to generate new metadata for a single recipe by passing . The existing local catalogue will be updated, not overwritten. It is quite easy to get rate limited by Docker Hub when running this command. If you have a Hub account you can have Abra log you in to avoid this. Pass "--user" and "--pass". Push your new release git.coopcloud.tech with "-p/--publish". This requires that you have permission to git push to these repositories and have your SSH keys configured on your account. `, ArgsUsage: "[]", Action: func(c *cli.Context) error { recipeName := c.Args().First() if recipeName != "" { internal.ValidateRecipe(c) } repos, err := recipe.ReadReposMetadata() if err != nil { logrus.Fatal(err) } var barLength int var logMsg string if recipeName != "" { barLength = 1 logMsg = fmt.Sprintf("ensuring %v recipe is cloned & up-to-date", barLength) } else { barLength = len(repos) logMsg = fmt.Sprintf("ensuring %v recipes are cloned & up-to-date, this could take some time...", barLength) } if !internal.SkipUpdates { logrus.Warn(logMsg) if err := updateRepositories(repos, recipeName); err != nil { logrus.Fatal(err) } } catl := make(recipe.RecipeCatalogue) catlBar := formatter.CreateProgressbar(barLength, "generating catalogue metadata...") for _, recipeMeta := range repos { if recipeName != "" && recipeName != recipeMeta.Name { catlBar.Add(1) continue } if _, exists := CatalogueSkipList[recipeMeta.Name]; exists { catlBar.Add(1) continue } versions, err := recipe.GetRecipeVersions( recipeMeta.Name, internal.RegistryUsername, internal.RegistryPassword, ) if err != nil { logrus.Fatal(err) } features, category, err := recipe.GetRecipeFeaturesAndCategory(recipeMeta.Name) if err != nil { logrus.Warn(err) } catl[recipeMeta.Name] = recipe.RecipeMeta{ Name: recipeMeta.Name, Repository: recipeMeta.CloneURL, SSHURL: recipeMeta.SSHURL, Icon: recipeMeta.AvatarURL, DefaultBranch: recipeMeta.DefaultBranch, Description: recipeMeta.Description, Website: recipeMeta.Website, Versions: versions, Category: category, Features: features, } catlBar.Add(1) } recipesJSON, err := json.MarshalIndent(catl, "", " ") if err != nil { logrus.Fatal(err) } if recipeName == "" { if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil { logrus.Fatal(err) } } else { catlFS, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } catlFS[recipeName] = catl[recipeName] updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", " ") if err != nil { logrus.Fatal(err) } if err := ioutil.WriteFile(config.RECIPES_JSON, updatedRecipesJSON, 0764); err != nil { logrus.Fatal(err) } } logrus.Infof("generated new recipe catalogue in %s", config.RECIPES_JSON) cataloguePath := path.Join(config.ABRA_DIR, "catalogue") if internal.Publish { isClean, err := gitPkg.IsClean(cataloguePath) if err != nil { logrus.Fatal(err) } if isClean { if !internal.Dry { logrus.Fatalf("no changes discovered in %s, nothing to publish?", cataloguePath) } } msg := "chore: publish new catalogue release changes" if err := gitPkg.Commit(cataloguePath, "**.json", msg, internal.Dry); err != nil { logrus.Fatal(err) } repo, err := git.PlainOpen(cataloguePath) if err != nil { logrus.Fatal(err) } sshURL := fmt.Sprintf(config.SSH_URL_TEMPLATE, "recipes") if err := gitPkg.CreateRemote(repo, "origin-ssh", sshURL, internal.Dry); err != nil { logrus.Fatal(err) } if err := gitPkg.Push(cataloguePath, "origin-ssh", false, internal.Dry); err != nil { logrus.Fatal(err) } } repo, err := git.PlainOpen(cataloguePath) if err != nil { logrus.Fatal(err) } head, err := repo.Head() if err != nil { logrus.Fatal(err) } if !internal.Dry && internal.Publish { url := fmt.Sprintf("%s/recipes/commit/%s", config.REPOS_BASE_URL, head.Hash()) logrus.Infof("new changes published: %s", url) } if internal.Dry { logrus.Info("dry run: no changes published") } return nil }, BashComplete: autocomplete.RecipeNameComplete, } // CatalogueCommand defines the `abra catalogue` command and sub-commands. var CatalogueCommand = cli.Command{ Name: "catalogue", Usage: "Manage the recipe catalogue", Aliases: []string{"c"}, ArgsUsage: "", Description: "This command helps recipe packagers interact with the recipe catalogue", Subcommands: []cli.Command{ catalogueGenerateCommand, }, } func updateRepositories(repos recipe.RepoCatalogue, recipeName string) error { var barLength int if recipeName != "" { barLength = 1 } else { barLength = len(repos) } cloneLimiter := limit.New(10) retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...") ch := make(chan string, barLength) for _, repoMeta := range repos { go func(rm recipe.RepoMeta) { cloneLimiter.Begin() defer cloneLimiter.End() if recipeName != "" && recipeName != rm.Name { ch <- rm.Name retrieveBar.Add(1) return } if _, exists := CatalogueSkipList[rm.Name]; exists { ch <- rm.Name retrieveBar.Add(1) return } recipeDir := path.Join(config.RECIPES_DIR, rm.Name) if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil { logrus.Fatal(err) } isClean, err := gitPkg.IsClean(recipeDir) if err != nil { logrus.Fatal(err) } if !isClean { logrus.Fatalf("%s has locally unstaged changes", rm.Name) } if err := recipe.EnsureUpToDate(rm.Name); err != nil { logrus.Fatal(err) } ch <- rm.Name retrieveBar.Add(1) }(repoMeta) } for range repos { <-ch // wait for everything } return nil }