abra/cli/catalogue/catalogue.go

package catalogue

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path"

	"coopcloud.tech/abra/cli/internal"
	"coopcloud.tech/abra/pkg/autocomplete"
	"coopcloud.tech/abra/pkg/config"
	"coopcloud.tech/abra/pkg/formatter"
	gitPkg "coopcloud.tech/abra/pkg/git"
	"coopcloud.tech/abra/pkg/limit"
	"coopcloud.tech/abra/pkg/recipe"
	"github.com/go-git/go-git/v5"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

// CatalogueSkipList is all the repos that are not recipes.
var CatalogueSkipList = map[string]bool{
	"abra":                  true,
	"abra-apps":             true,
	"abra-aur":              true,
	"abra-bash":             true,
	"abra-capsul":           true,
	"abra-gandi":            true,
	"abra-hetzner":          true,
	"apps":                  true,
	"aur-abra-git":          true,
	"auto-apps-json":        true,
	"auto-mirror":           true,
	"backup-bot":            true,
	"backup-bot-two":        true,
	"beta.coopcloud.tech":   true,
	"comrade-renovate-bot":  true,
	"coopcloud.tech":        true,
	"coturn":                true,
	"docker-cp-deploy":      true,
	"docker-dind-bats-kcov": true,
	"docs.coopcloud.tech":   true,
	"drone-abra":            true,
	"example":               true,
	"gardening":             true,
	"go-abra":               true,
	"organising":            true,
	"outline-with-patch":    true,
	"pyabra":                true,
	"radicle-seed-node":     true,
	"recipes":               true,
	"stack-ssh-deploy":      true,
	"swarm-cronjob":         true,
	"tagcmp":                true,
	"traefik-cert-dumper":   true,
	"tyop":                  true,
}

var catalogueGenerateCommand = cli.Command{
	Name:    "generate",
	Aliases: []string{"g"},
	Usage:   "Generate the recipe catalogue",
	Flags: []cli.Flag{
		internal.DebugFlag,
		internal.NoInputFlag,
		internal.PublishFlag,
		internal.DryFlag,
		internal.SkipUpdatesFlag,
		internal.RegistryUsernameFlag,
		internal.RegistryPasswordFlag,
	},
	Before: internal.SubCommandBefore,
	Description: `
This command generates a new copy of the recipe catalogue which can be found on:

    https://recipes.coopcloud.tech

It polls the entire git.coopcloud.tech/coop-cloud/... recipe repository
listing, parses README.md and git tags of those repositories to produce recipe
metadata and produces a recipes JSON file.

It is possible to generate new metadata for a single recipe by passing
<recipe>. The existing local catalogue will be updated, not overwritten.

It is quite easy to get rate limited by Docker Hub when running this command.
If you have a Hub account you can have Abra log you in to avoid this. Pass
"--user" and "--pass".

Push your new release git.coopcloud.tech with "-p/--publish". This requires
that you have permission to git push to these repositories and have your SSH
keys configured on your account.
`,
	ArgsUsage: "[<recipe>]",
	Action: func(c *cli.Context) error {
		recipeName := c.Args().First()
		if recipeName != "" {
			internal.ValidateRecipe(c)
		}

		repos, err := recipe.ReadReposMetadata()
		if err != nil {
			logrus.Fatal(err)
		}

		var barLength int
		var logMsg string
		if recipeName != "" {
			barLength = 1
			logMsg = fmt.Sprintf("ensuring %v recipe is cloned & up-to-date", barLength)
		} else {
			barLength = len(repos)
			logMsg = fmt.Sprintf("ensuring %v recipes are cloned & up-to-date, this could take some time...", barLength)
		}

		if !internal.SkipUpdates {
			logrus.Warn(logMsg)
			if err := updateRepositories(repos, recipeName); err != nil {
				logrus.Fatal(err)
			}
		}

		catl := make(recipe.RecipeCatalogue)
		catlBar := formatter.CreateProgressbar(barLength, "generating catalogue metadata...")
		for _, recipeMeta := range repos {
			if recipeName != "" && recipeName != recipeMeta.Name {
				catlBar.Add(1)
				continue
			}

			if _, exists := CatalogueSkipList[recipeMeta.Name]; exists {
				catlBar.Add(1)
				continue
			}

			versions, err := recipe.GetRecipeVersions(
				recipeMeta.Name,
				internal.RegistryUsername,
				internal.RegistryPassword,
			)
			if err != nil {
				logrus.Fatal(err)
			}

			features, category, err := recipe.GetRecipeFeaturesAndCategory(recipeMeta.Name)
			if err != nil {
				logrus.Warn(err)
			}

			catl[recipeMeta.Name] = recipe.RecipeMeta{
				Name:          recipeMeta.Name,
				Repository:    recipeMeta.CloneURL,
				SSHURL:        recipeMeta.SSHURL,
				Icon:          recipeMeta.AvatarURL,
				DefaultBranch: recipeMeta.DefaultBranch,
				Description:   recipeMeta.Description,
				Website:       recipeMeta.Website,
				Versions:      versions,
				Category:      category,
				Features:      features,
			}

			catlBar.Add(1)
		}

		recipesJSON, err := json.MarshalIndent(catl, "", "    ")
		if err != nil {
			logrus.Fatal(err)
		}

		if recipeName == "" {
			if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil {
				logrus.Fatal(err)
			}
		} else {
			catlFS, err := recipe.ReadRecipeCatalogue()
			if err != nil {
				logrus.Fatal(err)
			}

			catlFS[recipeName] = catl[recipeName]

			updatedRecipesJSON, err := json.MarshalIndent(catlFS, "", "    ")
			if err != nil {
				logrus.Fatal(err)
			}

			if err := ioutil.WriteFile(config.RECIPES_JSON, updatedRecipesJSON, 0764); err != nil {
				logrus.Fatal(err)
			}
		}

		logrus.Infof("generated new recipe catalogue in %s", config.RECIPES_JSON)

		cataloguePath := path.Join(config.ABRA_DIR, "catalogue")
		if internal.Publish {

			isClean, err := gitPkg.IsClean(cataloguePath)
			if err != nil {
				logrus.Fatal(err)
			}

			if isClean {
				if !internal.Dry {
					logrus.Fatalf("no changes discovered in %s, nothing to publish?", cataloguePath)
				}
			}

			msg := "chore: publish new catalogue release changes"
			if err := gitPkg.Commit(cataloguePath, "**.json", msg, internal.Dry); err != nil {
				logrus.Fatal(err)
			}

			repo, err := git.PlainOpen(cataloguePath)
			if err != nil {
				logrus.Fatal(err)
			}

			sshURL := fmt.Sprintf(config.SSH_URL_TEMPLATE, "recipes")
			if err := gitPkg.CreateRemote(repo, "origin-ssh", sshURL, internal.Dry); err != nil {
				logrus.Fatal(err)
			}

			if err := gitPkg.Push(cataloguePath, "origin-ssh", false, internal.Dry); err != nil {
				logrus.Fatal(err)
			}
		}

		repo, err := git.PlainOpen(cataloguePath)
		if err != nil {
			logrus.Fatal(err)
		}

		head, err := repo.Head()
		if err != nil {
			logrus.Fatal(err)
		}

		if !internal.Dry && internal.Publish {
			url := fmt.Sprintf("%s/recipes/commit/%s", config.REPOS_BASE_URL, head.Hash())
			logrus.Infof("new changes published: %s", url)
		}

		if internal.Dry {
			logrus.Info("dry run: no changes published")
		}

		return nil
	},
	BashComplete: autocomplete.RecipeNameComplete,
}

// CatalogueCommand defines the `abra catalogue` command and sub-commands.
var CatalogueCommand = cli.Command{
	Name:        "catalogue",
	Usage:       "Manage the recipe catalogue",
	Aliases:     []string{"c"},
	ArgsUsage:   "<recipe>",
	Description: "This command helps recipe packagers interact with the recipe catalogue",
	Subcommands: []cli.Command{
		catalogueGenerateCommand,
	},
}

func updateRepositories(repos recipe.RepoCatalogue, recipeName string) error {
	var barLength int
	if recipeName != "" {
		barLength = 1
	} else {
		barLength = len(repos)
	}

	cloneLimiter := limit.New(10)

	retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...")
	ch := make(chan string, barLength)
	for _, repoMeta := range repos {
		go func(rm recipe.RepoMeta) {
			cloneLimiter.Begin()
			defer cloneLimiter.End()

			if recipeName != "" && recipeName != rm.Name {
				ch <- rm.Name
				retrieveBar.Add(1)
				return
			}
			if _, exists := CatalogueSkipList[rm.Name]; exists {
				ch <- rm.Name
				retrieveBar.Add(1)
				return
			}

			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)

			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
				logrus.Fatal(err)
			}

			isClean, err := gitPkg.IsClean(recipeDir)
			if err != nil {
				logrus.Fatal(err)
			}

			if !isClean {
				logrus.Fatalf("%s has locally unstaged changes", rm.Name)
			}

			if err := recipe.EnsureUpToDate(rm.Name); err != nil {
				logrus.Fatal(err)
			}

			ch <- rm.Name
			retrieveBar.Add(1)
		}(repoMeta)
	}

	for range repos {
		<-ch // wait for everything
	}

	return nil
}