abra/pkg/recipe/recipe.go

package recipe

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"slices"
	"sort"
	"strconv"
	"strings"

	"coopcloud.tech/abra/pkg/catalogue"
	"coopcloud.tech/abra/pkg/config"
	"coopcloud.tech/abra/pkg/formatter"
	gitPkg "coopcloud.tech/abra/pkg/git"
	"coopcloud.tech/abra/pkg/limit"
	"coopcloud.tech/abra/pkg/log"
	"coopcloud.tech/abra/pkg/upstream/stack"
	loader "coopcloud.tech/abra/pkg/upstream/stack"
	"coopcloud.tech/abra/pkg/web"
	"coopcloud.tech/tagcmp"
	"github.com/distribution/reference"
	composetypes "github.com/docker/cli/cli/compose/types"
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing"
)

// RecipeCatalogueURL is the only current recipe catalogue available.
const RecipeCatalogueURL = "https://recipes.coopcloud.tech/recipes.json"

// ReposMetadataURL is the recipe repository metadata.
const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"

// tag represents a git tag.
type tag = string

// service represents a service within a recipe.
type service = string

// ServiceMeta represents meta info associated with a service.
type ServiceMeta struct {
	Image string `json:"image"`
	Tag   string `json:"tag"`
}

// RecipeVersions are the versions associated with a recipe.
type RecipeVersions []map[tag]map[service]ServiceMeta

// RecipeMeta represents metadata for a recipe in the abra catalogue.
type RecipeMeta struct {
	Category      string         `json:"category"`
	DefaultBranch string         `json:"default_branch"`
	Description   string         `json:"description"`
	Features      Features       `json:"features"`
	Icon          string         `json:"icon"`
	Name          string         `json:"name"`
	Repository    string         `json:"repository"`
	SSHURL        string         `json:"ssh_url"`
	Versions      RecipeVersions `json:"versions"`
	Website       string         `json:"website"`
}

// TopicMeta represents a list of topics for a repository.
type TopicMeta struct {
	Topics []string `json:"topics"`
}

// LatestVersion returns the latest version of a recipe.
func (r RecipeMeta) LatestVersion() string {
	var version string

	// apps.json versions are sorted so the last key is latest
	latest := r.Versions[len(r.Versions)-1]

	for tag := range latest {
		version = tag
	}

	log.Debugf("choosing %s as latest version of %s", version, r.Name)

	return version
}

// Name represents a recipe name.
type Name = string

// RecipeCatalogue represents the entire recipe catalogue.
type RecipeCatalogue map[Name]RecipeMeta

// Flatten converts AppCatalogue to slice
func (r RecipeCatalogue) Flatten() []RecipeMeta {
	recipes := make([]RecipeMeta, 0, len(r))

	for name := range r {
		recipes = append(recipes, r[name])
	}

	return recipes
}

// ByRecipeName sorts recipes by name.
type ByRecipeName []RecipeMeta

func (r ByRecipeName) Len() int      { return len(r) }
func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
func (r ByRecipeName) Less(i, j int) bool {
	return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name)
}

// Image represents a recipe container image.
type Image struct {
	Image  string `json:"image"`
	Rating string `json:"rating"`
	Source string `json:"source"`
	URL    string `json:"url"`
}

// Features represent what top-level features a recipe supports (e.g. does this recipe support backups?).
type Features struct {
	Backups     string `json:"backups"`
	Email       string `json:"email"`
	Healthcheck string `json:"healthcheck"`
	Image       Image  `json:"image"`
	Status      int    `json:"status"`
	Tests       string `json:"tests"`
	SSO         string `json:"sso"`
}

// Recipe represents a recipe.
type Recipe struct {
	Name   string
	Config *composetypes.Config
	Meta   RecipeMeta
}

// Get retrieves a recipe.
func Get(recipeName string, offline bool) (Recipe, error) {
	r := Get2(recipeName)
	if err := r.EnsureExists(); err != nil {
		return Recipe{}, err
	}

	pattern := fmt.Sprintf("%s/%s/compose**yml", config.RECIPES_DIR, r.Name)
	composeFiles, err := filepath.Glob(pattern)
	if err != nil {
		return Recipe{}, err
	}

	if len(composeFiles) == 0 {
		return Recipe{}, fmt.Errorf("%s is missing a compose.yml or compose.*.yml file?", r.Name)
	}

	sampleEnv, err := r.SampleEnv()
	if err != nil {
		return Recipe{}, err
	}

	opts := stack.Deploy{Composefiles: composeFiles}
	config, err := loader.LoadComposefile(opts, sampleEnv)
	if err != nil {
		return Recipe{}, err
	}

	meta, err := GetRecipeMeta(r.Name, offline)
	if err != nil {
		switch err.(type) {
		case RecipeMissingFromCatalogue:
			meta = RecipeMeta{}
		default:
			return Recipe{}, err
		}
	}

	return Recipe{
		Name:   recipeName,
		Config: config,
		Meta:   meta,
	}, nil
}

func Get2(name string) Recipe2 {
	dir := path.Join(config.RECIPES_DIR, name)
	return Recipe2{
		Name:   name,
		Dir:    dir,
		SSHURL: fmt.Sprintf(config.SSH_URL_TEMPLATE, name),

		ReadmePath:    path.Join(dir, "README.md"),
		SampleEnvPath: path.Join(dir, ".env.sample"),
	}
}

type Recipe2 struct {
	Name   string
	Dir    string
	SSHURL string

	ReadmePath    string
	SampleEnvPath string
}

// GetRecipesLocal retrieves all local recipe directories
func GetRecipesLocal() ([]string, error) {
	var recipes []string

	recipes, err := config.GetAllFoldersInDirectory(config.RECIPES_DIR)
	if err != nil {
		return recipes, err
	}

	return recipes, nil
}

// GetVersionLabelLocal retrieves the version label on the local recipe config
func GetVersionLabelLocal(recipe Recipe) (string, error) {
	var label string

	for _, service := range recipe.Config.Services {
		for label, value := range service.Deploy.Labels {
			if strings.HasPrefix(label, "coop-cloud") && strings.Contains(label, "version") {
				return value, nil
			}
		}
	}

	if label == "" {
		return label, fmt.Errorf("%s has no version label? try running \"abra recipe sync %s\" first?", recipe.Name, recipe.Name)
	}

	return label, nil
}

func GetRecipeFeaturesAndCategory(r Recipe2) (Features, string, error) {
	feat := Features{}

	var category string

	log.Debugf("attempting to open %s for recipe metadata parsing", r.ReadmePath)

	readmeFS, err := ioutil.ReadFile(r.ReadmePath)
	if err != nil {
		return feat, category, err
	}

	readmeMetadata, err := GetStringInBetween( // Find text between delimiters
		r.Name,
		string(readmeFS),
		"<!-- metadata -->", "<!-- endmetadata -->",
	)
	if err != nil {
		return feat, category, err
	}

	readmeLines := strings.Split( // Array item from lines
		strings.ReplaceAll( // Remove \t tabs
			readmeMetadata, "\t", "",
		),
		"\n")

	for _, val := range readmeLines {
		if strings.Contains(val, "**Status**") {
			feat.Status, _ = strconv.Atoi(strings.TrimSpace(strings.Split(strings.TrimPrefix(val, "* **Status**:"), ",")[0]))
		}
		if strings.Contains(val, "**Category**") {
			category = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Category**:"),
			)
		}
		if strings.Contains(val, "**Backups**") {
			feat.Backups = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Backups**:"),
			)
		}
		if strings.Contains(val, "**Email**") {
			feat.Email = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Email**:"),
			)
		}
		if strings.Contains(val, "**SSO**") {
			feat.SSO = strings.TrimSpace(
				strings.TrimPrefix(val, "* **SSO**:"),
			)
		}
		if strings.Contains(val, "**Healthcheck**") {
			feat.Healthcheck = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Healthcheck**:"),
			)
		}
		if strings.Contains(val, "**Tests**") {
			feat.Tests = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Tests**:"),
			)
		}
		if strings.Contains(val, "**Image**") {
			imageMetadata, err := GetImageMetadata(strings.TrimSpace(
				strings.TrimPrefix(val, "* **Image**:"),
			), r.Name)
			if err != nil {
				continue
			}
			feat.Image = imageMetadata
		}
	}

	return feat, category, nil
}

func GetImageMetadata(imageRowString, recipeName string) (Image, error) {
	img := Image{}

	imgFields := strings.Split(imageRowString, ",")

	for i, elem := range imgFields {
		imgFields[i] = strings.TrimSpace(elem)
	}

	if len(imgFields) < 3 {
		if imageRowString != "" {
			log.Warnf("%s image meta has incorrect format: %s", recipeName, imageRowString)
		} else {
			log.Warnf("%s image meta is empty?", recipeName)
		}
		return img, nil
	}

	img.Rating = imgFields[1]
	img.Source = imgFields[2]

	imgString := imgFields[0]

	imageName, err := GetStringInBetween(recipeName, imgString, "[", "]")
	if err != nil {
		log.Fatal(err)
	}
	img.Image = strings.ReplaceAll(imageName, "`", "")

	imageURL, err := GetStringInBetween(recipeName, imgString, "(", ")")
	if err != nil {
		log.Fatal(err)
	}
	img.URL = imageURL

	return img, nil
}

// GetStringInBetween returns empty string if no start or end string found
func GetStringInBetween(recipeName, str, start, end string) (result string, err error) {
	s := strings.Index(str, start)
	if s == -1 {
		return "", fmt.Errorf("%s: marker string %s not found", recipeName, start)
	}

	s += len(start)
	e := strings.Index(str[s:], end)

	if e == -1 {
		return "", fmt.Errorf("%s: end marker %s not found", recipeName, end)
	}

	return str[s : s+e], nil
}

// ReadRecipeCatalogue reads the recipe catalogue.
func ReadRecipeCatalogue(offline bool) (RecipeCatalogue, error) {
	recipes := make(RecipeCatalogue)

	if err := catalogue.EnsureCatalogue(); err != nil {
		return nil, err
	}

	if !offline {
		if err := catalogue.EnsureUpToDate(); err != nil {
			return nil, err
		}
	}

	if err := readRecipeCatalogueFS(&recipes); err != nil {
		return nil, err
	}

	return recipes, nil
}

// readRecipeCatalogueFS reads the catalogue from the file system.
func readRecipeCatalogueFS(target interface{}) error {
	recipesJSONFS, err := ioutil.ReadFile(config.RECIPES_JSON)
	if err != nil {
		return err
	}

	if err := json.Unmarshal(recipesJSONFS, &target); err != nil {
		return err
	}

	log.Debugf("read recipe catalogue from file system cache in %s", config.RECIPES_JSON)

	return nil
}

// VersionsOfService lists the version of a service.
func VersionsOfService(recipe, serviceName string, offline bool) ([]string, error) {
	var versions []string

	catalogue, err := ReadRecipeCatalogue(offline)
	if err != nil {
		return nil, err
	}

	rec, ok := catalogue[recipe]
	if !ok {
		return versions, nil
	}

	alreadySeen := make(map[string]bool)
	for _, serviceVersion := range rec.Versions {
		for tag := range serviceVersion {
			if _, ok := alreadySeen[tag]; !ok {
				alreadySeen[tag] = true
				versions = append(versions, tag)
			}
		}
	}

	log.Debugf("detected versions %s for %s", strings.Join(versions, ", "), recipe)

	return versions, nil
}

// RecipeMissingFromCatalogue signifies a recipe is not present in the catalogue.
type RecipeMissingFromCatalogue struct{ err string }

// Error outputs the error message.
func (r RecipeMissingFromCatalogue) Error() string {
	return r.err
}

// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
func GetRecipeMeta(recipeName string, offline bool) (RecipeMeta, error) {
	catl, err := ReadRecipeCatalogue(offline)
	if err != nil {
		return RecipeMeta{}, err
	}

	recipeMeta, ok := catl[recipeName]
	if !ok {
		return RecipeMeta{}, RecipeMissingFromCatalogue{
			err: fmt.Sprintf("recipe %s does not exist?", recipeName),
		}
	}

	log.Debugf("recipe metadata retrieved for %s", recipeName)

	return recipeMeta, nil
}

// RepoMeta is a single recipe repo metadata.
type RepoMeta struct {
	ID                        int `json:"id"`
	Owner                     Owner
	Name                      string      `json:"name"`
	FullName                  string      `json:"full_name"`
	Description               string      `json:"description"`
	Empty                     bool        `json:"empty"`
	Private                   bool        `json:"private"`
	Fork                      bool        `json:"fork"`
	Template                  bool        `json:"template"`
	Parent                    interface{} `json:"parent"`
	Mirror                    bool        `json:"mirror"`
	Size                      int         `json:"size"`
	HTMLURL                   string      `json:"html_url"`
	SSHURL                    string      `json:"ssh_url"`
	CloneURL                  string      `json:"clone_url"`
	OriginalURL               string      `json:"original_url"`
	Website                   string      `json:"website"`
	StarsCount                int         `json:"stars_count"`
	ForksCount                int         `json:"forks_count"`
	WatchersCount             int         `json:"watchers_count"`
	OpenIssuesCount           int         `json:"open_issues_count"`
	OpenPRCount               int         `json:"open_pr_counter"`
	ReleaseCounter            int         `json:"release_counter"`
	DefaultBranch             string      `json:"default_branch"`
	Archived                  bool        `json:"archived"`
	CreatedAt                 string      `json:"created_at"`
	UpdatedAt                 string      `json:"updated_at"`
	Permissions               Permissions
	HasIssues                 bool `json:"has_issues"`
	InternalTracker           InternalTracker
	HasWiki                   bool   `json:"has_wiki"`
	HasPullRequests           bool   `json:"has_pull_requests"`
	HasProjects               bool   `json:"has_projects"`
	IgnoreWhitespaceConflicts bool   `json:"ignore_whitespace_conflicts"`
	AllowMergeCommits         bool   `json:"allow_merge_commits"`
	AllowRebase               bool   `json:"allow_rebase"`
	AllowRebaseExplicit       bool   `json:"allow_rebase_explicit"`
	AllowSquashMerge          bool   `json:"allow_squash_merge"`
	AvatarURL                 string `json:"avatar_url"`
	Internal                  bool   `json:"internal"`
	MirrorInterval            string `json:"mirror_interval"`
}

// Owner is the repo organisation owner metadata.
type Owner struct {
	ID         int    `json:"id"`
	Login      string `json:"login"`
	FullName   string `json:"full_name"`
	Email      string `json:"email"`
	AvatarURL  string `json:"avatar_url"`
	Language   string `json:"language"`
	IsAdmin    bool   `json:"is_admin"`
	LastLogin  string `json:"last_login"`
	Created    string `json:"created"`
	Restricted bool   `json:"restricted"`
	Username   string `json:"username"`
}

// Permissions is perms metadata for a repo.
type Permissions struct {
	Admin bool `json:"admin"`
	Push  bool `json:"push"`
	Pull  bool `json:"pull"`
}

// InternalTracker is issue tracker metadata for a repo.
type InternalTracker struct {
	EnableTimeTracker                bool `json:"enable_time_tracker"`
	AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"`
	EnableIssuesDependencies         bool `json:"enable_issue_dependencies"`
}

// RepoCatalogue represents all the recipe repo metadata.
type RepoCatalogue map[string]RepoMeta

// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
func ReadReposMetadata() (RepoCatalogue, error) {
	reposMeta := make(RepoCatalogue)

	pageIdx := 1
	bar := formatter.CreateProgressbar(-1, "retrieving recipe repos list from git.coopcloud.tech...")
	for {
		var reposList []RepoMeta

		pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx)

		log.Debugf("fetching repo metadata from %s", pagedURL)

		if err := web.ReadJSON(pagedURL, &reposList); err != nil {
			return reposMeta, err
		}

		if len(reposList) == 0 {
			bar.Add(1)
			break
		}

		for idx, repo := range reposList {
			var topicMeta TopicMeta

			topicsURL := getReposTopicUrl(repo.Name)
			if err := web.ReadJSON(topicsURL, &topicMeta); err != nil {
				return reposMeta, err
			}

			if slices.Contains(topicMeta.Topics, "recipe") && repo.Name != "example" {
				reposMeta[repo.Name] = reposList[idx]
			}
		}

		pageIdx++
		bar.Add(1)
	}

	fmt.Println() // newline for spinner

	return reposMeta, nil
}

// GetRecipeVersions retrieves all recipe versions.
func GetRecipeVersions(recipeName string, offline bool) (RecipeVersions, error) {
	versions := RecipeVersions{}
	recipeDir := path.Join(config.RECIPES_DIR, recipeName)

	log.Debugf("attempting to open git repository in %s", recipeDir)

	repo, err := git.PlainOpen(recipeDir)
	if err != nil {
		return versions, err
	}

	worktree, err := repo.Worktree()
	if err != nil {
		return versions, err
	}

	gitTags, err := repo.Tags()
	if err != nil {
		return versions, err
	}

	if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
		tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/")

		log.Debugf("processing %s for %s", tag, recipeName)

		checkOutOpts := &git.CheckoutOptions{
			Create: false,
			Force:  true,
			Branch: plumbing.ReferenceName(ref.Name()),
		}
		if err := worktree.Checkout(checkOutOpts); err != nil {
			log.Debugf("failed to check out %s in %s", tag, recipeDir)
			return err
		}

		log.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir)

		recipe, err := Get(recipeName, offline)
		if err != nil {
			return err
		}

		versionMeta := make(map[string]ServiceMeta)
		for _, service := range recipe.Config.Services {

			img, err := reference.ParseNormalizedNamed(service.Image)
			if err != nil {
				return err
			}

			path := reference.Path(img)

			path = formatter.StripTagMeta(path)

			var tag string
			switch img.(type) {
			case reference.NamedTagged:
				tag = img.(reference.NamedTagged).Tag()
			case reference.Named:
				log.Warnf("%s service is missing image tag?", path)
				continue
			}

			versionMeta[service.Name] = ServiceMeta{
				Image: path,
				Tag:   tag,
			}
		}

		versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta})

		return nil
	}); err != nil {
		return versions, err
	}

	_, err = gitPkg.CheckoutDefaultBranch(repo, recipeDir)
	if err != nil {
		return versions, err
	}

	sortRecipeVersions(versions)

	log.Debugf("collected %s for %s", versions, recipeName)

	return versions, nil
}

// sortRecipeVersions sorts the recipe semver versions
func sortRecipeVersions(versions RecipeVersions) {
	sort.Slice(versions, func(i, j int) bool {
		version1, err := tagcmp.Parse(getVersionString(versions[i]))
		if err != nil {
			panic(err)
		}
		version2, err := tagcmp.Parse(getVersionString(versions[j]))
		if err != nil {
			panic(err)
		}
		return version1.IsLessThan(version2)
	})
}

// getVersionString returns the version string from RecipeVersions
func getVersionString(versionMap map[string]map[string]ServiceMeta) string {
	// Assuming there's only one key in versionMap
	for k := range versionMap {
		return k
	}
	return ""
}

// sortVersionStrings sorts a list of semver version strings
func sortVersionStrings(versions []string) {
	sort.Slice(versions, func(i, j int) bool {
		version1, err := tagcmp.Parse(versions[i])
		if err != nil {
			panic(err)
		}
		version2, err := tagcmp.Parse(versions[j])
		if err != nil {
			panic(err)
		}
		return version1.IsLessThan(version2)
	})
}

// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) {
	var versions []string

	if recipeMeta, exists := catl[recipeName]; exists {
		for _, versionMeta := range recipeMeta.Versions {
			for tag := range versionMeta {
				versions = append(versions, tag)
			}
		}
	}

	sortVersionStrings(versions)

	return versions, nil
}

// UpdateRepositories clones and updates all recipe repositories locally.
func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
	var barLength int
	if recipeName != "" {
		barLength = 1
	} else {
		barLength = len(repos)
	}

	cloneLimiter := limit.New(10)

	retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...")
	ch := make(chan string, barLength)
	for _, repoMeta := range repos {
		go func(rm RepoMeta) {
			cloneLimiter.Begin()
			defer cloneLimiter.End()

			if recipeName != "" && recipeName != rm.Name {
				ch <- rm.Name
				retrieveBar.Add(1)
				return
			}

			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
				log.Fatal(err)
			}

			ch <- rm.Name
			retrieveBar.Add(1)
		}(repoMeta)
	}

	for range repos {
		<-ch // wait for everything
	}

	return nil
}

// getReposTopicUrl retrieves the repository specific topic listing.
func getReposTopicUrl(repoName string) string {
	return fmt.Sprintf("https://git.coopcloud.tech/api/v1/repos/coop-cloud/%s/topics", repoName)
}

// ensurePathExists ensures that a path exists.
func ensurePathExists(path string) error {
	if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
		return err
	}
	return nil
}

// GetComposeFiles gets the list of compose files for an app (or recipe if you
// don't already have an app) which should be merged into a composetypes.Config
// while respecting the COMPOSE_FILE env var.
func GetComposeFiles(recipe string, appEnv map[string]string) ([]string, error) {
	var composeFiles []string

	composeFileEnvVar, ok := appEnv["COMPOSE_FILE"]
	if !ok {
		path := fmt.Sprintf("%s/%s/compose.yml", config.RECIPES_DIR, recipe)
		if err := ensurePathExists(path); err != nil {
			return composeFiles, err
		}
		log.Debugf("no COMPOSE_FILE detected, loading default: %s", path)
		composeFiles = append(composeFiles, path)
		return composeFiles, nil
	}

	if !strings.Contains(composeFileEnvVar, ":") {
		path := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipe, composeFileEnvVar)
		if err := ensurePathExists(path); err != nil {
			return composeFiles, err
		}
		log.Debugf("COMPOSE_FILE detected, loading %s", path)
		composeFiles = append(composeFiles, path)
		return composeFiles, nil
	}

	numComposeFiles := strings.Count(composeFileEnvVar, ":") + 1
	envVars := strings.SplitN(composeFileEnvVar, ":", numComposeFiles)
	if len(envVars) != numComposeFiles {
		return composeFiles, fmt.Errorf("COMPOSE_FILE (=\"%s\") parsing failed?", composeFileEnvVar)
	}

	for _, file := range envVars {
		path := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, recipe, file)
		if err := ensurePathExists(path); err != nil {
			return composeFiles, err
		}
		composeFiles = append(composeFiles, path)
	}

	log.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", "))
	log.Debugf("retrieved %s configs for %s", strings.Join(composeFiles, ", "), recipe)

	return composeFiles, nil
}