abra/pkg/recipe/recipe.go

package recipe

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"

	"coopcloud.tech/abra/pkg/catalogue"
	"coopcloud.tech/abra/pkg/config"
	"coopcloud.tech/abra/pkg/formatter"
	gitPkg "coopcloud.tech/abra/pkg/git"
	"coopcloud.tech/abra/pkg/limit"
	"coopcloud.tech/abra/pkg/log"
	"coopcloud.tech/abra/pkg/web"
	"coopcloud.tech/tagcmp"
	"github.com/go-git/go-git/v5"
)

// RecipeCatalogueURL is the only current recipe catalogue available.
const RecipeCatalogueURL = "https://recipes.coopcloud.tech/recipes.json"

// ReposMetadataURL is the recipe repository metadata.
const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"

// tag represents a git tag.
type tag = string

// service represents a service within a recipe.
type service = string

// ServiceMeta represents meta info associated with a service.
type ServiceMeta struct {
	Image string `json:"image"`
	Tag   string `json:"tag"`
}

// RecipeVersions are the versions associated with a recipe.
type RecipeVersions []map[tag]map[service]ServiceMeta

// RecipeMeta represents metadata for a recipe in the abra catalogue.
type RecipeMeta struct {
	Category      string         `json:"category"`
	DefaultBranch string         `json:"default_branch"`
	Description   string         `json:"description"`
	Features      Features       `json:"features"`
	Icon          string         `json:"icon"`
	Name          string         `json:"name"`
	Repository    string         `json:"repository"`
	SSHURL        string         `json:"ssh_url"`
	Versions      RecipeVersions `json:"versions"`
	Website       string         `json:"website"`
}

// LatestVersion returns the latest version of a recipe.
func (r RecipeMeta) LatestVersion() string {
	var version string

	// apps.json versions are sorted so the last key is latest
	latest := r.Versions[len(r.Versions)-1]

	for tag := range latest {
		version = tag
	}

	log.Debugf("choosing %s as latest version of %s", version, r.Name)

	return version
}

// Name represents a recipe name.
type Name = string

// RecipeCatalogue represents the entire recipe catalogue.
type RecipeCatalogue map[Name]RecipeMeta

// Flatten converts AppCatalogue to slice
func (r RecipeCatalogue) Flatten() []RecipeMeta {
	recipes := make([]RecipeMeta, 0, len(r))

	for name := range r {
		recipes = append(recipes, r[name])
	}

	return recipes
}

// ByRecipeName sorts recipes by name.
type ByRecipeName []RecipeMeta

func (r ByRecipeName) Len() int      { return len(r) }
func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
func (r ByRecipeName) Less(i, j int) bool {
	return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name)
}

// Image represents a recipe container image.
type Image struct {
	Image  string `json:"image"`
	Rating string `json:"rating"`
	Source string `json:"source"`
	URL    string `json:"url"`
}

// Features represent what top-level features a recipe supports (e.g. does this recipe support backups?).
type Features struct {
	Backups     string `json:"backups"`
	Email       string `json:"email"`
	Healthcheck string `json:"healthcheck"`
	Image       Image  `json:"image"`
	Status      int    `json:"status"`
	Tests       string `json:"tests"`
	SSO         string `json:"sso"`
}

func GetEnvVersionRaw(name string) (string, error) {
	var version string

	if strings.Contains(name, ":") {
		split := strings.Split(name, ":")
		if len(split) > 2 {
			return version, fmt.Errorf("version seems invalid: %s", name)
		}
		version = split[1]
	}

	return version, nil
}

func Get(name string) Recipe {
	version := ""
	if strings.Contains(name, ":") {
		split := strings.Split(name, ":")
		if len(split) > 2 {
			log.Fatalf("version seems invalid: %s", name)
		}
		name = split[0]

		version = split[1]
		if strings.HasSuffix(version, config.DIRTY_DEFAULT) {
			version = strings.Replace(split[1], config.DIRTY_DEFAULT, "", 1)
			log.Debugf("removed dirty suffix from .env version: %s -> %s", split[1], version)
		}
	}

	gitURL := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, name)
	sshURL := fmt.Sprintf(config.RECIPES_SSH_URL_TEMPLATE, name)
	if strings.Contains(name, "/") {
		u, err := url.Parse(name)
		if err != nil {
			log.Fatalf("invalid recipe: %s", err)
		}
		u.Scheme = "https"
		gitURL = u.String() + ".git"

		u.Scheme = "ssh"
		u.User = url.User("git")
		sshURL = u.String() + ".git"
	}

	dir := path.Join(config.RECIPES_DIR, escapeRecipeName(name))

	r := Recipe{
		Name:       name,
		EnvVersion: version,
		Dir:        dir,
		GitURL:     gitURL,
		SSHURL:     sshURL,

		ComposePath:   path.Join(dir, "compose.yml"),
		ReadmePath:    path.Join(dir, "README.md"),
		SampleEnvPath: path.Join(dir, ".env.sample"),
		AbraShPath:    path.Join(dir, "abra.sh"),
	}

	if err := r.IsDirty(); err != nil && !errors.Is(err, git.ErrRepositoryNotExists) {
		log.Fatalf("failed to check git status of %s: %s", r.Name, err)
	}

	return r
}

type Recipe struct {
	Name       string
	EnvVersion string
	Dirty      bool // NOTE(d1): git terminology for unstaged changes
	Dir        string
	GitURL     string
	SSHURL     string

	ComposePath   string
	ReadmePath    string
	SampleEnvPath string
	AbraShPath    string
}

// String outputs a human-friendly string representation.
func (r Recipe) String() string {
	out := fmt.Sprintf("{name: %s, ", r.Name)
	out += fmt.Sprintf("version : %s, ", r.EnvVersion)
	out += fmt.Sprintf("dirty: %v, ", r.Dirty)
	out += fmt.Sprintf("dir: %s, ", r.Dir)
	out += fmt.Sprintf("git url: %s, ", r.GitURL)
	out += fmt.Sprintf("ssh url: %s, ", r.SSHURL)
	out += fmt.Sprintf("compose: %s, ", r.ComposePath)
	out += fmt.Sprintf("readme: %s, ", r.ReadmePath)
	out += fmt.Sprintf("sample env: %s, ", r.SampleEnvPath)
	out += fmt.Sprintf("abra.sh: %s}", r.AbraShPath)
	return out
}

func escapeRecipeName(recipeName string) string {
	recipeName = strings.ReplaceAll(recipeName, "/", "_")
	recipeName = strings.ReplaceAll(recipeName, ".", "_")
	return recipeName
}

// GetRecipesLocal retrieves all local recipe directories
func GetRecipesLocal() ([]string, error) {
	var recipes []string

	recipes, err := config.GetAllFoldersInDirectory(config.RECIPES_DIR)
	if err != nil {
		return recipes, err
	}

	return recipes, nil
}

func GetRecipeFeaturesAndCategory(r Recipe) (Features, string, []string, error) {
	var (
		category string
		warnMsgs []string
		feat     = Features{}
	)

	log.Debugf("%s: attempt recipe metadata parse", r.ReadmePath)

	readmeFS, err := ioutil.ReadFile(r.ReadmePath)
	if err != nil {
		return feat, category, warnMsgs, err
	}

	readmeMetadata, err := GetStringInBetween( // Find text between delimiters
		r.Name,
		string(readmeFS),
		"<!-- metadata -->", "<!-- endmetadata -->",
	)
	if err != nil {
		return feat, category, warnMsgs, err
	}

	readmeLines := strings.Split( // Array item from lines
		strings.ReplaceAll( // Remove \t tabs
			readmeMetadata, "\t", "",
		),
		"\n")

	for _, val := range readmeLines {
		if strings.Contains(val, "**Status**") {
			feat.Status, _ = strconv.Atoi(strings.TrimSpace(strings.Split(strings.TrimPrefix(val, "* **Status**:"), ",")[0]))
		}
		if strings.Contains(val, "**Category**") {
			category = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Category**:"),
			)
		}
		if strings.Contains(val, "**Backups**") {
			feat.Backups = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Backups**:"),
			)
		}
		if strings.Contains(val, "**Email**") {
			feat.Email = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Email**:"),
			)
		}
		if strings.Contains(val, "**SSO**") {
			feat.SSO = strings.TrimSpace(
				strings.TrimPrefix(val, "* **SSO**:"),
			)
		}
		if strings.Contains(val, "**Healthcheck**") {
			feat.Healthcheck = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Healthcheck**:"),
			)
		}
		if strings.Contains(val, "**Tests**") {
			feat.Tests = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Tests**:"),
			)
		}
		if strings.Contains(val, "**Image**") {
			imageMetadata, warnings, err := GetImageMetadata(strings.TrimSpace(
				strings.TrimPrefix(val, "* **Image**:"),
			), r.Name)
			if err != nil {
				continue
			}
			if len(warnings) > 0 {
				warnMsgs = append(warnMsgs, warnings...)
			}
			feat.Image = imageMetadata
		}
	}

	return feat, category, warnMsgs, nil
}

func GetImageMetadata(imageRowString, recipeName string) (Image, []string, error) {
	var warnMsgs []string

	img := Image{}

	imgFields := strings.Split(imageRowString, ",")

	for i, elem := range imgFields {
		imgFields[i] = strings.TrimSpace(elem)
	}

	if len(imgFields) < 3 {
		if imageRowString != "" {
			warnMsgs = append(
				warnMsgs,
				fmt.Sprintf("%s: image meta has incorrect format: %s", recipeName, imageRowString),
			)
		} else {
			warnMsgs = append(
				warnMsgs,
				fmt.Sprintf("%s: image meta is empty?", recipeName),
			)
		}

		return img, warnMsgs, nil
	}

	img.Rating = imgFields[1]
	img.Source = imgFields[2]

	imgString := imgFields[0]

	imageName, err := GetStringInBetween(recipeName, imgString, "[", "]")
	if err != nil {
		return img, warnMsgs, err
	}
	img.Image = strings.ReplaceAll(imageName, "`", "")

	imageURL, err := GetStringInBetween(recipeName, imgString, "(", ")")
	if err != nil {
		return img, warnMsgs, err
	}
	img.URL = imageURL

	return img, warnMsgs, nil
}

// GetStringInBetween returns empty string if no start or end string found
func GetStringInBetween(recipeName, str, start, end string) (result string, err error) {
	s := strings.Index(str, start)
	if s == -1 {
		return "", fmt.Errorf("%s: marker string %s not found", recipeName, start)
	}

	s += len(start)
	e := strings.Index(str[s:], end)

	if e == -1 {
		return "", fmt.Errorf("%s: end marker %s not found", recipeName, end)
	}

	return str[s : s+e], nil
}

// ReadRecipeCatalogue reads the recipe catalogue.
func ReadRecipeCatalogue(offline bool) (RecipeCatalogue, error) {
	recipes := make(RecipeCatalogue)

	if err := catalogue.EnsureCatalogue(); err != nil {
		return nil, err
	}

	if !offline {
		if err := catalogue.EnsureUpToDate(); err != nil {
			return nil, err
		}
	}

	if err := readRecipeCatalogueFS(&recipes); err != nil {
		return nil, err
	}

	return recipes, nil
}

// readRecipeCatalogueFS reads the catalogue from the file system.
func readRecipeCatalogueFS(target interface{}) error {
	recipesJSONFS, err := ioutil.ReadFile(config.RECIPES_JSON)
	if err != nil {
		return err
	}

	if err := json.Unmarshal(recipesJSONFS, &target); err != nil {
		return err
	}

	log.Debugf("read recipe catalogue from file system cache in %s", config.RECIPES_JSON)

	return nil
}

// VersionsOfService lists the version of a service.
func VersionsOfService(recipe, serviceName string, offline bool) ([]string, error) {
	var versions []string

	catalogue, err := ReadRecipeCatalogue(offline)
	if err != nil {
		return nil, err
	}

	rec, ok := catalogue[recipe]
	if !ok {
		return versions, nil
	}

	alreadySeen := make(map[string]bool)
	for _, serviceVersion := range rec.Versions {
		for tag := range serviceVersion {
			if _, ok := alreadySeen[tag]; !ok {
				alreadySeen[tag] = true
				versions = append(versions, tag)
			}
		}
	}

	log.Debugf("detected versions %s for %s", strings.Join(versions, ", "), recipe)

	return versions, nil
}

// RecipeMissingFromCatalogue signifies a recipe is not present in the catalogue.
type RecipeMissingFromCatalogue struct{ err string }

// Error outputs the error message.
func (r RecipeMissingFromCatalogue) Error() string {
	return r.err
}

// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
func GetRecipeMeta(recipeName string, offline bool) (RecipeMeta, error) {
	catl, err := ReadRecipeCatalogue(offline)
	if err != nil {
		return RecipeMeta{}, err
	}

	recipeMeta, ok := catl[recipeName]
	if !ok {
		return RecipeMeta{}, RecipeMissingFromCatalogue{
			err: fmt.Sprintf("recipe %s does not exist?", recipeName),
		}
	}

	log.Debugf("recipe metadata retrieved for %s", recipeName)

	return recipeMeta, nil
}

// RepoMeta is a single recipe repo metadata.
type RepoMeta struct {
	ID                        int `json:"id"`
	Owner                     Owner
	Name                      string      `json:"name"`
	FullName                  string      `json:"full_name"`
	Description               string      `json:"description"`
	Empty                     bool        `json:"empty"`
	Private                   bool        `json:"private"`
	Fork                      bool        `json:"fork"`
	Template                  bool        `json:"template"`
	Parent                    interface{} `json:"parent"`
	Mirror                    bool        `json:"mirror"`
	Size                      int         `json:"size"`
	HTMLURL                   string      `json:"html_url"`
	SSHURL                    string      `json:"ssh_url"`
	CloneURL                  string      `json:"clone_url"`
	OriginalURL               string      `json:"original_url"`
	Website                   string      `json:"website"`
	StarsCount                int         `json:"stars_count"`
	ForksCount                int         `json:"forks_count"`
	WatchersCount             int         `json:"watchers_count"`
	OpenIssuesCount           int         `json:"open_issues_count"`
	OpenPRCount               int         `json:"open_pr_counter"`
	ReleaseCounter            int         `json:"release_counter"`
	DefaultBranch             string      `json:"default_branch"`
	Archived                  bool        `json:"archived"`
	CreatedAt                 string      `json:"created_at"`
	UpdatedAt                 string      `json:"updated_at"`
	Permissions               Permissions
	HasIssues                 bool `json:"has_issues"`
	InternalTracker           InternalTracker
	HasWiki                   bool   `json:"has_wiki"`
	HasPullRequests           bool   `json:"has_pull_requests"`
	HasProjects               bool   `json:"has_projects"`
	IgnoreWhitespaceConflicts bool   `json:"ignore_whitespace_conflicts"`
	AllowMergeCommits         bool   `json:"allow_merge_commits"`
	AllowRebase               bool   `json:"allow_rebase"`
	AllowRebaseExplicit       bool   `json:"allow_rebase_explicit"`
	AllowSquashMerge          bool   `json:"allow_squash_merge"`
	AvatarURL                 string `json:"avatar_url"`
	Internal                  bool   `json:"internal"`
	MirrorInterval            string `json:"mirror_interval"`
}

// Owner is the repo organisation owner metadata.
type Owner struct {
	ID         int    `json:"id"`
	Login      string `json:"login"`
	FullName   string `json:"full_name"`
	Email      string `json:"email"`
	AvatarURL  string `json:"avatar_url"`
	Language   string `json:"language"`
	IsAdmin    bool   `json:"is_admin"`
	LastLogin  string `json:"last_login"`
	Created    string `json:"created"`
	Restricted bool   `json:"restricted"`
	Username   string `json:"username"`
}

// Permissions is perms metadata for a repo.
type Permissions struct {
	Admin bool `json:"admin"`
	Push  bool `json:"push"`
	Pull  bool `json:"pull"`
}

// InternalTracker is issue tracker metadata for a repo.
type InternalTracker struct {
	EnableTimeTracker                bool `json:"enable_time_tracker"`
	AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"`
	EnableIssuesDependencies         bool `json:"enable_issue_dependencies"`
}

// RepoCatalogue represents all the recipe repo metadata.
type RepoCatalogue map[string]RepoMeta

// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
func ReadReposMetadata(debug bool) (RepoCatalogue, error) {
	reposMeta := make(RepoCatalogue)

	pageIdx := 1
	bar := formatter.CreateProgressbar(-1, "collecting recipe listing")
	for {
		var reposList []RepoMeta

		pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx)

		log.Debugf("fetching repo metadata from %s", pagedURL)

		if err := web.ReadJSON(pagedURL, &reposList); err != nil {
			return reposMeta, err
		}

		if len(reposList) == 0 {
			if !debug {
				bar.Add(1)
			}
			break
		}

		for idx, repo := range reposList {
			// NOTE(d1): the "example" recipe is a temporary special case
			// https://git.coopcloud.tech/toolshed/organising/issues/666
			if repo.Name == "example" {
				continue
			}

			reposMeta[repo.Name] = reposList[idx]
		}

		pageIdx++

		if !debug {
			bar.Add(1)
		}
	}

	if err := bar.Close(); err != nil {
		return reposMeta, err
	}

	return reposMeta, nil
}

// sortRecipeVersions sorts the recipe semver versions
func sortRecipeVersions(versions RecipeVersions) {
	sort.Slice(versions, func(i, j int) bool {
		version1, err := tagcmp.Parse(getVersionString(versions[i]))
		if err != nil {
			panic(err)
		}
		version2, err := tagcmp.Parse(getVersionString(versions[j]))
		if err != nil {
			panic(err)
		}
		return version1.IsLessThan(version2)
	})
}

// getVersionString returns the version string from RecipeVersions
func getVersionString(versionMap map[string]map[string]ServiceMeta) string {
	// Assuming there's only one key in versionMap
	for k := range versionMap {
		return k
	}
	return ""
}

// sortVersionStrings sorts a list of semver version strings
func sortVersionStrings(versions []string) {
	sort.Slice(versions, func(i, j int) bool {
		version1, err := tagcmp.Parse(versions[i])
		if err != nil {
			panic(err)
		}
		version2, err := tagcmp.Parse(versions[j])
		if err != nil {
			panic(err)
		}
		return version1.IsLessThan(version2)
	})
}

// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) {
	var versions []string

	if recipeMeta, exists := catl[recipeName]; exists {
		for _, versionMeta := range recipeMeta.Versions {
			for tag := range versionMeta {
				versions = append(versions, tag)
			}
		}
	}

	sortVersionStrings(versions)

	return versions, nil
}

// UpdateRepositories clones and updates all recipe repositories locally.
func UpdateRepositories(repos RepoCatalogue, recipeName string, debug bool) error {
	var barLength int
	if recipeName != "" {
		barLength = 1
	} else {
		barLength = len(repos)
	}

	cloneLimiter := limit.New(3)

	retrieveBar := formatter.CreateProgressbar(barLength, "retrieving recipes")
	ch := make(chan string, barLength)
	for _, repoMeta := range repos {
		go func(rm RepoMeta) {
			cloneLimiter.Begin()
			defer cloneLimiter.End()

			if recipeName != "" && recipeName != rm.Name {
				ch <- rm.Name
				if !debug {
					retrieveBar.Add(1)
				}
				return
			}

			if err := gitPkg.Clone(Get(rm.Name).Dir, rm.CloneURL); err != nil {
				log.Fatal(err)
			}

			ch <- rm.Name
			if !debug {
				retrieveBar.Add(1)
			}
		}(repoMeta)
	}

	for range repos {
		<-ch // wait for everything
	}

	if err := retrieveBar.Close(); err != nil {
		return err
	}

	return nil
}

// ensurePathExists ensures that a path exists.
func ensurePathExists(path string) error {
	if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
		return err
	}
	return nil
}