abra/pkg/catalogue/catalogue.go

// Package catalogue provides ways of interacting with recipe catalogues which
// are JSON data structures which contain meta information about recipes (e.g.
// what versions of the Nextcloud recipe are available?).
package catalogue

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"time"

	"coopcloud.tech/abra/cli/formatter"
	"coopcloud.tech/abra/pkg/client"
	"coopcloud.tech/abra/pkg/config"
	"coopcloud.tech/abra/pkg/recipe"
	"coopcloud.tech/abra/pkg/web"
	"github.com/docker/distribution/reference"
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing"
	"github.com/sirupsen/logrus"
)

// RecipeCatalogueURL is the only current recipe catalogue available.
const RecipeCatalogueURL = "https://apps.coopcloud.tech"

// ReposMetadataURL is the recipe repository metadata
const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"

// image represents a recipe container image.
type image struct {
	Image  string `json:"image"`
	Rating string `json:"rating"`
	Source string `json:"source"`
	URL    string `json:"url"`
}

// features represent what top-level features a recipe supports (e.g. does this
// recipe support backups?).
type features struct {
	Backups     string `json:"backups"`
	Email       string `json:"email"`
	Healthcheck string `json:"healthcheck"`
	Image       image  `json:"image"`
	Status      int    `json:"status"`
	Tests       string `json:"tests"`
	SSO         string `json:"sso"`
}

// tag represents a git tag.
type tag = string

// service represents a service within a recipe.
type service = string

// ServiceMeta represents meta info associated with a service.
type ServiceMeta struct {
	Digest string `json:"digest"`
	Image  string `json:"image"`
	Tag    string `json:"tag"`
}

// RecipeVersions are the versions associated with a recipe.
type RecipeVersions []map[tag]map[service]ServiceMeta

// RecipeMeta represents metadata for a recipe in the abra catalogue.
type RecipeMeta struct {
	Category      string         `json:"category"`
	DefaultBranch string         `json:"default_branch"`
	Description   string         `json:"description"`
	Features      features       `json:"features"`
	Icon          string         `json:"icon"`
	Name          string         `json:"name"`
	Repository    string         `json:"repository"`
	Versions      RecipeVersions `json:"versions"`
	Website       string         `json:"website"`
}

// LatestVersion returns the latest version of a recipe.
func (r RecipeMeta) LatestVersion() string {
	var version string

	// apps.json versions are sorted so the last key is latest
	latest := r.Versions[len(r.Versions)-1]

	for tag := range latest {
		version = tag
	}

	logrus.Debugf("choosing %s as latest version of %s", version, r.Name)

	return version
}

// Name represents a recipe name.
type Name = string

// RecipeCatalogue represents the entire recipe catalogue.
type RecipeCatalogue map[Name]RecipeMeta

// Flatten converts AppCatalogue to slice
func (r RecipeCatalogue) Flatten() []RecipeMeta {
	recipes := make([]RecipeMeta, 0, len(r))

	for name := range r {
		recipes = append(recipes, r[name])
	}

	return recipes
}

// ByRecipeName sorts recipes by name.
type ByRecipeName []RecipeMeta

func (r ByRecipeName) Len() int      { return len(r) }
func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
func (r ByRecipeName) Less(i, j int) bool {
	return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name)
}

// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
// is up to date.
func recipeCatalogueFSIsLatest() (bool, error) {
	httpClient := web.NewHTTPRetryClient()
	res, err := httpClient.Head(RecipeCatalogueURL)
	if err != nil {
		return false, err
	}

	lastModified := res.Header["Last-Modified"][0]
	parsed, err := time.Parse(time.RFC1123, lastModified)
	if err != nil {
		return false, err
	}

	info, err := os.Stat(config.APPS_JSON)
	if err != nil {
		if os.IsNotExist(err) {
			logrus.Debugf("no recipe catalogue found in file system cache")
			return false, nil
		}
		return false, err
	}

	localModifiedTime := info.ModTime().Unix()
	remoteModifiedTime := parsed.Unix()

	if localModifiedTime < remoteModifiedTime {
		logrus.Debug("file system cached recipe catalogue is out-of-date")
		return false, nil
	}

	logrus.Debug("file system cached recipe catalogue is now up-to-date")

	return true, nil
}

// ReadRecipeCatalogue reads the recipe catalogue.
func ReadRecipeCatalogue() (RecipeCatalogue, error) {
	recipes := make(RecipeCatalogue)

	recipeFSIsLatest, err := recipeCatalogueFSIsLatest()
	if err != nil {
		return nil, err
	}

	if !recipeFSIsLatest {
		logrus.Debugf("reading recipe catalogue from web to get latest")
		if err := readRecipeCatalogueWeb(&recipes); err != nil {
			return nil, err
		}
		return recipes, nil
	}

	logrus.Debugf("reading recipe catalogue from file system cache to get latest")
	if err := readRecipeCatalogueFS(&recipes); err != nil {
		return nil, err
	}

	return recipes, nil
}

// readRecipeCatalogueFS reads the catalogue from the file system.
func readRecipeCatalogueFS(target interface{}) error {
	recipesJSONFS, err := ioutil.ReadFile(config.APPS_JSON)
	if err != nil {
		return err
	}

	if err := json.Unmarshal(recipesJSONFS, &target); err != nil {
		return err
	}

	logrus.Debugf("read recipe catalogue from file system cache in %s", config.APPS_JSON)

	return nil
}

// readRecipeCatalogueWeb reads the catalogue from the web.
func readRecipeCatalogueWeb(target interface{}) error {
	if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil {
		return err
	}

	recipesJSON, err := json.MarshalIndent(target, "", "    ")
	if err != nil {
		return err
	}

	if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0764); err != nil {
		return err
	}

	logrus.Debugf("read recipe catalogue from web at %s", RecipeCatalogueURL)

	return nil
}

// VersionsOfService lists the version of a service.
func VersionsOfService(recipe, serviceName string) ([]string, error) {
	var versions []string

	catalogue, err := ReadRecipeCatalogue()
	if err != nil {
		return nil, err
	}

	rec, ok := catalogue[recipe]
	if !ok {
		return versions, nil
	}

	alreadySeen := make(map[string]bool)
	for _, serviceVersion := range rec.Versions {
		for tag := range serviceVersion {
			if _, ok := alreadySeen[tag]; !ok {
				alreadySeen[tag] = true
				versions = append(versions, tag)
			}
		}
	}

	logrus.Debugf("detected versions %s for %s", strings.Join(versions, ", "), recipe)

	return versions, nil
}

// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
	catl, err := ReadRecipeCatalogue()
	if err != nil {
		return RecipeMeta{}, err
	}

	recipeMeta, ok := catl[recipeName]
	if !ok {
		err := fmt.Errorf("recipe %s does not exist?", recipeName)
		return RecipeMeta{}, err
	}

	if err := recipe.EnsureExists(recipeName); err != nil {
		return RecipeMeta{}, err
	}

	logrus.Debugf("recipe metadata retrieved for %s", recipeName)

	return recipeMeta, nil
}

// RepoMeta is a single recipe repo metadata.
type RepoMeta struct {
	ID                        int `json:"id"`
	Owner                     Owner
	Name                      string      `json:"name"`
	FullName                  string      `json:"full_name"`
	Description               string      `json:"description"`
	Empty                     bool        `json:"empty"`
	Private                   bool        `json:"private"`
	Fork                      bool        `json:"fork"`
	Template                  bool        `json:"template"`
	Parent                    interface{} `json:"parent"`
	Mirror                    bool        `json:"mirror"`
	Size                      int         `json:"size"`
	HTMLURL                   string      `json:"html_url"`
	SSHURL                    string      `json:"ssh_url"`
	CloneURL                  string      `json:"clone_url"`
	OriginalURL               string      `json:"original_url"`
	Website                   string      `json:"website"`
	StarsCount                int         `json:"stars_count"`
	ForksCount                int         `json:"forks_count"`
	WatchersCount             int         `json:"watchers_count"`
	OpenIssuesCount           int         `json:"open_issues_count"`
	OpenPRCount               int         `json:"open_pr_counter"`
	ReleaseCounter            int         `json:"release_counter"`
	DefaultBranch             string      `json:"default_branch"`
	Archived                  bool        `json:"archived"`
	CreatedAt                 string      `json:"created_at"`
	UpdatedAt                 string      `json:"updated_at"`
	Permissions               Permissions
	HasIssues                 bool `json:"has_issues"`
	InternalTracker           InternalTracker
	HasWiki                   bool   `json:"has_wiki"`
	HasPullRequests           bool   `json:"has_pull_requests"`
	HasProjects               bool   `json:"has_projects"`
	IgnoreWhitespaceConflicts bool   `json:"ignore_whitespace_conflicts"`
	AllowMergeCommits         bool   `json:"allow_merge_commits"`
	AllowRebase               bool   `json:"allow_rebase"`
	AllowRebaseExplicit       bool   `json:"allow_rebase_explicit"`
	AllowSquashMerge          bool   `json:"allow_squash_merge"`
	AvatarURL                 string `json:"avatar_url"`
	Internal                  bool   `json:"internal"`
	MirrorInterval            string `json:"mirror_interval"`
}

// Owner is the repo organisation owner metadata.
type Owner struct {
	ID         int    `json:"id"`
	Login      string `json:"login"`
	FullName   string `json:"full_name"`
	Email      string `json:"email"`
	AvatarURL  string `json:"avatar_url"`
	Language   string `json:"language"`
	IsAdmin    bool   `json:"is_admin"`
	LastLogin  string `json:"last_login"`
	Created    string `json:"created"`
	Restricted bool   `json:"restricted"`
	Username   string `json:"username"`
}

// Permissions is perms metadata for a repo.
type Permissions struct {
	Admin bool `json:"admin"`
	Push  bool `json:"push"`
	Pull  bool `json:"pull"`
}

// InternalTracker is issue tracker metadata for a repo.
type InternalTracker struct {
	EnableTimeTracker                bool `json:"enable_time_tracker"`
	AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"`
	EnableIssuesDependencies         bool `json:"enable_issue_dependencies"`
}

// RepoCatalogue represents all the recipe repo metadata.
type RepoCatalogue map[string]RepoMeta

// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
func ReadReposMetadata() (RepoCatalogue, error) {
	reposMeta := make(RepoCatalogue)

	pageIdx := 1
	bar := formatter.CreateProgressbar(-1, "retrieving recipe repos list from git.coopcloud.tech...")
	for {
		var reposList []RepoMeta

		pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx)

		logrus.Debugf("fetching repo metadata from %s", pagedURL)

		if err := web.ReadJSON(pagedURL, &reposList); err != nil {
			return reposMeta, err
		}

		if len(reposList) == 0 {
			bar.Add(1)
			break
		}

		for idx, repo := range reposList {
			reposMeta[repo.Name] = reposList[idx]
		}

		pageIdx++
		bar.Add(1)
	}

	return reposMeta, nil
}

func GetStringInBetween(str, start, end string) (result string, err error) {
	// GetStringInBetween returns empty string if no start or end string found
	s := strings.Index(str, start)
	if s == -1 {
		return "", fmt.Errorf("marker string '%s' not found", start)
	}
	s += len(start)
	e := strings.Index(str[s:], end)
	if e == -1 {
		return "", fmt.Errorf("end marker '%s' not found", end)
	}
	return str[s : s+e], nil
}

func GetImageMetadata(imageRowString, recipeName string) (image, error) {
	img := image{}

	imgFields := strings.Split(imageRowString, ",")

	for i, elem := range imgFields {
		imgFields[i] = strings.TrimSpace(elem)
	}

	if len(imgFields) < 3 {
		if imageRowString != "" {
			logrus.Warnf("%s image meta has incorrect format: %s", recipeName, imageRowString)
		} else {
			logrus.Warnf("%s image meta is empty?", recipeName)
		}
		return img, nil
	}

	img.Rating = imgFields[1]
	img.Source = imgFields[2]

	imgString := imgFields[0]

	imageName, err := GetStringInBetween(imgString, "[", "]")
	if err != nil {
		logrus.Fatal(err)
	}
	img.Image = strings.ReplaceAll(imageName, "`", "")

	imageURL, err := GetStringInBetween(imgString, "(", ")")
	if err != nil {
		logrus.Fatal(err)
	}
	img.URL = imageURL

	return img, nil
}

func GetRecipeFeaturesAndCategory(recipeName string) (features, string, error) {
	feat := features{}

	var category string

	readmePath := path.Join(config.ABRA_DIR, "apps", recipeName, "README.md")

	logrus.Debugf("attempting to open %s for recipe metadata parsing", readmePath)

	readmeFS, err := ioutil.ReadFile(readmePath)
	if err != nil {
		return feat, category, err
	}

	readmeMetadata, err := GetStringInBetween( // Find text between delimiters
		string(readmeFS),
		"<!-- metadata -->", "<!-- endmetadata -->",
	)
	if err != nil {
		return feat, category, err
	}

	readmeLines := strings.Split( // Array item from lines
		strings.ReplaceAll( // Remove \t tabs
			readmeMetadata, "\t", "",
		),
		"\n")

	for _, val := range readmeLines {
		if strings.Contains(val, "**Category**") {
			category = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Category**:"),
			)
		}
		if strings.Contains(val, "**Backups**") {
			feat.Backups = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Backups**:"),
			)
		}
		if strings.Contains(val, "**Email**") {
			feat.Email = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Email**:"),
			)
		}
		if strings.Contains(val, "**SSO**") {
			feat.SSO = strings.TrimSpace(
				strings.TrimPrefix(val, "* **SSO**:"),
			)
		}
		if strings.Contains(val, "**Healthcheck**") {
			feat.Healthcheck = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Healthcheck**:"),
			)
		}
		if strings.Contains(val, "**Tests**") {
			feat.Tests = strings.TrimSpace(
				strings.TrimPrefix(val, "* **Tests**:"),
			)
		}
		if strings.Contains(val, "**Image**") {
			imageMetadata, err := GetImageMetadata(strings.TrimSpace(
				strings.TrimPrefix(val, "* **Image**:"),
			), recipeName)
			if err != nil {
				continue
			}
			feat.Image = imageMetadata
		}
	}

	return feat, category, nil
}

// GetRecipeVersions retrieves all recipe versions.
func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
	versions := RecipeVersions{}

	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)

	logrus.Debugf("attempting to open git repository in %s", recipeDir)

	repo, err := git.PlainOpen(recipeDir)
	if err != nil {
		return versions, err
	}

	worktree, err := repo.Worktree()
	if err != nil {
		logrus.Fatal(err)
	}

	gitTags, err := repo.Tags()
	if err != nil {
		return versions, err
	}

	if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
		tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/")

		logrus.Debugf("processing %s for %s", tag, recipeName)

		checkOutOpts := &git.CheckoutOptions{
			Create: false,
			Force:  true,
			Branch: plumbing.ReferenceName(ref.Name()),
		}
		if err := worktree.Checkout(checkOutOpts); err != nil {
			logrus.Debugf("failed to check out %s in %s", tag, recipeDir)
			return err
		}

		logrus.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir)

		recipe, err := recipe.Get(recipeName)
		if err != nil {
			return err
		}

		cl, err := client.New("default") // only required for docker.io registry calls
		if err != nil {
			logrus.Fatal(err)
		}

		versionMeta := make(map[string]ServiceMeta)
		for _, service := range recipe.Config.Services {

			img, err := reference.ParseNormalizedNamed(service.Image)
			if err != nil {
				return err
			}

			path := reference.Path(img)
			if strings.Contains(path, "library") {
				path = strings.Split(path, "/")[1]
			}

			var tag string
			switch img.(type) {
			case reference.NamedTagged:
				tag = img.(reference.NamedTagged).Tag()
			case reference.Named:
				logrus.Warnf("%s service is missing image tag?", path)
				continue
			}

			logrus.Debugf("looking up image: %s from %s", img, path)

			digest, err := client.GetTagDigest(cl, img)
			if err != nil {
				logrus.Warn(err)
				continue
			}

			versionMeta[service.Name] = ServiceMeta{
				Digest: digest,
				Image:  path,
				Tag:    img.(reference.NamedTagged).Tag(),
			}

			logrus.Debugf("collecting digest: %s, image: %s, tag: %s", digest, path, tag)
		}

		versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta})

		return nil
	}); err != nil {
		return versions, err
	}

	branch := "master"
	if _, err := repo.Branch("master"); err != nil {
		if _, err := repo.Branch("main"); err != nil {
			logrus.Debugf("failed to select branch in %s", recipeDir)
			logrus.Fatal(err)
		}
		branch = "main"
	}

	refName := fmt.Sprintf("refs/heads/%s", branch)
	checkOutOpts := &git.CheckoutOptions{
		Create: false,
		Force:  true,
		Branch: plumbing.ReferenceName(refName),
	}
	if err := worktree.Checkout(checkOutOpts); err != nil {
		logrus.Debugf("failed to check out %s in %s", branch, recipeDir)
		logrus.Fatal(err)
	}

	logrus.Debugf("switched back to %s in %s", branch, recipeDir)
	logrus.Debugf("collected %s for %s", versions, recipeName)

	return versions, nil
}

// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) {
	var versions []string

	if recipeMeta, exists := catl[recipeName]; exists {
		for _, versionMeta := range recipeMeta.Versions {
			for tag := range versionMeta {
				versions = append(versions, tag)
			}
		}
	}

	return versions, nil
}