forked from coop-cloud/abra
643 lines
17 KiB
Go
643 lines
17 KiB
Go
// Package catalogue provides ways of interacting with recipe catalogues which
|
|
// are JSON data structures which contain meta information about recipes (e.g.
|
|
// what versions of the Nextcloud recipe are available?).
|
|
package catalogue
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
"time"
|
|
|
|
"coopcloud.tech/abra/cli/formatter"
|
|
"coopcloud.tech/abra/pkg/client"
|
|
"coopcloud.tech/abra/pkg/config"
|
|
"coopcloud.tech/abra/pkg/recipe"
|
|
"coopcloud.tech/abra/pkg/web"
|
|
"github.com/docker/distribution/reference"
|
|
"github.com/go-git/go-git/v5"
|
|
"github.com/go-git/go-git/v5/plumbing"
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// RecipeCatalogueURL is the only current recipe catalogue available.
|
|
const RecipeCatalogueURL = "https://apps.coopcloud.tech"
|
|
|
|
// ReposMetadataURL is the recipe repository metadata
|
|
const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
|
|
|
|
// image represents a recipe container image.
|
|
type image struct {
|
|
Image string `json:"image"`
|
|
Rating string `json:"rating"`
|
|
Source string `json:"source"`
|
|
URL string `json:"url"`
|
|
}
|
|
|
|
// features represent what top-level features a recipe supports (e.g. does this
|
|
// recipe support backups?).
|
|
type features struct {
|
|
Backups string `json:"backups"`
|
|
Email string `json:"email"`
|
|
Healthcheck string `json:"healthcheck"`
|
|
Image image `json:"image"`
|
|
Status int `json:"status"`
|
|
Tests string `json:"tests"`
|
|
SSO string `json:"sso"`
|
|
}
|
|
|
|
// tag represents a git tag.
|
|
type tag = string
|
|
|
|
// service represents a service within a recipe.
|
|
type service = string
|
|
|
|
// ServiceMeta represents meta info associated with a service.
|
|
type ServiceMeta struct {
|
|
Digest string `json:"digest"`
|
|
Image string `json:"image"`
|
|
Tag string `json:"tag"`
|
|
}
|
|
|
|
// RecipeVersions are the versions associated with a recipe.
|
|
type RecipeVersions []map[tag]map[service]ServiceMeta
|
|
|
|
// RecipeMeta represents metadata for a recipe in the abra catalogue.
|
|
type RecipeMeta struct {
|
|
Category string `json:"category"`
|
|
DefaultBranch string `json:"default_branch"`
|
|
Description string `json:"description"`
|
|
Features features `json:"features"`
|
|
Icon string `json:"icon"`
|
|
Name string `json:"name"`
|
|
Repository string `json:"repository"`
|
|
Versions RecipeVersions `json:"versions"`
|
|
Website string `json:"website"`
|
|
}
|
|
|
|
// LatestVersion returns the latest version of a recipe.
|
|
func (r RecipeMeta) LatestVersion() string {
|
|
var version string
|
|
|
|
// apps.json versions are sorted so the last key is latest
|
|
latest := r.Versions[len(r.Versions)-1]
|
|
|
|
for tag := range latest {
|
|
version = tag
|
|
}
|
|
|
|
logrus.Debugf("choosing %s as latest version of %s", version, r.Name)
|
|
|
|
return version
|
|
}
|
|
|
|
// Name represents a recipe name.
|
|
type Name = string
|
|
|
|
// RecipeCatalogue represents the entire recipe catalogue.
|
|
type RecipeCatalogue map[Name]RecipeMeta
|
|
|
|
// Flatten converts AppCatalogue to slice
|
|
func (r RecipeCatalogue) Flatten() []RecipeMeta {
|
|
recipes := make([]RecipeMeta, 0, len(r))
|
|
|
|
for name := range r {
|
|
recipes = append(recipes, r[name])
|
|
}
|
|
|
|
return recipes
|
|
}
|
|
|
|
// ByRecipeName sorts recipes by name.
|
|
type ByRecipeName []RecipeMeta
|
|
|
|
func (r ByRecipeName) Len() int { return len(r) }
|
|
func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
|
|
func (r ByRecipeName) Less(i, j int) bool {
|
|
return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name)
|
|
}
|
|
|
|
// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
|
|
// is up to date.
|
|
func recipeCatalogueFSIsLatest() (bool, error) {
|
|
httpClient := web.NewHTTPRetryClient()
|
|
res, err := httpClient.Head(RecipeCatalogueURL)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
lastModified := res.Header["Last-Modified"][0]
|
|
parsed, err := time.Parse(time.RFC1123, lastModified)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
info, err := os.Stat(config.APPS_JSON)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
logrus.Debugf("no recipe catalogue found in file system cache")
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
|
|
localModifiedTime := info.ModTime().Unix()
|
|
remoteModifiedTime := parsed.Unix()
|
|
|
|
if localModifiedTime < remoteModifiedTime {
|
|
logrus.Debug("file system cached recipe catalogue is out-of-date")
|
|
return false, nil
|
|
}
|
|
|
|
logrus.Debug("file system cached recipe catalogue is now up-to-date")
|
|
|
|
return true, nil
|
|
}
|
|
|
|
// ReadRecipeCatalogue reads the recipe catalogue.
|
|
func ReadRecipeCatalogue() (RecipeCatalogue, error) {
|
|
recipes := make(RecipeCatalogue)
|
|
|
|
recipeFSIsLatest, err := recipeCatalogueFSIsLatest()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if !recipeFSIsLatest {
|
|
logrus.Debugf("reading recipe catalogue from web to get latest")
|
|
if err := readRecipeCatalogueWeb(&recipes); err != nil {
|
|
return nil, err
|
|
}
|
|
return recipes, nil
|
|
}
|
|
|
|
logrus.Debugf("reading recipe catalogue from file system cache to get latest")
|
|
if err := readRecipeCatalogueFS(&recipes); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return recipes, nil
|
|
}
|
|
|
|
// readRecipeCatalogueFS reads the catalogue from the file system.
|
|
func readRecipeCatalogueFS(target interface{}) error {
|
|
recipesJSONFS, err := ioutil.ReadFile(config.APPS_JSON)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := json.Unmarshal(recipesJSONFS, &target); err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("read recipe catalogue from file system cache in %s", config.APPS_JSON)
|
|
|
|
return nil
|
|
}
|
|
|
|
// readRecipeCatalogueWeb reads the catalogue from the web.
|
|
func readRecipeCatalogueWeb(target interface{}) error {
|
|
if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil {
|
|
return err
|
|
}
|
|
|
|
recipesJSON, err := json.MarshalIndent(target, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0764); err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("read recipe catalogue from web at %s", RecipeCatalogueURL)
|
|
|
|
return nil
|
|
}
|
|
|
|
// VersionsOfService lists the version of a service.
|
|
func VersionsOfService(recipe, serviceName string) ([]string, error) {
|
|
var versions []string
|
|
|
|
catalogue, err := ReadRecipeCatalogue()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rec, ok := catalogue[recipe]
|
|
if !ok {
|
|
return versions, nil
|
|
}
|
|
|
|
alreadySeen := make(map[string]bool)
|
|
for _, serviceVersion := range rec.Versions {
|
|
for tag := range serviceVersion {
|
|
if _, ok := alreadySeen[tag]; !ok {
|
|
alreadySeen[tag] = true
|
|
versions = append(versions, tag)
|
|
}
|
|
}
|
|
}
|
|
|
|
logrus.Debugf("detected versions %s for %s", strings.Join(versions, ", "), recipe)
|
|
|
|
return versions, nil
|
|
}
|
|
|
|
// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
|
|
func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
|
|
catl, err := ReadRecipeCatalogue()
|
|
if err != nil {
|
|
return RecipeMeta{}, err
|
|
}
|
|
|
|
recipeMeta, ok := catl[recipeName]
|
|
if !ok {
|
|
err := fmt.Errorf("recipe %s does not exist?", recipeName)
|
|
return RecipeMeta{}, err
|
|
}
|
|
|
|
if err := recipe.EnsureExists(recipeName); err != nil {
|
|
return RecipeMeta{}, err
|
|
}
|
|
|
|
logrus.Debugf("recipe metadata retrieved for %s", recipeName)
|
|
|
|
return recipeMeta, nil
|
|
}
|
|
|
|
// RepoMeta is a single recipe repo metadata.
|
|
type RepoMeta struct {
|
|
ID int `json:"id"`
|
|
Owner Owner
|
|
Name string `json:"name"`
|
|
FullName string `json:"full_name"`
|
|
Description string `json:"description"`
|
|
Empty bool `json:"empty"`
|
|
Private bool `json:"private"`
|
|
Fork bool `json:"fork"`
|
|
Template bool `json:"template"`
|
|
Parent interface{} `json:"parent"`
|
|
Mirror bool `json:"mirror"`
|
|
Size int `json:"size"`
|
|
HTMLURL string `json:"html_url"`
|
|
SSHURL string `json:"ssh_url"`
|
|
CloneURL string `json:"clone_url"`
|
|
OriginalURL string `json:"original_url"`
|
|
Website string `json:"website"`
|
|
StarsCount int `json:"stars_count"`
|
|
ForksCount int `json:"forks_count"`
|
|
WatchersCount int `json:"watchers_count"`
|
|
OpenIssuesCount int `json:"open_issues_count"`
|
|
OpenPRCount int `json:"open_pr_counter"`
|
|
ReleaseCounter int `json:"release_counter"`
|
|
DefaultBranch string `json:"default_branch"`
|
|
Archived bool `json:"archived"`
|
|
CreatedAt string `json:"created_at"`
|
|
UpdatedAt string `json:"updated_at"`
|
|
Permissions Permissions
|
|
HasIssues bool `json:"has_issues"`
|
|
InternalTracker InternalTracker
|
|
HasWiki bool `json:"has_wiki"`
|
|
HasPullRequests bool `json:"has_pull_requests"`
|
|
HasProjects bool `json:"has_projects"`
|
|
IgnoreWhitespaceConflicts bool `json:"ignore_whitespace_conflicts"`
|
|
AllowMergeCommits bool `json:"allow_merge_commits"`
|
|
AllowRebase bool `json:"allow_rebase"`
|
|
AllowRebaseExplicit bool `json:"allow_rebase_explicit"`
|
|
AllowSquashMerge bool `json:"allow_squash_merge"`
|
|
AvatarURL string `json:"avatar_url"`
|
|
Internal bool `json:"internal"`
|
|
MirrorInterval string `json:"mirror_interval"`
|
|
}
|
|
|
|
// Owner is the repo organisation owner metadata.
|
|
type Owner struct {
|
|
ID int `json:"id"`
|
|
Login string `json:"login"`
|
|
FullName string `json:"full_name"`
|
|
Email string `json:"email"`
|
|
AvatarURL string `json:"avatar_url"`
|
|
Language string `json:"language"`
|
|
IsAdmin bool `json:"is_admin"`
|
|
LastLogin string `json:"last_login"`
|
|
Created string `json:"created"`
|
|
Restricted bool `json:"restricted"`
|
|
Username string `json:"username"`
|
|
}
|
|
|
|
// Permissions is perms metadata for a repo.
|
|
type Permissions struct {
|
|
Admin bool `json:"admin"`
|
|
Push bool `json:"push"`
|
|
Pull bool `json:"pull"`
|
|
}
|
|
|
|
// InternalTracker is issue tracker metadata for a repo.
|
|
type InternalTracker struct {
|
|
EnableTimeTracker bool `json:"enable_time_tracker"`
|
|
AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"`
|
|
EnableIssuesDependencies bool `json:"enable_issue_dependencies"`
|
|
}
|
|
|
|
// RepoCatalogue represents all the recipe repo metadata.
|
|
type RepoCatalogue map[string]RepoMeta
|
|
|
|
// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
|
|
func ReadReposMetadata() (RepoCatalogue, error) {
|
|
reposMeta := make(RepoCatalogue)
|
|
|
|
pageIdx := 1
|
|
bar := formatter.CreateProgressbar(-1, "retrieving recipe repos list from git.coopcloud.tech...")
|
|
for {
|
|
var reposList []RepoMeta
|
|
|
|
pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx)
|
|
|
|
logrus.Debugf("fetching repo metadata from %s", pagedURL)
|
|
|
|
if err := web.ReadJSON(pagedURL, &reposList); err != nil {
|
|
return reposMeta, err
|
|
}
|
|
|
|
if len(reposList) == 0 {
|
|
bar.Add(1)
|
|
break
|
|
}
|
|
|
|
for idx, repo := range reposList {
|
|
reposMeta[repo.Name] = reposList[idx]
|
|
}
|
|
|
|
pageIdx++
|
|
bar.Add(1)
|
|
}
|
|
|
|
return reposMeta, nil
|
|
}
|
|
|
|
func GetStringInBetween(str, start, end string) (result string, err error) {
|
|
// GetStringInBetween returns empty string if no start or end string found
|
|
s := strings.Index(str, start)
|
|
if s == -1 {
|
|
return "", fmt.Errorf("marker string '%s' not found", start)
|
|
}
|
|
s += len(start)
|
|
e := strings.Index(str[s:], end)
|
|
if e == -1 {
|
|
return "", fmt.Errorf("end marker '%s' not found", end)
|
|
}
|
|
return str[s : s+e], nil
|
|
}
|
|
|
|
func GetImageMetadata(imageRowString, recipeName string) (image, error) {
|
|
img := image{}
|
|
|
|
imgFields := strings.Split(imageRowString, ",")
|
|
|
|
for i, elem := range imgFields {
|
|
imgFields[i] = strings.TrimSpace(elem)
|
|
}
|
|
|
|
if len(imgFields) < 3 {
|
|
if imageRowString != "" {
|
|
logrus.Warnf("%s image meta has incorrect format: %s", recipeName, imageRowString)
|
|
} else {
|
|
logrus.Warnf("%s image meta is empty?", recipeName)
|
|
}
|
|
return img, nil
|
|
}
|
|
|
|
img.Rating = imgFields[1]
|
|
img.Source = imgFields[2]
|
|
|
|
imgString := imgFields[0]
|
|
|
|
imageName, err := GetStringInBetween(imgString, "[", "]")
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
img.Image = strings.ReplaceAll(imageName, "`", "")
|
|
|
|
imageURL, err := GetStringInBetween(imgString, "(", ")")
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
img.URL = imageURL
|
|
|
|
return img, nil
|
|
}
|
|
|
|
func GetRecipeFeaturesAndCategory(recipeName string) (features, string, error) {
|
|
feat := features{}
|
|
|
|
var category string
|
|
|
|
readmePath := path.Join(config.ABRA_DIR, "apps", recipeName, "README.md")
|
|
|
|
logrus.Debugf("attempting to open %s for recipe metadata parsing", readmePath)
|
|
|
|
readmeFS, err := ioutil.ReadFile(readmePath)
|
|
if err != nil {
|
|
return feat, category, err
|
|
}
|
|
|
|
readmeMetadata, err := GetStringInBetween( // Find text between delimiters
|
|
string(readmeFS),
|
|
"<!-- metadata -->", "<!-- endmetadata -->",
|
|
)
|
|
if err != nil {
|
|
return feat, category, err
|
|
}
|
|
|
|
readmeLines := strings.Split( // Array item from lines
|
|
strings.ReplaceAll( // Remove \t tabs
|
|
readmeMetadata, "\t", "",
|
|
),
|
|
"\n")
|
|
|
|
for _, val := range readmeLines {
|
|
if strings.Contains(val, "**Category**") {
|
|
category = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Category**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Backups**") {
|
|
feat.Backups = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Backups**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Email**") {
|
|
feat.Email = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Email**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**SSO**") {
|
|
feat.SSO = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **SSO**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Healthcheck**") {
|
|
feat.Healthcheck = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Healthcheck**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Tests**") {
|
|
feat.Tests = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Tests**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Image**") {
|
|
imageMetadata, err := GetImageMetadata(strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Image**:"),
|
|
), recipeName)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
feat.Image = imageMetadata
|
|
}
|
|
}
|
|
|
|
return feat, category, nil
|
|
}
|
|
|
|
// GetRecipeVersions retrieves all recipe versions.
|
|
func GetRecipeVersions(recipeName string) (RecipeVersions, error) {
|
|
versions := RecipeVersions{}
|
|
|
|
recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
|
|
|
|
logrus.Debugf("attempting to open git repository in %s", recipeDir)
|
|
|
|
repo, err := git.PlainOpen(recipeDir)
|
|
if err != nil {
|
|
return versions, err
|
|
}
|
|
|
|
worktree, err := repo.Worktree()
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
gitTags, err := repo.Tags()
|
|
if err != nil {
|
|
return versions, err
|
|
}
|
|
|
|
if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
|
|
tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/")
|
|
|
|
logrus.Debugf("processing %s for %s", tag, recipeName)
|
|
|
|
checkOutOpts := &git.CheckoutOptions{
|
|
Create: false,
|
|
Force: true,
|
|
Branch: plumbing.ReferenceName(ref.Name()),
|
|
}
|
|
if err := worktree.Checkout(checkOutOpts); err != nil {
|
|
logrus.Debugf("failed to check out %s in %s", tag, recipeDir)
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir)
|
|
|
|
recipe, err := recipe.Get(recipeName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
cl, err := client.New("default") // only required for docker.io registry calls
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
versionMeta := make(map[string]ServiceMeta)
|
|
for _, service := range recipe.Config.Services {
|
|
|
|
img, err := reference.ParseNormalizedNamed(service.Image)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
path := reference.Path(img)
|
|
if strings.Contains(path, "library") {
|
|
path = strings.Split(path, "/")[1]
|
|
}
|
|
|
|
var tag string
|
|
switch img.(type) {
|
|
case reference.NamedTagged:
|
|
tag = img.(reference.NamedTagged).Tag()
|
|
case reference.Named:
|
|
logrus.Warnf("%s service is missing image tag?", path)
|
|
continue
|
|
}
|
|
|
|
logrus.Debugf("looking up image: %s from %s", img, path)
|
|
|
|
digest, err := client.GetTagDigest(cl, img)
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
continue
|
|
}
|
|
|
|
versionMeta[service.Name] = ServiceMeta{
|
|
Digest: digest,
|
|
Image: path,
|
|
Tag: img.(reference.NamedTagged).Tag(),
|
|
}
|
|
|
|
logrus.Debugf("collecting digest: %s, image: %s, tag: %s", digest, path, tag)
|
|
}
|
|
|
|
versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta})
|
|
|
|
return nil
|
|
}); err != nil {
|
|
return versions, err
|
|
}
|
|
|
|
branch := "master"
|
|
if _, err := repo.Branch("master"); err != nil {
|
|
if _, err := repo.Branch("main"); err != nil {
|
|
logrus.Debugf("failed to select branch in %s", recipeDir)
|
|
logrus.Fatal(err)
|
|
}
|
|
branch = "main"
|
|
}
|
|
|
|
refName := fmt.Sprintf("refs/heads/%s", branch)
|
|
checkOutOpts := &git.CheckoutOptions{
|
|
Create: false,
|
|
Force: true,
|
|
Branch: plumbing.ReferenceName(refName),
|
|
}
|
|
if err := worktree.Checkout(checkOutOpts); err != nil {
|
|
logrus.Debugf("failed to check out %s in %s", branch, recipeDir)
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
logrus.Debugf("switched back to %s in %s", branch, recipeDir)
|
|
logrus.Debugf("collected %s for %s", versions, recipeName)
|
|
|
|
return versions, nil
|
|
}
|
|
|
|
// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
|
|
func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) {
|
|
var versions []string
|
|
|
|
if recipeMeta, exists := catl[recipeName]; exists {
|
|
for _, versionMeta := range recipeMeta.Versions {
|
|
for tag := range versionMeta {
|
|
versions = append(versions, tag)
|
|
}
|
|
}
|
|
}
|
|
|
|
return versions, nil
|
|
}
|