cellarspoon
4c186678b8
Catalogue package had to be merged into the recipe package due to too many circular import errors. Also, use https url for cloning, assume folks don't have ssh setup by default (the whole reason for the refactor).
1022 lines
26 KiB
Go
1022 lines
26 KiB
Go
package recipe
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"coopcloud.tech/abra/cli/formatter"
|
|
"coopcloud.tech/abra/pkg/client"
|
|
"coopcloud.tech/abra/pkg/compose"
|
|
"coopcloud.tech/abra/pkg/config"
|
|
gitPkg "coopcloud.tech/abra/pkg/git"
|
|
"coopcloud.tech/abra/pkg/upstream/stack"
|
|
loader "coopcloud.tech/abra/pkg/upstream/stack"
|
|
"coopcloud.tech/abra/pkg/web"
|
|
composetypes "github.com/docker/cli/cli/compose/types"
|
|
"github.com/docker/distribution/reference"
|
|
"github.com/go-git/go-git/v5"
|
|
"github.com/go-git/go-git/v5/plumbing"
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// RecipeCatalogueURL is the only current recipe catalogue available.
|
|
const RecipeCatalogueURL = "https://apps.coopcloud.tech"
|
|
|
|
// ReposMetadataURL is the recipe repository metadata
|
|
const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
|
|
|
|
// tag represents a git tag.
|
|
type tag = string
|
|
|
|
// service represents a service within a recipe.
|
|
type service = string
|
|
|
|
// ServiceMeta represents meta info associated with a service.
|
|
type ServiceMeta struct {
|
|
Digest string `json:"digest"`
|
|
Image string `json:"image"`
|
|
Tag string `json:"tag"`
|
|
}
|
|
|
|
// RecipeVersions are the versions associated with a recipe.
|
|
type RecipeVersions []map[tag]map[service]ServiceMeta
|
|
|
|
// RecipeMeta represents metadata for a recipe in the abra catalogue.
|
|
type RecipeMeta struct {
|
|
Category string `json:"category"`
|
|
DefaultBranch string `json:"default_branch"`
|
|
Description string `json:"description"`
|
|
Features Features `json:"features"`
|
|
Icon string `json:"icon"`
|
|
Name string `json:"name"`
|
|
Repository string `json:"repository"`
|
|
Versions RecipeVersions `json:"versions"`
|
|
Website string `json:"website"`
|
|
}
|
|
|
|
// LatestVersion returns the latest version of a recipe.
|
|
func (r RecipeMeta) LatestVersion() string {
|
|
var version string
|
|
|
|
// apps.json versions are sorted so the last key is latest
|
|
latest := r.Versions[len(r.Versions)-1]
|
|
|
|
for tag := range latest {
|
|
version = tag
|
|
}
|
|
|
|
logrus.Debugf("choosing %s as latest version of %s", version, r.Name)
|
|
|
|
return version
|
|
}
|
|
|
|
// Name represents a recipe name.
|
|
type Name = string
|
|
|
|
// RecipeCatalogue represents the entire recipe catalogue.
|
|
type RecipeCatalogue map[Name]RecipeMeta
|
|
|
|
// Flatten converts AppCatalogue to slice
|
|
func (r RecipeCatalogue) Flatten() []RecipeMeta {
|
|
recipes := make([]RecipeMeta, 0, len(r))
|
|
|
|
for name := range r {
|
|
recipes = append(recipes, r[name])
|
|
}
|
|
|
|
return recipes
|
|
}
|
|
|
|
// ByRecipeName sorts recipes by name.
|
|
type ByRecipeName []RecipeMeta
|
|
|
|
func (r ByRecipeName) Len() int { return len(r) }
|
|
func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
|
|
func (r ByRecipeName) Less(i, j int) bool {
|
|
return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name)
|
|
}
|
|
|
|
// Image represents a recipe container image.
|
|
type Image struct {
|
|
Image string `json:"image"`
|
|
Rating string `json:"rating"`
|
|
Source string `json:"source"`
|
|
URL string `json:"url"`
|
|
}
|
|
|
|
// Features represent what top-level features a recipe supports (e.g. does this recipe support backups?).
|
|
type Features struct {
|
|
Backups string `json:"backups"`
|
|
Email string `json:"email"`
|
|
Healthcheck string `json:"healthcheck"`
|
|
Image Image `json:"image"`
|
|
Status int `json:"status"`
|
|
Tests string `json:"tests"`
|
|
SSO string `json:"sso"`
|
|
}
|
|
|
|
// Recipe represents a recipe.
|
|
type Recipe struct {
|
|
Name string
|
|
Config *composetypes.Config
|
|
Meta RecipeMeta
|
|
}
|
|
|
|
// Dir retrieves the recipe repository path
|
|
func (r Recipe) Dir() string {
|
|
return path.Join(config.RECIPES_DIR, r.Name)
|
|
}
|
|
|
|
// UpdateLabel updates a recipe label
|
|
func (r Recipe) UpdateLabel(pattern, serviceName, label string) error {
|
|
fullPattern := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, pattern)
|
|
if err := compose.UpdateLabel(fullPattern, serviceName, label, r.Name); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// UpdateTag updates a recipe tag
|
|
func (r Recipe) UpdateTag(image, tag string) error {
|
|
pattern := fmt.Sprintf("%s/%s/compose**yml", config.RECIPES_DIR, r.Name)
|
|
if err := compose.UpdateTag(pattern, image, tag, r.Name); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Tags list the recipe tags
|
|
func (r Recipe) Tags() ([]string, error) {
|
|
var tags []string
|
|
|
|
repo, err := git.PlainOpen(r.Dir())
|
|
if err != nil {
|
|
return tags, err
|
|
}
|
|
|
|
gitTags, err := repo.Tags()
|
|
if err != nil {
|
|
return tags, err
|
|
}
|
|
|
|
if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
|
|
tags = append(tags, strings.TrimPrefix(string(ref.Name()), "refs/tags/"))
|
|
return nil
|
|
}); err != nil {
|
|
return tags, err
|
|
}
|
|
|
|
logrus.Debugf("detected %s as tags for recipe %s", strings.Join(tags, ", "), r.Name)
|
|
|
|
return tags, nil
|
|
}
|
|
|
|
// Get retrieves a recipe.
|
|
func Get(recipeName string) (Recipe, error) {
|
|
if err := EnsureExists(recipeName); err != nil {
|
|
return Recipe{}, err
|
|
}
|
|
|
|
pattern := fmt.Sprintf("%s/%s/compose**yml", config.RECIPES_DIR, recipeName)
|
|
composeFiles, err := filepath.Glob(pattern)
|
|
if err != nil {
|
|
return Recipe{}, err
|
|
}
|
|
|
|
if len(composeFiles) == 0 {
|
|
return Recipe{}, fmt.Errorf("%s is missing a compose.yml or compose.*.yml file?", recipeName)
|
|
}
|
|
|
|
envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
|
|
sampleEnv, err := config.ReadEnv(envSamplePath)
|
|
if err != nil {
|
|
return Recipe{}, err
|
|
}
|
|
|
|
opts := stack.Deploy{Composefiles: composeFiles}
|
|
config, err := loader.LoadComposefile(opts, sampleEnv)
|
|
if err != nil {
|
|
return Recipe{}, err
|
|
}
|
|
|
|
meta, err := GetRecipeMeta(recipeName)
|
|
if err != nil {
|
|
return Recipe{}, err
|
|
}
|
|
|
|
return Recipe{
|
|
Name: recipeName,
|
|
Config: config,
|
|
Meta: meta,
|
|
}, nil
|
|
}
|
|
|
|
// EnsureExists ensures that a recipe is locally cloned
|
|
func EnsureExists(recipeName string) error {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
if _, err := os.Stat(recipeDir); os.IsNotExist(err) {
|
|
logrus.Debugf("%s does not exist, attemmpting to clone", recipeDir)
|
|
url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, recipeName)
|
|
if err := gitPkg.Clone(recipeDir, url); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// EnsureVersion checks whether a specific version exists for a recipe.
|
|
func EnsureVersion(recipeName, version string) error {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
isClean, err := gitPkg.IsClean(recipeName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !isClean {
|
|
return fmt.Errorf("%s has locally unstaged changes", recipeName)
|
|
}
|
|
|
|
if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
|
|
return err
|
|
}
|
|
|
|
repo, err := git.PlainOpen(recipeDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
tags, err := repo.Tags()
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
var parsedTags []string
|
|
var tagRef plumbing.ReferenceName
|
|
if err := tags.ForEach(func(ref *plumbing.Reference) (err error) {
|
|
parsedTags = append(parsedTags, ref.Name().Short())
|
|
if ref.Name().Short() == version {
|
|
tagRef = ref.Name()
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("read %s as tags for recipe %s", strings.Join(parsedTags, ", "), recipeName)
|
|
|
|
if tagRef.String() == "" {
|
|
logrus.Warnf("no git tag discovered for %s, assuming unreleased recipe", recipeName)
|
|
return nil
|
|
}
|
|
|
|
worktree, err := repo.Worktree()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
opts := &git.CheckoutOptions{
|
|
Branch: tagRef,
|
|
Create: false,
|
|
Force: true,
|
|
}
|
|
if err := worktree.Checkout(opts); err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("successfully checked %s out to %s in %s", recipeName, tagRef.Short(), recipeDir)
|
|
|
|
return nil
|
|
}
|
|
|
|
// EnsureLatest makes sure the latest commit is checked out for a local recipe repository
|
|
func EnsureLatest(recipeName string) error {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
isClean, err := gitPkg.IsClean(recipeName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !isClean {
|
|
return fmt.Errorf("%s has locally unstaged changes", recipeName)
|
|
}
|
|
|
|
if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("attempting to open git repository in %s", recipeDir)
|
|
|
|
repo, err := git.PlainOpen(recipeDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
worktree, err := repo.Worktree()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
branch, err := gitPkg.GetCurrentBranch(repo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
checkOutOpts := &git.CheckoutOptions{
|
|
Create: false,
|
|
Force: true,
|
|
Branch: plumbing.ReferenceName(branch),
|
|
}
|
|
|
|
if err := worktree.Checkout(checkOutOpts); err != nil {
|
|
logrus.Debugf("failed to check out %s in %s", branch, recipeDir)
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// ChaosVersion constructs a chaos mode recipe version.
|
|
func ChaosVersion(recipeName string) (string, error) {
|
|
var version string
|
|
|
|
head, err := gitPkg.GetRecipeHead(recipeName)
|
|
if err != nil {
|
|
return version, err
|
|
}
|
|
|
|
version = head.String()[:8]
|
|
|
|
isClean, err := gitPkg.IsClean(recipeName)
|
|
if err != nil {
|
|
return version, err
|
|
}
|
|
|
|
if !isClean {
|
|
version = fmt.Sprintf("%s + unstaged changes", version)
|
|
}
|
|
|
|
return version, nil
|
|
}
|
|
|
|
// GetRecipesLocal retrieves all local recipe directories
|
|
func GetRecipesLocal() ([]string, error) {
|
|
var recipes []string
|
|
|
|
recipes, err := config.GetAllFoldersInDirectory(config.RECIPES_DIR)
|
|
if err != nil {
|
|
return recipes, err
|
|
}
|
|
|
|
return recipes, nil
|
|
}
|
|
|
|
// GetVersionLabelLocal retrieves the version label on the local recipe config
|
|
func GetVersionLabelLocal(recipe Recipe) (string, error) {
|
|
var label string
|
|
|
|
for _, service := range recipe.Config.Services {
|
|
for label, value := range service.Deploy.Labels {
|
|
if strings.HasPrefix(label, "coop-cloud") {
|
|
return value, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
if label == "" {
|
|
return label, fmt.Errorf("%s has no version label? try running \"abra recipe sync %s\" first?", recipe.Name, recipe.Name)
|
|
}
|
|
|
|
return label, nil
|
|
}
|
|
|
|
func GetRecipeFeaturesAndCategory(recipeName string) (Features, string, error) {
|
|
feat := Features{}
|
|
|
|
var category string
|
|
|
|
readmePath := path.Join(config.RECIPES_DIR, recipeName, "README.md")
|
|
|
|
logrus.Debugf("attempting to open %s for recipe metadata parsing", readmePath)
|
|
|
|
readmeFS, err := ioutil.ReadFile(readmePath)
|
|
if err != nil {
|
|
return feat, category, err
|
|
}
|
|
|
|
readmeMetadata, err := GetStringInBetween( // Find text between delimiters
|
|
string(readmeFS),
|
|
"<!-- metadata -->", "<!-- endmetadata -->",
|
|
)
|
|
if err != nil {
|
|
return feat, category, err
|
|
}
|
|
|
|
readmeLines := strings.Split( // Array item from lines
|
|
strings.ReplaceAll( // Remove \t tabs
|
|
readmeMetadata, "\t", "",
|
|
),
|
|
"\n")
|
|
|
|
for _, val := range readmeLines {
|
|
if strings.Contains(val, "**Category**") {
|
|
category = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Category**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Backups**") {
|
|
feat.Backups = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Backups**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Email**") {
|
|
feat.Email = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Email**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**SSO**") {
|
|
feat.SSO = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **SSO**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Healthcheck**") {
|
|
feat.Healthcheck = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Healthcheck**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Tests**") {
|
|
feat.Tests = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Tests**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Image**") {
|
|
imageMetadata, err := GetImageMetadata(strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Image**:"),
|
|
), recipeName)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
feat.Image = imageMetadata
|
|
}
|
|
}
|
|
|
|
return feat, category, nil
|
|
}
|
|
|
|
func GetImageMetadata(imageRowString, recipeName string) (Image, error) {
|
|
img := Image{}
|
|
|
|
imgFields := strings.Split(imageRowString, ",")
|
|
|
|
for i, elem := range imgFields {
|
|
imgFields[i] = strings.TrimSpace(elem)
|
|
}
|
|
|
|
if len(imgFields) < 3 {
|
|
if imageRowString != "" {
|
|
logrus.Warnf("%s image meta has incorrect format: %s", recipeName, imageRowString)
|
|
} else {
|
|
logrus.Warnf("%s image meta is empty?", recipeName)
|
|
}
|
|
return img, nil
|
|
}
|
|
|
|
img.Rating = imgFields[1]
|
|
img.Source = imgFields[2]
|
|
|
|
imgString := imgFields[0]
|
|
|
|
imageName, err := GetStringInBetween(imgString, "[", "]")
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
img.Image = strings.ReplaceAll(imageName, "`", "")
|
|
|
|
imageURL, err := GetStringInBetween(imgString, "(", ")")
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
img.URL = imageURL
|
|
|
|
return img, nil
|
|
}
|
|
|
|
// GetStringInBetween returns empty string if no start or end string found
|
|
func GetStringInBetween(str, start, end string) (result string, err error) {
|
|
s := strings.Index(str, start)
|
|
if s == -1 {
|
|
return "", fmt.Errorf("marker string %s not found", start)
|
|
}
|
|
|
|
s += len(start)
|
|
e := strings.Index(str[s:], end)
|
|
|
|
if e == -1 {
|
|
return "", fmt.Errorf("end marker %s not found", end)
|
|
}
|
|
|
|
return str[s : s+e], nil
|
|
}
|
|
|
|
// EnsureUpToDate ensures that the local repo is synced to the remote
|
|
func EnsureUpToDate(recipeName string) error {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
isClean, err := gitPkg.IsClean(recipeName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !isClean {
|
|
return fmt.Errorf("%s has locally unstaged changes", recipeName)
|
|
}
|
|
|
|
repo, err := git.PlainOpen(recipeDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
remotes, err := repo.Remotes()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(remotes) == 0 {
|
|
logrus.Debugf("cannot ensure %s is up-to-date, no git remotes configured", recipeName)
|
|
return nil
|
|
}
|
|
|
|
worktree, err := repo.Worktree()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
branch, err := CheckoutDefaultBranch(repo, recipeName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
opts := &git.PullOptions{
|
|
Force: true,
|
|
ReferenceName: branch,
|
|
}
|
|
|
|
if err := worktree.Pull(opts); err != nil {
|
|
if !strings.Contains(err.Error(), "already up-to-date") {
|
|
return err
|
|
}
|
|
}
|
|
|
|
logrus.Debugf("fetched latest git changes for %s", recipeName)
|
|
|
|
return nil
|
|
}
|
|
|
|
func GetDefaultBranch(repo *git.Repository, recipeName string) (plumbing.ReferenceName, error) {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
branch := "master"
|
|
if _, err := repo.Branch("master"); err != nil {
|
|
if _, err := repo.Branch("main"); err != nil {
|
|
logrus.Debugf("failed to select branch in %s", recipeDir)
|
|
return "", err
|
|
}
|
|
branch = "main"
|
|
}
|
|
|
|
return plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)), nil
|
|
}
|
|
|
|
func CheckoutDefaultBranch(repo *git.Repository, recipeName string) (plumbing.ReferenceName, error) {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
branch, err := GetDefaultBranch(repo, recipeName)
|
|
if err != nil {
|
|
return plumbing.ReferenceName(""), err
|
|
}
|
|
|
|
worktree, err := repo.Worktree()
|
|
if err != nil {
|
|
return plumbing.ReferenceName(""), err
|
|
}
|
|
|
|
checkOutOpts := &git.CheckoutOptions{
|
|
Create: false,
|
|
Force: true,
|
|
Branch: branch,
|
|
}
|
|
|
|
if err := worktree.Checkout(checkOutOpts); err != nil {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
logrus.Debugf("failed to check out %s in %s", branch, recipeDir)
|
|
return branch, err
|
|
}
|
|
|
|
logrus.Debugf("successfully checked out %v in %s", branch, recipeDir)
|
|
|
|
return branch, nil
|
|
}
|
|
|
|
// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
|
|
// is up to date.
|
|
func recipeCatalogueFSIsLatest() (bool, error) {
|
|
httpClient := web.NewHTTPRetryClient()
|
|
res, err := httpClient.Head(RecipeCatalogueURL)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
lastModified := res.Header["Last-Modified"][0]
|
|
parsed, err := time.Parse(time.RFC1123, lastModified)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
info, err := os.Stat(config.RECIPES_JSON)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
logrus.Debugf("no recipe catalogue found in file system cache")
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
|
|
localModifiedTime := info.ModTime().Unix()
|
|
remoteModifiedTime := parsed.Unix()
|
|
|
|
if localModifiedTime < remoteModifiedTime {
|
|
logrus.Debug("file system cached recipe catalogue is out-of-date")
|
|
return false, nil
|
|
}
|
|
|
|
logrus.Debug("file system cached recipe catalogue is now up-to-date")
|
|
|
|
return true, nil
|
|
}
|
|
|
|
// ReadRecipeCatalogue reads the recipe catalogue.
|
|
func ReadRecipeCatalogue() (RecipeCatalogue, error) {
|
|
recipes := make(RecipeCatalogue)
|
|
|
|
recipeFSIsLatest, err := recipeCatalogueFSIsLatest()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if !recipeFSIsLatest {
|
|
logrus.Debugf("reading recipe catalogue from web to get latest")
|
|
if err := readRecipeCatalogueWeb(&recipes); err != nil {
|
|
return nil, err
|
|
}
|
|
return recipes, nil
|
|
}
|
|
|
|
logrus.Debugf("reading recipe catalogue from file system cache to get latest")
|
|
if err := readRecipeCatalogueFS(&recipes); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return recipes, nil
|
|
}
|
|
|
|
// readRecipeCatalogueFS reads the catalogue from the file system.
|
|
func readRecipeCatalogueFS(target interface{}) error {
|
|
recipesJSONFS, err := ioutil.ReadFile(config.RECIPES_JSON)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := json.Unmarshal(recipesJSONFS, &target); err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("read recipe catalogue from file system cache in %s", config.RECIPES_JSON)
|
|
|
|
return nil
|
|
}
|
|
|
|
// readRecipeCatalogueWeb reads the catalogue from the web.
|
|
func readRecipeCatalogueWeb(target interface{}) error {
|
|
if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil {
|
|
return err
|
|
}
|
|
|
|
recipesJSON, err := json.MarshalIndent(target, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("read recipe catalogue from web at %s", RecipeCatalogueURL)
|
|
|
|
return nil
|
|
}
|
|
|
|
// VersionsOfService lists the version of a service.
|
|
func VersionsOfService(recipe, serviceName string) ([]string, error) {
|
|
var versions []string
|
|
|
|
catalogue, err := ReadRecipeCatalogue()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rec, ok := catalogue[recipe]
|
|
if !ok {
|
|
return versions, nil
|
|
}
|
|
|
|
alreadySeen := make(map[string]bool)
|
|
for _, serviceVersion := range rec.Versions {
|
|
for tag := range serviceVersion {
|
|
if _, ok := alreadySeen[tag]; !ok {
|
|
alreadySeen[tag] = true
|
|
versions = append(versions, tag)
|
|
}
|
|
}
|
|
}
|
|
|
|
logrus.Debugf("detected versions %s for %s", strings.Join(versions, ", "), recipe)
|
|
|
|
return versions, nil
|
|
}
|
|
|
|
// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
|
|
func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
|
|
catl, err := ReadRecipeCatalogue()
|
|
if err != nil {
|
|
return RecipeMeta{}, err
|
|
}
|
|
|
|
recipeMeta, ok := catl[recipeName]
|
|
if !ok {
|
|
err := fmt.Errorf("recipe %s does not exist?", recipeName)
|
|
return RecipeMeta{}, err
|
|
}
|
|
|
|
if err := EnsureExists(recipeName); err != nil {
|
|
return RecipeMeta{}, err
|
|
}
|
|
|
|
logrus.Debugf("recipe metadata retrieved for %s", recipeName)
|
|
|
|
return recipeMeta, nil
|
|
}
|
|
|
|
// RepoMeta is a single recipe repo metadata.
|
|
type RepoMeta struct {
|
|
ID int `json:"id"`
|
|
Owner Owner
|
|
Name string `json:"name"`
|
|
FullName string `json:"full_name"`
|
|
Description string `json:"description"`
|
|
Empty bool `json:"empty"`
|
|
Private bool `json:"private"`
|
|
Fork bool `json:"fork"`
|
|
Template bool `json:"template"`
|
|
Parent interface{} `json:"parent"`
|
|
Mirror bool `json:"mirror"`
|
|
Size int `json:"size"`
|
|
HTMLURL string `json:"html_url"`
|
|
SSHURL string `json:"ssh_url"`
|
|
CloneURL string `json:"clone_url"`
|
|
OriginalURL string `json:"original_url"`
|
|
Website string `json:"website"`
|
|
StarsCount int `json:"stars_count"`
|
|
ForksCount int `json:"forks_count"`
|
|
WatchersCount int `json:"watchers_count"`
|
|
OpenIssuesCount int `json:"open_issues_count"`
|
|
OpenPRCount int `json:"open_pr_counter"`
|
|
ReleaseCounter int `json:"release_counter"`
|
|
DefaultBranch string `json:"default_branch"`
|
|
Archived bool `json:"archived"`
|
|
CreatedAt string `json:"created_at"`
|
|
UpdatedAt string `json:"updated_at"`
|
|
Permissions Permissions
|
|
HasIssues bool `json:"has_issues"`
|
|
InternalTracker InternalTracker
|
|
HasWiki bool `json:"has_wiki"`
|
|
HasPullRequests bool `json:"has_pull_requests"`
|
|
HasProjects bool `json:"has_projects"`
|
|
IgnoreWhitespaceConflicts bool `json:"ignore_whitespace_conflicts"`
|
|
AllowMergeCommits bool `json:"allow_merge_commits"`
|
|
AllowRebase bool `json:"allow_rebase"`
|
|
AllowRebaseExplicit bool `json:"allow_rebase_explicit"`
|
|
AllowSquashMerge bool `json:"allow_squash_merge"`
|
|
AvatarURL string `json:"avatar_url"`
|
|
Internal bool `json:"internal"`
|
|
MirrorInterval string `json:"mirror_interval"`
|
|
}
|
|
|
|
// Owner is the repo organisation owner metadata.
|
|
type Owner struct {
|
|
ID int `json:"id"`
|
|
Login string `json:"login"`
|
|
FullName string `json:"full_name"`
|
|
Email string `json:"email"`
|
|
AvatarURL string `json:"avatar_url"`
|
|
Language string `json:"language"`
|
|
IsAdmin bool `json:"is_admin"`
|
|
LastLogin string `json:"last_login"`
|
|
Created string `json:"created"`
|
|
Restricted bool `json:"restricted"`
|
|
Username string `json:"username"`
|
|
}
|
|
|
|
// Permissions is perms metadata for a repo.
|
|
type Permissions struct {
|
|
Admin bool `json:"admin"`
|
|
Push bool `json:"push"`
|
|
Pull bool `json:"pull"`
|
|
}
|
|
|
|
// InternalTracker is issue tracker metadata for a repo.
|
|
type InternalTracker struct {
|
|
EnableTimeTracker bool `json:"enable_time_tracker"`
|
|
AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"`
|
|
EnableIssuesDependencies bool `json:"enable_issue_dependencies"`
|
|
}
|
|
|
|
// RepoCatalogue represents all the recipe repo metadata.
|
|
type RepoCatalogue map[string]RepoMeta
|
|
|
|
// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
|
|
func ReadReposMetadata() (RepoCatalogue, error) {
|
|
reposMeta := make(RepoCatalogue)
|
|
|
|
pageIdx := 1
|
|
bar := formatter.CreateProgressbar(-1, "retrieving recipe repos list from git.coopcloud.tech...")
|
|
for {
|
|
var reposList []RepoMeta
|
|
|
|
pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx)
|
|
|
|
logrus.Debugf("fetching repo metadata from %s", pagedURL)
|
|
|
|
if err := web.ReadJSON(pagedURL, &reposList); err != nil {
|
|
return reposMeta, err
|
|
}
|
|
|
|
if len(reposList) == 0 {
|
|
bar.Add(1)
|
|
break
|
|
}
|
|
|
|
for idx, repo := range reposList {
|
|
reposMeta[repo.Name] = reposList[idx]
|
|
}
|
|
|
|
pageIdx++
|
|
bar.Add(1)
|
|
}
|
|
|
|
fmt.Println() // newline for spinner
|
|
|
|
return reposMeta, nil
|
|
}
|
|
|
|
// GetRecipeVersions retrieves all recipe versions.
|
|
func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (RecipeVersions, error) {
|
|
versions := RecipeVersions{}
|
|
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
logrus.Debugf("attempting to open git repository in %s", recipeDir)
|
|
|
|
repo, err := git.PlainOpen(recipeDir)
|
|
if err != nil {
|
|
return versions, err
|
|
}
|
|
|
|
worktree, err := repo.Worktree()
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
gitTags, err := repo.Tags()
|
|
if err != nil {
|
|
return versions, err
|
|
}
|
|
|
|
if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
|
|
tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/")
|
|
|
|
logrus.Debugf("processing %s for %s", tag, recipeName)
|
|
|
|
checkOutOpts := &git.CheckoutOptions{
|
|
Create: false,
|
|
Force: true,
|
|
Branch: plumbing.ReferenceName(ref.Name()),
|
|
}
|
|
if err := worktree.Checkout(checkOutOpts); err != nil {
|
|
logrus.Debugf("failed to check out %s in %s", tag, recipeDir)
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir)
|
|
|
|
recipe, err := Get(recipeName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
cl, err := client.New("default") // only required for docker.io registry calls
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
queryCache := make(map[reference.Named]string)
|
|
versionMeta := make(map[string]ServiceMeta)
|
|
for _, service := range recipe.Config.Services {
|
|
|
|
img, err := reference.ParseNormalizedNamed(service.Image)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
path := reference.Path(img)
|
|
if strings.Contains(path, "library") {
|
|
path = strings.Split(path, "/")[1]
|
|
}
|
|
|
|
var tag string
|
|
switch img.(type) {
|
|
case reference.NamedTagged:
|
|
tag = img.(reference.NamedTagged).Tag()
|
|
case reference.Named:
|
|
logrus.Warnf("%s service is missing image tag?", path)
|
|
continue
|
|
}
|
|
|
|
var exists bool
|
|
var digest string
|
|
if digest, exists = queryCache[img]; !exists {
|
|
logrus.Debugf("looking up image: %s from %s", img, path)
|
|
var err error
|
|
digest, err = client.GetTagDigest(cl, img, registryUsername, registryPassword)
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
continue
|
|
}
|
|
logrus.Debugf("queried for image: %s, tag: %s, digest: %s", path, tag, digest)
|
|
queryCache[img] = digest
|
|
logrus.Debugf("cached image: %s, tag: %s, digest: %s", path, tag, digest)
|
|
} else {
|
|
logrus.Debugf("reading image: %s, tag: %s, digest: %s from cache", path, tag, digest)
|
|
}
|
|
|
|
versionMeta[service.Name] = ServiceMeta{
|
|
Digest: digest,
|
|
Image: path,
|
|
Tag: img.(reference.NamedTagged).Tag(),
|
|
}
|
|
}
|
|
|
|
versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta})
|
|
|
|
return nil
|
|
}); err != nil {
|
|
return versions, err
|
|
}
|
|
|
|
_, err = CheckoutDefaultBranch(repo, recipeName)
|
|
if err != nil {
|
|
return versions, err
|
|
}
|
|
|
|
logrus.Debugf("collected %s for %s", versions, recipeName)
|
|
|
|
return versions, nil
|
|
}
|
|
|
|
// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
|
|
func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) {
|
|
var versions []string
|
|
|
|
if recipeMeta, exists := catl[recipeName]; exists {
|
|
for _, versionMeta := range recipeMeta.Versions {
|
|
for tag := range versionMeta {
|
|
versions = append(versions, tag)
|
|
}
|
|
}
|
|
}
|
|
|
|
return versions, nil
|
|
}
|