1065 lines
27 KiB
Go
1065 lines
27 KiB
Go
package recipe
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"coopcloud.tech/abra/pkg/client"
|
|
"coopcloud.tech/abra/pkg/compose"
|
|
"coopcloud.tech/abra/pkg/config"
|
|
"coopcloud.tech/abra/pkg/formatter"
|
|
gitPkg "coopcloud.tech/abra/pkg/git"
|
|
"coopcloud.tech/abra/pkg/upstream/stack"
|
|
loader "coopcloud.tech/abra/pkg/upstream/stack"
|
|
"coopcloud.tech/abra/pkg/web"
|
|
composetypes "github.com/docker/cli/cli/compose/types"
|
|
"github.com/docker/distribution/reference"
|
|
"github.com/go-git/go-git/v5"
|
|
"github.com/go-git/go-git/v5/plumbing"
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// RecipeCatalogueURL is the only current recipe catalogue available.
|
|
const RecipeCatalogueURL = "https://apps.coopcloud.tech"
|
|
|
|
// ReposMetadataURL is the recipe repository metadata
|
|
const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
|
|
|
|
// tag represents a git tag.
|
|
type tag = string
|
|
|
|
// service represents a service within a recipe.
|
|
type service = string
|
|
|
|
// ServiceMeta represents meta info associated with a service.
|
|
type ServiceMeta struct {
|
|
Digest string `json:"digest"`
|
|
Image string `json:"image"`
|
|
Tag string `json:"tag"`
|
|
}
|
|
|
|
// RecipeVersions are the versions associated with a recipe.
|
|
type RecipeVersions []map[tag]map[service]ServiceMeta
|
|
|
|
// RecipeMeta represents metadata for a recipe in the abra catalogue.
|
|
type RecipeMeta struct {
|
|
Category string `json:"category"`
|
|
DefaultBranch string `json:"default_branch"`
|
|
Description string `json:"description"`
|
|
Features Features `json:"features"`
|
|
Icon string `json:"icon"`
|
|
Name string `json:"name"`
|
|
Repository string `json:"repository"`
|
|
SSHURL string `json:"ssh_url"`
|
|
Versions RecipeVersions `json:"versions"`
|
|
Website string `json:"website"`
|
|
}
|
|
|
|
// LatestVersion returns the latest version of a recipe.
|
|
func (r RecipeMeta) LatestVersion() string {
|
|
var version string
|
|
|
|
// apps.json versions are sorted so the last key is latest
|
|
latest := r.Versions[len(r.Versions)-1]
|
|
|
|
for tag := range latest {
|
|
version = tag
|
|
}
|
|
|
|
logrus.Debugf("choosing %s as latest version of %s", version, r.Name)
|
|
|
|
return version
|
|
}
|
|
|
|
// Name represents a recipe name.
|
|
type Name = string
|
|
|
|
// RecipeCatalogue represents the entire recipe catalogue.
|
|
type RecipeCatalogue map[Name]RecipeMeta
|
|
|
|
// Flatten converts AppCatalogue to slice
|
|
func (r RecipeCatalogue) Flatten() []RecipeMeta {
|
|
recipes := make([]RecipeMeta, 0, len(r))
|
|
|
|
for name := range r {
|
|
recipes = append(recipes, r[name])
|
|
}
|
|
|
|
return recipes
|
|
}
|
|
|
|
// ByRecipeName sorts recipes by name.
|
|
type ByRecipeName []RecipeMeta
|
|
|
|
func (r ByRecipeName) Len() int { return len(r) }
|
|
func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
|
|
func (r ByRecipeName) Less(i, j int) bool {
|
|
return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name)
|
|
}
|
|
|
|
// Image represents a recipe container image.
|
|
type Image struct {
|
|
Image string `json:"image"`
|
|
Rating string `json:"rating"`
|
|
Source string `json:"source"`
|
|
URL string `json:"url"`
|
|
}
|
|
|
|
// Features represent what top-level features a recipe supports (e.g. does this recipe support backups?).
|
|
type Features struct {
|
|
Backups string `json:"backups"`
|
|
Email string `json:"email"`
|
|
Healthcheck string `json:"healthcheck"`
|
|
Image Image `json:"image"`
|
|
Status int `json:"status"`
|
|
Tests string `json:"tests"`
|
|
SSO string `json:"sso"`
|
|
}
|
|
|
|
// Recipe represents a recipe.
|
|
type Recipe struct {
|
|
Name string
|
|
Config *composetypes.Config
|
|
Meta RecipeMeta
|
|
}
|
|
|
|
// Push pushes the latest changes to a SSH URL remote. You need to have your
|
|
// local SSH configuration for git.coopcloud.tech working for this to work
|
|
func (r Recipe) Push(dryRun bool) error {
|
|
repo, err := git.PlainOpen(r.Dir())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := gitPkg.CreateRemote(repo, "origin-ssh", r.Meta.SSHURL, dryRun); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := gitPkg.Push(r.Dir(), "origin-ssh", true, dryRun); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Dir retrieves the recipe repository path
|
|
func (r Recipe) Dir() string {
|
|
return path.Join(config.RECIPES_DIR, r.Name)
|
|
}
|
|
|
|
// UpdateLabel updates a recipe label
|
|
func (r Recipe) UpdateLabel(pattern, serviceName, label string) error {
|
|
fullPattern := fmt.Sprintf("%s/%s/%s", config.RECIPES_DIR, r.Name, pattern)
|
|
if err := compose.UpdateLabel(fullPattern, serviceName, label, r.Name); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// UpdateTag updates a recipe tag
|
|
func (r Recipe) UpdateTag(image, tag string) error {
|
|
pattern := fmt.Sprintf("%s/%s/compose**yml", config.RECIPES_DIR, r.Name)
|
|
|
|
image = StripTagMeta(image)
|
|
|
|
if err := compose.UpdateTag(pattern, image, tag, r.Name); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Tags list the recipe tags
|
|
func (r Recipe) Tags() ([]string, error) {
|
|
var tags []string
|
|
|
|
repo, err := git.PlainOpen(r.Dir())
|
|
if err != nil {
|
|
return tags, err
|
|
}
|
|
|
|
gitTags, err := repo.Tags()
|
|
if err != nil {
|
|
return tags, err
|
|
}
|
|
|
|
if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
|
|
tags = append(tags, strings.TrimPrefix(string(ref.Name()), "refs/tags/"))
|
|
return nil
|
|
}); err != nil {
|
|
return tags, err
|
|
}
|
|
|
|
logrus.Debugf("detected %s as tags for recipe %s", strings.Join(tags, ", "), r.Name)
|
|
|
|
return tags, nil
|
|
}
|
|
|
|
// Get retrieves a recipe.
|
|
func Get(recipeName string) (Recipe, error) {
|
|
if err := EnsureExists(recipeName); err != nil {
|
|
return Recipe{}, err
|
|
}
|
|
|
|
pattern := fmt.Sprintf("%s/%s/compose**yml", config.RECIPES_DIR, recipeName)
|
|
composeFiles, err := filepath.Glob(pattern)
|
|
if err != nil {
|
|
return Recipe{}, err
|
|
}
|
|
|
|
if len(composeFiles) == 0 {
|
|
return Recipe{}, fmt.Errorf("%s is missing a compose.yml or compose.*.yml file?", recipeName)
|
|
}
|
|
|
|
envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
|
|
sampleEnv, err := config.ReadEnv(envSamplePath)
|
|
if err != nil {
|
|
return Recipe{}, err
|
|
}
|
|
|
|
opts := stack.Deploy{Composefiles: composeFiles}
|
|
config, err := loader.LoadComposefile(opts, sampleEnv)
|
|
if err != nil {
|
|
return Recipe{}, err
|
|
}
|
|
|
|
meta, err := GetRecipeMeta(recipeName)
|
|
if err != nil {
|
|
return Recipe{}, err
|
|
}
|
|
|
|
return Recipe{
|
|
Name: recipeName,
|
|
Config: config,
|
|
Meta: meta,
|
|
}, nil
|
|
}
|
|
|
|
// EnsureExists ensures that a recipe is locally cloned
|
|
func EnsureExists(recipeName string) error {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
if _, err := os.Stat(recipeDir); os.IsNotExist(err) {
|
|
logrus.Debugf("%s does not exist, attemmpting to clone", recipeDir)
|
|
url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, recipeName)
|
|
if err := gitPkg.Clone(recipeDir, url); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// EnsureVersion checks whether a specific version exists for a recipe.
|
|
func EnsureVersion(recipeName, version string) error {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
isClean, err := gitPkg.IsClean(recipeDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !isClean {
|
|
return fmt.Errorf("%s has locally unstaged changes", recipeName)
|
|
}
|
|
|
|
if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
|
|
return err
|
|
}
|
|
|
|
repo, err := git.PlainOpen(recipeDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
tags, err := repo.Tags()
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
var parsedTags []string
|
|
var tagRef plumbing.ReferenceName
|
|
if err := tags.ForEach(func(ref *plumbing.Reference) (err error) {
|
|
parsedTags = append(parsedTags, ref.Name().Short())
|
|
if ref.Name().Short() == version {
|
|
tagRef = ref.Name()
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("read %s as tags for recipe %s", strings.Join(parsedTags, ", "), recipeName)
|
|
|
|
if tagRef.String() == "" {
|
|
logrus.Warnf("no published release discovered for %s", recipeName)
|
|
return nil
|
|
}
|
|
|
|
worktree, err := repo.Worktree()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
opts := &git.CheckoutOptions{
|
|
Branch: tagRef,
|
|
Create: false,
|
|
Force: true,
|
|
}
|
|
if err := worktree.Checkout(opts); err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("successfully checked %s out to %s in %s", recipeName, tagRef.Short(), recipeDir)
|
|
|
|
return nil
|
|
}
|
|
|
|
// EnsureLatest makes sure the latest commit is checked out for a local recipe repository
|
|
func EnsureLatest(recipeName string) error {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
isClean, err := gitPkg.IsClean(recipeDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !isClean {
|
|
return fmt.Errorf("%s has locally unstaged changes", recipeName)
|
|
}
|
|
|
|
if err := gitPkg.EnsureGitRepo(recipeDir); err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("attempting to open git repository in %s", recipeDir)
|
|
|
|
repo, err := git.PlainOpen(recipeDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
worktree, err := repo.Worktree()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
branch, err := gitPkg.GetCurrentBranch(repo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
checkOutOpts := &git.CheckoutOptions{
|
|
Create: false,
|
|
Force: true,
|
|
Branch: plumbing.ReferenceName(branch),
|
|
}
|
|
|
|
if err := worktree.Checkout(checkOutOpts); err != nil {
|
|
logrus.Debugf("failed to check out %s in %s", branch, recipeDir)
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// ChaosVersion constructs a chaos mode recipe version.
|
|
func ChaosVersion(recipeName string) (string, error) {
|
|
var version string
|
|
|
|
head, err := gitPkg.GetRecipeHead(recipeName)
|
|
if err != nil {
|
|
return version, err
|
|
}
|
|
|
|
version = formatter.SmallSHA(head.String())
|
|
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
isClean, err := gitPkg.IsClean(recipeDir)
|
|
if err != nil {
|
|
return version, err
|
|
}
|
|
|
|
if !isClean {
|
|
version = fmt.Sprintf("%s + unstaged changes", version)
|
|
}
|
|
|
|
return version, nil
|
|
}
|
|
|
|
// GetRecipesLocal retrieves all local recipe directories
|
|
func GetRecipesLocal() ([]string, error) {
|
|
var recipes []string
|
|
|
|
recipes, err := config.GetAllFoldersInDirectory(config.RECIPES_DIR)
|
|
if err != nil {
|
|
return recipes, err
|
|
}
|
|
|
|
return recipes, nil
|
|
}
|
|
|
|
// GetVersionLabelLocal retrieves the version label on the local recipe config
|
|
func GetVersionLabelLocal(recipe Recipe) (string, error) {
|
|
var label string
|
|
|
|
for _, service := range recipe.Config.Services {
|
|
for label, value := range service.Deploy.Labels {
|
|
if strings.HasPrefix(label, "coop-cloud") {
|
|
return value, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
if label == "" {
|
|
return label, fmt.Errorf("%s has no version label? try running \"abra recipe sync %s\" first?", recipe.Name, recipe.Name)
|
|
}
|
|
|
|
return label, nil
|
|
}
|
|
|
|
func GetRecipeFeaturesAndCategory(recipeName string) (Features, string, error) {
|
|
feat := Features{}
|
|
|
|
var category string
|
|
|
|
readmePath := path.Join(config.RECIPES_DIR, recipeName, "README.md")
|
|
|
|
logrus.Debugf("attempting to open %s for recipe metadata parsing", readmePath)
|
|
|
|
readmeFS, err := ioutil.ReadFile(readmePath)
|
|
if err != nil {
|
|
return feat, category, err
|
|
}
|
|
|
|
readmeMetadata, err := GetStringInBetween( // Find text between delimiters
|
|
recipeName,
|
|
string(readmeFS),
|
|
"<!-- metadata -->", "<!-- endmetadata -->",
|
|
)
|
|
if err != nil {
|
|
return feat, category, err
|
|
}
|
|
|
|
readmeLines := strings.Split( // Array item from lines
|
|
strings.ReplaceAll( // Remove \t tabs
|
|
readmeMetadata, "\t", "",
|
|
),
|
|
"\n")
|
|
|
|
for _, val := range readmeLines {
|
|
if strings.Contains(val, "**Category**") {
|
|
category = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Category**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Backups**") {
|
|
feat.Backups = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Backups**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Email**") {
|
|
feat.Email = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Email**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**SSO**") {
|
|
feat.SSO = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **SSO**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Healthcheck**") {
|
|
feat.Healthcheck = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Healthcheck**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Tests**") {
|
|
feat.Tests = strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Tests**:"),
|
|
)
|
|
}
|
|
if strings.Contains(val, "**Image**") {
|
|
imageMetadata, err := GetImageMetadata(strings.TrimSpace(
|
|
strings.TrimPrefix(val, "* **Image**:"),
|
|
), recipeName)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
feat.Image = imageMetadata
|
|
}
|
|
}
|
|
|
|
return feat, category, nil
|
|
}
|
|
|
|
func GetImageMetadata(imageRowString, recipeName string) (Image, error) {
|
|
img := Image{}
|
|
|
|
imgFields := strings.Split(imageRowString, ",")
|
|
|
|
for i, elem := range imgFields {
|
|
imgFields[i] = strings.TrimSpace(elem)
|
|
}
|
|
|
|
if len(imgFields) < 3 {
|
|
if imageRowString != "" {
|
|
logrus.Warnf("%s image meta has incorrect format: %s", recipeName, imageRowString)
|
|
} else {
|
|
logrus.Warnf("%s image meta is empty?", recipeName)
|
|
}
|
|
return img, nil
|
|
}
|
|
|
|
img.Rating = imgFields[1]
|
|
img.Source = imgFields[2]
|
|
|
|
imgString := imgFields[0]
|
|
|
|
imageName, err := GetStringInBetween(recipeName, imgString, "[", "]")
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
img.Image = strings.ReplaceAll(imageName, "`", "")
|
|
|
|
imageURL, err := GetStringInBetween(recipeName, imgString, "(", ")")
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
img.URL = imageURL
|
|
|
|
return img, nil
|
|
}
|
|
|
|
// GetStringInBetween returns empty string if no start or end string found
|
|
func GetStringInBetween(recipeName, str, start, end string) (result string, err error) {
|
|
s := strings.Index(str, start)
|
|
if s == -1 {
|
|
return "", fmt.Errorf("%s: marker string %s not found", recipeName, start)
|
|
}
|
|
|
|
s += len(start)
|
|
e := strings.Index(str[s:], end)
|
|
|
|
if e == -1 {
|
|
return "", fmt.Errorf("%s: end marker %s not found", recipeName, end)
|
|
}
|
|
|
|
return str[s : s+e], nil
|
|
}
|
|
|
|
// EnsureUpToDate ensures that the local repo is synced to the remote
|
|
func EnsureUpToDate(recipeName string) error {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
isClean, err := gitPkg.IsClean(recipeDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !isClean {
|
|
return fmt.Errorf("%s has locally unstaged changes", recipeName)
|
|
}
|
|
|
|
repo, err := git.PlainOpen(recipeDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
remotes, err := repo.Remotes()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(remotes) == 0 {
|
|
logrus.Debugf("cannot ensure %s is up-to-date, no git remotes configured", recipeName)
|
|
return nil
|
|
}
|
|
|
|
worktree, err := repo.Worktree()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
branch, err := CheckoutDefaultBranch(repo, recipeName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
opts := &git.PullOptions{
|
|
Force: true,
|
|
ReferenceName: branch,
|
|
}
|
|
|
|
if err := worktree.Pull(opts); err != nil {
|
|
if !strings.Contains(err.Error(), "already up-to-date") {
|
|
return err
|
|
}
|
|
}
|
|
|
|
logrus.Debugf("fetched latest git changes for %s", recipeName)
|
|
|
|
return nil
|
|
}
|
|
|
|
func GetDefaultBranch(repo *git.Repository, recipeName string) (plumbing.ReferenceName, error) {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
branch := "master"
|
|
if _, err := repo.Branch("master"); err != nil {
|
|
if _, err := repo.Branch("main"); err != nil {
|
|
logrus.Debugf("failed to select branch in %s", recipeDir)
|
|
return "", err
|
|
}
|
|
branch = "main"
|
|
}
|
|
|
|
return plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)), nil
|
|
}
|
|
|
|
func CheckoutDefaultBranch(repo *git.Repository, recipeName string) (plumbing.ReferenceName, error) {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
branch, err := GetDefaultBranch(repo, recipeName)
|
|
if err != nil {
|
|
return plumbing.ReferenceName(""), err
|
|
}
|
|
|
|
worktree, err := repo.Worktree()
|
|
if err != nil {
|
|
return plumbing.ReferenceName(""), err
|
|
}
|
|
|
|
checkOutOpts := &git.CheckoutOptions{
|
|
Create: false,
|
|
Force: true,
|
|
Branch: branch,
|
|
}
|
|
|
|
if err := worktree.Checkout(checkOutOpts); err != nil {
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
logrus.Debugf("failed to check out %s in %s", branch, recipeDir)
|
|
return branch, err
|
|
}
|
|
|
|
logrus.Debugf("successfully checked out %v in %s", branch, recipeDir)
|
|
|
|
return branch, nil
|
|
}
|
|
|
|
// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
|
|
// is up to date.
|
|
func recipeCatalogueFSIsLatest() (bool, error) {
|
|
httpClient := web.NewHTTPRetryClient()
|
|
res, err := httpClient.Head(RecipeCatalogueURL)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
lastModified := res.Header["Last-Modified"][0]
|
|
parsed, err := time.Parse(time.RFC1123, lastModified)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
info, err := os.Stat(config.RECIPES_JSON)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
logrus.Debugf("no recipe catalogue found in file system cache")
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
|
|
localModifiedTime := info.ModTime().Unix()
|
|
remoteModifiedTime := parsed.Unix()
|
|
|
|
if localModifiedTime < remoteModifiedTime {
|
|
logrus.Debug("file system cached recipe catalogue is out-of-date")
|
|
return false, nil
|
|
}
|
|
|
|
logrus.Debug("file system cached recipe catalogue is now up-to-date")
|
|
|
|
return true, nil
|
|
}
|
|
|
|
// ReadRecipeCatalogue reads the recipe catalogue.
|
|
func ReadRecipeCatalogue() (RecipeCatalogue, error) {
|
|
recipes := make(RecipeCatalogue)
|
|
|
|
recipeFSIsLatest, err := recipeCatalogueFSIsLatest()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if !recipeFSIsLatest {
|
|
logrus.Debugf("reading recipe catalogue from web to get latest")
|
|
if err := readRecipeCatalogueWeb(&recipes); err != nil {
|
|
return nil, err
|
|
}
|
|
return recipes, nil
|
|
}
|
|
|
|
logrus.Debugf("reading recipe catalogue from file system cache to get latest")
|
|
if err := readRecipeCatalogueFS(&recipes); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return recipes, nil
|
|
}
|
|
|
|
// readRecipeCatalogueFS reads the catalogue from the file system.
|
|
func readRecipeCatalogueFS(target interface{}) error {
|
|
recipesJSONFS, err := ioutil.ReadFile(config.RECIPES_JSON)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := json.Unmarshal(recipesJSONFS, &target); err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("read recipe catalogue from file system cache in %s", config.RECIPES_JSON)
|
|
|
|
return nil
|
|
}
|
|
|
|
// readRecipeCatalogueWeb reads the catalogue from the web.
|
|
func readRecipeCatalogueWeb(target interface{}) error {
|
|
if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil {
|
|
return err
|
|
}
|
|
|
|
recipesJSON, err := json.MarshalIndent(target, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil {
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("read recipe catalogue from web at %s", RecipeCatalogueURL)
|
|
|
|
return nil
|
|
}
|
|
|
|
// VersionsOfService lists the version of a service.
|
|
func VersionsOfService(recipe, serviceName string) ([]string, error) {
|
|
var versions []string
|
|
|
|
catalogue, err := ReadRecipeCatalogue()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rec, ok := catalogue[recipe]
|
|
if !ok {
|
|
return versions, nil
|
|
}
|
|
|
|
alreadySeen := make(map[string]bool)
|
|
for _, serviceVersion := range rec.Versions {
|
|
for tag := range serviceVersion {
|
|
if _, ok := alreadySeen[tag]; !ok {
|
|
alreadySeen[tag] = true
|
|
versions = append(versions, tag)
|
|
}
|
|
}
|
|
}
|
|
|
|
logrus.Debugf("detected versions %s for %s", strings.Join(versions, ", "), recipe)
|
|
|
|
return versions, nil
|
|
}
|
|
|
|
// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue.
|
|
func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
|
|
catl, err := ReadRecipeCatalogue()
|
|
if err != nil {
|
|
return RecipeMeta{}, err
|
|
}
|
|
|
|
recipeMeta, ok := catl[recipeName]
|
|
if !ok {
|
|
err := fmt.Errorf("recipe %s does not exist?", recipeName)
|
|
return RecipeMeta{}, err
|
|
}
|
|
|
|
if err := EnsureExists(recipeName); err != nil {
|
|
return RecipeMeta{}, err
|
|
}
|
|
|
|
logrus.Debugf("recipe metadata retrieved for %s", recipeName)
|
|
|
|
return recipeMeta, nil
|
|
}
|
|
|
|
// RepoMeta is a single recipe repo metadata.
|
|
type RepoMeta struct {
|
|
ID int `json:"id"`
|
|
Owner Owner
|
|
Name string `json:"name"`
|
|
FullName string `json:"full_name"`
|
|
Description string `json:"description"`
|
|
Empty bool `json:"empty"`
|
|
Private bool `json:"private"`
|
|
Fork bool `json:"fork"`
|
|
Template bool `json:"template"`
|
|
Parent interface{} `json:"parent"`
|
|
Mirror bool `json:"mirror"`
|
|
Size int `json:"size"`
|
|
HTMLURL string `json:"html_url"`
|
|
SSHURL string `json:"ssh_url"`
|
|
CloneURL string `json:"clone_url"`
|
|
OriginalURL string `json:"original_url"`
|
|
Website string `json:"website"`
|
|
StarsCount int `json:"stars_count"`
|
|
ForksCount int `json:"forks_count"`
|
|
WatchersCount int `json:"watchers_count"`
|
|
OpenIssuesCount int `json:"open_issues_count"`
|
|
OpenPRCount int `json:"open_pr_counter"`
|
|
ReleaseCounter int `json:"release_counter"`
|
|
DefaultBranch string `json:"default_branch"`
|
|
Archived bool `json:"archived"`
|
|
CreatedAt string `json:"created_at"`
|
|
UpdatedAt string `json:"updated_at"`
|
|
Permissions Permissions
|
|
HasIssues bool `json:"has_issues"`
|
|
InternalTracker InternalTracker
|
|
HasWiki bool `json:"has_wiki"`
|
|
HasPullRequests bool `json:"has_pull_requests"`
|
|
HasProjects bool `json:"has_projects"`
|
|
IgnoreWhitespaceConflicts bool `json:"ignore_whitespace_conflicts"`
|
|
AllowMergeCommits bool `json:"allow_merge_commits"`
|
|
AllowRebase bool `json:"allow_rebase"`
|
|
AllowRebaseExplicit bool `json:"allow_rebase_explicit"`
|
|
AllowSquashMerge bool `json:"allow_squash_merge"`
|
|
AvatarURL string `json:"avatar_url"`
|
|
Internal bool `json:"internal"`
|
|
MirrorInterval string `json:"mirror_interval"`
|
|
}
|
|
|
|
// Owner is the repo organisation owner metadata.
|
|
type Owner struct {
|
|
ID int `json:"id"`
|
|
Login string `json:"login"`
|
|
FullName string `json:"full_name"`
|
|
Email string `json:"email"`
|
|
AvatarURL string `json:"avatar_url"`
|
|
Language string `json:"language"`
|
|
IsAdmin bool `json:"is_admin"`
|
|
LastLogin string `json:"last_login"`
|
|
Created string `json:"created"`
|
|
Restricted bool `json:"restricted"`
|
|
Username string `json:"username"`
|
|
}
|
|
|
|
// Permissions is perms metadata for a repo.
|
|
type Permissions struct {
|
|
Admin bool `json:"admin"`
|
|
Push bool `json:"push"`
|
|
Pull bool `json:"pull"`
|
|
}
|
|
|
|
// InternalTracker is issue tracker metadata for a repo.
|
|
type InternalTracker struct {
|
|
EnableTimeTracker bool `json:"enable_time_tracker"`
|
|
AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"`
|
|
EnableIssuesDependencies bool `json:"enable_issue_dependencies"`
|
|
}
|
|
|
|
// RepoCatalogue represents all the recipe repo metadata.
|
|
type RepoCatalogue map[string]RepoMeta
|
|
|
|
// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea.
|
|
func ReadReposMetadata() (RepoCatalogue, error) {
|
|
reposMeta := make(RepoCatalogue)
|
|
|
|
pageIdx := 1
|
|
bar := formatter.CreateProgressbar(-1, "retrieving recipe repos list from git.coopcloud.tech...")
|
|
for {
|
|
var reposList []RepoMeta
|
|
|
|
pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx)
|
|
|
|
logrus.Debugf("fetching repo metadata from %s", pagedURL)
|
|
|
|
if err := web.ReadJSON(pagedURL, &reposList); err != nil {
|
|
return reposMeta, err
|
|
}
|
|
|
|
if len(reposList) == 0 {
|
|
bar.Add(1)
|
|
break
|
|
}
|
|
|
|
for idx, repo := range reposList {
|
|
reposMeta[repo.Name] = reposList[idx]
|
|
}
|
|
|
|
pageIdx++
|
|
bar.Add(1)
|
|
}
|
|
|
|
fmt.Println() // newline for spinner
|
|
|
|
return reposMeta, nil
|
|
}
|
|
|
|
// GetRecipeVersions retrieves all recipe versions.
|
|
func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (RecipeVersions, error) {
|
|
versions := RecipeVersions{}
|
|
|
|
recipeDir := path.Join(config.RECIPES_DIR, recipeName)
|
|
|
|
logrus.Debugf("attempting to open git repository in %s", recipeDir)
|
|
|
|
repo, err := git.PlainOpen(recipeDir)
|
|
if err != nil {
|
|
return versions, err
|
|
}
|
|
|
|
worktree, err := repo.Worktree()
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
gitTags, err := repo.Tags()
|
|
if err != nil {
|
|
return versions, err
|
|
}
|
|
|
|
if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) {
|
|
tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/")
|
|
|
|
logrus.Debugf("processing %s for %s", tag, recipeName)
|
|
|
|
checkOutOpts := &git.CheckoutOptions{
|
|
Create: false,
|
|
Force: true,
|
|
Branch: plumbing.ReferenceName(ref.Name()),
|
|
}
|
|
if err := worktree.Checkout(checkOutOpts); err != nil {
|
|
logrus.Debugf("failed to check out %s in %s", tag, recipeDir)
|
|
return err
|
|
}
|
|
|
|
logrus.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir)
|
|
|
|
recipe, err := Get(recipeName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
cl, err := client.New("default") // only required for docker.io registry calls
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
queryCache := make(map[reference.Named]string)
|
|
versionMeta := make(map[string]ServiceMeta)
|
|
for _, service := range recipe.Config.Services {
|
|
|
|
img, err := reference.ParseNormalizedNamed(service.Image)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
path := reference.Path(img)
|
|
|
|
path = StripTagMeta(path)
|
|
|
|
var tag string
|
|
switch img.(type) {
|
|
case reference.NamedTagged:
|
|
tag = img.(reference.NamedTagged).Tag()
|
|
case reference.Named:
|
|
logrus.Warnf("%s service is missing image tag?", path)
|
|
continue
|
|
}
|
|
|
|
var exists bool
|
|
var digest string
|
|
if digest, exists = queryCache[img]; !exists {
|
|
logrus.Debugf("looking up image: %s from %s", img, path)
|
|
var err error
|
|
digest, err = client.GetTagDigest(cl, img, registryUsername, registryPassword)
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
continue
|
|
}
|
|
logrus.Debugf("queried for image: %s, tag: %s, digest: %s", path, tag, digest)
|
|
queryCache[img] = digest
|
|
logrus.Debugf("cached image: %s, tag: %s, digest: %s", path, tag, digest)
|
|
} else {
|
|
logrus.Debugf("reading image: %s, tag: %s, digest: %s from cache", path, tag, digest)
|
|
}
|
|
|
|
versionMeta[service.Name] = ServiceMeta{
|
|
Digest: digest,
|
|
Image: path,
|
|
Tag: img.(reference.NamedTagged).Tag(),
|
|
}
|
|
}
|
|
|
|
versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta})
|
|
|
|
return nil
|
|
}); err != nil {
|
|
return versions, err
|
|
}
|
|
|
|
_, err = CheckoutDefaultBranch(repo, recipeName)
|
|
if err != nil {
|
|
return versions, err
|
|
}
|
|
|
|
logrus.Debugf("collected %s for %s", versions, recipeName)
|
|
|
|
return versions, nil
|
|
}
|
|
|
|
// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue.
|
|
func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) {
|
|
var versions []string
|
|
|
|
if recipeMeta, exists := catl[recipeName]; exists {
|
|
for _, versionMeta := range recipeMeta.Versions {
|
|
for tag := range versionMeta {
|
|
versions = append(versions, tag)
|
|
}
|
|
}
|
|
}
|
|
|
|
return versions, nil
|
|
}
|
|
|
|
// StripTagMeta strips front-matter image tag data that we don't need for parsing.
|
|
func StripTagMeta(image string) string {
|
|
originalImage := image
|
|
|
|
if strings.Contains(image, "docker.io") {
|
|
image = strings.Split(image, "/")[1]
|
|
}
|
|
|
|
if strings.Contains(image, "library") {
|
|
image = strings.Split(image, "/")[1]
|
|
}
|
|
|
|
if originalImage != image {
|
|
logrus.Debugf("stripped %s to %s for parsing", originalImage, image)
|
|
}
|
|
|
|
return image
|
|
}
|