From 4c186678b82e005070d20c1d191f4bc2b8cf26d9 Mon Sep 17 00:00:00 2001 From: cellarspoon Date: Mon, 27 Dec 2021 16:40:59 +0100 Subject: [PATCH] fix: clone https url by default Catalogue package had to be merged into the recipe package due to too many circular import errors. Also, use https url for cloning, assume folks don't have ssh setup by default (the whole reason for the refactor). --- cli/app/list.go | 8 +- cli/app/rollback.go | 5 +- cli/app/upgrade.go | 5 +- cli/app/version.go | 6 +- cli/catalogue/catalogue.go | 21 +- cli/internal/deploy.go | 5 +- cli/internal/validate.go | 3 +- cli/recipe/list.go | 6 +- cli/recipe/release.go | 21 +- cli/recipe/upgrade.go | 4 +- cli/recipe/version.go | 4 +- pkg/autocomplete/autocomplete.go | 4 +- pkg/catalogue/catalogue.go | 491 ------------------------------- pkg/git/push.go | 22 +- pkg/lint/recipe.go | 6 +- pkg/recipe/recipe.go | 485 +++++++++++++++++++++++++++++- 16 files changed, 543 insertions(+), 553 deletions(-) delete mode 100644 pkg/catalogue/catalogue.go diff --git a/cli/app/list.go b/cli/app/list.go index 560813128..9e96e2ffb 100644 --- a/cli/app/list.go +++ b/cli/app/list.go @@ -7,8 +7,8 @@ import ( abraFormatter "coopcloud.tech/abra/cli/formatter" "coopcloud.tech/abra/cli/internal" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/ssh" "coopcloud.tech/tagcmp" "github.com/sirupsen/logrus" @@ -92,7 +92,7 @@ can take some time. sort.Sort(config.ByServerAndType(apps)) statuses := make(map[string]map[string]string) - var catl catalogue.RecipeCatalogue + var catl recipe.RecipeCatalogue if status { alreadySeen := make(map[string]bool) for _, app := range apps { @@ -110,7 +110,7 @@ can take some time. } var err error - catl, err = catalogue.ReadRecipeCatalogue() + catl, err = recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } @@ -153,7 +153,7 @@ can take some time. var newUpdates []string if version != "unknown" { - updates, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl) + updates, err := recipe.GetRecipeCatalogueVersions(app.Type, catl) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/rollback.go b/cli/app/rollback.go index 8e977c8bf..c1df7fb50 100644 --- a/cli/app/rollback.go +++ b/cli/app/rollback.go @@ -4,7 +4,6 @@ import ( "fmt" "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/lint" "coopcloud.tech/abra/pkg/recipe" @@ -75,12 +74,12 @@ recipes. logrus.Fatalf("%s is not deployed?", app.Name) } - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } - versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl) + versions, err := recipe.GetRecipeCatalogueVersions(app.Type, catl) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/upgrade.go b/cli/app/upgrade.go index 2e559ca70..dcdb8e659 100644 --- a/cli/app/upgrade.go +++ b/cli/app/upgrade.go @@ -5,7 +5,6 @@ import ( "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/lint" @@ -79,12 +78,12 @@ recipes. logrus.Fatalf("%s is not deployed?", app.Name) } - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } - versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl) + versions, err := recipe.GetRecipeCatalogueVersions(app.Type, catl) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/version.go b/cli/app/version.go index bbfd1d4a9..ef1ab0334 100644 --- a/cli/app/version.go +++ b/cli/app/version.go @@ -6,8 +6,8 @@ import ( abraFormatter "coopcloud.tech/abra/cli/formatter" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/client" + "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/upstream/stack" "github.com/docker/distribution/reference" "github.com/sirupsen/logrus" @@ -64,12 +64,12 @@ Cloud recipe version. logrus.Fatalf("%s is not deployed?", app.Name) } - recipeMeta, err := catalogue.GetRecipeMeta(app.Type) + recipeMeta, err := recipe.GetRecipeMeta(app.Type) if err != nil { logrus.Fatal(err) } - versionsMeta := make(map[string]catalogue.ServiceMeta) + versionsMeta := make(map[string]recipe.ServiceMeta) for _, recipeVersion := range recipeMeta.Versions { if currentVersion, exists := recipeVersion[deployedVersion]; exists { versionsMeta = currentVersion diff --git a/cli/catalogue/catalogue.go b/cli/catalogue/catalogue.go index ae428d13e..aa7aa24ed 100644 --- a/cli/catalogue/catalogue.go +++ b/cli/catalogue/catalogue.go @@ -9,7 +9,6 @@ import ( "coopcloud.tech/abra/cli/formatter" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/config" gitPkg "coopcloud.tech/abra/pkg/git" "coopcloud.tech/abra/pkg/limit" @@ -103,7 +102,7 @@ If you have a Hub account you can have Abra log you in to avoid this. Pass return err } - repos, err := catalogue.ReadReposMetadata() + repos, err := recipe.ReadReposMetadata() if err != nil { logrus.Fatal(err) } @@ -125,7 +124,7 @@ If you have a Hub account you can have Abra log you in to avoid this. Pass } } - catl := make(catalogue.RecipeCatalogue) + catl := make(recipe.RecipeCatalogue) catlBar := formatter.CreateProgressbar(barLength, "generating catalogue metadata...") for _, recipeMeta := range repos { if recipeName != "" && recipeName != recipeMeta.Name { @@ -138,7 +137,7 @@ If you have a Hub account you can have Abra log you in to avoid this. Pass continue } - versions, err := catalogue.GetRecipeVersions( + versions, err := recipe.GetRecipeVersions( recipeMeta.Name, internal.RegistryUsername, internal.RegistryPassword, @@ -152,7 +151,7 @@ If you have a Hub account you can have Abra log you in to avoid this. Pass logrus.Warn(err) } - catl[recipeMeta.Name] = catalogue.RecipeMeta{ + catl[recipeMeta.Name] = recipe.RecipeMeta{ Name: recipeMeta.Name, Repository: recipeMeta.CloneURL, Icon: recipeMeta.AvatarURL, @@ -177,7 +176,7 @@ If you have a Hub account you can have Abra log you in to avoid this. Pass logrus.Fatal(err) } } else { - catlFS, err := catalogue.ReadRecipeCatalogue() + catlFS, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } @@ -213,7 +212,7 @@ If you have a Hub account you can have Abra log you in to avoid this. Pass } if internal.Push { - if err := gitPkg.Push(cataloguePath); err != nil { + if err := gitPkg.Push(cataloguePath, false); err != nil { logrus.Fatal(err) } } @@ -236,7 +235,7 @@ var CatalogueCommand = &cli.Command{ }, } -func updateRepositories(repos catalogue.RepoCatalogue, recipeName string) error { +func updateRepositories(repos recipe.RepoCatalogue, recipeName string) error { var barLength int if recipeName != "" { barLength = 1 @@ -246,10 +245,10 @@ func updateRepositories(repos catalogue.RepoCatalogue, recipeName string) error cloneLimiter := limit.New(10) - retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are up-to-date...") + retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...") ch := make(chan string, barLength) for _, repoMeta := range repos { - go func(rm catalogue.RepoMeta) { + go func(rm recipe.RepoMeta) { cloneLimiter.Begin() defer cloneLimiter.End() @@ -266,7 +265,7 @@ func updateRepositories(repos catalogue.RepoCatalogue, recipeName string) error recipeDir := path.Join(config.RECIPES_DIR, rm.Name) - if err := gitPkg.Clone(recipeDir, rm.SSHURL); err != nil { + if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil { logrus.Fatal(err) } diff --git a/cli/internal/deploy.go b/cli/internal/deploy.go index 358ee3cb3..28fa6d69a 100644 --- a/cli/internal/deploy.go +++ b/cli/internal/deploy.go @@ -5,7 +5,6 @@ import ( "strings" abraFormatter "coopcloud.tech/abra/cli/formatter" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/dns" @@ -58,11 +57,11 @@ func DeployAction(c *cli.Context) error { version := deployedVersion if version == "" && !Chaos { - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } - versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl) + versions, err := recipe.GetRecipeCatalogueVersions(app.Type, catl) if err != nil { logrus.Fatal(err) } diff --git a/cli/internal/validate.go b/cli/internal/validate.go index 0e202ba2a..92701140f 100644 --- a/cli/internal/validate.go +++ b/cli/internal/validate.go @@ -5,7 +5,6 @@ import ( "strings" "coopcloud.tech/abra/pkg/app" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/ssh" @@ -50,7 +49,7 @@ func ValidateRecipeWithPrompt(c *cli.Context) recipe.Recipe { if recipeName == "" && !NoInput { var recipes []string - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } diff --git a/cli/recipe/list.go b/cli/recipe/list.go index c8d8d0a5f..55bb75541 100644 --- a/cli/recipe/list.go +++ b/cli/recipe/list.go @@ -7,7 +7,7 @@ import ( "strings" "coopcloud.tech/abra/cli/formatter" - "coopcloud.tech/abra/pkg/catalogue" + "coopcloud.tech/abra/pkg/recipe" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -29,13 +29,13 @@ var recipeListCommand = &cli.Command{ patternFlag, }, Action: func(c *cli.Context) error { - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err.Error()) } recipes := catl.Flatten() - sort.Sort(catalogue.ByRecipeName(recipes)) + sort.Sort(recipe.ByRecipeName(recipes)) tableCol := []string{"name", "category", "status", "healthcheck", "backups", "email", "tests", "SSO"} table := formatter.CreateTable(tableCol) diff --git a/cli/recipe/release.go b/cli/recipe/release.go index 70cd748a8..dbda67e4d 100644 --- a/cli/recipe/release.go +++ b/cli/recipe/release.go @@ -17,7 +17,6 @@ import ( "github.com/AlecAivazis/survey/v2" "github.com/docker/distribution/reference" "github.com/go-git/go-git/v5" - configPkg "github.com/go-git/go-git/v5/config" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -214,7 +213,7 @@ func createReleaseFromTag(recipe recipe.Recipe, tagString, mainAppVersion string logrus.Fatal(err) } - if err := pushRelease(tagString, repo); err != nil { + if err := pushRelease(recipe.Dir()); err != nil { logrus.Fatal(err) } @@ -309,7 +308,7 @@ func tagRelease(tagString string, repo *git.Repository) error { return nil } -func pushRelease(tagString string, repo *git.Repository) error { +func pushRelease(recipeDir string) error { if internal.Dry { logrus.Info("dry run: no changes pushed") return nil @@ -326,21 +325,9 @@ func pushRelease(tagString string, repo *git.Repository) error { } if internal.Push { - if err := repo.Push(&git.PushOptions{}); err != nil { + if err := gitPkg.Push(recipeDir, true); err != nil { return err } - - tagRef := fmt.Sprintf("+refs/tags/%s:refs/tags/%s", tagString, tagString) - pushOpts := &git.PushOptions{ - RefSpecs: []configPkg.RefSpec{ - configPkg.RefSpec(tagRef), - }, - } - if err := repo.Push(pushOpts); err != nil { - return err - } - - logrus.Info(fmt.Sprintf("pushed tag %s to remote", tagString)) } return nil @@ -416,7 +403,7 @@ func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recip logrus.Fatal(err) } - if err := pushRelease(newTagString, repo); err != nil { + if err := pushRelease(recipe.Dir()); err != nil { logrus.Fatal(err) } diff --git a/cli/recipe/upgrade.go b/cli/recipe/upgrade.go index 4ea0f7332..02e12a5bc 100644 --- a/cli/recipe/upgrade.go +++ b/cli/recipe/upgrade.go @@ -9,9 +9,9 @@ import ( "strings" "coopcloud.tech/abra/cli/internal" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" + recipePkg "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/tagcmp" "github.com/AlecAivazis/survey/v2" "github.com/docker/distribution/reference" @@ -147,7 +147,7 @@ You may invoke this command in "wizard" mode and be prompted for input: continue // skip on to the next tag and don't update any compose files } - catlVersions, err := catalogue.VersionsOfService(recipe.Name, service.Name) + catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name) if err != nil { logrus.Fatal(err) } diff --git a/cli/recipe/version.go b/cli/recipe/version.go index 040db877c..6d6686a2a 100644 --- a/cli/recipe/version.go +++ b/cli/recipe/version.go @@ -4,7 +4,7 @@ import ( "coopcloud.tech/abra/cli/formatter" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/catalogue" + recipePkg "coopcloud.tech/abra/pkg/recipe" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -18,7 +18,7 @@ var recipeVersionCommand = &cli.Command{ Action: func(c *cli.Context) error { recipe := internal.ValidateRecipe(c) - catalogue, err := catalogue.ReadRecipeCatalogue() + catalogue, err := recipePkg.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } diff --git a/pkg/autocomplete/autocomplete.go b/pkg/autocomplete/autocomplete.go index e8675680a..4d37b62c5 100644 --- a/pkg/autocomplete/autocomplete.go +++ b/pkg/autocomplete/autocomplete.go @@ -3,8 +3,8 @@ package autocomplete import ( "fmt" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/recipe" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -27,7 +27,7 @@ func AppNameComplete(c *cli.Context) { // RecipeNameComplete completes recipe names func RecipeNameComplete(c *cli.Context) { - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Warn(err) } diff --git a/pkg/catalogue/catalogue.go b/pkg/catalogue/catalogue.go deleted file mode 100644 index 39745131f..000000000 --- a/pkg/catalogue/catalogue.go +++ /dev/null @@ -1,491 +0,0 @@ -// Package catalogue provides ways of interacting with recipe catalogues which -// are JSON data structures which contain meta information about recipes (e.g. -// what versions of the Nextcloud recipe are available?). -package catalogue - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path" - "strings" - "time" - - "coopcloud.tech/abra/cli/formatter" - "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/config" - "coopcloud.tech/abra/pkg/recipe" - "coopcloud.tech/abra/pkg/web" - "github.com/docker/distribution/reference" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/sirupsen/logrus" -) - -// RecipeCatalogueURL is the only current recipe catalogue available. -const RecipeCatalogueURL = "https://apps.coopcloud.tech" - -// ReposMetadataURL is the recipe repository metadata -const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos" - -// tag represents a git tag. -type tag = string - -// service represents a service within a recipe. -type service = string - -// ServiceMeta represents meta info associated with a service. -type ServiceMeta struct { - Digest string `json:"digest"` - Image string `json:"image"` - Tag string `json:"tag"` -} - -// RecipeVersions are the versions associated with a recipe. -type RecipeVersions []map[tag]map[service]ServiceMeta - -// RecipeMeta represents metadata for a recipe in the abra catalogue. -type RecipeMeta struct { - Category string `json:"category"` - DefaultBranch string `json:"default_branch"` - Description string `json:"description"` - Features recipe.Features `json:"features"` - Icon string `json:"icon"` - Name string `json:"name"` - Repository string `json:"repository"` - Versions RecipeVersions `json:"versions"` - Website string `json:"website"` -} - -// LatestVersion returns the latest version of a recipe. -func (r RecipeMeta) LatestVersion() string { - var version string - - // apps.json versions are sorted so the last key is latest - latest := r.Versions[len(r.Versions)-1] - - for tag := range latest { - version = tag - } - - logrus.Debugf("choosing %s as latest version of %s", version, r.Name) - - return version -} - -// Name represents a recipe name. -type Name = string - -// RecipeCatalogue represents the entire recipe catalogue. -type RecipeCatalogue map[Name]RecipeMeta - -// Flatten converts AppCatalogue to slice -func (r RecipeCatalogue) Flatten() []RecipeMeta { - recipes := make([]RecipeMeta, 0, len(r)) - - for name := range r { - recipes = append(recipes, r[name]) - } - - return recipes -} - -// ByRecipeName sorts recipes by name. -type ByRecipeName []RecipeMeta - -func (r ByRecipeName) Len() int { return len(r) } -func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] } -func (r ByRecipeName) Less(i, j int) bool { - return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name) -} - -// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally -// is up to date. -func recipeCatalogueFSIsLatest() (bool, error) { - httpClient := web.NewHTTPRetryClient() - res, err := httpClient.Head(RecipeCatalogueURL) - if err != nil { - return false, err - } - - lastModified := res.Header["Last-Modified"][0] - parsed, err := time.Parse(time.RFC1123, lastModified) - if err != nil { - return false, err - } - - info, err := os.Stat(config.RECIPES_JSON) - if err != nil { - if os.IsNotExist(err) { - logrus.Debugf("no recipe catalogue found in file system cache") - return false, nil - } - return false, err - } - - localModifiedTime := info.ModTime().Unix() - remoteModifiedTime := parsed.Unix() - - if localModifiedTime < remoteModifiedTime { - logrus.Debug("file system cached recipe catalogue is out-of-date") - return false, nil - } - - logrus.Debug("file system cached recipe catalogue is now up-to-date") - - return true, nil -} - -// ReadRecipeCatalogue reads the recipe catalogue. -func ReadRecipeCatalogue() (RecipeCatalogue, error) { - recipes := make(RecipeCatalogue) - - recipeFSIsLatest, err := recipeCatalogueFSIsLatest() - if err != nil { - return nil, err - } - - if !recipeFSIsLatest { - logrus.Debugf("reading recipe catalogue from web to get latest") - if err := readRecipeCatalogueWeb(&recipes); err != nil { - return nil, err - } - return recipes, nil - } - - logrus.Debugf("reading recipe catalogue from file system cache to get latest") - if err := readRecipeCatalogueFS(&recipes); err != nil { - return nil, err - } - - return recipes, nil -} - -// readRecipeCatalogueFS reads the catalogue from the file system. -func readRecipeCatalogueFS(target interface{}) error { - recipesJSONFS, err := ioutil.ReadFile(config.RECIPES_JSON) - if err != nil { - return err - } - - if err := json.Unmarshal(recipesJSONFS, &target); err != nil { - return err - } - - logrus.Debugf("read recipe catalogue from file system cache in %s", config.RECIPES_JSON) - - return nil -} - -// readRecipeCatalogueWeb reads the catalogue from the web. -func readRecipeCatalogueWeb(target interface{}) error { - if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil { - return err - } - - recipesJSON, err := json.MarshalIndent(target, "", " ") - if err != nil { - return err - } - - if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil { - return err - } - - logrus.Debugf("read recipe catalogue from web at %s", RecipeCatalogueURL) - - return nil -} - -// VersionsOfService lists the version of a service. -func VersionsOfService(recipe, serviceName string) ([]string, error) { - var versions []string - - catalogue, err := ReadRecipeCatalogue() - if err != nil { - return nil, err - } - - rec, ok := catalogue[recipe] - if !ok { - return versions, nil - } - - alreadySeen := make(map[string]bool) - for _, serviceVersion := range rec.Versions { - for tag := range serviceVersion { - if _, ok := alreadySeen[tag]; !ok { - alreadySeen[tag] = true - versions = append(versions, tag) - } - } - } - - logrus.Debugf("detected versions %s for %s", strings.Join(versions, ", "), recipe) - - return versions, nil -} - -// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue. -func GetRecipeMeta(recipeName string) (RecipeMeta, error) { - catl, err := ReadRecipeCatalogue() - if err != nil { - return RecipeMeta{}, err - } - - recipeMeta, ok := catl[recipeName] - if !ok { - err := fmt.Errorf("recipe %s does not exist?", recipeName) - return RecipeMeta{}, err - } - - if err := recipe.EnsureExists(recipeName); err != nil { - return RecipeMeta{}, err - } - - logrus.Debugf("recipe metadata retrieved for %s", recipeName) - - return recipeMeta, nil -} - -// RepoMeta is a single recipe repo metadata. -type RepoMeta struct { - ID int `json:"id"` - Owner Owner - Name string `json:"name"` - FullName string `json:"full_name"` - Description string `json:"description"` - Empty bool `json:"empty"` - Private bool `json:"private"` - Fork bool `json:"fork"` - Template bool `json:"template"` - Parent interface{} `json:"parent"` - Mirror bool `json:"mirror"` - Size int `json:"size"` - HTMLURL string `json:"html_url"` - SSHURL string `json:"ssh_url"` - CloneURL string `json:"clone_url"` - OriginalURL string `json:"original_url"` - Website string `json:"website"` - StarsCount int `json:"stars_count"` - ForksCount int `json:"forks_count"` - WatchersCount int `json:"watchers_count"` - OpenIssuesCount int `json:"open_issues_count"` - OpenPRCount int `json:"open_pr_counter"` - ReleaseCounter int `json:"release_counter"` - DefaultBranch string `json:"default_branch"` - Archived bool `json:"archived"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - Permissions Permissions - HasIssues bool `json:"has_issues"` - InternalTracker InternalTracker - HasWiki bool `json:"has_wiki"` - HasPullRequests bool `json:"has_pull_requests"` - HasProjects bool `json:"has_projects"` - IgnoreWhitespaceConflicts bool `json:"ignore_whitespace_conflicts"` - AllowMergeCommits bool `json:"allow_merge_commits"` - AllowRebase bool `json:"allow_rebase"` - AllowRebaseExplicit bool `json:"allow_rebase_explicit"` - AllowSquashMerge bool `json:"allow_squash_merge"` - AvatarURL string `json:"avatar_url"` - Internal bool `json:"internal"` - MirrorInterval string `json:"mirror_interval"` -} - -// Owner is the repo organisation owner metadata. -type Owner struct { - ID int `json:"id"` - Login string `json:"login"` - FullName string `json:"full_name"` - Email string `json:"email"` - AvatarURL string `json:"avatar_url"` - Language string `json:"language"` - IsAdmin bool `json:"is_admin"` - LastLogin string `json:"last_login"` - Created string `json:"created"` - Restricted bool `json:"restricted"` - Username string `json:"username"` -} - -// Permissions is perms metadata for a repo. -type Permissions struct { - Admin bool `json:"admin"` - Push bool `json:"push"` - Pull bool `json:"pull"` -} - -// InternalTracker is issue tracker metadata for a repo. -type InternalTracker struct { - EnableTimeTracker bool `json:"enable_time_tracker"` - AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"` - EnableIssuesDependencies bool `json:"enable_issue_dependencies"` -} - -// RepoCatalogue represents all the recipe repo metadata. -type RepoCatalogue map[string]RepoMeta - -// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea. -func ReadReposMetadata() (RepoCatalogue, error) { - reposMeta := make(RepoCatalogue) - - pageIdx := 1 - bar := formatter.CreateProgressbar(-1, "retrieving recipe repos list from git.coopcloud.tech...") - for { - var reposList []RepoMeta - - pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx) - - logrus.Debugf("fetching repo metadata from %s", pagedURL) - - if err := web.ReadJSON(pagedURL, &reposList); err != nil { - return reposMeta, err - } - - if len(reposList) == 0 { - bar.Add(1) - break - } - - for idx, repo := range reposList { - reposMeta[repo.Name] = reposList[idx] - } - - pageIdx++ - bar.Add(1) - } - - fmt.Println() // newline for spinner - - return reposMeta, nil -} - -// GetRecipeVersions retrieves all recipe versions. -func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (RecipeVersions, error) { - versions := RecipeVersions{} - - recipeDir := path.Join(config.RECIPES_DIR, recipeName) - - logrus.Debugf("attempting to open git repository in %s", recipeDir) - - repo, err := git.PlainOpen(recipeDir) - if err != nil { - return versions, err - } - - worktree, err := repo.Worktree() - if err != nil { - logrus.Fatal(err) - } - - gitTags, err := repo.Tags() - if err != nil { - return versions, err - } - - if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) { - tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/") - - logrus.Debugf("processing %s for %s", tag, recipeName) - - checkOutOpts := &git.CheckoutOptions{ - Create: false, - Force: true, - Branch: plumbing.ReferenceName(ref.Name()), - } - if err := worktree.Checkout(checkOutOpts); err != nil { - logrus.Debugf("failed to check out %s in %s", tag, recipeDir) - return err - } - - logrus.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir) - - recipe, err := recipe.Get(recipeName) - if err != nil { - return err - } - - cl, err := client.New("default") // only required for docker.io registry calls - if err != nil { - logrus.Fatal(err) - } - - queryCache := make(map[reference.Named]string) - versionMeta := make(map[string]ServiceMeta) - for _, service := range recipe.Config.Services { - - img, err := reference.ParseNormalizedNamed(service.Image) - if err != nil { - return err - } - - path := reference.Path(img) - if strings.Contains(path, "library") { - path = strings.Split(path, "/")[1] - } - - var tag string - switch img.(type) { - case reference.NamedTagged: - tag = img.(reference.NamedTagged).Tag() - case reference.Named: - logrus.Warnf("%s service is missing image tag?", path) - continue - } - - var exists bool - var digest string - if digest, exists = queryCache[img]; !exists { - logrus.Debugf("looking up image: %s from %s", img, path) - var err error - digest, err = client.GetTagDigest(cl, img, registryUsername, registryPassword) - if err != nil { - logrus.Warn(err) - continue - } - logrus.Debugf("queried for image: %s, tag: %s, digest: %s", path, tag, digest) - queryCache[img] = digest - logrus.Debugf("cached image: %s, tag: %s, digest: %s", path, tag, digest) - } else { - logrus.Debugf("reading image: %s, tag: %s, digest: %s from cache", path, tag, digest) - } - - versionMeta[service.Name] = ServiceMeta{ - Digest: digest, - Image: path, - Tag: img.(reference.NamedTagged).Tag(), - } - } - - versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta}) - - return nil - }); err != nil { - return versions, err - } - - _, err = recipe.CheckoutDefaultBranch(repo, recipeName) - if err != nil { - return versions, err - } - - logrus.Debugf("collected %s for %s", versions, recipeName) - - return versions, nil -} - -// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue. -func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) { - var versions []string - - if recipeMeta, exists := catl[recipeName]; exists { - for _, versionMeta := range recipeMeta.Versions { - for tag := range versionMeta { - versions = append(versions, tag) - } - } - } - - return versions, nil -} diff --git a/pkg/git/push.go b/pkg/git/push.go index 68adf807b..cb8378f5f 100644 --- a/pkg/git/push.go +++ b/pkg/git/push.go @@ -1,13 +1,18 @@ package git import ( + "path" + + configPkg "coopcloud.tech/abra/pkg/config" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" "github.com/sirupsen/logrus" ) // Push pushes the latest changes -func Push(repoPath string) error { - commitRepo, err := git.PlainOpen(repoPath) +func Push(recipeName string, tags bool) error { + recipeDir := path.Join(configPkg.RECIPES_DIR, recipeName) + commitRepo, err := git.PlainOpen(recipeDir) if err != nil { return err } @@ -15,8 +20,19 @@ func Push(repoPath string) error { if err := commitRepo.Push(&git.PushOptions{}); err != nil { return err } + logrus.Info("git changes pushed") - logrus.Info("changes pushed") + if tags { + pushOpts := &git.PushOptions{ + RefSpecs: []config.RefSpec{ + config.RefSpec("+refs/tags/*:refs/tags/*"), + }, + } + if err := commitRepo.Push(pushOpts); err != nil { + return err + } + logrus.Info("git tags pushed") + } return nil } diff --git a/pkg/lint/recipe.go b/pkg/lint/recipe.go index 80a72c7c1..231a7eb4f 100644 --- a/pkg/lint/recipe.go +++ b/pkg/lint/recipe.go @@ -6,9 +6,9 @@ import ( "os" "path" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/recipe" + recipePkg "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/tagcmp" "github.com/docker/distribution/reference" "github.com/sirupsen/logrus" @@ -269,12 +269,12 @@ func LintImagePresent(recipe recipe.Recipe) (bool, error) { } func LintHasPublishedVersion(recipe recipe.Recipe) (bool, error) { - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipePkg.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } - versions, err := catalogue.GetRecipeCatalogueVersions(recipe.Name, catl) + versions, err := recipePkg.GetRecipeCatalogueVersions(recipe.Name, catl) if err != nil { logrus.Fatal(err) } diff --git a/pkg/recipe/recipe.go b/pkg/recipe/recipe.go index 41a94d6cf..53cbf82d0 100644 --- a/pkg/recipe/recipe.go +++ b/pkg/recipe/recipe.go @@ -1,24 +1,107 @@ package recipe import ( + "encoding/json" "fmt" "io/ioutil" "os" "path" "path/filepath" "strings" + "time" + "coopcloud.tech/abra/cli/formatter" + "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/compose" "coopcloud.tech/abra/pkg/config" gitPkg "coopcloud.tech/abra/pkg/git" "coopcloud.tech/abra/pkg/upstream/stack" loader "coopcloud.tech/abra/pkg/upstream/stack" + "coopcloud.tech/abra/pkg/web" composetypes "github.com/docker/cli/cli/compose/types" + "github.com/docker/distribution/reference" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/sirupsen/logrus" ) +// RecipeCatalogueURL is the only current recipe catalogue available. +const RecipeCatalogueURL = "https://apps.coopcloud.tech" + +// ReposMetadataURL is the recipe repository metadata +const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos" + +// tag represents a git tag. +type tag = string + +// service represents a service within a recipe. +type service = string + +// ServiceMeta represents meta info associated with a service. +type ServiceMeta struct { + Digest string `json:"digest"` + Image string `json:"image"` + Tag string `json:"tag"` +} + +// RecipeVersions are the versions associated with a recipe. +type RecipeVersions []map[tag]map[service]ServiceMeta + +// RecipeMeta represents metadata for a recipe in the abra catalogue. +type RecipeMeta struct { + Category string `json:"category"` + DefaultBranch string `json:"default_branch"` + Description string `json:"description"` + Features Features `json:"features"` + Icon string `json:"icon"` + Name string `json:"name"` + Repository string `json:"repository"` + Versions RecipeVersions `json:"versions"` + Website string `json:"website"` +} + +// LatestVersion returns the latest version of a recipe. +func (r RecipeMeta) LatestVersion() string { + var version string + + // apps.json versions are sorted so the last key is latest + latest := r.Versions[len(r.Versions)-1] + + for tag := range latest { + version = tag + } + + logrus.Debugf("choosing %s as latest version of %s", version, r.Name) + + return version +} + +// Name represents a recipe name. +type Name = string + +// RecipeCatalogue represents the entire recipe catalogue. +type RecipeCatalogue map[Name]RecipeMeta + +// Flatten converts AppCatalogue to slice +func (r RecipeCatalogue) Flatten() []RecipeMeta { + recipes := make([]RecipeMeta, 0, len(r)) + + for name := range r { + recipes = append(recipes, r[name]) + } + + return recipes +} + +// ByRecipeName sorts recipes by name. +type ByRecipeName []RecipeMeta + +func (r ByRecipeName) Len() int { return len(r) } +func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] } +func (r ByRecipeName) Less(i, j int) bool { + return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name) +} + // Image represents a recipe container image. type Image struct { Image string `json:"image"` @@ -42,6 +125,7 @@ type Features struct { type Recipe struct { Name string Config *composetypes.Config + Meta RecipeMeta } // Dir retrieves the recipe repository path @@ -121,7 +205,16 @@ func Get(recipeName string) (Recipe, error) { return Recipe{}, err } - return Recipe{Name: recipeName, Config: config}, nil + meta, err := GetRecipeMeta(recipeName) + if err != nil { + return Recipe{}, err + } + + return Recipe{ + Name: recipeName, + Config: config, + Meta: meta, + }, nil } // EnsureExists ensures that a recipe is locally cloned @@ -536,3 +629,393 @@ func CheckoutDefaultBranch(repo *git.Repository, recipeName string) (plumbing.Re return branch, nil } + +// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally +// is up to date. +func recipeCatalogueFSIsLatest() (bool, error) { + httpClient := web.NewHTTPRetryClient() + res, err := httpClient.Head(RecipeCatalogueURL) + if err != nil { + return false, err + } + + lastModified := res.Header["Last-Modified"][0] + parsed, err := time.Parse(time.RFC1123, lastModified) + if err != nil { + return false, err + } + + info, err := os.Stat(config.RECIPES_JSON) + if err != nil { + if os.IsNotExist(err) { + logrus.Debugf("no recipe catalogue found in file system cache") + return false, nil + } + return false, err + } + + localModifiedTime := info.ModTime().Unix() + remoteModifiedTime := parsed.Unix() + + if localModifiedTime < remoteModifiedTime { + logrus.Debug("file system cached recipe catalogue is out-of-date") + return false, nil + } + + logrus.Debug("file system cached recipe catalogue is now up-to-date") + + return true, nil +} + +// ReadRecipeCatalogue reads the recipe catalogue. +func ReadRecipeCatalogue() (RecipeCatalogue, error) { + recipes := make(RecipeCatalogue) + + recipeFSIsLatest, err := recipeCatalogueFSIsLatest() + if err != nil { + return nil, err + } + + if !recipeFSIsLatest { + logrus.Debugf("reading recipe catalogue from web to get latest") + if err := readRecipeCatalogueWeb(&recipes); err != nil { + return nil, err + } + return recipes, nil + } + + logrus.Debugf("reading recipe catalogue from file system cache to get latest") + if err := readRecipeCatalogueFS(&recipes); err != nil { + return nil, err + } + + return recipes, nil +} + +// readRecipeCatalogueFS reads the catalogue from the file system. +func readRecipeCatalogueFS(target interface{}) error { + recipesJSONFS, err := ioutil.ReadFile(config.RECIPES_JSON) + if err != nil { + return err + } + + if err := json.Unmarshal(recipesJSONFS, &target); err != nil { + return err + } + + logrus.Debugf("read recipe catalogue from file system cache in %s", config.RECIPES_JSON) + + return nil +} + +// readRecipeCatalogueWeb reads the catalogue from the web. +func readRecipeCatalogueWeb(target interface{}) error { + if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil { + return err + } + + recipesJSON, err := json.MarshalIndent(target, "", " ") + if err != nil { + return err + } + + if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil { + return err + } + + logrus.Debugf("read recipe catalogue from web at %s", RecipeCatalogueURL) + + return nil +} + +// VersionsOfService lists the version of a service. +func VersionsOfService(recipe, serviceName string) ([]string, error) { + var versions []string + + catalogue, err := ReadRecipeCatalogue() + if err != nil { + return nil, err + } + + rec, ok := catalogue[recipe] + if !ok { + return versions, nil + } + + alreadySeen := make(map[string]bool) + for _, serviceVersion := range rec.Versions { + for tag := range serviceVersion { + if _, ok := alreadySeen[tag]; !ok { + alreadySeen[tag] = true + versions = append(versions, tag) + } + } + } + + logrus.Debugf("detected versions %s for %s", strings.Join(versions, ", "), recipe) + + return versions, nil +} + +// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue. +func GetRecipeMeta(recipeName string) (RecipeMeta, error) { + catl, err := ReadRecipeCatalogue() + if err != nil { + return RecipeMeta{}, err + } + + recipeMeta, ok := catl[recipeName] + if !ok { + err := fmt.Errorf("recipe %s does not exist?", recipeName) + return RecipeMeta{}, err + } + + if err := EnsureExists(recipeName); err != nil { + return RecipeMeta{}, err + } + + logrus.Debugf("recipe metadata retrieved for %s", recipeName) + + return recipeMeta, nil +} + +// RepoMeta is a single recipe repo metadata. +type RepoMeta struct { + ID int `json:"id"` + Owner Owner + Name string `json:"name"` + FullName string `json:"full_name"` + Description string `json:"description"` + Empty bool `json:"empty"` + Private bool `json:"private"` + Fork bool `json:"fork"` + Template bool `json:"template"` + Parent interface{} `json:"parent"` + Mirror bool `json:"mirror"` + Size int `json:"size"` + HTMLURL string `json:"html_url"` + SSHURL string `json:"ssh_url"` + CloneURL string `json:"clone_url"` + OriginalURL string `json:"original_url"` + Website string `json:"website"` + StarsCount int `json:"stars_count"` + ForksCount int `json:"forks_count"` + WatchersCount int `json:"watchers_count"` + OpenIssuesCount int `json:"open_issues_count"` + OpenPRCount int `json:"open_pr_counter"` + ReleaseCounter int `json:"release_counter"` + DefaultBranch string `json:"default_branch"` + Archived bool `json:"archived"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + Permissions Permissions + HasIssues bool `json:"has_issues"` + InternalTracker InternalTracker + HasWiki bool `json:"has_wiki"` + HasPullRequests bool `json:"has_pull_requests"` + HasProjects bool `json:"has_projects"` + IgnoreWhitespaceConflicts bool `json:"ignore_whitespace_conflicts"` + AllowMergeCommits bool `json:"allow_merge_commits"` + AllowRebase bool `json:"allow_rebase"` + AllowRebaseExplicit bool `json:"allow_rebase_explicit"` + AllowSquashMerge bool `json:"allow_squash_merge"` + AvatarURL string `json:"avatar_url"` + Internal bool `json:"internal"` + MirrorInterval string `json:"mirror_interval"` +} + +// Owner is the repo organisation owner metadata. +type Owner struct { + ID int `json:"id"` + Login string `json:"login"` + FullName string `json:"full_name"` + Email string `json:"email"` + AvatarURL string `json:"avatar_url"` + Language string `json:"language"` + IsAdmin bool `json:"is_admin"` + LastLogin string `json:"last_login"` + Created string `json:"created"` + Restricted bool `json:"restricted"` + Username string `json:"username"` +} + +// Permissions is perms metadata for a repo. +type Permissions struct { + Admin bool `json:"admin"` + Push bool `json:"push"` + Pull bool `json:"pull"` +} + +// InternalTracker is issue tracker metadata for a repo. +type InternalTracker struct { + EnableTimeTracker bool `json:"enable_time_tracker"` + AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"` + EnableIssuesDependencies bool `json:"enable_issue_dependencies"` +} + +// RepoCatalogue represents all the recipe repo metadata. +type RepoCatalogue map[string]RepoMeta + +// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea. +func ReadReposMetadata() (RepoCatalogue, error) { + reposMeta := make(RepoCatalogue) + + pageIdx := 1 + bar := formatter.CreateProgressbar(-1, "retrieving recipe repos list from git.coopcloud.tech...") + for { + var reposList []RepoMeta + + pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx) + + logrus.Debugf("fetching repo metadata from %s", pagedURL) + + if err := web.ReadJSON(pagedURL, &reposList); err != nil { + return reposMeta, err + } + + if len(reposList) == 0 { + bar.Add(1) + break + } + + for idx, repo := range reposList { + reposMeta[repo.Name] = reposList[idx] + } + + pageIdx++ + bar.Add(1) + } + + fmt.Println() // newline for spinner + + return reposMeta, nil +} + +// GetRecipeVersions retrieves all recipe versions. +func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (RecipeVersions, error) { + versions := RecipeVersions{} + + recipeDir := path.Join(config.RECIPES_DIR, recipeName) + + logrus.Debugf("attempting to open git repository in %s", recipeDir) + + repo, err := git.PlainOpen(recipeDir) + if err != nil { + return versions, err + } + + worktree, err := repo.Worktree() + if err != nil { + logrus.Fatal(err) + } + + gitTags, err := repo.Tags() + if err != nil { + return versions, err + } + + if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) { + tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/") + + logrus.Debugf("processing %s for %s", tag, recipeName) + + checkOutOpts := &git.CheckoutOptions{ + Create: false, + Force: true, + Branch: plumbing.ReferenceName(ref.Name()), + } + if err := worktree.Checkout(checkOutOpts); err != nil { + logrus.Debugf("failed to check out %s in %s", tag, recipeDir) + return err + } + + logrus.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir) + + recipe, err := Get(recipeName) + if err != nil { + return err + } + + cl, err := client.New("default") // only required for docker.io registry calls + if err != nil { + logrus.Fatal(err) + } + + queryCache := make(map[reference.Named]string) + versionMeta := make(map[string]ServiceMeta) + for _, service := range recipe.Config.Services { + + img, err := reference.ParseNormalizedNamed(service.Image) + if err != nil { + return err + } + + path := reference.Path(img) + if strings.Contains(path, "library") { + path = strings.Split(path, "/")[1] + } + + var tag string + switch img.(type) { + case reference.NamedTagged: + tag = img.(reference.NamedTagged).Tag() + case reference.Named: + logrus.Warnf("%s service is missing image tag?", path) + continue + } + + var exists bool + var digest string + if digest, exists = queryCache[img]; !exists { + logrus.Debugf("looking up image: %s from %s", img, path) + var err error + digest, err = client.GetTagDigest(cl, img, registryUsername, registryPassword) + if err != nil { + logrus.Warn(err) + continue + } + logrus.Debugf("queried for image: %s, tag: %s, digest: %s", path, tag, digest) + queryCache[img] = digest + logrus.Debugf("cached image: %s, tag: %s, digest: %s", path, tag, digest) + } else { + logrus.Debugf("reading image: %s, tag: %s, digest: %s from cache", path, tag, digest) + } + + versionMeta[service.Name] = ServiceMeta{ + Digest: digest, + Image: path, + Tag: img.(reference.NamedTagged).Tag(), + } + } + + versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta}) + + return nil + }); err != nil { + return versions, err + } + + _, err = CheckoutDefaultBranch(repo, recipeName) + if err != nil { + return versions, err + } + + logrus.Debugf("collected %s for %s", versions, recipeName) + + return versions, nil +} + +// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue. +func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) { + var versions []string + + if recipeMeta, exists := catl[recipeName]; exists { + for _, versionMeta := range recipeMeta.Versions { + for tag := range versionMeta { + versions = append(versions, tag) + } + } + } + + return versions, nil +}