diff --git a/cli/app/list.go b/cli/app/list.go index 56081312..9e96e2ff 100644 --- a/cli/app/list.go +++ b/cli/app/list.go @@ -7,8 +7,8 @@ import ( abraFormatter "coopcloud.tech/abra/cli/formatter" "coopcloud.tech/abra/cli/internal" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/ssh" "coopcloud.tech/tagcmp" "github.com/sirupsen/logrus" @@ -92,7 +92,7 @@ can take some time. sort.Sort(config.ByServerAndType(apps)) statuses := make(map[string]map[string]string) - var catl catalogue.RecipeCatalogue + var catl recipe.RecipeCatalogue if status { alreadySeen := make(map[string]bool) for _, app := range apps { @@ -110,7 +110,7 @@ can take some time. } var err error - catl, err = catalogue.ReadRecipeCatalogue() + catl, err = recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } @@ -153,7 +153,7 @@ can take some time. var newUpdates []string if version != "unknown" { - updates, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl) + updates, err := recipe.GetRecipeCatalogueVersions(app.Type, catl) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/rollback.go b/cli/app/rollback.go index 8e977c8b..c1df7fb5 100644 --- a/cli/app/rollback.go +++ b/cli/app/rollback.go @@ -4,7 +4,6 @@ import ( "fmt" "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/lint" "coopcloud.tech/abra/pkg/recipe" @@ -75,12 +74,12 @@ recipes. logrus.Fatalf("%s is not deployed?", app.Name) } - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } - versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl) + versions, err := recipe.GetRecipeCatalogueVersions(app.Type, catl) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/upgrade.go b/cli/app/upgrade.go index 2e559ca7..dcdb8e65 100644 --- a/cli/app/upgrade.go +++ b/cli/app/upgrade.go @@ -5,7 +5,6 @@ import ( "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/lint" @@ -79,12 +78,12 @@ recipes. logrus.Fatalf("%s is not deployed?", app.Name) } - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } - versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl) + versions, err := recipe.GetRecipeCatalogueVersions(app.Type, catl) if err != nil { logrus.Fatal(err) } diff --git a/cli/app/version.go b/cli/app/version.go index bbfd1d4a..ef1ab033 100644 --- a/cli/app/version.go +++ b/cli/app/version.go @@ -6,8 +6,8 @@ import ( abraFormatter "coopcloud.tech/abra/cli/formatter" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/client" + "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/upstream/stack" "github.com/docker/distribution/reference" "github.com/sirupsen/logrus" @@ -64,12 +64,12 @@ Cloud recipe version. logrus.Fatalf("%s is not deployed?", app.Name) } - recipeMeta, err := catalogue.GetRecipeMeta(app.Type) + recipeMeta, err := recipe.GetRecipeMeta(app.Type) if err != nil { logrus.Fatal(err) } - versionsMeta := make(map[string]catalogue.ServiceMeta) + versionsMeta := make(map[string]recipe.ServiceMeta) for _, recipeVersion := range recipeMeta.Versions { if currentVersion, exists := recipeVersion[deployedVersion]; exists { versionsMeta = currentVersion diff --git a/cli/catalogue/catalogue.go b/cli/catalogue/catalogue.go index ae428d13..aa7aa24e 100644 --- a/cli/catalogue/catalogue.go +++ b/cli/catalogue/catalogue.go @@ -9,7 +9,6 @@ import ( "coopcloud.tech/abra/cli/formatter" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/config" gitPkg "coopcloud.tech/abra/pkg/git" "coopcloud.tech/abra/pkg/limit" @@ -103,7 +102,7 @@ If you have a Hub account you can have Abra log you in to avoid this. Pass return err } - repos, err := catalogue.ReadReposMetadata() + repos, err := recipe.ReadReposMetadata() if err != nil { logrus.Fatal(err) } @@ -125,7 +124,7 @@ If you have a Hub account you can have Abra log you in to avoid this. Pass } } - catl := make(catalogue.RecipeCatalogue) + catl := make(recipe.RecipeCatalogue) catlBar := formatter.CreateProgressbar(barLength, "generating catalogue metadata...") for _, recipeMeta := range repos { if recipeName != "" && recipeName != recipeMeta.Name { @@ -138,7 +137,7 @@ If you have a Hub account you can have Abra log you in to avoid this. Pass continue } - versions, err := catalogue.GetRecipeVersions( + versions, err := recipe.GetRecipeVersions( recipeMeta.Name, internal.RegistryUsername, internal.RegistryPassword, @@ -152,7 +151,7 @@ If you have a Hub account you can have Abra log you in to avoid this. Pass logrus.Warn(err) } - catl[recipeMeta.Name] = catalogue.RecipeMeta{ + catl[recipeMeta.Name] = recipe.RecipeMeta{ Name: recipeMeta.Name, Repository: recipeMeta.CloneURL, Icon: recipeMeta.AvatarURL, @@ -177,7 +176,7 @@ If you have a Hub account you can have Abra log you in to avoid this. Pass logrus.Fatal(err) } } else { - catlFS, err := catalogue.ReadRecipeCatalogue() + catlFS, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } @@ -213,7 +212,7 @@ If you have a Hub account you can have Abra log you in to avoid this. Pass } if internal.Push { - if err := gitPkg.Push(cataloguePath); err != nil { + if err := gitPkg.Push(cataloguePath, false); err != nil { logrus.Fatal(err) } } @@ -236,7 +235,7 @@ var CatalogueCommand = &cli.Command{ }, } -func updateRepositories(repos catalogue.RepoCatalogue, recipeName string) error { +func updateRepositories(repos recipe.RepoCatalogue, recipeName string) error { var barLength int if recipeName != "" { barLength = 1 @@ -246,10 +245,10 @@ func updateRepositories(repos catalogue.RepoCatalogue, recipeName string) error cloneLimiter := limit.New(10) - retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are up-to-date...") + retrieveBar := formatter.CreateProgressbar(barLength, "ensuring recipes are cloned & up-to-date...") ch := make(chan string, barLength) for _, repoMeta := range repos { - go func(rm catalogue.RepoMeta) { + go func(rm recipe.RepoMeta) { cloneLimiter.Begin() defer cloneLimiter.End() @@ -266,7 +265,7 @@ func updateRepositories(repos catalogue.RepoCatalogue, recipeName string) error recipeDir := path.Join(config.RECIPES_DIR, rm.Name) - if err := gitPkg.Clone(recipeDir, rm.SSHURL); err != nil { + if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil { logrus.Fatal(err) } diff --git a/cli/internal/deploy.go b/cli/internal/deploy.go index 358ee3cb..28fa6d69 100644 --- a/cli/internal/deploy.go +++ b/cli/internal/deploy.go @@ -5,7 +5,6 @@ import ( "strings" abraFormatter "coopcloud.tech/abra/cli/formatter" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/dns" @@ -58,11 +57,11 @@ func DeployAction(c *cli.Context) error { version := deployedVersion if version == "" && !Chaos { - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } - versions, err := catalogue.GetRecipeCatalogueVersions(app.Type, catl) + versions, err := recipe.GetRecipeCatalogueVersions(app.Type, catl) if err != nil { logrus.Fatal(err) } diff --git a/cli/internal/validate.go b/cli/internal/validate.go index 0e202ba2..92701140 100644 --- a/cli/internal/validate.go +++ b/cli/internal/validate.go @@ -5,7 +5,6 @@ import ( "strings" "coopcloud.tech/abra/pkg/app" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/abra/pkg/ssh" @@ -50,7 +49,7 @@ func ValidateRecipeWithPrompt(c *cli.Context) recipe.Recipe { if recipeName == "" && !NoInput { var recipes []string - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } diff --git a/cli/recipe/list.go b/cli/recipe/list.go index c8d8d0a5..55bb7554 100644 --- a/cli/recipe/list.go +++ b/cli/recipe/list.go @@ -7,7 +7,7 @@ import ( "strings" "coopcloud.tech/abra/cli/formatter" - "coopcloud.tech/abra/pkg/catalogue" + "coopcloud.tech/abra/pkg/recipe" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -29,13 +29,13 @@ var recipeListCommand = &cli.Command{ patternFlag, }, Action: func(c *cli.Context) error { - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err.Error()) } recipes := catl.Flatten() - sort.Sort(catalogue.ByRecipeName(recipes)) + sort.Sort(recipe.ByRecipeName(recipes)) tableCol := []string{"name", "category", "status", "healthcheck", "backups", "email", "tests", "SSO"} table := formatter.CreateTable(tableCol) diff --git a/cli/recipe/release.go b/cli/recipe/release.go index 70cd748a..dbda67e4 100644 --- a/cli/recipe/release.go +++ b/cli/recipe/release.go @@ -17,7 +17,6 @@ import ( "github.com/AlecAivazis/survey/v2" "github.com/docker/distribution/reference" "github.com/go-git/go-git/v5" - configPkg "github.com/go-git/go-git/v5/config" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -214,7 +213,7 @@ func createReleaseFromTag(recipe recipe.Recipe, tagString, mainAppVersion string logrus.Fatal(err) } - if err := pushRelease(tagString, repo); err != nil { + if err := pushRelease(recipe.Dir()); err != nil { logrus.Fatal(err) } @@ -309,7 +308,7 @@ func tagRelease(tagString string, repo *git.Repository) error { return nil } -func pushRelease(tagString string, repo *git.Repository) error { +func pushRelease(recipeDir string) error { if internal.Dry { logrus.Info("dry run: no changes pushed") return nil @@ -326,21 +325,9 @@ func pushRelease(tagString string, repo *git.Repository) error { } if internal.Push { - if err := repo.Push(&git.PushOptions{}); err != nil { + if err := gitPkg.Push(recipeDir, true); err != nil { return err } - - tagRef := fmt.Sprintf("+refs/tags/%s:refs/tags/%s", tagString, tagString) - pushOpts := &git.PushOptions{ - RefSpecs: []configPkg.RefSpec{ - configPkg.RefSpec(tagRef), - }, - } - if err := repo.Push(pushOpts); err != nil { - return err - } - - logrus.Info(fmt.Sprintf("pushed tag %s to remote", tagString)) } return nil @@ -416,7 +403,7 @@ func createReleaseFromPreviousTag(tagString, mainAppVersion string, recipe recip logrus.Fatal(err) } - if err := pushRelease(newTagString, repo); err != nil { + if err := pushRelease(recipe.Dir()); err != nil { logrus.Fatal(err) } diff --git a/cli/recipe/upgrade.go b/cli/recipe/upgrade.go index 4ea0f733..02e12a5b 100644 --- a/cli/recipe/upgrade.go +++ b/cli/recipe/upgrade.go @@ -9,9 +9,9 @@ import ( "strings" "coopcloud.tech/abra/cli/internal" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/config" + recipePkg "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/tagcmp" "github.com/AlecAivazis/survey/v2" "github.com/docker/distribution/reference" @@ -147,7 +147,7 @@ You may invoke this command in "wizard" mode and be prompted for input: continue // skip on to the next tag and don't update any compose files } - catlVersions, err := catalogue.VersionsOfService(recipe.Name, service.Name) + catlVersions, err := recipePkg.VersionsOfService(recipe.Name, service.Name) if err != nil { logrus.Fatal(err) } diff --git a/cli/recipe/version.go b/cli/recipe/version.go index 040db877..6d6686a2 100644 --- a/cli/recipe/version.go +++ b/cli/recipe/version.go @@ -4,7 +4,7 @@ import ( "coopcloud.tech/abra/cli/formatter" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" - "coopcloud.tech/abra/pkg/catalogue" + recipePkg "coopcloud.tech/abra/pkg/recipe" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -18,7 +18,7 @@ var recipeVersionCommand = &cli.Command{ Action: func(c *cli.Context) error { recipe := internal.ValidateRecipe(c) - catalogue, err := catalogue.ReadRecipeCatalogue() + catalogue, err := recipePkg.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } diff --git a/pkg/autocomplete/autocomplete.go b/pkg/autocomplete/autocomplete.go index e8675680..4d37b62c 100644 --- a/pkg/autocomplete/autocomplete.go +++ b/pkg/autocomplete/autocomplete.go @@ -3,8 +3,8 @@ package autocomplete import ( "fmt" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/recipe" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -27,7 +27,7 @@ func AppNameComplete(c *cli.Context) { // RecipeNameComplete completes recipe names func RecipeNameComplete(c *cli.Context) { - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipe.ReadRecipeCatalogue() if err != nil { logrus.Warn(err) } diff --git a/pkg/catalogue/catalogue.go b/pkg/catalogue/catalogue.go deleted file mode 100644 index 39745131..00000000 --- a/pkg/catalogue/catalogue.go +++ /dev/null @@ -1,491 +0,0 @@ -// Package catalogue provides ways of interacting with recipe catalogues which -// are JSON data structures which contain meta information about recipes (e.g. -// what versions of the Nextcloud recipe are available?). -package catalogue - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path" - "strings" - "time" - - "coopcloud.tech/abra/cli/formatter" - "coopcloud.tech/abra/pkg/client" - "coopcloud.tech/abra/pkg/config" - "coopcloud.tech/abra/pkg/recipe" - "coopcloud.tech/abra/pkg/web" - "github.com/docker/distribution/reference" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/sirupsen/logrus" -) - -// RecipeCatalogueURL is the only current recipe catalogue available. -const RecipeCatalogueURL = "https://apps.coopcloud.tech" - -// ReposMetadataURL is the recipe repository metadata -const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos" - -// tag represents a git tag. -type tag = string - -// service represents a service within a recipe. -type service = string - -// ServiceMeta represents meta info associated with a service. -type ServiceMeta struct { - Digest string `json:"digest"` - Image string `json:"image"` - Tag string `json:"tag"` -} - -// RecipeVersions are the versions associated with a recipe. -type RecipeVersions []map[tag]map[service]ServiceMeta - -// RecipeMeta represents metadata for a recipe in the abra catalogue. -type RecipeMeta struct { - Category string `json:"category"` - DefaultBranch string `json:"default_branch"` - Description string `json:"description"` - Features recipe.Features `json:"features"` - Icon string `json:"icon"` - Name string `json:"name"` - Repository string `json:"repository"` - Versions RecipeVersions `json:"versions"` - Website string `json:"website"` -} - -// LatestVersion returns the latest version of a recipe. -func (r RecipeMeta) LatestVersion() string { - var version string - - // apps.json versions are sorted so the last key is latest - latest := r.Versions[len(r.Versions)-1] - - for tag := range latest { - version = tag - } - - logrus.Debugf("choosing %s as latest version of %s", version, r.Name) - - return version -} - -// Name represents a recipe name. -type Name = string - -// RecipeCatalogue represents the entire recipe catalogue. -type RecipeCatalogue map[Name]RecipeMeta - -// Flatten converts AppCatalogue to slice -func (r RecipeCatalogue) Flatten() []RecipeMeta { - recipes := make([]RecipeMeta, 0, len(r)) - - for name := range r { - recipes = append(recipes, r[name]) - } - - return recipes -} - -// ByRecipeName sorts recipes by name. -type ByRecipeName []RecipeMeta - -func (r ByRecipeName) Len() int { return len(r) } -func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] } -func (r ByRecipeName) Less(i, j int) bool { - return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name) -} - -// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally -// is up to date. -func recipeCatalogueFSIsLatest() (bool, error) { - httpClient := web.NewHTTPRetryClient() - res, err := httpClient.Head(RecipeCatalogueURL) - if err != nil { - return false, err - } - - lastModified := res.Header["Last-Modified"][0] - parsed, err := time.Parse(time.RFC1123, lastModified) - if err != nil { - return false, err - } - - info, err := os.Stat(config.RECIPES_JSON) - if err != nil { - if os.IsNotExist(err) { - logrus.Debugf("no recipe catalogue found in file system cache") - return false, nil - } - return false, err - } - - localModifiedTime := info.ModTime().Unix() - remoteModifiedTime := parsed.Unix() - - if localModifiedTime < remoteModifiedTime { - logrus.Debug("file system cached recipe catalogue is out-of-date") - return false, nil - } - - logrus.Debug("file system cached recipe catalogue is now up-to-date") - - return true, nil -} - -// ReadRecipeCatalogue reads the recipe catalogue. -func ReadRecipeCatalogue() (RecipeCatalogue, error) { - recipes := make(RecipeCatalogue) - - recipeFSIsLatest, err := recipeCatalogueFSIsLatest() - if err != nil { - return nil, err - } - - if !recipeFSIsLatest { - logrus.Debugf("reading recipe catalogue from web to get latest") - if err := readRecipeCatalogueWeb(&recipes); err != nil { - return nil, err - } - return recipes, nil - } - - logrus.Debugf("reading recipe catalogue from file system cache to get latest") - if err := readRecipeCatalogueFS(&recipes); err != nil { - return nil, err - } - - return recipes, nil -} - -// readRecipeCatalogueFS reads the catalogue from the file system. -func readRecipeCatalogueFS(target interface{}) error { - recipesJSONFS, err := ioutil.ReadFile(config.RECIPES_JSON) - if err != nil { - return err - } - - if err := json.Unmarshal(recipesJSONFS, &target); err != nil { - return err - } - - logrus.Debugf("read recipe catalogue from file system cache in %s", config.RECIPES_JSON) - - return nil -} - -// readRecipeCatalogueWeb reads the catalogue from the web. -func readRecipeCatalogueWeb(target interface{}) error { - if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil { - return err - } - - recipesJSON, err := json.MarshalIndent(target, "", " ") - if err != nil { - return err - } - - if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil { - return err - } - - logrus.Debugf("read recipe catalogue from web at %s", RecipeCatalogueURL) - - return nil -} - -// VersionsOfService lists the version of a service. -func VersionsOfService(recipe, serviceName string) ([]string, error) { - var versions []string - - catalogue, err := ReadRecipeCatalogue() - if err != nil { - return nil, err - } - - rec, ok := catalogue[recipe] - if !ok { - return versions, nil - } - - alreadySeen := make(map[string]bool) - for _, serviceVersion := range rec.Versions { - for tag := range serviceVersion { - if _, ok := alreadySeen[tag]; !ok { - alreadySeen[tag] = true - versions = append(versions, tag) - } - } - } - - logrus.Debugf("detected versions %s for %s", strings.Join(versions, ", "), recipe) - - return versions, nil -} - -// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue. -func GetRecipeMeta(recipeName string) (RecipeMeta, error) { - catl, err := ReadRecipeCatalogue() - if err != nil { - return RecipeMeta{}, err - } - - recipeMeta, ok := catl[recipeName] - if !ok { - err := fmt.Errorf("recipe %s does not exist?", recipeName) - return RecipeMeta{}, err - } - - if err := recipe.EnsureExists(recipeName); err != nil { - return RecipeMeta{}, err - } - - logrus.Debugf("recipe metadata retrieved for %s", recipeName) - - return recipeMeta, nil -} - -// RepoMeta is a single recipe repo metadata. -type RepoMeta struct { - ID int `json:"id"` - Owner Owner - Name string `json:"name"` - FullName string `json:"full_name"` - Description string `json:"description"` - Empty bool `json:"empty"` - Private bool `json:"private"` - Fork bool `json:"fork"` - Template bool `json:"template"` - Parent interface{} `json:"parent"` - Mirror bool `json:"mirror"` - Size int `json:"size"` - HTMLURL string `json:"html_url"` - SSHURL string `json:"ssh_url"` - CloneURL string `json:"clone_url"` - OriginalURL string `json:"original_url"` - Website string `json:"website"` - StarsCount int `json:"stars_count"` - ForksCount int `json:"forks_count"` - WatchersCount int `json:"watchers_count"` - OpenIssuesCount int `json:"open_issues_count"` - OpenPRCount int `json:"open_pr_counter"` - ReleaseCounter int `json:"release_counter"` - DefaultBranch string `json:"default_branch"` - Archived bool `json:"archived"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - Permissions Permissions - HasIssues bool `json:"has_issues"` - InternalTracker InternalTracker - HasWiki bool `json:"has_wiki"` - HasPullRequests bool `json:"has_pull_requests"` - HasProjects bool `json:"has_projects"` - IgnoreWhitespaceConflicts bool `json:"ignore_whitespace_conflicts"` - AllowMergeCommits bool `json:"allow_merge_commits"` - AllowRebase bool `json:"allow_rebase"` - AllowRebaseExplicit bool `json:"allow_rebase_explicit"` - AllowSquashMerge bool `json:"allow_squash_merge"` - AvatarURL string `json:"avatar_url"` - Internal bool `json:"internal"` - MirrorInterval string `json:"mirror_interval"` -} - -// Owner is the repo organisation owner metadata. -type Owner struct { - ID int `json:"id"` - Login string `json:"login"` - FullName string `json:"full_name"` - Email string `json:"email"` - AvatarURL string `json:"avatar_url"` - Language string `json:"language"` - IsAdmin bool `json:"is_admin"` - LastLogin string `json:"last_login"` - Created string `json:"created"` - Restricted bool `json:"restricted"` - Username string `json:"username"` -} - -// Permissions is perms metadata for a repo. -type Permissions struct { - Admin bool `json:"admin"` - Push bool `json:"push"` - Pull bool `json:"pull"` -} - -// InternalTracker is issue tracker metadata for a repo. -type InternalTracker struct { - EnableTimeTracker bool `json:"enable_time_tracker"` - AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"` - EnableIssuesDependencies bool `json:"enable_issue_dependencies"` -} - -// RepoCatalogue represents all the recipe repo metadata. -type RepoCatalogue map[string]RepoMeta - -// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea. -func ReadReposMetadata() (RepoCatalogue, error) { - reposMeta := make(RepoCatalogue) - - pageIdx := 1 - bar := formatter.CreateProgressbar(-1, "retrieving recipe repos list from git.coopcloud.tech...") - for { - var reposList []RepoMeta - - pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx) - - logrus.Debugf("fetching repo metadata from %s", pagedURL) - - if err := web.ReadJSON(pagedURL, &reposList); err != nil { - return reposMeta, err - } - - if len(reposList) == 0 { - bar.Add(1) - break - } - - for idx, repo := range reposList { - reposMeta[repo.Name] = reposList[idx] - } - - pageIdx++ - bar.Add(1) - } - - fmt.Println() // newline for spinner - - return reposMeta, nil -} - -// GetRecipeVersions retrieves all recipe versions. -func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (RecipeVersions, error) { - versions := RecipeVersions{} - - recipeDir := path.Join(config.RECIPES_DIR, recipeName) - - logrus.Debugf("attempting to open git repository in %s", recipeDir) - - repo, err := git.PlainOpen(recipeDir) - if err != nil { - return versions, err - } - - worktree, err := repo.Worktree() - if err != nil { - logrus.Fatal(err) - } - - gitTags, err := repo.Tags() - if err != nil { - return versions, err - } - - if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) { - tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/") - - logrus.Debugf("processing %s for %s", tag, recipeName) - - checkOutOpts := &git.CheckoutOptions{ - Create: false, - Force: true, - Branch: plumbing.ReferenceName(ref.Name()), - } - if err := worktree.Checkout(checkOutOpts); err != nil { - logrus.Debugf("failed to check out %s in %s", tag, recipeDir) - return err - } - - logrus.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir) - - recipe, err := recipe.Get(recipeName) - if err != nil { - return err - } - - cl, err := client.New("default") // only required for docker.io registry calls - if err != nil { - logrus.Fatal(err) - } - - queryCache := make(map[reference.Named]string) - versionMeta := make(map[string]ServiceMeta) - for _, service := range recipe.Config.Services { - - img, err := reference.ParseNormalizedNamed(service.Image) - if err != nil { - return err - } - - path := reference.Path(img) - if strings.Contains(path, "library") { - path = strings.Split(path, "/")[1] - } - - var tag string - switch img.(type) { - case reference.NamedTagged: - tag = img.(reference.NamedTagged).Tag() - case reference.Named: - logrus.Warnf("%s service is missing image tag?", path) - continue - } - - var exists bool - var digest string - if digest, exists = queryCache[img]; !exists { - logrus.Debugf("looking up image: %s from %s", img, path) - var err error - digest, err = client.GetTagDigest(cl, img, registryUsername, registryPassword) - if err != nil { - logrus.Warn(err) - continue - } - logrus.Debugf("queried for image: %s, tag: %s, digest: %s", path, tag, digest) - queryCache[img] = digest - logrus.Debugf("cached image: %s, tag: %s, digest: %s", path, tag, digest) - } else { - logrus.Debugf("reading image: %s, tag: %s, digest: %s from cache", path, tag, digest) - } - - versionMeta[service.Name] = ServiceMeta{ - Digest: digest, - Image: path, - Tag: img.(reference.NamedTagged).Tag(), - } - } - - versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta}) - - return nil - }); err != nil { - return versions, err - } - - _, err = recipe.CheckoutDefaultBranch(repo, recipeName) - if err != nil { - return versions, err - } - - logrus.Debugf("collected %s for %s", versions, recipeName) - - return versions, nil -} - -// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue. -func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) { - var versions []string - - if recipeMeta, exists := catl[recipeName]; exists { - for _, versionMeta := range recipeMeta.Versions { - for tag := range versionMeta { - versions = append(versions, tag) - } - } - } - - return versions, nil -} diff --git a/pkg/git/push.go b/pkg/git/push.go index 68adf807..cb8378f5 100644 --- a/pkg/git/push.go +++ b/pkg/git/push.go @@ -1,13 +1,18 @@ package git import ( + "path" + + configPkg "coopcloud.tech/abra/pkg/config" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" "github.com/sirupsen/logrus" ) // Push pushes the latest changes -func Push(repoPath string) error { - commitRepo, err := git.PlainOpen(repoPath) +func Push(recipeName string, tags bool) error { + recipeDir := path.Join(configPkg.RECIPES_DIR, recipeName) + commitRepo, err := git.PlainOpen(recipeDir) if err != nil { return err } @@ -15,8 +20,19 @@ func Push(repoPath string) error { if err := commitRepo.Push(&git.PushOptions{}); err != nil { return err } + logrus.Info("git changes pushed") - logrus.Info("changes pushed") + if tags { + pushOpts := &git.PushOptions{ + RefSpecs: []config.RefSpec{ + config.RefSpec("+refs/tags/*:refs/tags/*"), + }, + } + if err := commitRepo.Push(pushOpts); err != nil { + return err + } + logrus.Info("git tags pushed") + } return nil } diff --git a/pkg/lint/recipe.go b/pkg/lint/recipe.go index 80a72c7c..231a7eb4 100644 --- a/pkg/lint/recipe.go +++ b/pkg/lint/recipe.go @@ -6,9 +6,9 @@ import ( "os" "path" - "coopcloud.tech/abra/pkg/catalogue" "coopcloud.tech/abra/pkg/config" "coopcloud.tech/abra/pkg/recipe" + recipePkg "coopcloud.tech/abra/pkg/recipe" "coopcloud.tech/tagcmp" "github.com/docker/distribution/reference" "github.com/sirupsen/logrus" @@ -269,12 +269,12 @@ func LintImagePresent(recipe recipe.Recipe) (bool, error) { } func LintHasPublishedVersion(recipe recipe.Recipe) (bool, error) { - catl, err := catalogue.ReadRecipeCatalogue() + catl, err := recipePkg.ReadRecipeCatalogue() if err != nil { logrus.Fatal(err) } - versions, err := catalogue.GetRecipeCatalogueVersions(recipe.Name, catl) + versions, err := recipePkg.GetRecipeCatalogueVersions(recipe.Name, catl) if err != nil { logrus.Fatal(err) } diff --git a/pkg/recipe/recipe.go b/pkg/recipe/recipe.go index 41a94d6c..53cbf82d 100644 --- a/pkg/recipe/recipe.go +++ b/pkg/recipe/recipe.go @@ -1,24 +1,107 @@ package recipe import ( + "encoding/json" "fmt" "io/ioutil" "os" "path" "path/filepath" "strings" + "time" + "coopcloud.tech/abra/cli/formatter" + "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/compose" "coopcloud.tech/abra/pkg/config" gitPkg "coopcloud.tech/abra/pkg/git" "coopcloud.tech/abra/pkg/upstream/stack" loader "coopcloud.tech/abra/pkg/upstream/stack" + "coopcloud.tech/abra/pkg/web" composetypes "github.com/docker/cli/cli/compose/types" + "github.com/docker/distribution/reference" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/sirupsen/logrus" ) +// RecipeCatalogueURL is the only current recipe catalogue available. +const RecipeCatalogueURL = "https://apps.coopcloud.tech" + +// ReposMetadataURL is the recipe repository metadata +const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos" + +// tag represents a git tag. +type tag = string + +// service represents a service within a recipe. +type service = string + +// ServiceMeta represents meta info associated with a service. +type ServiceMeta struct { + Digest string `json:"digest"` + Image string `json:"image"` + Tag string `json:"tag"` +} + +// RecipeVersions are the versions associated with a recipe. +type RecipeVersions []map[tag]map[service]ServiceMeta + +// RecipeMeta represents metadata for a recipe in the abra catalogue. +type RecipeMeta struct { + Category string `json:"category"` + DefaultBranch string `json:"default_branch"` + Description string `json:"description"` + Features Features `json:"features"` + Icon string `json:"icon"` + Name string `json:"name"` + Repository string `json:"repository"` + Versions RecipeVersions `json:"versions"` + Website string `json:"website"` +} + +// LatestVersion returns the latest version of a recipe. +func (r RecipeMeta) LatestVersion() string { + var version string + + // apps.json versions are sorted so the last key is latest + latest := r.Versions[len(r.Versions)-1] + + for tag := range latest { + version = tag + } + + logrus.Debugf("choosing %s as latest version of %s", version, r.Name) + + return version +} + +// Name represents a recipe name. +type Name = string + +// RecipeCatalogue represents the entire recipe catalogue. +type RecipeCatalogue map[Name]RecipeMeta + +// Flatten converts AppCatalogue to slice +func (r RecipeCatalogue) Flatten() []RecipeMeta { + recipes := make([]RecipeMeta, 0, len(r)) + + for name := range r { + recipes = append(recipes, r[name]) + } + + return recipes +} + +// ByRecipeName sorts recipes by name. +type ByRecipeName []RecipeMeta + +func (r ByRecipeName) Len() int { return len(r) } +func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] } +func (r ByRecipeName) Less(i, j int) bool { + return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name) +} + // Image represents a recipe container image. type Image struct { Image string `json:"image"` @@ -42,6 +125,7 @@ type Features struct { type Recipe struct { Name string Config *composetypes.Config + Meta RecipeMeta } // Dir retrieves the recipe repository path @@ -121,7 +205,16 @@ func Get(recipeName string) (Recipe, error) { return Recipe{}, err } - return Recipe{Name: recipeName, Config: config}, nil + meta, err := GetRecipeMeta(recipeName) + if err != nil { + return Recipe{}, err + } + + return Recipe{ + Name: recipeName, + Config: config, + Meta: meta, + }, nil } // EnsureExists ensures that a recipe is locally cloned @@ -536,3 +629,393 @@ func CheckoutDefaultBranch(repo *git.Repository, recipeName string) (plumbing.Re return branch, nil } + +// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally +// is up to date. +func recipeCatalogueFSIsLatest() (bool, error) { + httpClient := web.NewHTTPRetryClient() + res, err := httpClient.Head(RecipeCatalogueURL) + if err != nil { + return false, err + } + + lastModified := res.Header["Last-Modified"][0] + parsed, err := time.Parse(time.RFC1123, lastModified) + if err != nil { + return false, err + } + + info, err := os.Stat(config.RECIPES_JSON) + if err != nil { + if os.IsNotExist(err) { + logrus.Debugf("no recipe catalogue found in file system cache") + return false, nil + } + return false, err + } + + localModifiedTime := info.ModTime().Unix() + remoteModifiedTime := parsed.Unix() + + if localModifiedTime < remoteModifiedTime { + logrus.Debug("file system cached recipe catalogue is out-of-date") + return false, nil + } + + logrus.Debug("file system cached recipe catalogue is now up-to-date") + + return true, nil +} + +// ReadRecipeCatalogue reads the recipe catalogue. +func ReadRecipeCatalogue() (RecipeCatalogue, error) { + recipes := make(RecipeCatalogue) + + recipeFSIsLatest, err := recipeCatalogueFSIsLatest() + if err != nil { + return nil, err + } + + if !recipeFSIsLatest { + logrus.Debugf("reading recipe catalogue from web to get latest") + if err := readRecipeCatalogueWeb(&recipes); err != nil { + return nil, err + } + return recipes, nil + } + + logrus.Debugf("reading recipe catalogue from file system cache to get latest") + if err := readRecipeCatalogueFS(&recipes); err != nil { + return nil, err + } + + return recipes, nil +} + +// readRecipeCatalogueFS reads the catalogue from the file system. +func readRecipeCatalogueFS(target interface{}) error { + recipesJSONFS, err := ioutil.ReadFile(config.RECIPES_JSON) + if err != nil { + return err + } + + if err := json.Unmarshal(recipesJSONFS, &target); err != nil { + return err + } + + logrus.Debugf("read recipe catalogue from file system cache in %s", config.RECIPES_JSON) + + return nil +} + +// readRecipeCatalogueWeb reads the catalogue from the web. +func readRecipeCatalogueWeb(target interface{}) error { + if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil { + return err + } + + recipesJSON, err := json.MarshalIndent(target, "", " ") + if err != nil { + return err + } + + if err := ioutil.WriteFile(config.RECIPES_JSON, recipesJSON, 0764); err != nil { + return err + } + + logrus.Debugf("read recipe catalogue from web at %s", RecipeCatalogueURL) + + return nil +} + +// VersionsOfService lists the version of a service. +func VersionsOfService(recipe, serviceName string) ([]string, error) { + var versions []string + + catalogue, err := ReadRecipeCatalogue() + if err != nil { + return nil, err + } + + rec, ok := catalogue[recipe] + if !ok { + return versions, nil + } + + alreadySeen := make(map[string]bool) + for _, serviceVersion := range rec.Versions { + for tag := range serviceVersion { + if _, ok := alreadySeen[tag]; !ok { + alreadySeen[tag] = true + versions = append(versions, tag) + } + } + } + + logrus.Debugf("detected versions %s for %s", strings.Join(versions, ", "), recipe) + + return versions, nil +} + +// GetRecipeMeta retrieves the recipe metadata from the recipe catalogue. +func GetRecipeMeta(recipeName string) (RecipeMeta, error) { + catl, err := ReadRecipeCatalogue() + if err != nil { + return RecipeMeta{}, err + } + + recipeMeta, ok := catl[recipeName] + if !ok { + err := fmt.Errorf("recipe %s does not exist?", recipeName) + return RecipeMeta{}, err + } + + if err := EnsureExists(recipeName); err != nil { + return RecipeMeta{}, err + } + + logrus.Debugf("recipe metadata retrieved for %s", recipeName) + + return recipeMeta, nil +} + +// RepoMeta is a single recipe repo metadata. +type RepoMeta struct { + ID int `json:"id"` + Owner Owner + Name string `json:"name"` + FullName string `json:"full_name"` + Description string `json:"description"` + Empty bool `json:"empty"` + Private bool `json:"private"` + Fork bool `json:"fork"` + Template bool `json:"template"` + Parent interface{} `json:"parent"` + Mirror bool `json:"mirror"` + Size int `json:"size"` + HTMLURL string `json:"html_url"` + SSHURL string `json:"ssh_url"` + CloneURL string `json:"clone_url"` + OriginalURL string `json:"original_url"` + Website string `json:"website"` + StarsCount int `json:"stars_count"` + ForksCount int `json:"forks_count"` + WatchersCount int `json:"watchers_count"` + OpenIssuesCount int `json:"open_issues_count"` + OpenPRCount int `json:"open_pr_counter"` + ReleaseCounter int `json:"release_counter"` + DefaultBranch string `json:"default_branch"` + Archived bool `json:"archived"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + Permissions Permissions + HasIssues bool `json:"has_issues"` + InternalTracker InternalTracker + HasWiki bool `json:"has_wiki"` + HasPullRequests bool `json:"has_pull_requests"` + HasProjects bool `json:"has_projects"` + IgnoreWhitespaceConflicts bool `json:"ignore_whitespace_conflicts"` + AllowMergeCommits bool `json:"allow_merge_commits"` + AllowRebase bool `json:"allow_rebase"` + AllowRebaseExplicit bool `json:"allow_rebase_explicit"` + AllowSquashMerge bool `json:"allow_squash_merge"` + AvatarURL string `json:"avatar_url"` + Internal bool `json:"internal"` + MirrorInterval string `json:"mirror_interval"` +} + +// Owner is the repo organisation owner metadata. +type Owner struct { + ID int `json:"id"` + Login string `json:"login"` + FullName string `json:"full_name"` + Email string `json:"email"` + AvatarURL string `json:"avatar_url"` + Language string `json:"language"` + IsAdmin bool `json:"is_admin"` + LastLogin string `json:"last_login"` + Created string `json:"created"` + Restricted bool `json:"restricted"` + Username string `json:"username"` +} + +// Permissions is perms metadata for a repo. +type Permissions struct { + Admin bool `json:"admin"` + Push bool `json:"push"` + Pull bool `json:"pull"` +} + +// InternalTracker is issue tracker metadata for a repo. +type InternalTracker struct { + EnableTimeTracker bool `json:"enable_time_tracker"` + AllowOnlyContributorsToTrackTime bool `json:"allow_only_contributors_to_track_time"` + EnableIssuesDependencies bool `json:"enable_issue_dependencies"` +} + +// RepoCatalogue represents all the recipe repo metadata. +type RepoCatalogue map[string]RepoMeta + +// ReadReposMetadata retrieves coop-cloud/... repo metadata from Gitea. +func ReadReposMetadata() (RepoCatalogue, error) { + reposMeta := make(RepoCatalogue) + + pageIdx := 1 + bar := formatter.CreateProgressbar(-1, "retrieving recipe repos list from git.coopcloud.tech...") + for { + var reposList []RepoMeta + + pagedURL := fmt.Sprintf("%s?page=%v", ReposMetadataURL, pageIdx) + + logrus.Debugf("fetching repo metadata from %s", pagedURL) + + if err := web.ReadJSON(pagedURL, &reposList); err != nil { + return reposMeta, err + } + + if len(reposList) == 0 { + bar.Add(1) + break + } + + for idx, repo := range reposList { + reposMeta[repo.Name] = reposList[idx] + } + + pageIdx++ + bar.Add(1) + } + + fmt.Println() // newline for spinner + + return reposMeta, nil +} + +// GetRecipeVersions retrieves all recipe versions. +func GetRecipeVersions(recipeName, registryUsername, registryPassword string) (RecipeVersions, error) { + versions := RecipeVersions{} + + recipeDir := path.Join(config.RECIPES_DIR, recipeName) + + logrus.Debugf("attempting to open git repository in %s", recipeDir) + + repo, err := git.PlainOpen(recipeDir) + if err != nil { + return versions, err + } + + worktree, err := repo.Worktree() + if err != nil { + logrus.Fatal(err) + } + + gitTags, err := repo.Tags() + if err != nil { + return versions, err + } + + if err := gitTags.ForEach(func(ref *plumbing.Reference) (err error) { + tag := strings.TrimPrefix(string(ref.Name()), "refs/tags/") + + logrus.Debugf("processing %s for %s", tag, recipeName) + + checkOutOpts := &git.CheckoutOptions{ + Create: false, + Force: true, + Branch: plumbing.ReferenceName(ref.Name()), + } + if err := worktree.Checkout(checkOutOpts); err != nil { + logrus.Debugf("failed to check out %s in %s", tag, recipeDir) + return err + } + + logrus.Debugf("successfully checked out %s in %s", ref.Name(), recipeDir) + + recipe, err := Get(recipeName) + if err != nil { + return err + } + + cl, err := client.New("default") // only required for docker.io registry calls + if err != nil { + logrus.Fatal(err) + } + + queryCache := make(map[reference.Named]string) + versionMeta := make(map[string]ServiceMeta) + for _, service := range recipe.Config.Services { + + img, err := reference.ParseNormalizedNamed(service.Image) + if err != nil { + return err + } + + path := reference.Path(img) + if strings.Contains(path, "library") { + path = strings.Split(path, "/")[1] + } + + var tag string + switch img.(type) { + case reference.NamedTagged: + tag = img.(reference.NamedTagged).Tag() + case reference.Named: + logrus.Warnf("%s service is missing image tag?", path) + continue + } + + var exists bool + var digest string + if digest, exists = queryCache[img]; !exists { + logrus.Debugf("looking up image: %s from %s", img, path) + var err error + digest, err = client.GetTagDigest(cl, img, registryUsername, registryPassword) + if err != nil { + logrus.Warn(err) + continue + } + logrus.Debugf("queried for image: %s, tag: %s, digest: %s", path, tag, digest) + queryCache[img] = digest + logrus.Debugf("cached image: %s, tag: %s, digest: %s", path, tag, digest) + } else { + logrus.Debugf("reading image: %s, tag: %s, digest: %s from cache", path, tag, digest) + } + + versionMeta[service.Name] = ServiceMeta{ + Digest: digest, + Image: path, + Tag: img.(reference.NamedTagged).Tag(), + } + } + + versions = append(versions, map[string]map[string]ServiceMeta{tag: versionMeta}) + + return nil + }); err != nil { + return versions, err + } + + _, err = CheckoutDefaultBranch(repo, recipeName) + if err != nil { + return versions, err + } + + logrus.Debugf("collected %s for %s", versions, recipeName) + + return versions, nil +} + +// GetRecipeCatalogueVersions list the recipe versions listed in the recipe catalogue. +func GetRecipeCatalogueVersions(recipeName string, catl RecipeCatalogue) ([]string, error) { + var versions []string + + if recipeMeta, exists := catl[recipeName]; exists { + for _, versionMeta := range recipeMeta.Versions { + for tag := range versionMeta { + versions = append(versions, tag) + } + } + } + + return versions, nil +}