WIP: filter by topic when building catalogue

See coop-cloud/organising#377

cli/catalogue/catalogue.go

@@ -98,11 +98,6 @@ keys configured on your account.
 				continue
 			}
 
-			if _, exists := catalogue.CatalogueSkipList[recipeMeta.Name]; exists {
-				catlBar.Add(1)
-				continue
-			}
-
 			versions, err := recipe.GetRecipeVersions(recipeMeta.Name, internal.Offline)
 			if err != nil {
 				logrus.Warn(err)
@@ -173,7 +168,7 @@ keys configured on your account.
 			}
 
 			msg := "chore: publish new catalogue release changes"
-			if err := gitPkg.Commit(cataloguePath, msg, internal.Dry); err != nil {
+			if err := gitPkg.Commit(cataloguePath, "**.json", msg, internal.Dry); err != nil {
 				logrus.Fatal(err)
 			}
 

cli/recipe/diff.go

@@ -1,40 +0,0 @@
-package recipe
-
-import (
-	"path"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/config"
-	gitPkg "coopcloud.tech/abra/pkg/git"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-var recipeDiffCommand = cli.Command{
-	Name:        "diff",
-	Usage:       "Show unstaged changes in recipe config",
-	Description: "Due to limitations in our underlying Git dependency, this command requires /usr/bin/git.",
-	Aliases:     []string{"d"},
-	ArgsUsage:   "<recipe>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.RecipeNameComplete,
-	Action: func(c *cli.Context) error {
-		recipeName := c.Args().First()
-
-		if recipeName != "" {
-			internal.ValidateRecipe(c)
-		}
-
-		recipeDir := path.Join(config.RECIPES_DIR, recipeName)
-		if err := gitPkg.DiffUnstaged(recipeDir); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
-}

cli/recipe/recipe.go

@@ -30,7 +30,5 @@ manner. Abra supports convenient automation for recipe maintainenace, see the
 		recipeSyncCommand,
 		recipeUpgradeCommand,
 		recipeVersionCommand,
-		recipeResetCommand,
-		recipeDiffCommand,
 	},
 }

cli/recipe/release.go

@@ -106,18 +106,6 @@ your SSH keys configured on your account.
 			}
 		}
 
-		isClean, err := gitPkg.IsClean(recipe.Dir())
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		if !isClean {
-			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
-			if err := gitPkg.DiffUnstaged(recipe.Dir()); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
 		if len(tags) > 0 {
 			logrus.Warnf("previous git tags detected, assuming this is a new semver release")
 			if err := createReleaseFromPreviousTag(tagString, mainAppVersion, recipe, tags); err != nil {
@@ -256,7 +244,7 @@ func commitRelease(recipe recipe.Recipe, tag string) error {
 
 	msg := fmt.Sprintf("chore: publish %s release", tag)
 	repoPath := path.Join(config.RECIPES_DIR, recipe.Name)
-	if err := gitPkg.Commit(repoPath, msg, internal.Dry); err != nil {
+	if err := gitPkg.Commit(repoPath, ".", msg, internal.Dry); err != nil {
 		return err
 	}
 

cli/recipe/reset.go

@@ -1,56 +0,0 @@
-package recipe
-
-import (
-	"path"
-
-	"coopcloud.tech/abra/cli/internal"
-	"coopcloud.tech/abra/pkg/autocomplete"
-	"coopcloud.tech/abra/pkg/config"
-	"github.com/go-git/go-git/v5"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-var recipeResetCommand = cli.Command{
-	Name:        "reset",
-	Usage:       "Remove all unstaged changes from recipe config",
-	Description: "WARNING, this will delete your changes. Be Careful.",
-	Aliases:     []string{"rs"},
-	ArgsUsage:   "<recipe>",
-	Flags: []cli.Flag{
-		internal.DebugFlag,
-		internal.NoInputFlag,
-	},
-	Before:       internal.SubCommandBefore,
-	BashComplete: autocomplete.RecipeNameComplete,
-	Action: func(c *cli.Context) error {
-		recipeName := c.Args().First()
-
-		if recipeName != "" {
-			internal.ValidateRecipe(c)
-		}
-
-		repoPath := path.Join(config.RECIPES_DIR, recipeName)
-		repo, err := git.PlainOpen(repoPath)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		ref, err := repo.Head()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		worktree, err := repo.Worktree()
-		if err != nil {
-			logrus.Fatal(err)
-		}
-
-		opts := &git.ResetOptions{Commit: ref.Hash(), Mode: git.HardReset}
-		if err := worktree.Reset(opts); err != nil {
-			logrus.Fatal(err)
-		}
-
-		return nil
-	},
-}

cli/recipe/sync.go

@@ -8,7 +8,6 @@ import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
 	"coopcloud.tech/abra/pkg/config"
-	gitPkg "coopcloud.tech/abra/pkg/git"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/go-git/go-git/v5"
@@ -199,17 +198,6 @@ likely to change.
 			logrus.Infof("dry run: not syncing label %s for recipe %s", nextTag, recipe.Name)
 		}
 
-		isClean, err := gitPkg.IsClean(recipe.Dir())
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		if !isClean {
-			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
-			if err := gitPkg.DiffUnstaged(recipe.Dir()); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
 		return nil
 	},
 }

cli/recipe/upgrade.go

@@ -14,7 +14,6 @@ import (
 	"coopcloud.tech/abra/pkg/client"
 	"coopcloud.tech/abra/pkg/config"
 	"coopcloud.tech/abra/pkg/formatter"
-	gitPkg "coopcloud.tech/abra/pkg/git"
 	recipePkg "coopcloud.tech/abra/pkg/recipe"
 	"coopcloud.tech/tagcmp"
 	"github.com/AlecAivazis/survey/v2"
@@ -327,7 +326,6 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				}
 
 				fmt.Println(string(jsonstring))
-
 				return nil
 			}
 
@@ -338,18 +336,6 @@ You may invoke this command in "wizard" mode and be prompted for input:
 				}
 			}
 		}
-
-		isClean, err := gitPkg.IsClean(recipeDir)
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		if !isClean {
-			logrus.Infof("%s currently has these unstaged changes 👇", recipe.Name)
-			if err := gitPkg.DiffUnstaged(recipeDir); err != nil {
-				logrus.Fatal(err)
-			}
-		}
-
 		return nil
 	},
 }

pkg/catalogue/catalogue.go

@@ -12,46 +12,6 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// CatalogueSkipList is all the repos that are not recipes.
-var CatalogueSkipList = map[string]bool{
-	"abra":                        true,
-	"abra-apps":                   true,
-	"abra-aur":                    true,
-	"abra-bash":                   true,
-	"abra-capsul":                 true,
-	"abra-gandi":                  true,
-	"abra-hetzner":                true,
-	"abra-test-recipe":            true,
-	"apps":                        true,
-	"aur-abra-git":                true,
-	"auto-mirror":                 true,
-	"auto-recipes-catalogue-json": true,
-	"backup-bot":                  true,
-	"backup-bot-two":              true,
-	"beta.coopcloud.tech":         true,
-	"comrade-renovate-bot":        true,
-	"coopcloud.tech":              true,
-	"coturn":                      true,
-	"docker-cp-deploy":            true,
-	"docker-dind-bats-kcov":       true,
-	"docs.coopcloud.tech":         true,
-	"drone-abra":                  true,
-	"example":                     true,
-	"gardening":                   true,
-	"go-abra":                     true,
-	"organising":                  true,
-	"pyabra":                      true,
-	"radicle-seed-node":           true,
-	"recipes-catalogue-json":      true,
-	"recipes-wishlist":            true,
-	"recipes.coopcloud.tech":      true,
-	"stack-ssh-deploy":            true,
-	"swarm-cronjob":               true,
-	"tagcmp":                      true,
-	"traefik-cert-dumper":         true,
-	"tyop":                        true,
-}
-
 // EnsureCatalogue ensures that the catalogue is cloned locally & present.
 func EnsureCatalogue() error {
 	catalogueDir := path.Join(config.ABRA_DIR, "catalogue")

pkg/git/commit.go

@@ -8,7 +8,7 @@ import (
 )
 
 // Commit runs a git commit
-func Commit(repoPath, commitMessage string, dryRun bool) error {
+func Commit(repoPath, glob, commitMessage string, dryRun bool) error {
 	if commitMessage == "" {
 		return fmt.Errorf("no commit message specified?")
 	}
@@ -33,8 +33,17 @@ func Commit(repoPath, commitMessage string, dryRun bool) error {
 	}
 
 	if !dryRun {
-		// NOTE(d1): `All: true` does not include untracked files
-		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{All: true})
+		err = commitWorktree.AddGlob(glob)
+		if err != nil {
+			return err
+		}
+		logrus.Debugf("staged %s for commit", glob)
+	} else {
+		logrus.Debugf("dry run: did not stage %s for commit", glob)
+	}
+
+	if !dryRun {
+		_, err = commitWorktree.Commit(commitMessage, &git.CommitOptions{})
 		if err != nil {
 			return err
 		}

pkg/git/diff.go

@@ -1,42 +0,0 @@
-package git
-
-import (
-	"fmt"
-	"os/exec"
-
-	"github.com/sirupsen/logrus"
-)
-
-// getGitDiffArgs builds the `git diff` invocation args. It removes the usage
-// of a pager and ensures that colours are specified even when Git might detect
-// otherwise.
-func getGitDiffArgs(repoPath string) []string {
-	return []string{
-		"-C",
-		repoPath,
-		"--no-pager",
-		"-c",
-		"color.diff=always",
-		"diff",
-	}
-}
-
-// DiffUnstaged shows a `git diff`. Due to limitations in the underlying go-git
-// library, this implementation requires the /usr/bin/git binary. It gracefully
-// skips if it cannot find the command on the system.
-func DiffUnstaged(path string) error {
-	if _, err := exec.LookPath("git"); err != nil {
-		logrus.Warnf("unable to locate git command, cannot output diff")
-		return nil
-	}
-
-	gitDiffArgs := getGitDiffArgs(path)
-	diff, err := exec.Command("git", gitDiffArgs...).Output()
-	if err != nil {
-		return nil
-	}
-
-	fmt.Print(string(diff))
-
-	return nil
-}

pkg/recipe/recipe.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -31,7 +32,7 @@ import (
 // RecipeCatalogueURL is the only current recipe catalogue available.
 const RecipeCatalogueURL = "https://recipes.coopcloud.tech/recipes.json"
 
-// ReposMetadataURL is the recipe repository metadata
+// ReposMetadataURL is the recipe repository metadata.
 const ReposMetadataURL = "https://git.coopcloud.tech/api/v1/orgs/coop-cloud/repos"
 
 // tag represents a git tag.
@@ -63,6 +64,11 @@ type RecipeMeta struct {
 	Website       string         `json:"website"`
 }
 
+// TopicMeta represents a list of topics for a repository.
+type TopicMeta struct {
+	Topics []string `json:"topics"`
+}
+
 // LatestVersion returns the latest version of a recipe.
 func (r RecipeMeta) LatestVersion() string {
 	var version string
@@ -822,7 +828,16 @@ func ReadReposMetadata() (RepoCatalogue, error) {
 		}
 
 		for idx, repo := range reposList {
-			reposMeta[repo.Name] = reposList[idx]
+			var topicMeta TopicMeta
+
+			topicsURL := getReposTopicUrl(repo.Name)
+			if err := web.ReadJSON(topicsURL, &topicMeta); err != nil {
+				return reposMeta, err
+			}
+
+			if slices.Contains(topicMeta.Topics, "recipe") && repo.Name != "example" {
+				reposMeta[repo.Name] = reposList[idx]
+			}
 		}
 
 		pageIdx++
@@ -1002,14 +1017,8 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 				retrieveBar.Add(1)
 				return
 			}
-			if _, exists := catalogue.CatalogueSkipList[rm.Name]; exists {
-				ch <- rm.Name
-				retrieveBar.Add(1)
-				return
-			}
 
 			recipeDir := path.Join(config.RECIPES_DIR, rm.Name)
-
 			if err := gitPkg.Clone(recipeDir, rm.CloneURL); err != nil {
 				logrus.Fatal(err)
 			}
@@ -1025,3 +1034,8 @@ func UpdateRepositories(repos RepoCatalogue, recipeName string) error {
 
 	return nil
 }
+
+// getReposTopicUrl retrieves the repository specific topic listing.
+func getReposTopicUrl(repoName string) string {
+	return fmt.Sprintf("https://git.coopcloud.tech/api/v1/repos/coop-cloud/%s/topics", repoName)
+}

scripts/installer/installer

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-ABRA_VERSION="0.8.1-beta"
+ABRA_VERSION="0.8.0-beta"
 ABRA_RELEASE_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$ABRA_VERSION"
-RC_VERSION="0.8.1-beta"
+RC_VERSION="0.8.0-beta"
 RC_VERSION_URL="https://git.coopcloud.tech/api/v1/repos/coop-cloud/abra/releases/tags/$RC_VERSION"
 
 for arg in "$@"; do

tests/integration/app_check.bats

@@ -35,7 +35,7 @@ setup(){
 
   run $ABRA app check "$TEST_APP_DOMAIN"
   assert_success
-  refute_output --partial '❌'
+  assert_output --partial 'all necessary environment variables defined'
 
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
 }
@@ -111,16 +111,12 @@ setup(){
 }
 
 @test "error if missing env var" {
-  run $ABRA app check "$TEST_APP_DOMAIN"
-  assert_success
-  refute_output --partial '❌'
-
   run bash -c 'echo "NEW_VAR=foo" >> "$ABRA_DIR/recipes/$TEST_RECIPE/.env.sample"'
   assert_success
 
   run $ABRA app check "$TEST_APP_DOMAIN" --chaos
-  assert_success
-  assert_output --partial '❌'
+  assert_failure
+  assert_output --partial "NEW_VAR"
 
   _checkout_recipe
 }

tests/integration/app_deploy.bats

@@ -321,8 +321,6 @@ teardown(){
 
 # bats test_tags=slow
 @test "COMPOSE_FILE with \$COMPOSE_FILE override works" {
-  _reset_recipe
-
   run sed -i 's/#COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/g' \
     "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
   assert_success

tests/integration/app_secret.bats

@@ -110,11 +110,10 @@ setup(){
   assert_success
   assert_output --partial 'test_pass_one'
 
-  run bash -c '$ABRA app secret ls $TEST_APP_DOMAIN --machine | \
-    jq -r ".[] | select(.name==\"test_pass_one\") | .version"'
+  run docker -c "$TEST_SERVER" secret ls
   assert_success
-  assert_output --partial 'v2'
-  refute_output --partial 'v1'
+  assert_output --regexp ".*_test_pass_one_v2"
+  refute_output --regexp ".*_test_pass_one_v1"
 
   run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
   assert_success

tests/integration/recipe_diff.bats

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-setup() {
-  load "$PWD/tests/integration/helpers/common"
-  _common_setup
-}
-
-@test "show unstaged changes" {
-  run $ABRA recipe diff "$TEST_RECIPE"
-  assert_success
-  refute_output --partial 'traefik.enable'
-
-  run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
-  assert_success
-
-  run $ABRA recipe diff "$TEST_RECIPE"
-  assert_success
-  assert_output --partial 'traefik.enable'
-
-  _reset_recipe
-}

tests/integration/recipe_reset.bats

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-setup() {
-  load "$PWD/tests/integration/helpers/common"
-  _common_setup
-}
-
-@test "reset unstaged changes" {
-  run $ABRA recipe fetch "$TEST_RECIPE"
-  assert_success
-
-  run sed -i '/traefik.enable=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
-  assert_success
-
-  run $ABRA recipe diff "$TEST_RECIPE"
-  assert_success
-  assert_output --partial 'traefik.enable'
-
-  run $ABRA recipe reset "$TEST_RECIPE"
-  assert_success
-
-  run $ABRA recipe diff "$TEST_RECIPE"
-  assert_success
-  refute_output --partial 'traefik.enable'
-}