feat: fetch all recipes when no recipe is specified (!401)

Closes coop-cloud/organising#530

Reviewed-on: coop-cloud/abra#401
Reviewed-by: decentral1se <decentral1se@noreply.git.coopcloud.tech>
Co-authored-by: p4u1 <p4u1_f4u1@riseup.net>
Co-committed-by: p4u1 <p4u1_f4u1@riseup.net>

cli/app/new.go

@@ -55,8 +55,16 @@ var appNewCommand = cli.Command{
 		internal.ChaosFlag,
 	},
 	Before:    internal.SubCommandBefore,
-	ArgsUsage:    "[<recipe>]",
-	BashComplete: autocomplete.RecipeNameComplete,
+	ArgsUsage: "[<recipe>] [<version>]",
+	BashComplete: func(ctx *cli.Context) {
+		args := ctx.Args()
+		switch len(args) {
+		case 0:
+			autocomplete.RecipeNameComplete(ctx)
+		case 1:
+			autocomplete.RecipeVersionComplete(ctx.Args().Get(0))
+		}
+	},
 	Action: func(c *cli.Context) error {
 		recipe := internal.ValidateRecipe(c)
 
@@ -69,9 +77,15 @@ var appNewCommand = cli.Command{
 					logrus.Fatal(err)
 				}
 			}
+			if c.Args().Get(1) == "" {
 				if err := recipePkg.EnsureLatest(recipe.Name); err != nil {
 					logrus.Fatal(err)
 				}
+			} else {
+				if err := recipePkg.EnsureVersion(recipe.Name, c.Args().Get(1)); err != nil {
+					logrus.Fatal(err)
+				}
+			}
 		}
 
 		if err := ensureServerFlag(); err != nil {

cli/app/remove.go

@@ -3,7 +3,9 @@ package app
 import (
 	"context"
 	"fmt"
+	"log"
 	"os"
+	"time"
 
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
@@ -124,9 +126,11 @@ flag.
 
 		if len(vols) > 0 {
 			for _, vol := range vols {
-				err := cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
+				err = retryFunc(5, func() error {
+					return cl.VolumeRemove(context.Background(), vol, internal.Force) // last argument is for force removing
+				})
 				if err != nil {
-					logrus.Fatal(err)
+					log.Fatalf("removing volumes failed: %s", err)
 				}
 				logrus.Info(fmt.Sprintf("volume %s removed", vol))
 			}
@@ -143,3 +147,21 @@ flag.
 		return nil
 	},
 }
+
+// retryFunc retries the given function for the given retries. After the nth
+// retry it waits (n + 1)^2 seconds before the next retry (starting with n=0).
+// It returns an error if the function still failed after the last retry.
+func retryFunc(retries int, fn func() error) error {
+	for i := 0; i < retries; i++ {
+		err := fn()
+		if err == nil {
+			return nil
+		}
+		if i+1 < retries {
+			sleep := time.Duration(i+1) * time.Duration(i+1)
+			logrus.Infof("%s: waiting %d seconds before next retry", err, sleep)
+			time.Sleep(sleep * time.Second)
+		}
+	}
+	return fmt.Errorf("%d retries failed", retries)
+}

cli/app/remove_test.go

@@ -0,0 +1,26 @@
+package app
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestRetryFunc(t *testing.T) {
+	err := retryFunc(1, func() error { return nil })
+	if err != nil {
+		t.Errorf("should not return an error: %s", err)
+	}
+
+	i := 0
+	fn := func() error {
+		i++
+		return fmt.Errorf("oh no, something went wrong!")
+	}
+	err = retryFunc(2, fn)
+	if err == nil {
+		t.Error("should return an error")
+	}
+	if i != 2 {
+		t.Errorf("The function should have been called 1 times, got %d", i)
+	}
+}

cli/recipe/fetch.go

@@ -3,6 +3,7 @@ package recipe
 import (
 	"coopcloud.tech/abra/cli/internal"
 	"coopcloud.tech/abra/pkg/autocomplete"
+	"coopcloud.tech/abra/pkg/formatter"
 	"coopcloud.tech/abra/pkg/recipe"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -17,26 +18,31 @@ var recipeFetchCommand = cli.Command{
 	Flags: []cli.Flag{
 		internal.DebugFlag,
 		internal.NoInputFlag,
+		internal.OfflineFlag,
 	},
 	Before:       internal.SubCommandBefore,
 	BashComplete: autocomplete.RecipeNameComplete,
 	Action: func(c *cli.Context) error {
 		recipeName := c.Args().First()
-
 		if recipeName != "" {
 			internal.ValidateRecipe(c)
+			if err := recipe.Ensure(recipeName); err != nil {
+				logrus.Fatal(err)
+			}
+			return nil
 		}
 
-		if err := recipe.EnsureExists(recipeName); err != nil {
+		catalogue, err := recipe.ReadRecipeCatalogue(internal.Offline)
+		if err != nil {
 			logrus.Fatal(err)
 		}
 
-		if err := recipe.EnsureUpToDate(recipeName); err != nil {
-			logrus.Fatal(err)
+		catlBar := formatter.CreateProgressbar(len(catalogue), "fetching latest recipes...")
+		for recipeName := range catalogue {
+			if err := recipe.Ensure(recipeName); err != nil {
+				logrus.Error(err)
 			}
-
-		if err := recipe.EnsureLatest(recipeName); err != nil {
-			logrus.Fatal(err)
+			catlBar.Add(1)
 		}
 
 		return nil

pkg/autocomplete/autocomplete.go

@@ -51,6 +51,20 @@ func RecipeNameComplete(c *cli.Context) {
 	}
 }
 
+// RecipeVersionComplete completes versions for the recipe.
+func RecipeVersionComplete(recipeName string) {
+	catl, err := recipe.ReadRecipeCatalogue(false)
+	if err != nil {
+		logrus.Warn(err)
+	}
+
+	for _, v := range catl[recipeName].Versions {
+		for v2 := range v {
+			fmt.Println(v2)
+		}
+	}
+}
+
 // ServerNameComplete completes server names.
 func ServerNameComplete(c *cli.Context) {
 	files, err := config.LoadAppFiles("")

pkg/recipe/recipe.go

@@ -264,6 +264,20 @@ func (r Recipe) SampleEnv() (map[string]string, error) {
 	return sampleEnv, nil
 }
 
+// Ensure makes sure the recipe exists, is up to date and has the latest version checked out.
+func Ensure(recipeName string) error {
+	if err := EnsureExists(recipeName); err != nil {
+		return err
+	}
+	if err := EnsureUpToDate(recipeName); err != nil {
+		return err
+	}
+	if err := EnsureLatest(recipeName); err != nil {
+		return err
+	}
+	return nil
+}
+
 // EnsureExists ensures that a recipe is locally cloned
 func EnsureExists(recipeName string) error {
 	recipeDir := path.Join(config.RECIPES_DIR, recipeName)

scripts/installer/installer

@@ -65,17 +65,19 @@ function install_abra_release {
 
   checksums=$(wget -q -O- $checksums_url)
   checksum=$(echo "$checksums" | grep "$FILENAME" - | sed -En 's/([0-9a-f]{64})\s+'"$FILENAME"'.*/\1/p')
+  abra_download="/tmp/abra-download"
 
   echo "downloading $ABRA_VERSION $PLATFORM binary release for abra..."
-  wget -q "$release_url" -O "$HOME/.local/bin/.abra-download"
-  localsum=$(sha256sum $HOME/.local/bin/.abra-download | sed -En 's/([0-9a-f]{64})\s+.*/\1/p')
+
+  wget -q "$release_url" -O $abra_download 
+  localsum=$(sha256sum $abra_download | sed -En 's/([0-9a-f]{64})\s+.*/\1/p')
   echo "checking if checksums match..."
   if [[ "$localsum" != "$checksum" ]]; then
       print_checksum_error
       exit 1
   fi
   echo "$(tput setaf 2)check successful!$(tput sgr0)"
-  mv "$HOME/.local/bin/.abra-download" "$HOME/.local/bin/abra"
+  mv "$abra_download" "$HOME/.local/bin/abra"
   chmod +x "$HOME/.local/bin/abra"
 
   x=$(echo $PATH | grep $HOME/.local/bin)

tests/integration/app_new.bats

@@ -18,9 +18,24 @@ setup(){
 }
 
 teardown(){
+  load "$PWD/tests/integration/helpers/common"
   _rm_app
 }
 
+@test "autocomplete" {
+  run $ABRA app new --generate-bash-completion
+  assert_success
+  assert_output --partial "traefik"
+  assert_output --partial "abra-test-recipe"
+
+  # Note: this test needs to be updated when a new version of the test recipe is published.
+  run $ABRA app new abra-test-recipe --generate-bash-completion
+  assert_success
+  assert_output "0.1.0+1.20.0
+0.1.1+1.20.2
+0.2.0+1.21.0"
+}
+
 @test "create new app" {
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
@@ -28,10 +43,29 @@ teardown(){
     --domain "$TEST_APP_DOMAIN"
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
+  assert_output --partial "Your branch is up to date with 'origin/main'."
+}
+
+@test "create new app with version" {
+  run $ABRA app new "$TEST_RECIPE" 0.1.1+1.20.2 \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$TEST_APP_DOMAIN"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" log -1
+  assert_output --partial "453db7121c0a56a7a8f15378f18fe3bf21ccfdef"
 }
 
 @test "does not overwrite existing env files" {
-  _new_app
+  run $ABRA app new "$TEST_RECIPE" \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "$TEST_APP_DOMAIN"
+  assert_success
 
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
@@ -74,8 +108,7 @@ teardown(){
     --no-input \
     --chaos \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN" \
-    --secrets
+    --domain "$TEST_APP_DOMAIN"
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
@@ -88,18 +121,17 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --partial "Your branch is behind 'origin/main' by 3 commits, and can be fast-forwarded."
 
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN" \
-    --secrets
+    --domain "$TEST_APP_DOMAIN"
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --partial 'behind 3'
+  assert_output --partial "Your branch is up to date with 'origin/main'."
 
   _reset_recipe
 }
@@ -109,7 +141,7 @@ teardown(){
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --partial "Your branch is behind 'origin/main' by 3 commits, and can be fast-forwarded."
 
   # NOTE(d1): need to use --chaos to force same commit
   run $ABRA app new "$TEST_RECIPE" \
@@ -117,13 +149,12 @@ teardown(){
     --offline \
     --chaos \
     --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN" \
-    --secrets
+    --domain "$TEST_APP_DOMAIN"
   assert_success
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial 'behind 3'
+  assert_output --partial "Your branch is behind 'origin/main' by 3 commits, and can be fast-forwarded."
 
   _reset_recipe
 }

tests/integration/app_remove.bats

@@ -104,10 +104,7 @@ teardown(){
 
   _undeploy_app
 
-  # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 5
-
-  run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
+  run $ABRA app volume rm "$TEST_APP_DOMAIN"
   assert_success
 
   run $ABRA app volume ls "$TEST_APP_DOMAIN"
@@ -132,9 +129,6 @@ teardown(){
 
   _undeploy_app
 
-  # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 5
-
   run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
   assert_success
   assert_output --partial 'test-volume'

tests/integration/recipe_fetch.bats

@@ -5,7 +5,17 @@ setup() {
   _common_setup
 }
 
-@test "recipe fetch" {
+@test "recipe fetch all" {
+  run rm -rf "$ABRA_DIR/recipes/matrix-synapse"
+  assert_success
+  assert_not_exists "$ABRA_DIR/recipes/matrix-synapse"
+
+  run $ABRA recipe fetch
+  assert_success
+  assert_exists "$ABRA_DIR/recipes/matrix-synapse"
+}
+
+@test "recipe fetch single recipe" {
   run rm -rf "$ABRA_DIR/recipes/matrix-synapse"
   assert_success
   assert_not_exists "$ABRA_DIR/recipes/matrix-synapse"