From 2014cd66229d8d8d5013af55a25c96b36bf7279b Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Sat, 22 Jun 2024 16:40:07 +0200
Subject: [PATCH 1/3] test: less fragile integration suite [ci skip]

See https://git.coopcloud.tech/coop-cloud/organising/issues/584
See https://git.coopcloud.tech/coop-cloud/organising/issues/595
---
 tests/integration/app_backup.bats     | 23 +++++--
 tests/integration/app_check.bats      | 30 ++++-----
 tests/integration/app_cmd.bats        | 32 ++++-----
 tests/integration/app_cp.bats         | 36 +++-------
 tests/integration/app_deploy.bats     | 97 +++++++--------------------
 tests/integration/app_errors.bats     | 10 ---
 tests/integration/app_list.bats       |  7 +-
 tests/integration/app_new.bats        |  9 +--
 tests/integration/app_ps.bats         |  7 +-
 tests/integration/app_remove.bats     | 21 ++----
 tests/integration/app_restart.bats    |  7 +-
 tests/integration/app_restore.bats    | 29 ++++++--
 tests/integration/app_rollback.bats   | 44 +++---------
 tests/integration/app_run.bats        |  9 +--
 tests/integration/app_secret.bats     | 35 ++++------
 tests/integration/app_services.bats   |  7 +-
 tests/integration/app_undeploy.bats   |  5 +-
 tests/integration/app_upgrade.bats    | 29 +-------
 tests/integration/app_version.bats    | 16 +----
 tests/integration/app_volume.bats     | 15 ++---
 tests/integration/dirs.bats           | 16 ++---
 tests/integration/helpers/app.bash    |  7 +-
 tests/integration/helpers/common.bash |  3 +
 tests/integration/helpers/docker.bash | 13 ++++
 tests/integration/helpers/git.bash    | 16 ++---
 tests/integration/helpers/recipe.bash |  9 +++
 tests/integration/install.bats        |  2 +
 tests/integration/recipe_diff.bats    |  6 +-
 tests/integration/recipe_fetch.bats   |  1 +
 tests/integration/recipe_lint.bats    | 30 ++++-----
 tests/integration/recipe_new.bats     |  1 -
 tests/integration/recipe_release.bats | 25 ++++---
 tests/integration/recipe_sync.bats    | 26 +++----
 tests/integration/recipe_upgrade.bats | 24 ++++---
 tests/integration/recipe_version.bats |  2 -
 tests/integration/server_add.bats     | 11 +--
 tests/integration/server_list.bats    |  8 ---
 tests/integration/server_remove.bats  |  4 ++
 tests/integration/upgrade.bats        |  2 +
 39 files changed, 249 insertions(+), 425 deletions(-)
 create mode 100644 tests/integration/helpers/docker.bash

diff --git a/tests/integration/app_backup.bats b/tests/integration/app_backup.bats
index 2e46fd29..c0435dff 100644
--- a/tests/integration/app_backup.bats
+++ b/tests/integration/app_backup.bats
@@ -19,13 +19,12 @@ setup() {
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
 }
 
 @test "retrieve recipe if missing" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
 
   run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
@@ -39,6 +38,8 @@ teardown(){
 }
 
 @test "bail if unstaged changes and no --chaos" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
   assert_success
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
@@ -59,6 +60,8 @@ teardown(){
 
 # bats test_tags=slow
 @test "do not bail if unstaged changes and --chaos" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
   assert_success
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
@@ -81,6 +84,8 @@ teardown(){
 }
 
 @test "ensure recipe up to date if no --offline" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
@@ -98,6 +103,8 @@ teardown(){
 }
 
 @test "ensure recipe not up to date if --offline" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
   refute [ -z "$latestCommit" ];
 
@@ -124,6 +131,8 @@ teardown(){
 }
 
 @test "detect backup labels" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run $ABRA app backup "$TEST_APP_DOMAIN" --debug
   assert_failure
   assert_output --partial 'no containers matching'
@@ -134,6 +143,8 @@ teardown(){
 }
 
 @test "error if backups not enabled" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run sed -i '/backupbot.backup=true/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
   assert_success
 
@@ -145,6 +156,8 @@ teardown(){
 }
 
 @test "error if backup paths not configured" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run sed -i '/backupbot.backup.path=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
   assert_success
 
@@ -157,6 +170,8 @@ teardown(){
 
 # bats test_tags=slow
 @test "backup single service" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   _deploy_app
 
   run $ABRA app backup "$TEST_APP_DOMAIN" app
diff --git a/tests/integration/app_check.bats b/tests/integration/app_check.bats
index e296e3cb..425162eb 100644
--- a/tests/integration/app_check.bats
+++ b/tests/integration/app_check.bats
@@ -18,6 +18,10 @@ setup(){
   _common_setup
 }
 
+teardown(){
+  _reset_recipe
+}
+
 @test "validate app argument" {
   run $ABRA app check
   assert_failure
@@ -35,8 +39,6 @@ setup(){
 
   run $ABRA app check "$TEST_APP_DOMAIN"
   assert_success
-  refute_output --partial '❌'
-
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
 }
 
@@ -66,37 +68,33 @@ setup(){
 }
 
 @test "ensure recipe up to date if no --offline" {
+  wantHash=$(_get_n_hash 3)
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal $(_get_current_hash) "$wantHash"
 
   run $ABRA app check "$TEST_APP_DOMAIN"
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
-
-  _reset_recipe
+  assert_equal $(_get_head_hash) $(_get_current_hash)
 }
 
 @test "ensure recipe not up to date if --offline" {
+  wantHash=$(_get_n_hash 1)
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~1
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "Your branch is behind 'origin/main' by 1 commit"
+  assert_equal $(_get_current_hash) "$wantHash"
 
   # NOTE(d1): we can't quite tell if this will fail or not in the future, so,
   # since it isn't an important part of what we're testing here, we don't check
   # it
   run $ABRA app check "$TEST_APP_DOMAIN" --offline
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "Your branch is behind 'origin/main' by 1 commit"
-
-  _reset_recipe
+  assert_equal $(_get_current_hash) "$wantHash"
 }
 
 @test "error if missing .env.sample" {
@@ -106,8 +104,6 @@ setup(){
   run $ABRA app check "$TEST_APP_DOMAIN" --chaos
   assert_failure
   assert_output --partial '.env.sample does not exist?'
-
-  _checkout_recipe
 }
 
 @test "error if missing env var" {
@@ -121,6 +117,4 @@ setup(){
   run $ABRA app check "$TEST_APP_DOMAIN" --chaos
   assert_success
   assert_output --partial '❌'
-
-  _checkout_recipe
 }
diff --git a/tests/integration/app_cmd.bats b/tests/integration/app_cmd.bats
index fd5e6695..1d4abc89 100644
--- a/tests/integration/app_cmd.bats
+++ b/tests/integration/app_cmd.bats
@@ -19,10 +19,8 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
+  _reset_recipe
 }
 
 # bats test_tags=slow
@@ -92,35 +90,33 @@ test_cmd_export"
 }
 
 @test "ensure recipe up to date if no --offline" {
+  wantHash=$(_get_n_hash 3)
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal $(_get_current_hash) "$wantHash"
 
   run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd
   assert_success
   assert_output --partial 'baz'
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "up to date"
-
-  _reset_recipe "$TEST_RECIPE"
+  assert_equal $(_get_head_hash) $(_get_current_hash)
 }
 
 @test "ensure recipe not up to date if --offline" {
+  wantHash=$(_get_n_hash 3)
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal $(_get_current_hash) "$wantHash"
 
   run $ABRA app cmd --local --offline "$TEST_APP_DOMAIN" test_cmd
   assert_success
   assert_output --partial 'baz'
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal $(_get_current_hash) $wantHash
 
   _reset_recipe "$TEST_RECIPE"
 }
@@ -167,12 +163,16 @@ test_cmd_export"
 }
 
 @test "run command with single arg" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/581"
+
   run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd_arg -- bing
   assert_success
   assert_output --partial 'bing'
 }
 
 @test "run command with several args" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/581"
+
   run $ABRA app cmd --local "$TEST_APP_DOMAIN" test_cmd_args -- bong bang
   assert_success
   assert_output --partial 'bong bang'
@@ -185,8 +185,6 @@ test_cmd_export"
   run $ABRA app cmd "$TEST_APP_DOMAIN" app test_cmd
   assert_success
   assert_output --partial 'baz'
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -196,6 +194,4 @@ test_cmd_export"
   run $ABRA app cmd "$TEST_APP_DOMAIN" doesnt_exist test_cmd
   assert_failure
   assert_output --partial 'no service doesnt_exist'
-
-  _undeploy_app
 }
diff --git a/tests/integration/app_cp.bats b/tests/integration/app_cp.bats
index 524ff866..62c99348 100644
--- a/tests/integration/app_cp.bats
+++ b/tests/integration/app_cp.bats
@@ -19,6 +19,13 @@ setup(){
   _common_setup
 }
 
+teardown(){
+  _rm "$BATS_TMPDIR/*.txt"
+  _rm_remote "/etc/*.txt"
+
+  _rm "$BATS_TMPDIR/mydir"
+}
+
 @test "validate app argument" {
   run $ABRA app cp
   assert_failure
@@ -49,6 +56,7 @@ setup(){
   assert_output --partial 'arguments must take $SERVICE:$PATH form'
 }
 
+# bats test_tags=slow
 @test "error if local file missing" {
   run $ABRA app cp "$TEST_APP_DOMAIN" thisfileshouldnotexist.txt app:/somewhere
   assert_failure
@@ -62,8 +70,6 @@ setup(){
   run $ABRA app cp "$TEST_APP_DOMAIN" "$BATS_TMPDIR/myfile.txt" doesnt_exist:/ --debug
   assert_failure
   assert_output --partial 'no containers matching'
-
-  _rm "$BATS_TMPDIR/myfile.txt"
 }
 
 # bats test_tags=slow
@@ -76,9 +82,6 @@ setup(){
   run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
   assert_success
   assert_output --partial "foo"
-
-  _rm "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile.txt"
 }
 
 # bats test_tags=slow
@@ -102,9 +105,6 @@ setup(){
   run $ABRA app run "$TEST_APP_DOMAIN" app cat /etc/myfile.txt
   assert_success
   assert_output --partial "bar"
-
-  _rm "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile.txt"
 }
 
 # bats test_tags=slow
@@ -155,9 +155,6 @@ setup(){
   run $ABRA app run "$TEST_APP_DOMAIN" app ls /etc/myfile2.txt
   assert_success
   assert_output --partial "myfile2.txt"
-
-  _rm "$BATS_TMPDIR/mydir"
-  _rm_remote "/etc/myfile*"
 }
 
 # bats test_tags=slow
@@ -169,9 +166,6 @@ setup(){
   assert_success
   assert_exists "$BATS_TMPDIR/myfile.txt"
   assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo"
-
-  _rm "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile.txt"
 }
 
 # bats test_tags=slow
@@ -183,9 +177,6 @@ setup(){
   assert_success
   assert_exists "$BATS_TMPDIR/myfile.txt"
   assert bash -c "cat $BATS_TMPDIR/myfile.txt | grep -q foo"
-
-  _rm "$BATS_TMPDIR/myfile.txt"
-  _rm_remote "/etc/myfile.txt"
 }
 
 # bats test_tags=slow
@@ -197,9 +188,6 @@ setup(){
   assert_success
   assert_exists "$BATS_TMPDIR/myfile2.txt"
   assert bash -c "cat $BATS_TMPDIR/myfile2.txt | grep -q foo"
-
-  _rm "$BATS_TMPDIR/myfile2.txt"
-  _rm_remote "/etc/myfile.txt"
 }
 
 # bats test_tags=slow
@@ -217,10 +205,6 @@ setup(){
   assert_exists "$BATS_TMPDIR/mydir/etc/myfile.txt"
   assert_success
   assert_exists "$BATS_TMPDIR/mydir/etc/myfile2.txt"
-
-  _rm "$BATS_TMPDIR/mydir"
-  _rm_remote "/etc/myfile.txt"
-  _rm_remote "/etc/myfile2.txt"
 }
 
 # bats test_tags=slow
@@ -238,8 +222,4 @@ setup(){
   assert_exists "$BATS_TMPDIR/mydir/myfile.txt"
   assert_success
   assert_exists "$BATS_TMPDIR/mydir/myfile2.txt"
-
-  _rm "$BATS_TMPDIR/mydir"
-  _rm_remote "/etc/myfile.txt"
-  _rm_remote "/etc/myfile2.txt"
 }
diff --git a/tests/integration/app_deploy.bats b/tests/integration/app_deploy.bats
index ca546b51..541c8405 100644
--- a/tests/integration/app_deploy.bats
+++ b/tests/integration/app_deploy.bats
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
 setup_file(){
-  load "$PWD/tests/integration/helpers/git"
   load "$PWD/tests/integration/helpers/common"
   _common_setup
   _add_server
@@ -17,14 +16,16 @@ teardown_file(){
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
-  _reset_recipe
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
+  _reset_recipe
+  _reset_app
+  _reset_tags
+
+  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
 }
 
 @test "validate app argument" {
@@ -50,11 +51,6 @@ teardown(){
   assert_failure
   assert_output --partial 'locally unstaged changes'
   refute_output --partial 'chaos'
-
-  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
-  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
-
-  _checkout_recipe
 }
 
 # bats test_tags=slow
@@ -71,56 +67,42 @@ teardown(){
     --chaos --no-input --no-converge-checks
   assert_success
   assert_output --partial 'chaos'
-
-  _undeploy_app
-
-  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
-  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
 }
 
 # bats test_tags=slow
 @test "ensure recipe up to date if no --offline" {
+  wantHash=$(_get_n_hash 3)
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal $(_get_current_hash) "$wantHash"
 
   run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  refute_output --regexp 'behind .* 3 commits'
-
-  _reset_recipe
-  _undeploy_app
+  assert_equal $(_get_head_hash) $(_get_current_hash)
 }
 
 # bats test_tags=slow
 @test "ensure recipe not up to date if --offline" {
+  wantHash=$(_get_n_hash 3)
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal $(_get_current_hash) "$wantHash"
 
   # NOTE(d1): need to use --chaos to force same commit
   run $ABRA app deploy "$TEST_APP_DOMAIN" \
     --no-input --no-converge-checks --chaos --offline
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
-
-  _undeploy_app
-  _reset_recipe
+  assert_equal $(_get_current_hash) "$wantHash"
 }
 
 # bats test_tags=slow
 @test "deploy latest commit if no published versions and no --chaos" {
-  # TODO(d1): fix with a new test recipe which has no published versions?
-  skip "known issue, abra-test-recipe has published versions now"
-
   latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
 
   _remove_tags
@@ -132,32 +114,22 @@ teardown(){
   assert_output --partial "$latestCommit"
   assert_output --partial 'using latest commit'
   refute_output --partial 'chaos'
-
-  _undeploy_app
-  _reset_tags
 }
 
 # bats test_tags=slow
 @test "ensure same commit if --chaos" {
-  latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+  wantHash=$(_get_n_hash 3)
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
-
-  threeCommitsBack="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
+  assert_equal $(_get_current_hash) "$wantHash"
 
   run $ABRA app deploy "$TEST_APP_DOMAIN" \
     --no-input --no-converge-checks --chaos
   assert_success
-  refute_output --partial "$latestCommit"
-  assert_output --partial "$threeCommitsBack"
+  assert_output --partial "${wantHash:0:8}"
   assert_output --partial 'chaos'
-
-  _undeploy_app
-  _reset_recipe
 }
 
 # bats test_tags=slow
@@ -169,8 +141,6 @@ teardown(){
   run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_success
 
-  _undeploy_app
-
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
 }
 
@@ -182,8 +152,6 @@ teardown(){
     --no-input --no-converge-checks --chaos
   assert_failure
   assert_output --partial 'failed lint checks'
-
-  _checkout_recipe
 }
 
 # bats test_tags=slow
@@ -194,8 +162,6 @@ teardown(){
     --no-input --no-converge-checks
   assert_failure
   assert_output --partial 'already deployed'
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -213,8 +179,6 @@ teardown(){
   assert_success
   assert_output --partial 'already deployed but continuing'
   assert_output --partial '--chaos'
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -240,9 +204,6 @@ teardown(){
   run $ABRA app secret remove "gitea.$TEST_SERVER" --all --no-input
   assert_success
 
-  # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 5
-
   run $ABRA app volume remove "gitea.$TEST_SERVER" --no-input
   assert_success
 
@@ -277,6 +238,7 @@ teardown(){
   assert_success
 }
 
+# bats test_tags=dns
 @test "ensure domain is checked" {
   if [[ "$TEST_SERVER" == "default" ]]; then
       skip "domain checks are disabled for local server"
@@ -302,8 +264,6 @@ teardown(){
     --no-input --no-converge-checks --no-domain-checks
   assert_success
   assert_output --partial 'skipping domain checks as requested'
-
-  _undeploy_app
 }
 
 @test "error if specific version does not exist" {
@@ -317,8 +277,6 @@ teardown(){
   run $ABRA app deploy "$TEST_APP_DOMAIN" "0.2.0+1.21.0" --no-input
   assert_success
   assert_output --partial "0.2.0+1.21.0"
-
-  _undeploy_app
 }
 
 @test "bail out if specific version and chaos" {
@@ -330,8 +288,6 @@ teardown(){
 
 # bats test_tags=slow
 @test "COMPOSE_FILE with \$COMPOSE_FILE override works" {
-  _reset_recipe
-
   run sed -i 's/#COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/COMPOSE_FILE="$COMPOSE_FILE:compose.extra_env.yml"/g' \
     "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
   assert_success
@@ -342,9 +298,6 @@ teardown(){
   assert_success
   assert_output --partial "compose.yml"
   assert_output --partial "compose.extra_env.yml"
-
-  _undeploy_app
-  _reset_app
 }
 
 @test "error if no secrets generated" {
@@ -359,8 +312,6 @@ teardown(){
   run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_failure
   assert_output --partial 'unable to deploy, secrets not generated'
-
-  _reset_app
 }
 
 # bats test_tags=slow
@@ -393,15 +344,15 @@ teardown(){
   _git_commit
 
   # NOTE(d1): 0.1.1+1.20.2 is a previous version which includes compose.extra_secret.yml
-  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.1+1.20.2" --no-input --no-converge-checks
+  run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.1+1.20.2" \
+    --no-input --no-converge-checks
   assert_success
   refute_output --partial 'no such file or directory'
 
   _undeploy_app
-  _reset_app
 
-  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
+  # TODO(d1): use of `--chaos` is a hack while the following is not fixed
+  # https://git.coopcloud.tech/coop-cloud/organising/issues/620
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all --chaos
   assert_success
-
-  _reset_recipe
 }
diff --git a/tests/integration/app_errors.bats b/tests/integration/app_errors.bats
index 91450cf3..6aab4c47 100644
--- a/tests/integration/app_errors.bats
+++ b/tests/integration/app_errors.bats
@@ -32,13 +32,3 @@ setup(){
   assert_failure
   assert_output --partial 'is not deployed'
 }
-
-# bats test_tags=slow
-@test "report errors" {
-  _deploy_app
-
-  run $ABRA app errors "$TEST_APP_DOMAIN"
-  assert_success
-
-  _undeploy_app
-}
diff --git a/tests/integration/app_list.bats b/tests/integration/app_list.bats
index dd38d10f..9067581c 100644
--- a/tests/integration/app_list.bats
+++ b/tests/integration/app_list.bats
@@ -18,10 +18,7 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
 }
 
 @test "list without status" {
@@ -46,8 +43,6 @@ teardown(){
   assert_output --partial "$TEST_SERVER"
   assert_output --partial "$TEST_APP_DOMAIN"
   assert_output --partial "deployed"
-
-  _undeploy_app
 }
 
 @test "filter by server" {
diff --git a/tests/integration/app_new.bats b/tests/integration/app_new.bats
index 7127df14..9646b57a 100644
--- a/tests/integration/app_new.bats
+++ b/tests/integration/app_new.bats
@@ -13,14 +13,13 @@ teardown_file(){
 
 setup(){
   load "$PWD/tests/integration/helpers/common"
-  load "$PWD/tests/integration/helpers/git"
   _common_setup
   _fetch_recipe
 }
 
 teardown(){
-  load "$PWD/tests/integration/helpers/common"
   _rm_app
+  _reset_recipe
 }
 
 @test "autocomplete" {
@@ -110,7 +109,6 @@ teardown(){
 }
 
 @test "ensure recipe up to date if no --offline" {
-  _reset_recipe
   wantHash=$(_get_n_hash 3)
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
@@ -126,12 +124,9 @@ teardown(){
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
   assert_equal $(_get_head_hash) $(_get_current_hash)
-
-  _reset_recipe
 }
 
 @test "ensure recipe not up to date if --offline" {
-  _reset_recipe
   wantHash=$(_get_n_hash 3)
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
@@ -150,8 +145,6 @@ teardown(){
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
   assert_equal $(_get_current_hash) "$wantHash"
-
-  _reset_recipe
 }
 
 # bats test_tags=slow
diff --git a/tests/integration/app_ps.bats b/tests/integration/app_ps.bats
index b2a9987c..0040965d 100644
--- a/tests/integration/app_ps.bats
+++ b/tests/integration/app_ps.bats
@@ -18,10 +18,7 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
 }
 
 @test "validate app argument" {
@@ -48,6 +45,4 @@ teardown(){
   assert_success
   assert_output --partial 'app'
   assert_output --partial 'healthy'
-
-  _undeploy_app
 }
diff --git a/tests/integration/app_remove.bats b/tests/integration/app_remove.bats
index c0be90da..c4ab0f7d 100644
--- a/tests/integration/app_remove.bats
+++ b/tests/integration/app_remove.bats
@@ -18,10 +18,11 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
+  if [[ ! -f "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env" ]]; then
+    _new_app
   fi
+
+  _undeploy_app
 }
 
 @test "validate app argument" {
@@ -40,8 +41,6 @@ teardown(){
 
   run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
   refute_output --partial 'ALERTA'
-
-  _new_app
 }
 
 # bats test_tags=slow
@@ -51,8 +50,6 @@ teardown(){
   run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
   assert_failure
   assert_output --partial 'is still deployed'
-
-  _undeploy_app
 }
 
 @test "detect no secrets to remove" {
@@ -71,8 +68,6 @@ teardown(){
   run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
   assert_success
   assert_output --partial 'no secrets to remove'
-
-  _new_app
 }
 
 @test "remove secrets" {
@@ -90,8 +85,6 @@ teardown(){
 
   sanitisedDomainName="${TEST_APP_DOMAIN//./_}"
   assert_output --partial "$sanitisedDomainName_test_pass_one_v1 removed"
-
-  _new_app
 }
 
 # bats test_tags=slow
@@ -115,8 +108,6 @@ teardown(){
   run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
   assert_success
   assert_output --partial 'no volumes to remove'
-
-  _new_app
 }
 
 # bats test_tags=slow
@@ -133,8 +124,6 @@ teardown(){
   assert_success
   assert_output --partial 'test-volume'
   assert_output --partial 'removed'
-
-  _new_app
 }
 
 @test "remove .env file" {
@@ -144,6 +133,4 @@ teardown(){
   assert_success
 
   assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
-
-  _new_app
 }
diff --git a/tests/integration/app_restart.bats b/tests/integration/app_restart.bats
index b01bb7b3..ceac66d9 100644
--- a/tests/integration/app_restart.bats
+++ b/tests/integration/app_restart.bats
@@ -18,10 +18,7 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
 }
 
 @test "validate app argument" {
@@ -55,6 +52,4 @@ teardown(){
   assert_output --regexp 'attempting to scale .* to 0'
   assert_output --regexp 'attempting to scale .* to 1'
   assert_output --partial 'service successfully restarted'
-
-  _undeploy_app
 }
diff --git a/tests/integration/app_restore.bats b/tests/integration/app_restore.bats
index f2cf52e8..2500d420 100644
--- a/tests/integration/app_restore.bats
+++ b/tests/integration/app_restore.bats
@@ -19,13 +19,12 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
 }
 
 @test "validate app argument" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run $ABRA app restore
   assert_failure
   assert_output --partial 'no app provided'
@@ -36,6 +35,8 @@ teardown(){
 }
 
 @test "retrieve recipe if missing" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE"
   assert_success
   assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE"
@@ -46,6 +47,8 @@ teardown(){
 }
 
 @test "bail if unstaged changes and no --chaos" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run bash -c "echo foo >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
   assert_success
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
@@ -66,6 +69,8 @@ teardown(){
 
 # bats test_tags=slow
 @test "do not bail if unstaged changes and --chaos" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run bash -c 'echo "unstaged changes" >> "$ABRA_DIR/recipes/$TEST_RECIPE/foo"'
   assert_success
   assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
@@ -105,6 +110,8 @@ teardown(){
 }
 
 @test "ensure recipe up to date if no --offline" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
@@ -119,6 +126,8 @@ teardown(){
 }
 
 @test "ensure recipe not up to date if --offline" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
   refute [ -z "$latestCommit" ];
 
@@ -142,6 +151,8 @@ teardown(){
 }
 
 @test "error if missing service" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run $ABRA app restore "$TEST_APP_DOMAIN"
   assert_failure
   assert_output --partial 'missing <service>'
@@ -152,6 +163,8 @@ teardown(){
 }
 
 @test "error if file doesn't exist" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run $ABRA app restore "$TEST_APP_DOMAIN" app DOESNTEXIST
   assert_failure
   assert_output --partial "doesn't exist"
@@ -159,6 +172,8 @@ teardown(){
 
 # bats test_tags=slow
 @test "detect labels if restore enabled" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run touch "$BATS_TMPDIR/foo.txt"
   assert_success
   assert_exists "$BATS_TMPDIR/foo.txt"
@@ -180,6 +195,8 @@ teardown(){
 
 # bats test_tags=slow
 @test "no error if restore not enabled" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run sed -i '/backupbot.restore=.*/d' "$ABRA_DIR/recipes/$TEST_RECIPE/compose.yml"
   assert_success
 
@@ -208,6 +225,8 @@ teardown(){
 
 # bats test_tags=slow
 @test "error if service doesn't exist" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run touch "$BATS_TMPDIR/foo.txt"
   assert_success
   assert_exists "$BATS_TMPDIR/foo.txt"
@@ -227,6 +246,8 @@ teardown(){
 
 # bats test_tags=slow
 @test "restore backup" {
+  skip "https://git.coopcloud.tech/coop-cloud/organising/issues/583"
+
   run touch "$BATS_TMPDIR/foo.txt"
   assert_success
   assert_exists "$BATS_TMPDIR/foo.txt"
diff --git a/tests/integration/app_rollback.bats b/tests/integration/app_rollback.bats
index b8200bbb..3b23c57b 100644
--- a/tests/integration/app_rollback.bats
+++ b/tests/integration/app_rollback.bats
@@ -19,10 +19,8 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
+  _reset_recipe
 }
 
 @test "validate app argument" {
@@ -46,40 +44,32 @@ teardown(){
 }
 
 @test "ensure recipe up to date if no --offline" {
+  wantHash=$(_get_n_hash 3)
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal $(_get_current_hash) "$wantHash"
 
   run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_failure
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --partial "up to date"
+  assert_equal $(_get_current_hash) $(_get_head_hash)
 }
 
 @test "ensure recipe not up to date if --offline" {
-  latestCommit="$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-parse --short HEAD)"
-  refute [ -z "$latestCommit" ];
+  wantHash=$(_get_n_hash 3)
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal $(_get_current_hash) "$wantHash"
 
   run $ABRA app rollback "$TEST_APP_DOMAIN" \
     --no-input --no-converge-checks --offline
   assert_failure
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
-
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "$latestCommit"
-  assert_success
-
-  _reset_recipe "$TEST_RECIPE"
+  assert_equal $(_get_current_hash) $wantHash
 }
 
 @test "bail if unstaged changes and no --chaos" {
@@ -133,7 +123,7 @@ teardown(){
   run $ABRA app deploy "$TEST_APP_DOMAIN" \
     --no-input --chaos
   assert_success
-  assert_output --partial "$latestCommit"
+  assert_output --partial "${latestCommit}"
   assert_output --partial 'chaos'
 
   run $ABRA app rollback "$TEST_APP_DOMAIN" \
@@ -141,8 +131,6 @@ teardown(){
   assert_success
   assert_output --partial "$latestCommit"
   assert_output --partial 'chaos'
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -156,10 +144,6 @@ teardown(){
     --no-input --chaos --chaos --no-converge-checks
   assert_failure
   assert_output --partial 'failed lint checks'
-
-  _undeploy_app
-
-  _checkout_recipe
 }
 
 @test "error if not already deployed" {
@@ -177,8 +161,6 @@ teardown(){
   run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_success
   assert_output --partial "you're on oldest"
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -190,8 +172,6 @@ teardown(){
   run $ABRA app rollback "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input --no-converge-checks
   assert_failure
   assert_output --partial 'is not a downgrade'
-
-  _undeploy_app
 }
 
 @test "bail out if specific version and chaos" {
@@ -210,8 +190,6 @@ teardown(){
   run $ABRA app rollback "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_success
   assert_output --partial "0.1.0+1.20.0"
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -223,6 +201,4 @@ teardown(){
   run $ABRA app rollback "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input --no-converge-checks
   assert_success
   assert_output --partial "0.1.0+1.20.0"
-
-  _undeploy_app
 }
diff --git a/tests/integration/app_run.bats b/tests/integration/app_run.bats
index abcd1aa0..d9e51d40 100644
--- a/tests/integration/app_run.bats
+++ b/tests/integration/app_run.bats
@@ -18,10 +18,7 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
 }
 
 @test "validate app argument" {
@@ -51,8 +48,6 @@ teardown(){
   run $ABRA app run "$TEST_APP_DOMAIN" DOESNTEXIST ls
   assert_failure
   assert_output --partial 'no containers matching'
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -62,6 +57,4 @@ teardown(){
   run $ABRA app run "$TEST_APP_DOMAIN" app ls /
   assert_success
   assert_output --partial 'root'
-
-  _undeploy_app
 }
diff --git a/tests/integration/app_secret.bats b/tests/integration/app_secret.bats
index bcb5e057..5e2d55f3 100644
--- a/tests/integration/app_secret.bats
+++ b/tests/integration/app_secret.bats
@@ -5,6 +5,7 @@ setup_file(){
   _common_setup
   _add_server
 
+  # NOTE(d1): create new app without secrets
   run $ABRA app new "$TEST_RECIPE" \
     --no-input \
     --server "$TEST_SERVER" \
@@ -19,18 +20,18 @@ teardown_file(){
   _reset_recipe
 }
 
-teardown() {
-  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
-  _reset_app
-  _reset_recipe
-  _checkout_recipe
-}
-
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
 }
 
+teardown(){
+  _reset_recipe
+  _reset_app
+
+  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all --no-input
+}
+
 @test "generate: validate arguments" {
   run $ABRA app secret generate
   assert_failure
@@ -61,29 +62,23 @@ setup(){
   assert_success
   assert_output --partial 'test_pass_one'
   assert_output --partial 'test_pass_two'
-  refute_output --partial 'extra_pass'
   assert_output --partial 'false'
-  refute_output --partial 'true'
 
   run $ABRA app secret generate "$TEST_APP_DOMAIN" --all
   assert_success
   assert_output --partial 'test_pass_one'
   assert_output --partial 'test_pass_two'
-  refute_output --partial 'extra_pass'
 
   run $ABRA app secret ls "$TEST_APP_DOMAIN"
   assert_success
   assert_output --partial 'test_pass_one'
   assert_output --partial 'test_pass_two'
-  refute_output --partial 'extra_pass'
-  refute_output --partial 'false'
   assert_output --partial 'true'
 
   run docker -c "$TEST_SERVER" secret ls
   assert_success
   assert_output --partial 'test_pass_one'
   assert_output --partial 'test_pass_two'
-  refute_output --partial 'extra_pass'
 }
 
 @test "generate: broken if missing version" {
@@ -150,11 +145,7 @@ setup(){
 
   run $ABRA app secret generate "$TEST_APP_DOMAIN" --all --chaos
   assert_success
-  assert_output --partial 'test_pass_one not enabled in recipe config'
-  assert_output --partial 'test_pass_two'
-
-  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all --chaos
-  assert_success
+  assert_output --partial 'test_pass_one not enabled'
 }
 
 @test "generate: ensure secret name uses trimmed stack name" {
@@ -184,9 +175,6 @@ setup(){
     | jq -r ".[] | select(.name==\"test_pass_two\") | .value" | awk "{print length}"'
   assert_success
   assert_output --partial '10'  # NOTE(d1): hardcoded # length=10 in recipe config
-
-  run $ABRA app secret rm "$TEST_APP_DOMAIN" --all
-  assert_success
 }
 
 @test "insert: validate arguments" {
@@ -228,11 +216,12 @@ setup(){
 
   run bash -c "echo bar >> $ABRA_DIR/recipes/$TEST_RECIPE/foo"
 
-  run $ABRA app secret insert --file "$TEST_APP_DOMAIN" test_pass_one v1 "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  run $ABRA app secret insert \
+    --file "$TEST_APP_DOMAIN" test_pass_one v1 "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
   assert_success
   assert_output --partial 'successfully stored on server'
 
-  run $ABRA app secret ls "$TEST_APP_DOMAIN"
+  run $ABRA app secret ls "$TEST_APP_DOMAIN" --chaos
   assert_success
   assert_output --partial 'true'
 }
diff --git a/tests/integration/app_services.bats b/tests/integration/app_services.bats
index 6776092e..3cab9bce 100644
--- a/tests/integration/app_services.bats
+++ b/tests/integration/app_services.bats
@@ -19,10 +19,7 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
 }
 
 @test "validate app argument" {
@@ -51,6 +48,4 @@ teardown(){
   sanitisedDomainName="${TEST_APP_DOMAIN//./_}"
   assert_output --partial "$sanitisedDomainName_app"
   assert_output --partial "nginx"
-
-  _undeploy_app
 }
diff --git a/tests/integration/app_undeploy.bats b/tests/integration/app_undeploy.bats
index 9d435203..385b705b 100644
--- a/tests/integration/app_undeploy.bats
+++ b/tests/integration/app_undeploy.bats
@@ -18,10 +18,7 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
 }
 
 @test "validate app argument" {
diff --git a/tests/integration/app_upgrade.bats b/tests/integration/app_upgrade.bats
index 0b1dafb8..03625311 100644
--- a/tests/integration/app_upgrade.bats
+++ b/tests/integration/app_upgrade.bats
@@ -18,10 +18,8 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
+  _reset_recipe
 }
 
 @test "validate app argument" {
@@ -43,8 +41,6 @@ teardown(){
   run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input
   assert_failure
   assert_output --partial 'is not an upgrade'
-
-  _undeploy_app
 }
 
 @test "bail out if specific version and chaos" {
@@ -65,10 +61,6 @@ teardown(){
     --no-input --no-converge-checks --chaos
   assert_failure
   assert_output --partial 'failed lint checks'
-
-  _checkout_recipe
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -78,8 +70,6 @@ teardown(){
   run $ABRA app upgrade "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_success
   assert_output --partial "you're on latest"
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -108,9 +98,6 @@ teardown(){
   run $ABRA app undeploy "$appDomain" --no-input
   assert_success
 
- # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 3
-
   run $ABRA app volume remove "$appDomain" --no-input
   assert_success
 
@@ -128,8 +115,6 @@ teardown(){
   run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.2.0+1.21.0" --no-input --no-converge-checks
   assert_success
   assert_output --partial '0.2.0+1.21.0'
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -143,8 +128,6 @@ teardown(){
   run $ABRA app upgrade "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_success
   assert_output --partial "$latestRelease"
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -160,8 +143,6 @@ teardown(){
   assert_output --partial "$latestRelease"
   assert_output --partial 'release notes baz' # 0.2.0+1.21.0
   refute_output --partial 'release notes bar' # 0.1.1+1.20.2
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -175,8 +156,6 @@ teardown(){
   assert_output --partial '0.1.1+1.20.2'
   assert_output --partial 'release notes bar' # 0.1.1+1.20.2
   refute_output --partial 'release notes baz' # 0.2.0+1.21.0
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -192,8 +171,6 @@ teardown(){
   assert_output --partial "$latestRelease"
   assert_output --partial 'release notes bar' # 0.1.1+1.20.2
   assert_output --partial 'release notes baz' # 0.2.0+1.21.0
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -207,6 +184,4 @@ teardown(){
   assert_output --partial '0.2.0+1.21.0'
   assert_output --partial 'release notes bar' # 0.1.1+1.20.2
   assert_output --partial 'release notes baz' # 0.2.0+1.21.0
-
-  _undeploy_app
 }
diff --git a/tests/integration/app_version.bats b/tests/integration/app_version.bats
index b11f58ef..48a940ee 100644
--- a/tests/integration/app_version.bats
+++ b/tests/integration/app_version.bats
@@ -18,10 +18,8 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
+  _reset_recipe
 }
 
 @test "validate app argument" {
@@ -52,22 +50,15 @@ teardown(){
   run $ABRA app version "$TEST_APP_DOMAIN"
   assert_failure
   assert_output --partial 'failed to determine'
-
-  _checkout_recipe
-  _undeploy_app
 }
 
 # bats test_tags=slow
 @test "error if not in catalogue" {
-  skip "known issue, see https://git.coopcloud.tech/coop-cloud/recipes-catalogue-json/issues/6"
-
   _deploy_app
 
   run $ABRA app version "$TEST_APP_DOMAIN"
   assert_failure
   assert_output --partial 'does not exist'
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
@@ -93,9 +84,6 @@ teardown(){
   run $ABRA app undeploy "$appDomain" --no-input
   assert_success
 
- # NOTE(d1): to let the stack come down before nuking volumes
-  sleep 5
-
   run $ABRA app volume remove "$appDomain" --no-input
   assert_success
 
diff --git a/tests/integration/app_volume.bats b/tests/integration/app_volume.bats
index 6327821c..cbd92961 100644
--- a/tests/integration/app_volume.bats
+++ b/tests/integration/app_volume.bats
@@ -18,10 +18,7 @@ setup(){
 }
 
 teardown(){
-  # https://github.com/bats-core/bats-core/issues/383#issuecomment-738628888
-  if [[ -z "${BATS_TEST_COMPLETED}" ]]; then
-    _undeploy_app
-  fi
+  _undeploy_app
 }
 
 @test "ls validate app argument" {
@@ -47,8 +44,6 @@ teardown(){
   run $ABRA app volume ls "$TEST_APP_DOMAIN"
   assert_success
   assert_output --partial 'test-volume'
-
-  _undeploy_app
 }
 
 @test "rm validate app argument" {
@@ -68,14 +63,13 @@ teardown(){
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
   assert_failure
   assert_output --partial 'is still deployed'
-
-  _undeploy_app
 }
 
 # bats test_tags=slow
 @test "remove volumes" {
-  _deploy_app
+  sleep 3 # NOTE(d1): hack to avoid "network not found"
 
+  _deploy_app
   _undeploy_app
 
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
@@ -85,8 +79,9 @@ teardown(){
 
 # bats test_tags=slow
 @test "remove no volumes" {
-  _deploy_app
+  sleep 3 # NOTE(d1): hack to avoid "network not found"
 
+  _deploy_app
   _undeploy_app
 
   run $ABRA app volume rm "$TEST_APP_DOMAIN" --force
diff --git a/tests/integration/dirs.bats b/tests/integration/dirs.bats
index dfe0a71d..894d7e94 100644
--- a/tests/integration/dirs.bats
+++ b/tests/integration/dirs.bats
@@ -27,8 +27,9 @@ setup(){
 @test "abra directory is created" {
   run $ABRA app ls
 
-  # no servers yet, so will fail. however, it will run the required code which
-  # checks if it should create these base directories and that is what we want
+  # NOTE(d1): no servers yet, so will fail. however, it will run the required
+  # code which checks if it should create these base directories and that is
+  # what we want
   assert_failure
 
   assert_exists "$ABRA_DIR"
@@ -36,16 +37,13 @@ setup(){
   assert_exists "$ABRA_DIR/recipes"
   assert_exists "$ABRA_DIR/backups"
   assert_exists "$ABRA_DIR/vendor"
-  assert_exists "$ABRA_DIR/catalogue"
+
+  assert_not_exists "$ABRA_DIR/catalogue"
 }
 
 @test "catalogue recipe is a git repository" {
-  run $ABRA app ls
-
-  # no servers yet, so will fail. however, it will run the required code which
-  # checks if it should create these base directories and that is what we want
-  assert_failure
-
+  run $ABRA recipe fetch "$TEST_RECIPE"
+  assert_success
   assert_output --partial 'local recipe catalogue is missing'
 
   assert_exists "$ABRA_DIR/catalogue"
diff --git a/tests/integration/helpers/app.bash b/tests/integration/helpers/app.bash
index 32208b84..89958586 100644
--- a/tests/integration/helpers/app.bash
+++ b/tests/integration/helpers/app.bash
@@ -46,10 +46,5 @@ _reset_app(){
   assert_success
   assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
 
-  run $ABRA app new "$TEST_RECIPE" \
-    --no-input \
-    --server "$TEST_SERVER" \
-    --domain "$TEST_APP_DOMAIN"
-  assert_success
-  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  _new_app
 }
diff --git a/tests/integration/helpers/common.bash b/tests/integration/helpers/common.bash
index 9f434436..892be7b2 100644
--- a/tests/integration/helpers/common.bash
+++ b/tests/integration/helpers/common.bash
@@ -10,6 +10,7 @@ _common_setup() {
   load "$PWD/tests/integration/helpers/git"
   load "$PWD/tests/integration/helpers/recipe"
   load "$PWD/tests/integration/helpers/server"
+  load "$PWD/tests/integration/helpers/docker"
 
   export ABRA="$PWD/abra"
   export KADABRA="$PWD/kadabra"
@@ -17,4 +18,6 @@ _common_setup() {
   export TEST_APP_NAME="$(basename "${BATS_TEST_FILENAME//./_}")"
   export TEST_APP_DOMAIN="$TEST_APP_NAME.$TEST_SERVER"
   export TEST_RECIPE="abra-test-recipe"
+
+  _ensure_swarm
 }
diff --git a/tests/integration/helpers/docker.bash b/tests/integration/helpers/docker.bash
new file mode 100644
index 00000000..52d5b9c9
--- /dev/null
+++ b/tests/integration/helpers/docker.bash
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+_ensure_swarm() {
+  if [ "$(docker info | grep Swarm | sed 's/Swarm: //g' | tr -d ' ')" == "inactive" ]; then
+    run docker swarm init
+    assert_success
+  fi
+
+  if ! $(docker network ls | grep -q 'proxy'); then
+    run docker network create -d overlay proxy
+    assert_success
+  fi
+}
diff --git a/tests/integration/helpers/git.bash b/tests/integration/helpers/git.bash
index 5f3c5b1a..2c15bf96 100644
--- a/tests/integration/helpers/git.bash
+++ b/tests/integration/helpers/git.bash
@@ -46,25 +46,17 @@ _git_commit() {
 }
 
 _get_tag_hash() {
-  tagHash=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-list -n 1 "$1")
-  assert_success
-  echo "$tagHash"
+  echo $(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" rev-list -n 1 "$1")
 }
 
 _get_head_hash() {
-  headHash=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H" HEAD)
-  assert_success
-  echo "$headHash"
+  echo $(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H" HEAD)
 }
 
 _get_current_hash() {
-  currentHash=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H")
-  assert_success
-  echo "$currentHash"
+  echo $(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H")
 }
 
 _get_n_hash() {
-  nHash=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H" "HEAD~$1")
-  assert_success
-  echo "$nHash"
+  echo $(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" show -s --format="%H" "HEAD~$1")
 }
diff --git a/tests/integration/helpers/recipe.bash b/tests/integration/helpers/recipe.bash
index 5b5ec3f6..17e48f19 100644
--- a/tests/integration/helpers/recipe.bash
+++ b/tests/integration/helpers/recipe.bash
@@ -17,3 +17,12 @@ _reset_recipe(){
 
   _fetch_recipe
 }
+
+_ensure_latest_version(){
+  latestRelease=$(git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag -l | tail -n 1)
+
+  if [ ! $latestRelease = "$1" ]; then
+    echo "expected latest recipe version of '$1', saw: $latestRelease"
+    return 1
+  fi
+}
diff --git a/tests/integration/install.bats b/tests/integration/install.bats
index a92ce234..d0d5fa9f 100644
--- a/tests/integration/install.bats
+++ b/tests/integration/install.bats
@@ -28,6 +28,8 @@ teardown(){
 
 # bats test_tags=slow
 @test "install release candidate from script" {
+  skip "current RC is brokenly specified in the installer script"
+
   run bash -c 'curl https://install.abra.coopcloud.tech | bash -s -- --rc'
   assert_success
 
diff --git a/tests/integration/recipe_diff.bats b/tests/integration/recipe_diff.bats
index fd384e35..85f7307a 100644
--- a/tests/integration/recipe_diff.bats
+++ b/tests/integration/recipe_diff.bats
@@ -5,6 +5,10 @@ setup() {
   _common_setup
 }
 
+teardown(){
+  _reset_recipe
+}
+
 @test "show unstaged changes" {
   run $ABRA recipe diff "$TEST_RECIPE"
   assert_success
@@ -16,6 +20,4 @@ setup() {
   run $ABRA recipe diff "$TEST_RECIPE"
   assert_success
   assert_output --partial 'traefik.enable'
-
-  _reset_recipe
 }
diff --git a/tests/integration/recipe_fetch.bats b/tests/integration/recipe_fetch.bats
index 1cc61e7c..d2415355 100644
--- a/tests/integration/recipe_fetch.bats
+++ b/tests/integration/recipe_fetch.bats
@@ -5,6 +5,7 @@ setup() {
   _common_setup
 }
 
+# bats test_tags=slow
 @test "recipe fetch all" {
   run rm -rf "$ABRA_DIR/recipes/matrix-synapse"
   assert_success
diff --git a/tests/integration/recipe_lint.bats b/tests/integration/recipe_lint.bats
index 401ab8fd..85657154 100644
--- a/tests/integration/recipe_lint.bats
+++ b/tests/integration/recipe_lint.bats
@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 
-teardown_file() {
+setup_file(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
-  _reset_recipe
+  _fetch_recipe
 }
 
 setup() {
@@ -11,6 +11,10 @@ setup() {
   _common_setup
 }
 
+teardown(){
+  _reset_recipe
+}
+
 @test "recipe lint" {
   run $ABRA recipe lint gitea
   assert_success
@@ -62,35 +66,31 @@ setup() {
 }
 
 @test "ensure recipe up to date if no --offline" {
+  wantHash=$(_get_n_hash 3)
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal $(_get_current_hash) "$wantHash"
 
   run $ABRA recipe lint "$TEST_RECIPE"
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
-
-  _reset_recipe
+  assert_equal $(_get_head_hash) $(_get_current_hash)
 }
 
 @test "ensure recipe not up to date if --offline" {
+  wantHash=$(_get_n_hash 3)
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal $(_get_current_hash) "$wantHash"
 
   run $ABRA recipe lint "$TEST_RECIPE" --offline
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
-
-  _reset_recipe
+  assert_equal $(_get_current_hash) "$wantHash"
 }
 
 @test "recipe lint warns on error" {
@@ -103,6 +103,4 @@ setup() {
 
   run $ABRA recipe lint "$TEST_RECIPE" --chaos
   assert_success --partial 'watch out, some critical errors are present'
-
-  _checkout_recipe
 }
diff --git a/tests/integration/recipe_new.bats b/tests/integration/recipe_new.bats
index c0c8da20..3a269be6 100644
--- a/tests/integration/recipe_new.bats
+++ b/tests/integration/recipe_new.bats
@@ -52,4 +52,3 @@ teardown(){
   assert_output --partial 'fooUser'
   assert_output --partial 'foo@example.com'
 }
-
diff --git a/tests/integration/recipe_release.bats b/tests/integration/recipe_release.bats
index 4c10f691..7bcb2822 100644
--- a/tests/integration/recipe_release.bats
+++ b/tests/integration/recipe_release.bats
@@ -32,8 +32,9 @@ teardown() {
   assert_output --partial 'unable to validate recipe'
 }
 
-# NOTE(d1): relies on only 3 versions being published for the $TEST_RECIPE.
 @test "release patch bump" {
+  _ensure_latest_version "0.3.0+1.21.0"
+
   run $ABRA recipe upgrade "$TEST_RECIPE" --no-input --patch
   assert_success
 
@@ -47,7 +48,7 @@ teardown() {
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" diff
   assert_success
-  assert_output --partial 'coop-cloud.${STACK_NAME}.version=0.2.1+1.21.6'
+  assert_output --partial 'coop-cloud.${STACK_NAME}.version=0.3.1+1.21.6'
 
   run $ABRA recipe release "$TEST_RECIPE" --no-input --patch
   assert_success
@@ -55,13 +56,12 @@ teardown() {
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag --list
   assert_success
-  assert_output --partial '0.2.1+1.21.6'
+  assert_output --partial '0.3.1+1.21.6'
 }
 
-# NOTE(d1): this test can't assert hardcoded versions since we upgrade a minor
-# version which could be anything in the future. so, we do our best with
-# --regexp.
 @test "release minor bump" {
+  _ensure_latest_version "0.3.0+1.21.0"
+
   run $ABRA recipe upgrade "$TEST_RECIPE" --no-input --minor
   assert_success
 
@@ -75,7 +75,7 @@ teardown() {
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" diff
   assert_success
-  assert_output --regexp 'coop-cloud\.\$\{STACK_NAME\}\.version=0\.3\.0\+1\.2.*'
+  assert_output --regexp 'coop-cloud\.\$\{STACK_NAME\}\.version=0\.4\.0\+1\.2.*'
 
   run $ABRA recipe release "$TEST_RECIPE" --no-input --minor
   assert_success
@@ -83,7 +83,7 @@ teardown() {
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" tag --list
   assert_success
-  assert_output --regexp '0\.3\.0\+1\.2.*'
+  assert_output --regexp '0\.4\.0\+1\.2.*'
 }
 
 @test "unknown files not committed" {
@@ -103,11 +103,14 @@ teardown() {
   assert_output --partial "fatal: pathspec 'foo' did not match any files"
 }
 
-# NOTE: relies on 0.2.x being the last minor version
 @test "release with next release note" {
+  _ensure_latest_version "0.3.0+1.21.0"
+
   _mkfile "$ABRA_DIR/recipes/$TEST_RECIPE/release/next" "those are some release notes for the next release"
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" add release/next
   assert_success
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" commit -m "added some release notes"
   assert_success
 
@@ -116,6 +119,6 @@ teardown() {
   assert_output --partial 'no -p/--publish passed, not publishing'
 
   assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/release/next"
-  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/release/0.3.0+1.21.0"
-  assert_file_contains "$ABRA_DIR/recipes/$TEST_RECIPE/release/0.3.0+1.21.0" "those are some release notes for the next release"
+  assert_exists "$ABRA_DIR/recipes/$TEST_RECIPE/release/0.4.0+1.21.0"
+  assert_file_contains "$ABRA_DIR/recipes/$TEST_RECIPE/release/0.4.0+1.21.0" "those are some release notes for the next release"
 }
diff --git a/tests/integration/recipe_sync.bats b/tests/integration/recipe_sync.bats
index f04735f8..39a4c262 100644
--- a/tests/integration/recipe_sync.bats
+++ b/tests/integration/recipe_sync.bats
@@ -16,6 +16,11 @@ setup(){
   _common_setup
 }
 
+teardown(){
+  _reset_recipe
+  _reset_tags
+}
+
 @test "validate recipe argument" {
   run $ABRA recipe sync --no-input
   assert_failure
@@ -40,8 +45,6 @@ setup(){
   run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
   assert_success
   assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
-
-  _checkout_recipe
 }
 
 @test "detect unstaged label changes" {
@@ -54,12 +57,11 @@ setup(){
   run $ABRA recipe sync "$TEST_RECIPE" --patch
   assert_success
   assert_output --partial 'is already set, nothing to do?'
-
-  _checkout_recipe
 }
 
-# NOTE(d1): relies on only 3 versions being published for the $TEST_RECIPE.
 @test "sync patch label bump" {
+  _ensure_latest_version "0.3.0+1.21.0"
+
   run $ABRA recipe upgrade "$TEST_RECIPE" --no-input --patch
   assert_success
 
@@ -72,12 +74,12 @@ setup(){
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" diff
   assert_success
-  assert_output --partial 'coop-cloud.${STACK_NAME}.version=0.2.1+1.21.6'
-
-  _checkout_recipe
+  assert_output --regexp 'coop-cloud\.\$\{STACK_NAME\}\.version=0\.3\.1\+1\.2.*'
 }
 
 @test "sync minor label bump" {
+  _ensure_latest_version "0.3.0+1.21.0"
+
   run $ABRA recipe upgrade "$TEST_RECIPE" --no-input --minor
   assert_success
 
@@ -90,9 +92,7 @@ setup(){
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" diff
   assert_success
-  assert_output --regexp 'coop-cloud\.\$\{STACK_NAME\}\.version=0\.2\.0\+1\.2.*'
-
-  _checkout_recipe
+  assert_output --regexp 'coop-cloud\.\$\{STACK_NAME\}\.version=0\.4\.0\+1\.2.*'
 }
 
 @test "error if --no-input and no initial version" {
@@ -102,8 +102,6 @@ setup(){
   assert_failure
   assert_output --partial 'unable to continue'
   assert_output --partial 'initial version'
-
-  _reset_tags
 }
 
 @test "output label sync only once" {
@@ -118,6 +116,4 @@ setup(){
   assert_success
   assert_line --index 0 --partial 'synced label'
   refute_line --index 1 --partial 'synced label'
-
-  _checkout_recipe
 }
diff --git a/tests/integration/recipe_upgrade.bats b/tests/integration/recipe_upgrade.bats
index a36efe26..13d9fb4c 100644
--- a/tests/integration/recipe_upgrade.bats
+++ b/tests/integration/recipe_upgrade.bats
@@ -17,6 +17,10 @@ setup(){
   _common_setup
 }
 
+teardown(){
+  _reset_recipe
+}
+
 @test "validate recipe argument" {
   run $ABRA recipe upgrade --no-input
   assert_failure
@@ -57,20 +61,18 @@ setup(){
 }
 
 @test "ensure up to date" {
+  wantHash=$(_get_n_hash 3)
+
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" reset --hard HEAD~3
   assert_success
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
+  assert_equal $(_get_current_hash) "$wantHash"
 
   run $ABRA recipe upgrade "$TEST_RECIPE" --no-input
   assert_success
   assert_output --partial 'can upgrade service: app'
 
-  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" status
-  assert_output --regexp 'behind .* 3 commits'
-
-  _reset_recipe
+  assert_equal $(_get_head_hash) $(_get_current_hash)
 }
 
 @test "only one of -x/y/z flags" {
@@ -79,16 +81,18 @@ setup(){
   assert_output --partial 'you can only use one of'
 }
 
-# NOTE(d1): relies on only 3 versions being published for the $TEST_RECIPE.
 @test "upgrade patch" {
+  run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" checkout "0.2.0+1.21.0"
+  assert_success
+
+  assert_equal $(_get_tag_hash 0.2.0+1.21.0) $(_get_current_hash)
+
   run $ABRA recipe upgrade "$TEST_RECIPE" --no-input --patch
   assert_success
 
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" diff
   assert_success
   assert_output --partial 'image: nginx:1.21.6'
-
-  _checkout_recipe
 }
 
 @test "upgrade minor" {
@@ -98,6 +102,4 @@ setup(){
   run git -C "$ABRA_DIR/recipes/$TEST_RECIPE" diff
   assert_success
   assert_output --regexp 'image: nginx:1.2.*'
-
-  _checkout_recipe
 }
diff --git a/tests/integration/recipe_version.bats b/tests/integration/recipe_version.bats
index ebc5bad9..42d2e4d0 100644
--- a/tests/integration/recipe_version.bats
+++ b/tests/integration/recipe_version.bats
@@ -12,8 +12,6 @@ setup() {
 }
 
 @test "error if not present in catalogue" {
-  skip "known issue, see https://git.coopcloud.tech/coop-cloud/recipes-catalogue-json/issues/6"
-
   run $ABRA recipe versions "$TEST_RECIPE"
   assert_failure
   assert_output --partial "is not published on the catalogue"
diff --git a/tests/integration/server_add.bats b/tests/integration/server_add.bats
index 0d0086d7..2e5ba378 100644
--- a/tests/integration/server_add.bats
+++ b/tests/integration/server_add.bats
@@ -3,25 +3,19 @@
 setup_file(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
-
-  run docker swarm leave --force
 }
 
 setup(){
   load "$PWD/tests/integration/helpers/common"
   _common_setup
-
-  run docker swarm init
-  assert_success
 }
 
 teardown(){
-  run docker swarm leave --force
-  assert_success
-
   _rm_server
+  _ensure_swarm
 }
 
+# bats test_tags=slow,dns
 @test "add new server" {
   run $ABRA server add "$TEST_SERVER"
   assert_success
@@ -57,6 +51,7 @@ teardown(){
   assert_output --partial 'swarm mode not enabled on local server'
 }
 
+# bats test_tags=slow
 @test "cleanup when cannot add server" {
   run $ABRA server add example.com
   assert_failure
diff --git a/tests/integration/server_list.bats b/tests/integration/server_list.bats
index 8396c215..28c47bd3 100644
--- a/tests/integration/server_list.bats
+++ b/tests/integration/server_list.bats
@@ -22,9 +22,6 @@ setup(){
 }
 
 @test "show 'local' when --local server created" {
-  run docker swarm init
-  assert_success
-
   run $ABRA server add --local
   assert_success
   assert_exists "$ABRA_DIR/servers/default"
@@ -34,11 +31,6 @@ setup(){
   assert_output --partial 'default'
   assert_output --partial 'local'
   assert_output --partial 'n/a'
-
-  run docker swarm leave --force
-  assert_success
-
-  _rm_default_server
 }
 
 @test "filter by problem" {
diff --git a/tests/integration/server_remove.bats b/tests/integration/server_remove.bats
index 2d3b7f8a..4c64da5e 100644
--- a/tests/integration/server_remove.bats
+++ b/tests/integration/server_remove.bats
@@ -26,6 +26,10 @@ setup(){
 }
 
 @test "remove server" {
+  if [ "$TEST_SERVER" = "default" ]; then
+    skip "cannot delete 'default' server (local)"
+  fi
+
   assert_exists "$ABRA_DIR/servers/$TEST_SERVER"
   assert bash -c "docker context ls | grep -q $TEST_SERVER"
 
diff --git a/tests/integration/upgrade.bats b/tests/integration/upgrade.bats
index 5dbe65f2..0b49040a 100644
--- a/tests/integration/upgrade.bats
+++ b/tests/integration/upgrade.bats
@@ -29,6 +29,8 @@ teardown(){
 
 # bats test_tags=slow
 @test "abra upgrade release candidate" {
+  skip "TODO: RC publishing broke somehow, needs investigation"
+
   run $ABRA upgrade --rc
   assert_success
   assert_output --partial 'Public interest infrastructure'
-- 
2.47.2


From 9554ad40c8060b00d081ceae33dae14a28bbe83a Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Tue, 2 Jul 2024 12:54:48 +0200
Subject: [PATCH 2/3] refactor: use adapted upstream detach=false logic [ci
 skip]

See https://git.coopcloud.tech/coop-cloud/organising/issues/607.
---
 cli/app/deploy.go             |  1 +
 cli/app/rollback.go           |  1 +
 cli/app/undeploy.go           |  5 ++-
 cli/app/upgrade.go            |  1 +
 cli/updater/updater.go        |  1 +
 go.mod                        |  3 +-
 go.sum                        |  2 -
 pkg/upstream/stack/options.go |  3 ++
 pkg/upstream/stack/remove.go  | 54 ++++++++++++++++++++++
 pkg/upstream/stack/stack.go   | 84 ++++++++++++++++++-----------------
 10 files changed, 110 insertions(+), 45 deletions(-)

diff --git a/cli/app/deploy.go b/cli/app/deploy.go
index fb72c0ec..87feae8f 100644
--- a/cli/app/deploy.go
+++ b/cli/app/deploy.go
@@ -196,6 +196,7 @@ recipes.
 			Namespace:    stackName,
 			Prune:        false,
 			ResolveImage: stack.ResolveImageAlways,
+			Detach:       false,
 		}
 		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
 		if err != nil {
diff --git a/cli/app/rollback.go b/cli/app/rollback.go
index f39d114b..43b1fbba 100644
--- a/cli/app/rollback.go
+++ b/cli/app/rollback.go
@@ -215,6 +215,7 @@ recipes.
 			Namespace:    stackName,
 			Prune:        false,
 			ResolveImage: stack.ResolveImageAlways,
+			Detach:       false,
 		}
 		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
 		if err != nil {
diff --git a/cli/app/undeploy.go b/cli/app/undeploy.go
index 3a1c2ffa..7b50908d 100644
--- a/cli/app/undeploy.go
+++ b/cli/app/undeploy.go
@@ -122,7 +122,10 @@ Passing "-p/--prune" does not remove those volumes.
 			logrus.Fatal(err)
 		}
 
-		rmOpts := stack.Remove{Namespaces: []string{app.StackName()}}
+		rmOpts := stack.Remove{
+			Namespaces: []string{app.StackName()},
+			Detach:     false,
+		}
 		if err := stack.RunRemove(context.Background(), cl, rmOpts); err != nil {
 			logrus.Fatal(err)
 		}
diff --git a/cli/app/upgrade.go b/cli/app/upgrade.go
index 366a5932..e2f6df6c 100644
--- a/cli/app/upgrade.go
+++ b/cli/app/upgrade.go
@@ -249,6 +249,7 @@ recipes.
 			Namespace:    stackName,
 			Prune:        false,
 			ResolveImage: stack.ResolveImageAlways,
+			Detach:       false,
 		}
 		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
 		if err != nil {
diff --git a/cli/updater/updater.go b/cli/updater/updater.go
index 4b036132..74266d8b 100644
--- a/cli/updater/updater.go
+++ b/cli/updater/updater.go
@@ -362,6 +362,7 @@ func createDeployConfig(recipeName string, stackName string, env config.AppEnv)
 		Namespace:    stackName,
 		Prune:        false,
 		ResolveImage: stack.ResolveImageAlways,
+		Detach:       false,
 	}
 
 	composeFiles, err := config.GetComposeFiles(recipeName, env)
diff --git a/go.mod b/go.mod
index 7347c815..a7e1c3a1 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ require (
 	git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
-	github.com/distribution/distribution v2.8.3+incompatible
+	github.com/distribution/reference v0.6.0
 	github.com/docker/cli v26.1.4+incompatible
 	github.com/docker/docker v26.1.4+incompatible
 	github.com/docker/go-units v0.5.0
@@ -36,7 +36,6 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
diff --git a/go.sum b/go.sum
index be044761..fff7c0bf 100644
--- a/go.sum
+++ b/go.sum
@@ -297,8 +297,6 @@ github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8l
 github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/distribution/distribution v2.8.3+incompatible h1:RlpEXBLq/WPXYvBYMDAmBX/SnhD67qwtvW/DzKc8pAo=
-github.com/distribution/distribution v2.8.3+incompatible/go.mod h1:EgLm2NgWtdKgzF9NpMzUKgzmR7AMmb0VQi2B+ZzDRjc=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
diff --git a/pkg/upstream/stack/options.go b/pkg/upstream/stack/options.go
index 14ca364e..39245eb4 100644
--- a/pkg/upstream/stack/options.go
+++ b/pkg/upstream/stack/options.go
@@ -7,9 +7,12 @@ type Deploy struct {
 	ResolveImage     string
 	SendRegistryAuth bool
 	Prune            bool
+	Detach           bool
+	Quiet            bool
 }
 
 // Remove holds docker stack remove options
 type Remove struct {
 	Namespaces []string
+	Detach     bool
 }
diff --git a/pkg/upstream/stack/remove.go b/pkg/upstream/stack/remove.go
index 6b0f043c..7c211b56 100644
--- a/pkg/upstream/stack/remove.go
+++ b/pkg/upstream/stack/remove.go
@@ -9,6 +9,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/api/types/versions"
+	"github.com/docker/docker/client"
 	apiclient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -56,6 +57,12 @@ func RunRemove(ctx context.Context, client *apiclient.Client, opts Remove) error
 
 		if hasError {
 			errs = append(errs, fmt.Sprintf("failed to remove some resources from stack: %s", namespace))
+			continue
+		}
+
+		err = waitOnTasks(ctx, client, namespace)
+		if err != nil {
+			errs = append(errs, fmt.Sprintf("failed to wait on tasks of stack: %s: %s", namespace, err))
 		}
 	}
 
@@ -136,3 +143,50 @@ func removeConfigs(
 	}
 	return hasError
 }
+
+// https://github.com/docker/cli/pull/4259
+func getStackTasks(ctx context.Context, apiclient client.APIClient, namespace string) ([]swarm.Task, error) {
+	return apiclient.TaskList(ctx, types.TaskListOptions{Filters: getStackFilter(namespace)})
+}
+
+var numberedStates = map[swarm.TaskState]int64{
+	swarm.TaskStateNew:       1,
+	swarm.TaskStateAllocated: 2,
+	swarm.TaskStatePending:   3,
+	swarm.TaskStateAssigned:  4,
+	swarm.TaskStateAccepted:  5,
+	swarm.TaskStatePreparing: 6,
+	swarm.TaskStateReady:     7,
+	swarm.TaskStateStarting:  8,
+	swarm.TaskStateRunning:   9,
+	swarm.TaskStateComplete:  10,
+	swarm.TaskStateShutdown:  11,
+	swarm.TaskStateFailed:    12,
+	swarm.TaskStateRejected:  13,
+}
+
+func terminalState(state swarm.TaskState) bool {
+	return numberedStates[state] > numberedStates[swarm.TaskStateRunning]
+}
+
+func waitOnTasks(ctx context.Context, client apiclient.APIClient, namespace string) error {
+	terminalStatesReached := 0
+	for {
+		tasks, err := getStackTasks(ctx, client, namespace)
+		if err != nil {
+			return fmt.Errorf("failed to get tasks: %w", err)
+		}
+
+		for _, task := range tasks {
+			if terminalState(task.Status.State) {
+				terminalStatesReached++
+				break
+			}
+		}
+
+		if terminalStatesReached == len(tasks) {
+			break
+		}
+	}
+	return nil
+}
diff --git a/pkg/upstream/stack/stack.go b/pkg/upstream/stack/stack.go
index d8e7bdd9..d9dc94f3 100644
--- a/pkg/upstream/stack/stack.go
+++ b/pkg/upstream/stack/stack.go
@@ -9,6 +9,8 @@ import (
 	"os/signal"
 	"time"
 
+	stdlibErr "errors"
+
 	"coopcloud.tech/abra/pkg/upstream/convert"
 	"github.com/docker/cli/cli/command/service/progress"
 	"github.com/docker/cli/cli/command/stack/formatter"
@@ -129,7 +131,7 @@ func IsDeployed(ctx context.Context, cl *dockerClient.Client, stackName string)
 func pruneServices(ctx context.Context, cl *dockerClient.Client, namespace convert.Namespace, services map[string]struct{}) {
 	oldServices, err := GetStackServices(ctx, cl, namespace.Name())
 	if err != nil {
-		logrus.Infof("Failed to list services: %s\n", err)
+		logrus.Infof("failed to list services: %s\n", err)
 	}
 
 	pruneServices := []swarm.Service{}
@@ -161,7 +163,7 @@ func validateResolveImageFlag(opts *Deploy) error {
 	case ResolveImageAlways, ResolveImageChanged, ResolveImageNever:
 		return nil
 	default:
-		return errors.Errorf("Invalid option %s for flag --resolve-image", opts.ResolveImage)
+		return errors.Errorf("invalid option %s for flag --resolve-image", opts.ResolveImage)
 	}
 }
 
@@ -206,7 +208,16 @@ func deployCompose(ctx context.Context, cl *dockerClient.Client, opts Deploy, co
 		return err
 	}
 
-	return deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage, appName, dontWait)
+	serviceIDs, err := deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage, appName, dontWait)
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("waiting for %s to deploy... please hold 🤚", appName)
+	if err := waitOnServices(ctx, cl, serviceIDs, appName); err == nil {
+		logrus.Infof("Successfully deployed %s", appName)
+	}
+	return err
 }
 
 func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) map[string]struct{} {
@@ -276,7 +287,7 @@ func createConfigs(ctx context.Context, cl *dockerClient.Client, configs []swarm
 			}
 		case dockerClient.IsErrNotFound(err):
 			// config does not exist, then we create a new one.
-			logrus.Infof("Creating config %s\n", configSpec.Name)
+			logrus.Infof("creating config %s\n", configSpec.Name)
 			if _, err := cl.ConfigCreate(ctx, configSpec); err != nil {
 				return errors.Wrapf(err, "failed to create config %s", configSpec.Name)
 			}
@@ -307,7 +318,7 @@ func createNetworks(ctx context.Context, cl *dockerClient.Client, namespace conv
 			createOpts.Driver = defaultNetworkDriver
 		}
 
-		logrus.Infof("Creating network %s\n", name)
+		logrus.Infof("creating network %s\n", name)
 		if _, err := cl.NetworkCreate(ctx, name, createOpts); err != nil {
 			return errors.Wrapf(err, "failed to create network %s", name)
 		}
@@ -323,10 +334,10 @@ func deployServices(
 	sendAuth bool,
 	resolveImage string,
 	appName string,
-	dontWait bool) error {
+	dontWait bool) ([]string, error) {
 	existingServices, err := GetStackServices(ctx, cl, namespace.Name())
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	existingServiceMap := make(map[string]swarm.Service)
@@ -334,7 +345,8 @@ func deployServices(
 		existingServiceMap[service.Spec.Name] = service
 	}
 
-	serviceIDs := make(map[string]string)
+	var serviceIDs []string
+
 	for internalName, serviceSpec := range services {
 		var (
 			name        = namespace.Scope(internalName)
@@ -378,16 +390,16 @@ func deployServices(
 
 			response, err := cl.ServiceUpdate(ctx, service.ID, service.Version, serviceSpec, updateOpts)
 			if err != nil {
-				return errors.Wrapf(err, "failed to update service %s", name)
+				return nil, errors.Wrapf(err, "failed to update service %s", name)
 			}
 
-			serviceIDs[service.ID] = name
-
 			for _, warning := range response.Warnings {
 				logrus.Warn(warning)
 			}
+
+			serviceIDs = append(serviceIDs, service.ID)
 		} else {
-			logrus.Infof("Creating service %s\n", name)
+			logrus.Infof("creating service %s\n", name)
 
 			createOpts := types.ServiceCreateOptions{EncodedRegistryAuth: encodedAuth}
 
@@ -398,43 +410,19 @@ func deployServices(
 
 			serviceCreateResponse, err := cl.ServiceCreate(ctx, serviceSpec, createOpts)
 			if err != nil {
-				return errors.Wrapf(err, "failed to create service %s", name)
+				return nil, errors.Wrapf(err, "failed to create service %s", name)
 			}
 
-			serviceIDs[serviceCreateResponse.ID] = name
+			serviceIDs = append(serviceIDs, serviceCreateResponse.ID)
 		}
 	}
 
-	var serviceNames []string
-	for _, serviceName := range serviceIDs {
-		serviceNames = append(serviceNames, serviceName)
-	}
-
 	if dontWait {
 		logrus.Warn("skipping converge logic checks")
-		return nil
+		return nil, nil
 	}
 
-	logrus.Infof("Waiting for %s to deploy... please hold 🤚", appName)
-	ch := make(chan error, len(serviceIDs))
-	for serviceID, serviceName := range serviceIDs {
-		logrus.Debugf("waiting on %s to converge", serviceName)
-		go func(sID, sName, aName string) {
-			ch <- WaitOnService(ctx, cl, sID, aName)
-		}(serviceID, serviceName, appName)
-	}
-
-	for _, serviceID := range serviceIDs {
-		err := <-ch
-		if err != nil {
-			return err
-		}
-		logrus.Debugf("assuming %s converged successfully", serviceID)
-	}
-
-	logrus.Infof("Successfully deployed %s", appName)
-
-	return nil
+	return serviceIDs, nil
 }
 
 func getStackNetworks(ctx context.Context, dockerclient client.APIClient, namespace string) ([]types.NetworkResource, error) {
@@ -449,6 +437,22 @@ func getStackConfigs(ctx context.Context, dockerclient client.APIClient, namespa
 	return dockerclient.ConfigList(ctx, types.ConfigListOptions{Filters: getStackFilter(namespace)})
 }
 
+func waitOnServices(ctx context.Context, cl *dockerClient.Client, serviceIDs []string, appName string) error {
+	var errs []error
+
+	for _, serviceID := range serviceIDs {
+		if err := WaitOnService(ctx, cl, serviceID, appName); err != nil {
+			errs = append(errs, fmt.Errorf("%s: %w", serviceID, err))
+		}
+	}
+
+	if len(errs) > 0 {
+		return stdlibErr.Join(errs...)
+	}
+
+	return nil
+}
+
 // https://github.com/docker/cli/blob/master/cli/command/service/helpers.go
 // https://github.com/docker/cli/blob/master/cli/command/service/progress/progress.go
 func WaitOnService(ctx context.Context, cl *dockerClient.Client, serviceID, appName string) error {
-- 
2.47.2


From 6a2db1abaae4248ebe6e9973d3e814f142beab16 Mon Sep 17 00:00:00 2001
From: decentral1se <cellarspoon@riseup.net>
Date: Tue, 2 Jul 2024 15:51:20 +0200
Subject: [PATCH 3/3] test: run int suite on remote server via cron

---
 .drone.yml | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/.drone.yml b/.drone.yml
index 4d4bef93..b1ab1989 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -57,6 +57,30 @@ steps:
       - make check
       - make test
 
+  - name: integration test
+    image: appleboy/drone-ssh
+    settings:
+      host:
+        - int.coopcloud.tech
+      username: abra
+      key:
+        from_secret: abra_int_private_key
+      port: 22
+      command_timeout: 60m
+      script_stop: true
+      envs: [ DRONE_SOURCE_BRANCH ]
+      request_pty: true
+      script:
+        - |
+          set +eux
+          sh run-int
+    when:
+      event:
+        - cron:
+      cron:
+        # @daily https://docs.drone.io/cron/
+        - integration 
+
 volumes:
   - name: deps
     temp: {}
-- 
2.47.2