diff --git a/tests/integration/app_deploy.bats b/tests/integration/app_deploy.bats
index 0d94a4a836..40af851c8c 100644
--- a/tests/integration/app_deploy.bats
+++ b/tests/integration/app_deploy.bats
@@ -367,6 +367,21 @@ teardown(){
 
   run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
   assert_failure
+  assert_output --partial "secret not generated"
+}
+
+@test "error if secret not inserted" {
+  run sed -i 's/COMPOSE_FILE="compose.yml"/COMPOSE_FILE="compose.yml:compose.skip_pass.yml"/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run sed -i 's/#SECRET_TEST_SKIP_PASS_VERSION=v1/SECRET_TEST_SKIP_PASS_VERSION=v1/g' \
+    "$ABRA_DIR/servers/$TEST_SERVER/$TEST_APP_DOMAIN.env"
+  assert_success
+
+  run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --no-converge-checks
+  assert_failure
+  assert_output --partial "secret not inserted"
 }
 
 # bats test_tags=slow
diff --git a/tests/integration/app_new.bats b/tests/integration/app_new.bats
index 92ccf4ee67..53bc3ca009 100644
--- a/tests/integration/app_new.bats
+++ b/tests/integration/app_new.bats
@@ -22,8 +22,15 @@ teardown(){
   _reset_recipe
   _reset_tags
 
-  run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
-  assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  if [[ -f "$ABRA_DIR/recipes/$TEST_RECIPE/foo" ]]; then
+    run rm -rf "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+    assert_not_exists "$ABRA_DIR/recipes/$TEST_RECIPE/foo"
+  fi
+
+  if [[ -f "$ABRA_DIR/servers/$TEST_SERVER/rauthy.$TEST_APP_DOMAIN.env" ]]; then
+    run rm -rf "$ABRA_DIR/servers/$TEST_SERVER/rauthy.$TEST_APP_DOMAIN.env"
+    assert_not_exists "$ABRA_DIR/servers/$TEST_SERVER/rauthy.$TEST_APP_DOMAIN.env"
+  fi
 }
 
 @test "create new app" {
@@ -270,3 +277,32 @@ teardown(){
   assert_success
   refute_output --partial "requires secret generation"
 }
+
+@test "do not warn about generation when generate=false" {
+  run $ABRA app new --domain "$TEST_APP_DOMAIN" renovate "1.0.1+41-full"
+  assert_success
+  refute_output --partial "requires secret generation"
+}
+
+@test "warn about insertion when generate=false" {
+  run $ABRA app new --domain "$TEST_APP_DOMAIN" renovate "1.0.1+41-full"
+  assert_success
+  assert_output --partial "requires secret insertion"
+}
+
+@test "warn about both insert/generate when generate=false/true" {
+  run $ABRA app new rauthy "1.0.0+0.32.3" \
+    --no-input \
+    --server "$TEST_SERVER" \
+    --domain "rauthy.$TEST_APP_DOMAIN"
+  assert_success
+  assert_exists "$ABRA_DIR/servers/$TEST_SERVER/rauthy.$TEST_APP_DOMAIN.env"
+  assert_output --partial "requires secret generation"
+  assert_output --partial "requires secret insertion"
+}
+
+@test "no warn about generation if already generated" {
+  run $ABRA app new "$TEST_RECIPE" --domain "$TEST_APP_DOMAIN" --secrets
+  assert_success
+  refute_output --partial "requires secret generation"
+}