fix: wait for containers to go away

See #564

pkg/upstream/stack/remove.go

@@ -12,6 +12,7 @@ import (
 	"coopcloud.tech/abra/pkg/i18n"
 	"coopcloud.tech/abra/pkg/log"
 	"github.com/docker/docker/api/types"
+	containerPkg "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/api/types/versions"
@@ -79,7 +80,7 @@ func RunRemove(ctx context.Context, client *apiclient.Client, opts Remove) error
 			hasError = removeNetworks(ctx, client, networks) || hasError
 
 			if hasError {
-				errs = append(errs, fmt.Sprint(i18n.G("failed to remove some resources from stack: %s", namespace)))
+				errs = append(errs, i18n.G("failed to remove some resources from stack: %s", namespace))
 				continue
 			}
 
@@ -190,6 +191,10 @@ func getStackTasks(ctx context.Context, apiclient client.APIClient, namespace st
 	return apiclient.TaskList(ctx, types.TaskListOptions{Filters: getStackFilter(namespace)})
 }
 
+func getStackContainers(ctx context.Context, apiclient client.APIClient, namespace string) ([]containerPkg.Summary, error) {
+	return apiclient.ContainerList(ctx, containerPkg.ListOptions{Filters: getStackFilter(namespace)})
+}
+
 var numberedStates = map[swarm.TaskState]int64{
 	swarm.TaskStateNew:       1,
 	swarm.TaskStateAllocated: 2,
@@ -236,12 +241,14 @@ func waitOnTasks(ctx context.Context, client apiclient.APIClient, namespace stri
 
 		for _, task := range tasks {
 			if terminalState(task.Status.State) {
+				log.Debug(i18n.G("waiting for %d task(s) to reach terminal state", len(tasks)-terminalStatesReached))
 				terminalStatesReached++
 				break
 			}
 		}
 
 		if terminalStatesReached == len(tasks) {
+			log.Debug(i18n.G("all tasks reached terminal state"))
 			break
 		}
 
@@ -250,5 +257,25 @@ func waitOnTasks(ctx context.Context, client apiclient.APIClient, namespace stri
 		}
 	}
 
+	lastSeenCount := -1
+	for {
+		containers, err := getStackContainers(ctx, client, namespace)
+		if err != nil {
+			return false, errors.New(i18n.G("failed to list containers of stack: %s", namespace))
+		}
+
+		numContainers := len(containers)
+
+		if numContainers == 0 {
+			log.Debug(i18n.G("all containers did really go away"))
+			break
+		}
+
+		if numContainers != lastSeenCount {
+			log.Debug(i18n.G("waiting for %d container(s) to really go away", numContainers))
+			lastSeenCount = numContainers
+		}
+	}
+
 	return false, nil
 }