feat: improved deploy progress reporting

See toolshed/abra#478
2025-03-20 14:23:09 +01:00
parent d0f982456e
commit 47045ca8f1
85 changed files with 8828 additions and 360 deletions
--- a/pkg/upstream/stack/remove.go
+++ b/pkg/upstream/stack/remove.go
@ -3,8 +3,11 @@ package stack // https://github.com/docker/cli/blob/master/cli/command/stack/rem
 import (
 	"context"
 	"fmt"
+	"os"
+	"os/signal"
 	"sort"
 	"strings"
+	"time"

 	"coopcloud.tech/abra/pkg/log"
 	"github.com/docker/docker/api/types"
@ -18,57 +21,87 @@ import (

 // RunRemove is the swarm implementation of docker stack remove
 func RunRemove(ctx context.Context, client *apiclient.Client, opts Remove) error {
-	var errs []string
-	for _, namespace := range opts.Namespaces {
-		services, err := GetStackServices(ctx, client, namespace)
-		if err != nil {
-			return err
-		}
+	sigIntCh := make(chan os.Signal, 1)
+	signal.Notify(sigIntCh, os.Interrupt)
+	defer signal.Stop(sigIntCh)

-		networks, err := getStackNetworks(ctx, client, namespace)
-		if err != nil {
-			return err
-		}
+	waitCh := make(chan struct{})
+	errCh := make(chan error)

-		var secrets []swarm.Secret
-		if versions.GreaterThanOrEqualTo(client.ClientVersion(), "1.25") {
-			secrets, err = getStackSecrets(ctx, client, namespace)
+	go func() {
+		var errs []string
+		for _, namespace := range opts.Namespaces {
+			services, err := GetStackServices(ctx, client, namespace)
 			if err != nil {
-				return err
+				errCh <- err
+				return
+			}
+
+			networks, err := getStackNetworks(ctx, client, namespace)
+			if err != nil {
+				errCh <- err
+				return
+			}
+
+			var secrets []swarm.Secret
+			if versions.GreaterThanOrEqualTo(client.ClientVersion(), "1.25") {
+				secrets, err = getStackSecrets(ctx, client, namespace)
+				if err != nil {
+					errCh <- err
+					return
+				}
+			}
+
+			var configs []swarm.Config
+			if versions.GreaterThanOrEqualTo(client.ClientVersion(), "1.30") {
+				configs, err = getStackConfigs(ctx, client, namespace)
+				if err != nil {
+					errCh <- err
+					return
+				}
+			}
+
+			if len(services)+len(networks)+len(secrets)+len(configs) == 0 {
+				log.Warnf("nothing found in stack: %s", namespace)
+				continue
+			}
+
+			hasError := removeServices(ctx, client, services)
+			hasError = removeSecrets(ctx, client, secrets) || hasError
+			hasError = removeConfigs(ctx, client, configs) || hasError
+			hasError = removeNetworks(ctx, client, networks) || hasError
+
+			if hasError {
+				errs = append(errs, fmt.Sprintf("failed to remove some resources from stack: %s", namespace))
+				continue
+			}
+
+			log.Info("polling undeploy status")
+			timeout, err := waitOnTasks(ctx, client, namespace)
+			if timeout {
+				errs = append(errs, err.Error())
+			} else {
+				if err != nil {
+					errs = append(errs, fmt.Sprintf("failed to wait on tasks of stack: %s: %s", namespace, err))
+				}
 			}
 		}

-		var configs []swarm.Config
-		if versions.GreaterThanOrEqualTo(client.ClientVersion(), "1.30") {
-			configs, err = getStackConfigs(ctx, client, namespace)
-			if err != nil {
-				return err
-			}
+		if len(errs) > 0 {
+			errCh <- errors.Errorf(strings.Join(errs, "\n"))
+			return
 		}

-		if len(services)+len(networks)+len(secrets)+len(configs) == 0 {
-			log.Warnf("nothing found in stack: %s", namespace)
-			continue
-		}
+		close(waitCh)
+	}()

-		hasError := removeServices(ctx, client, services)
-		hasError = removeSecrets(ctx, client, secrets) || hasError
-		hasError = removeConfigs(ctx, client, configs) || hasError
-		hasError = removeNetworks(ctx, client, networks) || hasError
-
-		if hasError {
-			errs = append(errs, fmt.Sprintf("failed to remove some resources from stack: %s", namespace))
-			continue
-		}
-
-		err = waitOnTasks(ctx, client, namespace)
-		if err != nil {
-			errs = append(errs, fmt.Sprintf("failed to wait on tasks of stack: %s: %s", namespace, err))
-		}
-	}
-
-	if len(errs) > 0 {
-		return errors.Errorf(strings.Join(errs, "\n"))
+	select {
+	case <-waitCh:
+		return nil
+	case <-sigIntCh:
+		return fmt.Errorf("skipping as requested, undeploy still in progress 🟠")
+	case err := <-errCh:
+		return err
 	}

 	return nil
@ -88,7 +121,7 @@ func removeServices(
 	var hasError bool
 	sort.Slice(services, sortServiceByName(services))
 	for _, service := range services {
-		log.Infof("removing service %s", service.Spec.Name)
+		log.Debugf("removing service %s", service.Spec.Name)
 		if err := client.ServiceRemove(ctx, service.ID); err != nil {
 			hasError = true
 			log.Fatalf("failed to remove service %s: %s", service.ID, err)
@ -104,7 +137,7 @@ func removeNetworks(
 ) bool {
 	var hasError bool
 	for _, network := range networks {
-		log.Infof("removing network %s", network.Name)
+		log.Debugf("removing network %s", network.Name)
 		if err := client.NetworkRemove(ctx, network.ID); err != nil {
 			hasError = true
 			log.Fatalf("failed to remove network %s: %s", network.ID, err)
@ -120,7 +153,7 @@ func removeSecrets(
 ) bool {
 	var hasError bool
 	for _, secret := range secrets {
-		log.Infof("removing secret %s", secret.Spec.Name)
+		log.Debugf("removing secret %s", secret.Spec.Name)
 		if err := client.SecretRemove(ctx, secret.ID); err != nil {
 			hasError = true
 			log.Fatalf("Failed to remove secret %s: %s", secret.ID, err)
@ -136,7 +169,7 @@ func removeConfigs(
 ) bool {
 	var hasError bool
 	for _, config := range configs {
-		log.Infof("removing config %s", config.Spec.Name)
+		log.Debugf("removing config %s", config.Spec.Name)
 		if err := client.ConfigRemove(ctx, config.ID); err != nil {
 			hasError = true
 			log.Fatalf("failed to remove config %s: %s", config.ID, err)
@ -170,12 +203,23 @@ func terminalState(state swarm.TaskState) bool {
 	return numberedStates[state] > numberedStates[swarm.TaskStateRunning]
 }

-func waitOnTasks(ctx context.Context, client apiclient.APIClient, namespace string) error {
+func waitOnTasks(ctx context.Context, client apiclient.APIClient, namespace string) (bool, error) {
+	var timedOut bool
+
+	log.Debugf("waiting on undeploy tasks (timeout=%v secs)", WaitTimeout)
+
+	go func() {
+		t := time.Duration(WaitTimeout) * time.Second
+		<-time.After(t)
+		log.Debug("timed out on undeploy")
+		timedOut = true
+	}()
+
 	terminalStatesReached := 0
 	for {
 		tasks, err := getStackTasks(ctx, client, namespace)
 		if err != nil {
-			return fmt.Errorf("failed to get tasks: %w", err)
+			return false, fmt.Errorf("failed to get tasks: %w", err)
 		}

 		for _, task := range tasks {
@ -188,6 +232,11 @@ func waitOnTasks(ctx context.Context, client apiclient.APIClient, namespace stri
 		if terminalStatesReached == len(tasks) {
 			break
 		}
+
+		if timedOut {
+			return true, fmt.Errorf("deployment timed out 🟠")
+		}
 	}
-	return nil
+
+	return false, nil
 }