diff --git a/cli/app/deploy.go b/cli/app/deploy.go index fb72c0ec..87feae8f 100644 --- a/cli/app/deploy.go +++ b/cli/app/deploy.go @@ -196,6 +196,7 @@ recipes. Namespace: stackName, Prune: false, ResolveImage: stack.ResolveImageAlways, + Detach: false, } compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env) if err != nil { diff --git a/cli/app/rollback.go b/cli/app/rollback.go index f39d114b..43b1fbba 100644 --- a/cli/app/rollback.go +++ b/cli/app/rollback.go @@ -215,6 +215,7 @@ recipes. Namespace: stackName, Prune: false, ResolveImage: stack.ResolveImageAlways, + Detach: false, } compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env) if err != nil { diff --git a/cli/app/undeploy.go b/cli/app/undeploy.go index 3a1c2ffa..7b50908d 100644 --- a/cli/app/undeploy.go +++ b/cli/app/undeploy.go @@ -122,7 +122,10 @@ Passing "-p/--prune" does not remove those volumes. logrus.Fatal(err) } - rmOpts := stack.Remove{Namespaces: []string{app.StackName()}} + rmOpts := stack.Remove{ + Namespaces: []string{app.StackName()}, + Detach: false, + } if err := stack.RunRemove(context.Background(), cl, rmOpts); err != nil { logrus.Fatal(err) } diff --git a/cli/app/upgrade.go b/cli/app/upgrade.go index 366a5932..e2f6df6c 100644 --- a/cli/app/upgrade.go +++ b/cli/app/upgrade.go @@ -249,6 +249,7 @@ recipes. Namespace: stackName, Prune: false, ResolveImage: stack.ResolveImageAlways, + Detach: false, } compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env) if err != nil { diff --git a/cli/updater/updater.go b/cli/updater/updater.go index 4b036132..74266d8b 100644 --- a/cli/updater/updater.go +++ b/cli/updater/updater.go @@ -362,6 +362,7 @@ func createDeployConfig(recipeName string, stackName string, env config.AppEnv) Namespace: stackName, Prune: false, ResolveImage: stack.ResolveImageAlways, + Detach: false, } composeFiles, err := config.GetComposeFiles(recipeName, env) diff --git a/go.mod b/go.mod index 7347c815..a7e1c3a1 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100509-01bff8284355 github.com/AlecAivazis/survey/v2 v2.3.7 github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4 - github.com/distribution/distribution v2.8.3+incompatible + github.com/distribution/reference v0.6.0 github.com/docker/cli v26.1.4+incompatible github.com/docker/docker v26.1.4+incompatible github.com/docker/go-units v0.5.0 @@ -36,7 +36,6 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/cyphar/filepath-securejoin v0.2.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/distribution/reference v0.6.0 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect github.com/docker/go-connections v0.5.0 // indirect diff --git a/go.sum b/go.sum index be044761..fff7c0bf 100644 --- a/go.sum +++ b/go.sum @@ -297,8 +297,6 @@ github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8l github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/distribution/distribution v2.8.3+incompatible h1:RlpEXBLq/WPXYvBYMDAmBX/SnhD67qwtvW/DzKc8pAo= -github.com/distribution/distribution v2.8.3+incompatible/go.mod h1:EgLm2NgWtdKgzF9NpMzUKgzmR7AMmb0VQi2B+ZzDRjc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= diff --git a/pkg/upstream/stack/options.go b/pkg/upstream/stack/options.go index 14ca364e..39245eb4 100644 --- a/pkg/upstream/stack/options.go +++ b/pkg/upstream/stack/options.go @@ -7,9 +7,12 @@ type Deploy struct { ResolveImage string SendRegistryAuth bool Prune bool + Detach bool + Quiet bool } // Remove holds docker stack remove options type Remove struct { Namespaces []string + Detach bool } diff --git a/pkg/upstream/stack/remove.go b/pkg/upstream/stack/remove.go index 6b0f043c..7c211b56 100644 --- a/pkg/upstream/stack/remove.go +++ b/pkg/upstream/stack/remove.go @@ -9,6 +9,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/api/types/versions" + "github.com/docker/docker/client" apiclient "github.com/docker/docker/client" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -56,6 +57,12 @@ func RunRemove(ctx context.Context, client *apiclient.Client, opts Remove) error if hasError { errs = append(errs, fmt.Sprintf("failed to remove some resources from stack: %s", namespace)) + continue + } + + err = waitOnTasks(ctx, client, namespace) + if err != nil { + errs = append(errs, fmt.Sprintf("failed to wait on tasks of stack: %s: %s", namespace, err)) } } @@ -136,3 +143,50 @@ func removeConfigs( } return hasError } + +// https://github.com/docker/cli/pull/4259 +func getStackTasks(ctx context.Context, apiclient client.APIClient, namespace string) ([]swarm.Task, error) { + return apiclient.TaskList(ctx, types.TaskListOptions{Filters: getStackFilter(namespace)}) +} + +var numberedStates = map[swarm.TaskState]int64{ + swarm.TaskStateNew: 1, + swarm.TaskStateAllocated: 2, + swarm.TaskStatePending: 3, + swarm.TaskStateAssigned: 4, + swarm.TaskStateAccepted: 5, + swarm.TaskStatePreparing: 6, + swarm.TaskStateReady: 7, + swarm.TaskStateStarting: 8, + swarm.TaskStateRunning: 9, + swarm.TaskStateComplete: 10, + swarm.TaskStateShutdown: 11, + swarm.TaskStateFailed: 12, + swarm.TaskStateRejected: 13, +} + +func terminalState(state swarm.TaskState) bool { + return numberedStates[state] > numberedStates[swarm.TaskStateRunning] +} + +func waitOnTasks(ctx context.Context, client apiclient.APIClient, namespace string) error { + terminalStatesReached := 0 + for { + tasks, err := getStackTasks(ctx, client, namespace) + if err != nil { + return fmt.Errorf("failed to get tasks: %w", err) + } + + for _, task := range tasks { + if terminalState(task.Status.State) { + terminalStatesReached++ + break + } + } + + if terminalStatesReached == len(tasks) { + break + } + } + return nil +} diff --git a/pkg/upstream/stack/stack.go b/pkg/upstream/stack/stack.go index d8e7bdd9..d9dc94f3 100644 --- a/pkg/upstream/stack/stack.go +++ b/pkg/upstream/stack/stack.go @@ -9,6 +9,8 @@ import ( "os/signal" "time" + stdlibErr "errors" + "coopcloud.tech/abra/pkg/upstream/convert" "github.com/docker/cli/cli/command/service/progress" "github.com/docker/cli/cli/command/stack/formatter" @@ -129,7 +131,7 @@ func IsDeployed(ctx context.Context, cl *dockerClient.Client, stackName string) func pruneServices(ctx context.Context, cl *dockerClient.Client, namespace convert.Namespace, services map[string]struct{}) { oldServices, err := GetStackServices(ctx, cl, namespace.Name()) if err != nil { - logrus.Infof("Failed to list services: %s\n", err) + logrus.Infof("failed to list services: %s\n", err) } pruneServices := []swarm.Service{} @@ -161,7 +163,7 @@ func validateResolveImageFlag(opts *Deploy) error { case ResolveImageAlways, ResolveImageChanged, ResolveImageNever: return nil default: - return errors.Errorf("Invalid option %s for flag --resolve-image", opts.ResolveImage) + return errors.Errorf("invalid option %s for flag --resolve-image", opts.ResolveImage) } } @@ -206,7 +208,16 @@ func deployCompose(ctx context.Context, cl *dockerClient.Client, opts Deploy, co return err } - return deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage, appName, dontWait) + serviceIDs, err := deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage, appName, dontWait) + if err != nil { + return err + } + + logrus.Infof("waiting for %s to deploy... please hold 🤚", appName) + if err := waitOnServices(ctx, cl, serviceIDs, appName); err == nil { + logrus.Infof("Successfully deployed %s", appName) + } + return err } func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) map[string]struct{} { @@ -276,7 +287,7 @@ func createConfigs(ctx context.Context, cl *dockerClient.Client, configs []swarm } case dockerClient.IsErrNotFound(err): // config does not exist, then we create a new one. - logrus.Infof("Creating config %s\n", configSpec.Name) + logrus.Infof("creating config %s\n", configSpec.Name) if _, err := cl.ConfigCreate(ctx, configSpec); err != nil { return errors.Wrapf(err, "failed to create config %s", configSpec.Name) } @@ -307,7 +318,7 @@ func createNetworks(ctx context.Context, cl *dockerClient.Client, namespace conv createOpts.Driver = defaultNetworkDriver } - logrus.Infof("Creating network %s\n", name) + logrus.Infof("creating network %s\n", name) if _, err := cl.NetworkCreate(ctx, name, createOpts); err != nil { return errors.Wrapf(err, "failed to create network %s", name) } @@ -323,10 +334,10 @@ func deployServices( sendAuth bool, resolveImage string, appName string, - dontWait bool) error { + dontWait bool) ([]string, error) { existingServices, err := GetStackServices(ctx, cl, namespace.Name()) if err != nil { - return err + return nil, err } existingServiceMap := make(map[string]swarm.Service) @@ -334,7 +345,8 @@ func deployServices( existingServiceMap[service.Spec.Name] = service } - serviceIDs := make(map[string]string) + var serviceIDs []string + for internalName, serviceSpec := range services { var ( name = namespace.Scope(internalName) @@ -378,16 +390,16 @@ func deployServices( response, err := cl.ServiceUpdate(ctx, service.ID, service.Version, serviceSpec, updateOpts) if err != nil { - return errors.Wrapf(err, "failed to update service %s", name) + return nil, errors.Wrapf(err, "failed to update service %s", name) } - serviceIDs[service.ID] = name - for _, warning := range response.Warnings { logrus.Warn(warning) } + + serviceIDs = append(serviceIDs, service.ID) } else { - logrus.Infof("Creating service %s\n", name) + logrus.Infof("creating service %s\n", name) createOpts := types.ServiceCreateOptions{EncodedRegistryAuth: encodedAuth} @@ -398,43 +410,19 @@ func deployServices( serviceCreateResponse, err := cl.ServiceCreate(ctx, serviceSpec, createOpts) if err != nil { - return errors.Wrapf(err, "failed to create service %s", name) + return nil, errors.Wrapf(err, "failed to create service %s", name) } - serviceIDs[serviceCreateResponse.ID] = name + serviceIDs = append(serviceIDs, serviceCreateResponse.ID) } } - var serviceNames []string - for _, serviceName := range serviceIDs { - serviceNames = append(serviceNames, serviceName) - } - if dontWait { logrus.Warn("skipping converge logic checks") - return nil + return nil, nil } - logrus.Infof("Waiting for %s to deploy... please hold 🤚", appName) - ch := make(chan error, len(serviceIDs)) - for serviceID, serviceName := range serviceIDs { - logrus.Debugf("waiting on %s to converge", serviceName) - go func(sID, sName, aName string) { - ch <- WaitOnService(ctx, cl, sID, aName) - }(serviceID, serviceName, appName) - } - - for _, serviceID := range serviceIDs { - err := <-ch - if err != nil { - return err - } - logrus.Debugf("assuming %s converged successfully", serviceID) - } - - logrus.Infof("Successfully deployed %s", appName) - - return nil + return serviceIDs, nil } func getStackNetworks(ctx context.Context, dockerclient client.APIClient, namespace string) ([]types.NetworkResource, error) { @@ -449,6 +437,22 @@ func getStackConfigs(ctx context.Context, dockerclient client.APIClient, namespa return dockerclient.ConfigList(ctx, types.ConfigListOptions{Filters: getStackFilter(namespace)}) } +func waitOnServices(ctx context.Context, cl *dockerClient.Client, serviceIDs []string, appName string) error { + var errs []error + + for _, serviceID := range serviceIDs { + if err := WaitOnService(ctx, cl, serviceID, appName); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", serviceID, err)) + } + } + + if len(errs) > 0 { + return stdlibErr.Join(errs...) + } + + return nil +} + // https://github.com/docker/cli/blob/master/cli/command/service/helpers.go // https://github.com/docker/cli/blob/master/cli/command/service/progress/progress.go func WaitOnService(ctx context.Context, cl *dockerClient.Client, serviceID, appName string) error {