From ab8db8df64962fa0af3d842510a3ad54be17eeef Mon Sep 17 00:00:00 2001 From: cellarspoon Date: Fri, 24 Dec 2021 02:23:46 +0100 Subject: [PATCH] feat: deploy --no-converge-checks & finish app errors --- cli/app/deploy.go | 1 + cli/app/errors.go | 100 +++++++++++++++++++++--------------- cli/app/ps.go | 13 +---- cli/app/rollback.go | 3 +- cli/app/upgrade.go | 3 +- cli/internal/common.go | 18 +++++++ cli/internal/deploy.go | 2 +- pkg/upstream/stack/stack.go | 18 ++++--- 8 files changed, 97 insertions(+), 61 deletions(-) diff --git a/cli/app/deploy.go b/cli/app/deploy.go index b305de23..e3c6079b 100644 --- a/cli/app/deploy.go +++ b/cli/app/deploy.go @@ -14,6 +14,7 @@ var appDeployCommand = &cli.Command{ internal.ForceFlag, internal.ChaosFlag, internal.NoDomainChecksFlag, + internal.DontWaitConvergeFlag, }, Description: ` This command deploys a new instance of an app. It does not support changing the diff --git a/cli/app/errors.go b/cli/app/errors.go index 3bb4308d..02594ce4 100644 --- a/cli/app/errors.go +++ b/cli/app/errors.go @@ -3,14 +3,17 @@ package app import ( "strconv" "strings" + "time" - abraFormatter "coopcloud.tech/abra/cli/formatter" "coopcloud.tech/abra/cli/internal" "coopcloud.tech/abra/pkg/autocomplete" "coopcloud.tech/abra/pkg/client" + "coopcloud.tech/abra/pkg/config" + "coopcloud.tech/abra/pkg/recipe" stack "coopcloud.tech/abra/pkg/upstream/stack" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + dockerClient "github.com/docker/docker/client" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" ) @@ -26,7 +29,7 @@ or having issues, it could be a lot of things. This command is best accompanied by "abra app logs ". `, Aliases: []string{"e"}, - Flags: []cli.Flag{}, + Flags: []cli.Flag{internal.WatchFlag}, BashComplete: autocomplete.AppNameComplete, Action: func(c *cli.Context) error { app := internal.ValidateApp(c) @@ -45,54 +48,69 @@ by "abra app logs ". logrus.Fatalf("%s is not deployed?", app.Name) } - filters := filters.NewArgs() - filters.Add("name", app.StackName()) + if !internal.Watch { + if err := checkErrors(c, cl, app); err != nil { + logrus.Fatal(err) + } + return nil + } + for { + if err := checkErrors(c, cl, app); err != nil { + logrus.Fatal(err) + } + time.Sleep(2 * time.Second) + } + + return nil + }, +} + +func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error { + recipe, err := recipe.Get(app.Type) + if err != nil { + return err + } + + for _, service := range recipe.Config.Services { + filters := filters.NewArgs() + filters.Add("name", service.Name) containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters}) + if err != nil { + return err + } + + if len(containers) == 0 { + logrus.Warnf("%s is not up, something seems wrong", service.Name) + continue + } + + container := containers[0] + containerState, err := cl.ContainerInspect(c.Context, container.ID) if err != nil { logrus.Fatal(err) } - tableCol := []string{"app name", "status", "error", "out of memory", "restart count", "healthcheck"} - table := abraFormatter.CreateTable(tableCol) - - for _, container := range containers { - serviceName := getServiceName(container.Names) - - containerState, err := cl.ContainerInspect(c.Context, container.ID) - if err != nil { - logrus.Fatal(err) - } - - errMsg := "N/A" - hcStat := "N/A" - var hcLogs []string - if containerState.State.Health != nil { - hcStat = containerState.State.Health.Status - for _, log := range containerState.State.Health.Log { - hcLogs = append(hcLogs, log.Output) - } - } - - if containerState.State.Error != "" { - errMsg = containerState.State.Error - } - - table.Append([]string{ - serviceName, - containerState.State.Status, - errMsg, - strconv.FormatBool(containerState.State.OOMKilled), - strconv.Itoa(containerState.RestartCount), - hcStat, - strings.Join(hcLogs, "\n"), - }) + if containerState.State.OOMKilled { + logrus.Warnf("%s has been killed due to an out of memory error", service.Name) } - table.Render() + if containerState.State.Error != "" { + logrus.Warnf("%s reports this error: ", containerState.State.Error) + } - return nil - }, + if containerState.State.Health != nil { + if containerState.State.Health.Status != "healthy" { + logrus.Warnf("%s healthcheck status is %s", service.Name, containerState.State.Health.Status) + logrus.Warnf("%s healthcheck has failed %s times", service.Name, strconv.Itoa(containerState.State.Health.FailingStreak)) + for _, log := range containerState.State.Health.Log { + logrus.Warnf("%s healthcheck logs: %s", service.Name, strings.TrimSpace(log.Output)) + } + } + } + } + + return nil } func getServiceName(names []string) string { diff --git a/cli/app/ps.go b/cli/app/ps.go index 84170243..1c2172d8 100644 --- a/cli/app/ps.go +++ b/cli/app/ps.go @@ -15,26 +15,17 @@ import ( "github.com/urfave/cli/v2" ) -var watch bool -var watchFlag = &cli.BoolFlag{ - Name: "watch", - Aliases: []string{"w"}, - Value: false, - Usage: "Watch status by polling repeatedly", - Destination: &watch, -} - var appPsCommand = &cli.Command{ Name: "ps", Usage: "Check app status", Description: "This command shows a more detailed status output of a specific deployed app.", Aliases: []string{"p"}, Flags: []cli.Flag{ - watchFlag, + internal.WatchFlag, }, BashComplete: autocomplete.AppNameComplete, Action: func(c *cli.Context) error { - if !watch { + if !internal.Watch { showPSOutput(c) return nil } diff --git a/cli/app/rollback.go b/cli/app/rollback.go index aff0a01c..68330023 100644 --- a/cli/app/rollback.go +++ b/cli/app/rollback.go @@ -25,6 +25,7 @@ var appRollbackCommand = &cli.Command{ Flags: []cli.Flag{ internal.ForceFlag, internal.ChaosFlag, + internal.DontWaitConvergeFlag, }, Description: ` This command rolls an app back to a previous version if one exists. @@ -164,7 +165,7 @@ recipes. } } - if err := stack.RunDeploy(cl, deployOpts, compose, app.Type); err != nil { + if err := stack.RunDeploy(cl, deployOpts, compose, app.Type, internal.DontWaitConverge); err != nil { logrus.Fatal(err) } diff --git a/cli/app/upgrade.go b/cli/app/upgrade.go index 573729bc..463c2c48 100644 --- a/cli/app/upgrade.go +++ b/cli/app/upgrade.go @@ -25,6 +25,7 @@ var appUpgradeCommand = &cli.Command{ internal.ForceFlag, internal.ChaosFlag, internal.NoDomainChecksFlag, + internal.NoDomainChecksFlag, }, Description: ` This command supports upgrading an app. You can use it to choose and roll out a @@ -165,7 +166,7 @@ recipes. logrus.Fatal(err) } - if err := stack.RunDeploy(cl, deployOpts, compose, app.Type); err != nil { + if err := stack.RunDeploy(cl, deployOpts, compose, app.Type, internal.DontWaitConverge); err != nil { logrus.Fatal(err) } diff --git a/cli/internal/common.go b/cli/internal/common.go index 92242b94..fe76421b 100644 --- a/cli/internal/common.go +++ b/cli/internal/common.go @@ -418,6 +418,24 @@ var AutoDNSRecordFlag = &cli.BoolFlag{ Destination: &AutoDNSRecord, } +var DontWaitConverge bool +var DontWaitConvergeFlag = &cli.BoolFlag{ + Name: "no-converge-checks", + Aliases: []string{"nc"}, + Value: false, + Usage: "Don't wait for converge logic checks", + Destination: &DontWaitConverge, +} + +var Watch bool +var WatchFlag = &cli.BoolFlag{ + Name: "watch", + Aliases: []string{"w"}, + Value: false, + Usage: "Watch status by polling repeatedly", + Destination: &Watch, +} + // SSHFailMsg is a hopefully helpful SSH failure message var SSHFailMsg = ` Woops, Abra is unable to connect to connect to %s. diff --git a/cli/internal/deploy.go b/cli/internal/deploy.go index cf6db9ec..4d464baa 100644 --- a/cli/internal/deploy.go +++ b/cli/internal/deploy.go @@ -135,7 +135,7 @@ func DeployAction(c *cli.Context) error { logrus.Warn("skipping domain checks as requested") } - if err := stack.RunDeploy(cl, deployOpts, compose, app.Env["TYPE"]); err != nil { + if err := stack.RunDeploy(cl, deployOpts, compose, app.Env["TYPE"], DontWaitConverge); err != nil { logrus.Fatal(err) } diff --git a/pkg/upstream/stack/stack.go b/pkg/upstream/stack/stack.go index 9361894e..b5ab59bf 100644 --- a/pkg/upstream/stack/stack.go +++ b/pkg/upstream/stack/stack.go @@ -158,7 +158,7 @@ func pruneServices(ctx context.Context, cl *dockerclient.Client, namespace conve } // RunDeploy is the swarm implementation of docker stack deploy -func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, recipeName string) error { +func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, recipeName string, dontWait bool) error { ctx := context.Background() if err := validateResolveImageFlag(&opts); err != nil { @@ -170,7 +170,7 @@ func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, r opts.ResolveImage = ResolveImageNever } - return deployCompose(ctx, cl, opts, cfg, recipeName) + return deployCompose(ctx, cl, opts, cfg, recipeName, dontWait) } // validateResolveImageFlag validates the opts.resolveImage command line option @@ -183,7 +183,7 @@ func validateResolveImageFlag(opts *Deploy) error { } } -func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, config *composetypes.Config, recipeName string) error { +func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, config *composetypes.Config, recipeName string, dontWait bool) error { namespace := convert.NewNamespace(opts.Namespace) if opts.Prune { @@ -224,7 +224,7 @@ func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, co return err } - return deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage, recipeName) + return deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage, recipeName, dontWait) } func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) map[string]struct{} { @@ -340,7 +340,8 @@ func deployServices( namespace convert.Namespace, sendAuth bool, resolveImage string, - recipeName string) error { + recipeName string, + dontWait bool) error { existingServices, err := GetStackServices(ctx, cl, namespace.Name()) if err != nil { return err @@ -439,8 +440,13 @@ func deployServices( for _, serviceName := range serviceIDs { serviceNames = append(serviceNames, serviceName) } - logrus.Infof("waiting for services to converge: %s", strings.Join(serviceNames, ", ")) + if dontWait { + logrus.Warn("skipping converge logic checks") + return nil + } + + logrus.Infof("waiting for services to converge: %s", strings.Join(serviceNames, ", ")) ch := make(chan error, len(serviceIDs)) for serviceID, serviceName := range serviceIDs { logrus.Debugf("waiting on %s to converge", serviceName)