feat: deploy --no-converge-checks & finish app errors
This commit is contained in:
parent
20f7a18caa
commit
ab8db8df64
|
@ -14,6 +14,7 @@ var appDeployCommand = &cli.Command{
|
|||
internal.ForceFlag,
|
||||
internal.ChaosFlag,
|
||||
internal.NoDomainChecksFlag,
|
||||
internal.DontWaitConvergeFlag,
|
||||
},
|
||||
Description: `
|
||||
This command deploys a new instance of an app. It does not support changing the
|
||||
|
|
|
@ -3,14 +3,17 @@ package app
|
|||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
abraFormatter "coopcloud.tech/abra/cli/formatter"
|
||||
"coopcloud.tech/abra/cli/internal"
|
||||
"coopcloud.tech/abra/pkg/autocomplete"
|
||||
"coopcloud.tech/abra/pkg/client"
|
||||
"coopcloud.tech/abra/pkg/config"
|
||||
"coopcloud.tech/abra/pkg/recipe"
|
||||
stack "coopcloud.tech/abra/pkg/upstream/stack"
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/filters"
|
||||
dockerClient "github.com/docker/docker/client"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
@ -26,7 +29,7 @@ or having issues, it could be a lot of things. This command is best accompanied
|
|||
by "abra app logs <app>".
|
||||
`,
|
||||
Aliases: []string{"e"},
|
||||
Flags: []cli.Flag{},
|
||||
Flags: []cli.Flag{internal.WatchFlag},
|
||||
BashComplete: autocomplete.AppNameComplete,
|
||||
Action: func(c *cli.Context) error {
|
||||
app := internal.ValidateApp(c)
|
||||
|
@ -45,54 +48,69 @@ by "abra app logs <app>".
|
|||
logrus.Fatalf("%s is not deployed?", app.Name)
|
||||
}
|
||||
|
||||
filters := filters.NewArgs()
|
||||
filters.Add("name", app.StackName())
|
||||
if !internal.Watch {
|
||||
if err := checkErrors(c, cl, app); err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
for {
|
||||
if err := checkErrors(c, cl, app); err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error {
|
||||
recipe, err := recipe.Get(app.Type)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, service := range recipe.Config.Services {
|
||||
filters := filters.NewArgs()
|
||||
filters.Add("name", service.Name)
|
||||
containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(containers) == 0 {
|
||||
logrus.Warnf("%s is not up, something seems wrong", service.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
container := containers[0]
|
||||
containerState, err := cl.ContainerInspect(c.Context, container.ID)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
tableCol := []string{"app name", "status", "error", "out of memory", "restart count", "healthcheck"}
|
||||
table := abraFormatter.CreateTable(tableCol)
|
||||
|
||||
for _, container := range containers {
|
||||
serviceName := getServiceName(container.Names)
|
||||
|
||||
containerState, err := cl.ContainerInspect(c.Context, container.ID)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
errMsg := "N/A"
|
||||
hcStat := "N/A"
|
||||
var hcLogs []string
|
||||
if containerState.State.Health != nil {
|
||||
hcStat = containerState.State.Health.Status
|
||||
for _, log := range containerState.State.Health.Log {
|
||||
hcLogs = append(hcLogs, log.Output)
|
||||
}
|
||||
}
|
||||
|
||||
if containerState.State.Error != "" {
|
||||
errMsg = containerState.State.Error
|
||||
}
|
||||
|
||||
table.Append([]string{
|
||||
serviceName,
|
||||
containerState.State.Status,
|
||||
errMsg,
|
||||
strconv.FormatBool(containerState.State.OOMKilled),
|
||||
strconv.Itoa(containerState.RestartCount),
|
||||
hcStat,
|
||||
strings.Join(hcLogs, "\n"),
|
||||
})
|
||||
if containerState.State.OOMKilled {
|
||||
logrus.Warnf("%s has been killed due to an out of memory error", service.Name)
|
||||
}
|
||||
|
||||
table.Render()
|
||||
if containerState.State.Error != "" {
|
||||
logrus.Warnf("%s reports this error: ", containerState.State.Error)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
if containerState.State.Health != nil {
|
||||
if containerState.State.Health.Status != "healthy" {
|
||||
logrus.Warnf("%s healthcheck status is %s", service.Name, containerState.State.Health.Status)
|
||||
logrus.Warnf("%s healthcheck has failed %s times", service.Name, strconv.Itoa(containerState.State.Health.FailingStreak))
|
||||
for _, log := range containerState.State.Health.Log {
|
||||
logrus.Warnf("%s healthcheck logs: %s", service.Name, strings.TrimSpace(log.Output))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getServiceName(names []string) string {
|
||||
|
|
|
@ -15,26 +15,17 @@ import (
|
|||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
var watch bool
|
||||
var watchFlag = &cli.BoolFlag{
|
||||
Name: "watch",
|
||||
Aliases: []string{"w"},
|
||||
Value: false,
|
||||
Usage: "Watch status by polling repeatedly",
|
||||
Destination: &watch,
|
||||
}
|
||||
|
||||
var appPsCommand = &cli.Command{
|
||||
Name: "ps",
|
||||
Usage: "Check app status",
|
||||
Description: "This command shows a more detailed status output of a specific deployed app.",
|
||||
Aliases: []string{"p"},
|
||||
Flags: []cli.Flag{
|
||||
watchFlag,
|
||||
internal.WatchFlag,
|
||||
},
|
||||
BashComplete: autocomplete.AppNameComplete,
|
||||
Action: func(c *cli.Context) error {
|
||||
if !watch {
|
||||
if !internal.Watch {
|
||||
showPSOutput(c)
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -25,6 +25,7 @@ var appRollbackCommand = &cli.Command{
|
|||
Flags: []cli.Flag{
|
||||
internal.ForceFlag,
|
||||
internal.ChaosFlag,
|
||||
internal.DontWaitConvergeFlag,
|
||||
},
|
||||
Description: `
|
||||
This command rolls an app back to a previous version if one exists.
|
||||
|
@ -164,7 +165,7 @@ recipes.
|
|||
}
|
||||
}
|
||||
|
||||
if err := stack.RunDeploy(cl, deployOpts, compose, app.Type); err != nil {
|
||||
if err := stack.RunDeploy(cl, deployOpts, compose, app.Type, internal.DontWaitConverge); err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
|
|
|
@ -25,6 +25,7 @@ var appUpgradeCommand = &cli.Command{
|
|||
internal.ForceFlag,
|
||||
internal.ChaosFlag,
|
||||
internal.NoDomainChecksFlag,
|
||||
internal.NoDomainChecksFlag,
|
||||
},
|
||||
Description: `
|
||||
This command supports upgrading an app. You can use it to choose and roll out a
|
||||
|
@ -165,7 +166,7 @@ recipes.
|
|||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
if err := stack.RunDeploy(cl, deployOpts, compose, app.Type); err != nil {
|
||||
if err := stack.RunDeploy(cl, deployOpts, compose, app.Type, internal.DontWaitConverge); err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
|
|
|
@ -418,6 +418,24 @@ var AutoDNSRecordFlag = &cli.BoolFlag{
|
|||
Destination: &AutoDNSRecord,
|
||||
}
|
||||
|
||||
var DontWaitConverge bool
|
||||
var DontWaitConvergeFlag = &cli.BoolFlag{
|
||||
Name: "no-converge-checks",
|
||||
Aliases: []string{"nc"},
|
||||
Value: false,
|
||||
Usage: "Don't wait for converge logic checks",
|
||||
Destination: &DontWaitConverge,
|
||||
}
|
||||
|
||||
var Watch bool
|
||||
var WatchFlag = &cli.BoolFlag{
|
||||
Name: "watch",
|
||||
Aliases: []string{"w"},
|
||||
Value: false,
|
||||
Usage: "Watch status by polling repeatedly",
|
||||
Destination: &Watch,
|
||||
}
|
||||
|
||||
// SSHFailMsg is a hopefully helpful SSH failure message
|
||||
var SSHFailMsg = `
|
||||
Woops, Abra is unable to connect to connect to %s.
|
||||
|
|
|
@ -135,7 +135,7 @@ func DeployAction(c *cli.Context) error {
|
|||
logrus.Warn("skipping domain checks as requested")
|
||||
}
|
||||
|
||||
if err := stack.RunDeploy(cl, deployOpts, compose, app.Env["TYPE"]); err != nil {
|
||||
if err := stack.RunDeploy(cl, deployOpts, compose, app.Env["TYPE"], DontWaitConverge); err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
|
|
|
@ -158,7 +158,7 @@ func pruneServices(ctx context.Context, cl *dockerclient.Client, namespace conve
|
|||
}
|
||||
|
||||
// RunDeploy is the swarm implementation of docker stack deploy
|
||||
func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, recipeName string) error {
|
||||
func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, recipeName string, dontWait bool) error {
|
||||
ctx := context.Background()
|
||||
|
||||
if err := validateResolveImageFlag(&opts); err != nil {
|
||||
|
@ -170,7 +170,7 @@ func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, r
|
|||
opts.ResolveImage = ResolveImageNever
|
||||
}
|
||||
|
||||
return deployCompose(ctx, cl, opts, cfg, recipeName)
|
||||
return deployCompose(ctx, cl, opts, cfg, recipeName, dontWait)
|
||||
}
|
||||
|
||||
// validateResolveImageFlag validates the opts.resolveImage command line option
|
||||
|
@ -183,7 +183,7 @@ func validateResolveImageFlag(opts *Deploy) error {
|
|||
}
|
||||
}
|
||||
|
||||
func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, config *composetypes.Config, recipeName string) error {
|
||||
func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, config *composetypes.Config, recipeName string, dontWait bool) error {
|
||||
namespace := convert.NewNamespace(opts.Namespace)
|
||||
|
||||
if opts.Prune {
|
||||
|
@ -224,7 +224,7 @@ func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, co
|
|||
return err
|
||||
}
|
||||
|
||||
return deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage, recipeName)
|
||||
return deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage, recipeName, dontWait)
|
||||
}
|
||||
|
||||
func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) map[string]struct{} {
|
||||
|
@ -340,7 +340,8 @@ func deployServices(
|
|||
namespace convert.Namespace,
|
||||
sendAuth bool,
|
||||
resolveImage string,
|
||||
recipeName string) error {
|
||||
recipeName string,
|
||||
dontWait bool) error {
|
||||
existingServices, err := GetStackServices(ctx, cl, namespace.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -439,8 +440,13 @@ func deployServices(
|
|||
for _, serviceName := range serviceIDs {
|
||||
serviceNames = append(serviceNames, serviceName)
|
||||
}
|
||||
logrus.Infof("waiting for services to converge: %s", strings.Join(serviceNames, ", "))
|
||||
|
||||
if dontWait {
|
||||
logrus.Warn("skipping converge logic checks")
|
||||
return nil
|
||||
}
|
||||
|
||||
logrus.Infof("waiting for services to converge: %s", strings.Join(serviceNames, ", "))
|
||||
ch := make(chan error, len(serviceIDs))
|
||||
for serviceID, serviceName := range serviceIDs {
|
||||
logrus.Debugf("waiting on %s to converge", serviceName)
|
||||
|
|
Loading…
Reference in New Issue