feat: deploy --no-converge-checks & finish app errors
This commit is contained in:
parent
20f7a18caa
commit
ab8db8df64
|
@ -14,6 +14,7 @@ var appDeployCommand = &cli.Command{
|
||||||
internal.ForceFlag,
|
internal.ForceFlag,
|
||||||
internal.ChaosFlag,
|
internal.ChaosFlag,
|
||||||
internal.NoDomainChecksFlag,
|
internal.NoDomainChecksFlag,
|
||||||
|
internal.DontWaitConvergeFlag,
|
||||||
},
|
},
|
||||||
Description: `
|
Description: `
|
||||||
This command deploys a new instance of an app. It does not support changing the
|
This command deploys a new instance of an app. It does not support changing the
|
||||||
|
|
|
@ -3,14 +3,17 @@ package app
|
||||||
import (
|
import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
abraFormatter "coopcloud.tech/abra/cli/formatter"
|
|
||||||
"coopcloud.tech/abra/cli/internal"
|
"coopcloud.tech/abra/cli/internal"
|
||||||
"coopcloud.tech/abra/pkg/autocomplete"
|
"coopcloud.tech/abra/pkg/autocomplete"
|
||||||
"coopcloud.tech/abra/pkg/client"
|
"coopcloud.tech/abra/pkg/client"
|
||||||
|
"coopcloud.tech/abra/pkg/config"
|
||||||
|
"coopcloud.tech/abra/pkg/recipe"
|
||||||
stack "coopcloud.tech/abra/pkg/upstream/stack"
|
stack "coopcloud.tech/abra/pkg/upstream/stack"
|
||||||
"github.com/docker/docker/api/types"
|
"github.com/docker/docker/api/types"
|
||||||
"github.com/docker/docker/api/types/filters"
|
"github.com/docker/docker/api/types/filters"
|
||||||
|
dockerClient "github.com/docker/docker/client"
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
"github.com/urfave/cli/v2"
|
"github.com/urfave/cli/v2"
|
||||||
)
|
)
|
||||||
|
@ -26,7 +29,7 @@ or having issues, it could be a lot of things. This command is best accompanied
|
||||||
by "abra app logs <app>".
|
by "abra app logs <app>".
|
||||||
`,
|
`,
|
||||||
Aliases: []string{"e"},
|
Aliases: []string{"e"},
|
||||||
Flags: []cli.Flag{},
|
Flags: []cli.Flag{internal.WatchFlag},
|
||||||
BashComplete: autocomplete.AppNameComplete,
|
BashComplete: autocomplete.AppNameComplete,
|
||||||
Action: func(c *cli.Context) error {
|
Action: func(c *cli.Context) error {
|
||||||
app := internal.ValidateApp(c)
|
app := internal.ValidateApp(c)
|
||||||
|
@ -45,54 +48,69 @@ by "abra app logs <app>".
|
||||||
logrus.Fatalf("%s is not deployed?", app.Name)
|
logrus.Fatalf("%s is not deployed?", app.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
filters := filters.NewArgs()
|
if !internal.Watch {
|
||||||
filters.Add("name", app.StackName())
|
if err := checkErrors(c, cl, app); err != nil {
|
||||||
|
logrus.Fatal(err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for {
|
||||||
|
if err := checkErrors(c, cl, app); err != nil {
|
||||||
|
logrus.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkErrors(c *cli.Context, cl *dockerClient.Client, app config.App) error {
|
||||||
|
recipe, err := recipe.Get(app.Type)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, service := range recipe.Config.Services {
|
||||||
|
filters := filters.NewArgs()
|
||||||
|
filters.Add("name", service.Name)
|
||||||
containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
|
containers, err := cl.ContainerList(c.Context, types.ContainerListOptions{Filters: filters})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(containers) == 0 {
|
||||||
|
logrus.Warnf("%s is not up, something seems wrong", service.Name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
container := containers[0]
|
||||||
|
containerState, err := cl.ContainerInspect(c.Context, container.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
tableCol := []string{"app name", "status", "error", "out of memory", "restart count", "healthcheck"}
|
if containerState.State.OOMKilled {
|
||||||
table := abraFormatter.CreateTable(tableCol)
|
logrus.Warnf("%s has been killed due to an out of memory error", service.Name)
|
||||||
|
|
||||||
for _, container := range containers {
|
|
||||||
serviceName := getServiceName(container.Names)
|
|
||||||
|
|
||||||
containerState, err := cl.ContainerInspect(c.Context, container.ID)
|
|
||||||
if err != nil {
|
|
||||||
logrus.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
errMsg := "N/A"
|
|
||||||
hcStat := "N/A"
|
|
||||||
var hcLogs []string
|
|
||||||
if containerState.State.Health != nil {
|
|
||||||
hcStat = containerState.State.Health.Status
|
|
||||||
for _, log := range containerState.State.Health.Log {
|
|
||||||
hcLogs = append(hcLogs, log.Output)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if containerState.State.Error != "" {
|
|
||||||
errMsg = containerState.State.Error
|
|
||||||
}
|
|
||||||
|
|
||||||
table.Append([]string{
|
|
||||||
serviceName,
|
|
||||||
containerState.State.Status,
|
|
||||||
errMsg,
|
|
||||||
strconv.FormatBool(containerState.State.OOMKilled),
|
|
||||||
strconv.Itoa(containerState.RestartCount),
|
|
||||||
hcStat,
|
|
||||||
strings.Join(hcLogs, "\n"),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
table.Render()
|
if containerState.State.Error != "" {
|
||||||
|
logrus.Warnf("%s reports this error: ", containerState.State.Error)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
if containerState.State.Health != nil {
|
||||||
},
|
if containerState.State.Health.Status != "healthy" {
|
||||||
|
logrus.Warnf("%s healthcheck status is %s", service.Name, containerState.State.Health.Status)
|
||||||
|
logrus.Warnf("%s healthcheck has failed %s times", service.Name, strconv.Itoa(containerState.State.Health.FailingStreak))
|
||||||
|
for _, log := range containerState.State.Health.Log {
|
||||||
|
logrus.Warnf("%s healthcheck logs: %s", service.Name, strings.TrimSpace(log.Output))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getServiceName(names []string) string {
|
func getServiceName(names []string) string {
|
||||||
|
|
|
@ -15,26 +15,17 @@ import (
|
||||||
"github.com/urfave/cli/v2"
|
"github.com/urfave/cli/v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
var watch bool
|
|
||||||
var watchFlag = &cli.BoolFlag{
|
|
||||||
Name: "watch",
|
|
||||||
Aliases: []string{"w"},
|
|
||||||
Value: false,
|
|
||||||
Usage: "Watch status by polling repeatedly",
|
|
||||||
Destination: &watch,
|
|
||||||
}
|
|
||||||
|
|
||||||
var appPsCommand = &cli.Command{
|
var appPsCommand = &cli.Command{
|
||||||
Name: "ps",
|
Name: "ps",
|
||||||
Usage: "Check app status",
|
Usage: "Check app status",
|
||||||
Description: "This command shows a more detailed status output of a specific deployed app.",
|
Description: "This command shows a more detailed status output of a specific deployed app.",
|
||||||
Aliases: []string{"p"},
|
Aliases: []string{"p"},
|
||||||
Flags: []cli.Flag{
|
Flags: []cli.Flag{
|
||||||
watchFlag,
|
internal.WatchFlag,
|
||||||
},
|
},
|
||||||
BashComplete: autocomplete.AppNameComplete,
|
BashComplete: autocomplete.AppNameComplete,
|
||||||
Action: func(c *cli.Context) error {
|
Action: func(c *cli.Context) error {
|
||||||
if !watch {
|
if !internal.Watch {
|
||||||
showPSOutput(c)
|
showPSOutput(c)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,6 +25,7 @@ var appRollbackCommand = &cli.Command{
|
||||||
Flags: []cli.Flag{
|
Flags: []cli.Flag{
|
||||||
internal.ForceFlag,
|
internal.ForceFlag,
|
||||||
internal.ChaosFlag,
|
internal.ChaosFlag,
|
||||||
|
internal.DontWaitConvergeFlag,
|
||||||
},
|
},
|
||||||
Description: `
|
Description: `
|
||||||
This command rolls an app back to a previous version if one exists.
|
This command rolls an app back to a previous version if one exists.
|
||||||
|
@ -164,7 +165,7 @@ recipes.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := stack.RunDeploy(cl, deployOpts, compose, app.Type); err != nil {
|
if err := stack.RunDeploy(cl, deployOpts, compose, app.Type, internal.DontWaitConverge); err != nil {
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -25,6 +25,7 @@ var appUpgradeCommand = &cli.Command{
|
||||||
internal.ForceFlag,
|
internal.ForceFlag,
|
||||||
internal.ChaosFlag,
|
internal.ChaosFlag,
|
||||||
internal.NoDomainChecksFlag,
|
internal.NoDomainChecksFlag,
|
||||||
|
internal.NoDomainChecksFlag,
|
||||||
},
|
},
|
||||||
Description: `
|
Description: `
|
||||||
This command supports upgrading an app. You can use it to choose and roll out a
|
This command supports upgrading an app. You can use it to choose and roll out a
|
||||||
|
@ -165,7 +166,7 @@ recipes.
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := stack.RunDeploy(cl, deployOpts, compose, app.Type); err != nil {
|
if err := stack.RunDeploy(cl, deployOpts, compose, app.Type, internal.DontWaitConverge); err != nil {
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -418,6 +418,24 @@ var AutoDNSRecordFlag = &cli.BoolFlag{
|
||||||
Destination: &AutoDNSRecord,
|
Destination: &AutoDNSRecord,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var DontWaitConverge bool
|
||||||
|
var DontWaitConvergeFlag = &cli.BoolFlag{
|
||||||
|
Name: "no-converge-checks",
|
||||||
|
Aliases: []string{"nc"},
|
||||||
|
Value: false,
|
||||||
|
Usage: "Don't wait for converge logic checks",
|
||||||
|
Destination: &DontWaitConverge,
|
||||||
|
}
|
||||||
|
|
||||||
|
var Watch bool
|
||||||
|
var WatchFlag = &cli.BoolFlag{
|
||||||
|
Name: "watch",
|
||||||
|
Aliases: []string{"w"},
|
||||||
|
Value: false,
|
||||||
|
Usage: "Watch status by polling repeatedly",
|
||||||
|
Destination: &Watch,
|
||||||
|
}
|
||||||
|
|
||||||
// SSHFailMsg is a hopefully helpful SSH failure message
|
// SSHFailMsg is a hopefully helpful SSH failure message
|
||||||
var SSHFailMsg = `
|
var SSHFailMsg = `
|
||||||
Woops, Abra is unable to connect to connect to %s.
|
Woops, Abra is unable to connect to connect to %s.
|
||||||
|
|
|
@ -135,7 +135,7 @@ func DeployAction(c *cli.Context) error {
|
||||||
logrus.Warn("skipping domain checks as requested")
|
logrus.Warn("skipping domain checks as requested")
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := stack.RunDeploy(cl, deployOpts, compose, app.Env["TYPE"]); err != nil {
|
if err := stack.RunDeploy(cl, deployOpts, compose, app.Env["TYPE"], DontWaitConverge); err != nil {
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -158,7 +158,7 @@ func pruneServices(ctx context.Context, cl *dockerclient.Client, namespace conve
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunDeploy is the swarm implementation of docker stack deploy
|
// RunDeploy is the swarm implementation of docker stack deploy
|
||||||
func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, recipeName string) error {
|
func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, recipeName string, dontWait bool) error {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
if err := validateResolveImageFlag(&opts); err != nil {
|
if err := validateResolveImageFlag(&opts); err != nil {
|
||||||
|
@ -170,7 +170,7 @@ func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, r
|
||||||
opts.ResolveImage = ResolveImageNever
|
opts.ResolveImage = ResolveImageNever
|
||||||
}
|
}
|
||||||
|
|
||||||
return deployCompose(ctx, cl, opts, cfg, recipeName)
|
return deployCompose(ctx, cl, opts, cfg, recipeName, dontWait)
|
||||||
}
|
}
|
||||||
|
|
||||||
// validateResolveImageFlag validates the opts.resolveImage command line option
|
// validateResolveImageFlag validates the opts.resolveImage command line option
|
||||||
|
@ -183,7 +183,7 @@ func validateResolveImageFlag(opts *Deploy) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, config *composetypes.Config, recipeName string) error {
|
func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, config *composetypes.Config, recipeName string, dontWait bool) error {
|
||||||
namespace := convert.NewNamespace(opts.Namespace)
|
namespace := convert.NewNamespace(opts.Namespace)
|
||||||
|
|
||||||
if opts.Prune {
|
if opts.Prune {
|
||||||
|
@ -224,7 +224,7 @@ func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, co
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage, recipeName)
|
return deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage, recipeName, dontWait)
|
||||||
}
|
}
|
||||||
|
|
||||||
func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) map[string]struct{} {
|
func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) map[string]struct{} {
|
||||||
|
@ -340,7 +340,8 @@ func deployServices(
|
||||||
namespace convert.Namespace,
|
namespace convert.Namespace,
|
||||||
sendAuth bool,
|
sendAuth bool,
|
||||||
resolveImage string,
|
resolveImage string,
|
||||||
recipeName string) error {
|
recipeName string,
|
||||||
|
dontWait bool) error {
|
||||||
existingServices, err := GetStackServices(ctx, cl, namespace.Name())
|
existingServices, err := GetStackServices(ctx, cl, namespace.Name())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -439,8 +440,13 @@ func deployServices(
|
||||||
for _, serviceName := range serviceIDs {
|
for _, serviceName := range serviceIDs {
|
||||||
serviceNames = append(serviceNames, serviceName)
|
serviceNames = append(serviceNames, serviceName)
|
||||||
}
|
}
|
||||||
logrus.Infof("waiting for services to converge: %s", strings.Join(serviceNames, ", "))
|
|
||||||
|
|
||||||
|
if dontWait {
|
||||||
|
logrus.Warn("skipping converge logic checks")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
logrus.Infof("waiting for services to converge: %s", strings.Join(serviceNames, ", "))
|
||||||
ch := make(chan error, len(serviceIDs))
|
ch := make(chan error, len(serviceIDs))
|
||||||
for serviceID, serviceName := range serviceIDs {
|
for serviceID, serviceName := range serviceIDs {
|
||||||
logrus.Debugf("waiting on %s to converge", serviceName)
|
logrus.Debugf("waiting on %s to converge", serviceName)
|
||||||
|
|
Loading…
Reference in New Issue