ensure_stack_deployed is now somewhat more reliable

Closes https://git.autonomic.zone/coop-cloud/abra/issues/165.

abra

@@ -169,54 +169,54 @@ printf -- "cat <<'EOM' >&2\n%s\nEOM\n" "$1"; }; error() {
 [[ -n $1 ]] && stderr "$1"; stderr "$usage"; _return 1; }; _return() {
 printf -- "exit %d\n" "$1"; exit "$1"; }; set -e; trimmed_doc=${DOC:1:2451}
 usage=${DOC:40:1842}; digest=c7bae
-shorts=(-e -b -s -C -U -h -d -v -n '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '')
+shorts=(-b -d -v -h -U -e -n -s -C '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '' '')
-longs=(--env --branch --stack --skip-check --skip-update --help --debug --verbose --no-prompt --status --server --type --domain --app-name --pass --secrets --all --update --force --fast --chaos --volumes --no-tty --user --bump --dev)
+longs=(--branch --debug --verbose --help --skip-update --env --no-prompt --stack --skip-check --status --server --type --domain --app-name --pass --secrets --all --update --force --fast --chaos --volumes --no-tty --user --bump --dev)
-argcounts=(1 1 1 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0); node_0(){
+argcounts=(1 0 0 0 0 1 0 1 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0); node_0(){
-value __env 0; }; node_1(){ value __branch 1; }; node_2(){ value __stack 2; }
+value __branch 0; }; node_1(){ switch __debug 1; }; node_2(){ switch __verbose 2
-node_3(){ switch __skip_check 3; }; node_4(){ switch __skip_update 4; }
+}; node_3(){ switch __help 3; }; node_4(){ switch __skip_update 4; }; node_5(){
-node_5(){ switch __help 5; }; node_6(){ switch __debug 6; }; node_7(){
+value __env 5; }; node_6(){ switch __no_prompt 6; }; node_7(){ value __stack 7
-switch __verbose 7; }; node_8(){ switch __no_prompt 8; }; node_9(){
+}; node_8(){ switch __skip_check 8; }; node_9(){ switch __status 9; }
-switch __status 9; }; node_10(){ value __server 10; }; node_11(){
+node_10(){ value __server 10; }; node_11(){ value __type 11; }; node_12(){
-value __type 11; }; node_12(){ value __domain 12; }; node_13(){
+value __domain 12; }; node_13(){ value __app_name 13; }; node_14(){
-value __app_name 13; }; node_14(){ switch __pass 14; }; node_15(){
+switch __pass 14; }; node_15(){ switch __secrets 15; }; node_16(){
-switch __secrets 15; }; node_16(){ switch __all 16; }; node_17(){
+switch __all 16; }; node_17(){ switch __update 17; }; node_18(){
-switch __update 17; }; node_18(){ switch __force 18; }; node_19(){
+switch __force 18; }; node_19(){ switch __fast 19; }; node_20(){
-switch __fast 19; }; node_20(){ switch __chaos 20; }; node_21(){
+switch __chaos 20; }; node_21(){ switch __volumes 21; }; node_22(){
-switch __volumes 21; }; node_22(){ switch __no_tty 22; }; node_23(){
+switch __no_tty 22; }; node_23(){ value __user 23; }; node_24(){
-value __user 23; }; node_24(){ switch __bump 24; }; node_25(){ switch __dev 25
+switch __bump 24; }; node_25(){ switch __dev 25; }; node_26(){ value _type_ a; }
-}; node_26(){ value _type_ a; }; node_27(){ value _app_ a; }; node_28(){
+node_27(){ value _app_ a; }; node_28(){ value _service_ a; }; node_29(){
-value _service_ a; }; node_29(){ value _version_ a; }; node_30(){ value _src_ a
+value _version_ a; }; node_30(){ value _src_ a; }; node_31(){ value _dst_ a; }
-}; node_31(){ value _dst_ a; }; node_32(){ value _backup_file_ a; }; node_33(){
+node_32(){ value _backup_file_ a; }; node_33(){ value _args_ a true; }
-value _args_ a true; }; node_34(){ value _secret_ a; }; node_35(){ value _cmd_ a
+node_34(){ value _secret_ a; }; node_35(){ value _cmd_ a; }; node_36(){
-}; node_36(){ value _data_ a; }; node_37(){ value _volume_ a; }; node_38(){
+value _data_ a; }; node_37(){ value _volume_ a; }; node_38(){ value _command_ a
-value _command_ a; }; node_39(){ value _recipe_ a; }; node_40(){ value _host_ a
+}; node_39(){ value _recipe_ a; }; node_40(){ value _host_ a; }; node_41(){
-}; node_41(){ value _user_ a; }; node_42(){ value _port_ a; }; node_43(){
+value _user_ a; }; node_42(){ value _port_ a; }; node_43(){ value _provider_ a
-value _provider_ a; }; node_44(){ value _subcommands_ a true; }; node_45(){
+}; node_44(){ value _subcommands_ a true; }; node_45(){ _command app; }
-_command app; }; node_46(){ _command list; }; node_47(){ _command ls; }
+node_46(){ _command list; }; node_47(){ _command ls; }; node_48(){ _command new
-node_48(){ _command new; }; node_49(){ _command backup; }; node_50(){
+}; node_49(){ _command backup; }; node_50(){ _command deploy; }; node_51(){
-_command deploy; }; node_51(){ _command check; }; node_52(){ _command version; }
+_command check; }; node_52(){ _command version; }; node_53(){ _command config; }
-node_53(){ _command config; }; node_54(){ _command cp; }; node_55(){
+node_54(){ _command cp; }; node_55(){ _command logs; }; node_56(){ _command ps
-_command logs; }; node_56(){ _command ps; }; node_57(){ _command restore; }
+}; node_57(){ _command restore; }; node_58(){ _command rm; }; node_59(){
-node_58(){ _command rm; }; node_59(){ _command delete; }; node_60(){
+_command delete; }; node_60(){ _command run; }; node_61(){ _command rollback; }
-_command run; }; node_61(){ _command rollback; }; node_62(){ _command secret; }
+node_62(){ _command secret; }; node_63(){ _command generate; }; node_64(){
-node_63(){ _command generate; }; node_64(){ _command insert; }; node_65(){
+_command insert; }; node_65(){ _command undeploy; }; node_66(){ _command volume
-_command undeploy; }; node_66(){ _command volume; }; node_67(){ _command recipe
+}; node_67(){ _command recipe; }; node_68(){ _command create; }; node_69(){
-}; node_68(){ _command create; }; node_69(){ _command release; }; node_70(){
+_command release; }; node_70(){ _command versions; }; node_71(){ _command server
-_command versions; }; node_71(){ _command server; }; node_72(){ _command add; }
+}; node_72(){ _command add; }; node_73(){ _command __ --; }; node_74(){
-node_73(){ _command __ --; }; node_74(){ _command init; }; node_75(){
+_command init; }; node_75(){ _command apps; }; node_76(){ _command upgrade; }
-_command apps; }; node_76(){ _command upgrade; }; node_77(){ _command doctor; }
+node_77(){ _command doctor; }; node_78(){ _command help; }; node_79(){
-node_78(){ _command help; }; node_79(){ optional 0 1 2 3 4 5 6 7 8; }
+optional 0 1 2 3 4 5 6 7 8; }; node_80(){ optional 79; }; node_81(){
-node_80(){ optional 79; }; node_81(){ either 46 47; }; node_82(){ required 81; }
+either 46 47; }; node_82(){ required 81; }; node_83(){ optional 9; }; node_84(){
-node_83(){ optional 9; }; node_84(){ optional 10; }; node_85(){ optional 11; }
+optional 10; }; node_85(){ optional 11; }; node_86(){ required 80 45 82 83 84 85
-node_86(){ required 80 45 82 83 84 85; }; node_87(){ optional 12; }; node_88(){
+}; node_87(){ optional 12; }; node_88(){ optional 13; }; node_89(){ optional 14
-optional 13; }; node_89(){ optional 14; }; node_90(){ optional 15; }; node_91(){
+}; node_90(){ optional 15; }; node_91(){ required 80 45 48 84 87 88 89 90 26; }
-required 80 45 48 84 87 88 89 90 26; }; node_92(){ either 28 16; }; node_93(){
+node_92(){ either 28 16; }; node_93(){ required 92; }; node_94(){
-required 92; }; node_94(){ required 80 45 27 49 93; }; node_95(){ optional 17; }
+required 80 45 27 49 93; }; node_95(){ optional 17; }; node_96(){ optional 18; }
-node_96(){ optional 18; }; node_97(){ optional 19; }; node_98(){ optional 20; }
+node_97(){ optional 19; }; node_98(){ optional 20; }; node_99(){ optional 29; }
-node_99(){ optional 29; }; node_100(){ required 80 45 27 50 95 96 97 98 99; }
+node_100(){ required 80 45 27 50 95 96 97 98 99; }; node_101(){
-node_101(){ required 80 45 27 51; }; node_102(){ required 80 45 27 52; }
+required 80 45 27 51; }; node_102(){ required 80 45 27 52; }; node_103(){
-node_103(){ required 80 45 27 53; }; node_104(){ required 80 45 27 54 30 31; }
+required 80 45 27 53; }; node_104(){ required 80 45 27 54 30 31; }; node_105(){
-node_105(){ optional 28; }; node_106(){ required 80 45 27 55 105; }; node_107(){
+optional 28; }; node_106(){ required 80 45 27 55 105; }; node_107(){
 required 80 45 27 56; }; node_108(){ required 80 45 27 57 93; }; node_109(){
 either 58 59; }; node_110(){ required 109; }; node_111(){ optional 21; }
 node_112(){ required 80 45 27 110 111 90; }; node_113(){ optional 32; }
@@ -245,8 +245,8 @@ required 80; }; node_157(){
 either 86 91 94 100 101 102 103 104 106 107 108 112 114 118 119 124 125 128 129 130 133 135 136 137 139 140 143 144 145 146 147 148 150 151 152 155 156
 }; node_158(){ required 157; }; cat <<<' docopt_exit() {
 [[ -n $1 ]] && printf "%s\n" "$1" >&2; printf "%s\n" "${DOC:40:1842}" >&2
-exit 1; }'; unset var___env var___branch var___stack var___skip_check \
+exit 1; }'; unset var___branch var___debug var___verbose var___help \
-var___skip_update var___help var___debug var___verbose var___no_prompt \
+var___skip_update var___env var___no_prompt var___stack var___skip_check \
 var___status var___server var___type var___domain var___app_name var___pass \
 var___secrets var___all var___update var___force var___fast var___chaos \
 var___volumes var___no_tty var___user var___bump var___dev var__type_ \
@@ -258,36 +258,37 @@ var_config var_cp var_logs var_ps var_restore var_rm var_delete var_run \
 var_rollback var_secret var_generate var_insert var_undeploy var_volume \
 var_recipe var_create var_release var_versions var_server var_add var___ \
 var_init var_apps var_upgrade var_doctor var_help; parse 158 "$@"
-local prefix=${DOCOPT_PREFIX:-''}; unset "${prefix}__env" "${prefix}__branch" \
+local prefix=${DOCOPT_PREFIX:-''}; unset "${prefix}__branch" \
-"${prefix}__stack" "${prefix}__skip_check" "${prefix}__skip_update" \
+"${prefix}__debug" "${prefix}__verbose" "${prefix}__help" \
-"${prefix}__help" "${prefix}__debug" "${prefix}__verbose" \
+"${prefix}__skip_update" "${prefix}__env" "${prefix}__no_prompt" \
-"${prefix}__no_prompt" "${prefix}__status" "${prefix}__server" \
+"${prefix}__stack" "${prefix}__skip_check" "${prefix}__status" \
-"${prefix}__type" "${prefix}__domain" "${prefix}__app_name" "${prefix}__pass" \
+"${prefix}__server" "${prefix}__type" "${prefix}__domain" \
-"${prefix}__secrets" "${prefix}__all" "${prefix}__update" "${prefix}__force" \
+"${prefix}__app_name" "${prefix}__pass" "${prefix}__secrets" "${prefix}__all" \
-"${prefix}__fast" "${prefix}__chaos" "${prefix}__volumes" "${prefix}__no_tty" \
+"${prefix}__update" "${prefix}__force" "${prefix}__fast" "${prefix}__chaos" \
-"${prefix}__user" "${prefix}__bump" "${prefix}__dev" "${prefix}_type_" \
+"${prefix}__volumes" "${prefix}__no_tty" "${prefix}__user" "${prefix}__bump" \
-"${prefix}_app_" "${prefix}_service_" "${prefix}_version_" "${prefix}_src_" \
+"${prefix}__dev" "${prefix}_type_" "${prefix}_app_" "${prefix}_service_" \
-"${prefix}_dst_" "${prefix}_backup_file_" "${prefix}_args_" \
+"${prefix}_version_" "${prefix}_src_" "${prefix}_dst_" \
-"${prefix}_secret_" "${prefix}_cmd_" "${prefix}_data_" "${prefix}_volume_" \
+"${prefix}_backup_file_" "${prefix}_args_" "${prefix}_secret_" \
-"${prefix}_command_" "${prefix}_recipe_" "${prefix}_host_" "${prefix}_user_" \
+"${prefix}_cmd_" "${prefix}_data_" "${prefix}_volume_" "${prefix}_command_" \
-"${prefix}_port_" "${prefix}_provider_" "${prefix}_subcommands_" \
+"${prefix}_recipe_" "${prefix}_host_" "${prefix}_user_" "${prefix}_port_" \
-"${prefix}app" "${prefix}list" "${prefix}ls" "${prefix}new" "${prefix}backup" \
+"${prefix}_provider_" "${prefix}_subcommands_" "${prefix}app" "${prefix}list" \
-"${prefix}deploy" "${prefix}check" "${prefix}version" "${prefix}config" \
+"${prefix}ls" "${prefix}new" "${prefix}backup" "${prefix}deploy" \
-"${prefix}cp" "${prefix}logs" "${prefix}ps" "${prefix}restore" "${prefix}rm" \
+"${prefix}check" "${prefix}version" "${prefix}config" "${prefix}cp" \
+"${prefix}logs" "${prefix}ps" "${prefix}restore" "${prefix}rm" \
 "${prefix}delete" "${prefix}run" "${prefix}rollback" "${prefix}secret" \
 "${prefix}generate" "${prefix}insert" "${prefix}undeploy" "${prefix}volume" \
 "${prefix}recipe" "${prefix}create" "${prefix}release" "${prefix}versions" \
 "${prefix}server" "${prefix}add" "${prefix}__" "${prefix}init" "${prefix}apps" \
 "${prefix}upgrade" "${prefix}doctor" "${prefix}help"
-eval "${prefix}"'__env=${var___env:-}'
 eval "${prefix}"'__branch=${var___branch:-}'
-eval "${prefix}"'__stack=${var___stack:-}'
-eval "${prefix}"'__skip_check=${var___skip_check:-false}'
-eval "${prefix}"'__skip_update=${var___skip_update:-false}'
-eval "${prefix}"'__help=${var___help:-false}'
 eval "${prefix}"'__debug=${var___debug:-false}'
 eval "${prefix}"'__verbose=${var___verbose:-false}'
+eval "${prefix}"'__help=${var___help:-false}'
+eval "${prefix}"'__skip_update=${var___skip_update:-false}'
+eval "${prefix}"'__env=${var___env:-}'
 eval "${prefix}"'__no_prompt=${var___no_prompt:-false}'
+eval "${prefix}"'__stack=${var___stack:-}'
+eval "${prefix}"'__skip_check=${var___skip_check:-false}'
 eval "${prefix}"'__status=${var___status:-false}'
 eval "${prefix}"'__server=${var___server:-}'
 eval "${prefix}"'__type=${var___type:-}'
@@ -355,9 +356,9 @@ eval "${prefix}"'upgrade=${var_upgrade:-false}'
 eval "${prefix}"'doctor=${var_doctor:-false}'
 eval "${prefix}"'help=${var_help:-false}'; local docopt_i=1
 [[ $BASH_VERSION =~ ^4.3 ]] && docopt_i=2; for ((;docopt_i>0;docopt_i--)); do
-declare -p "${prefix}__env" "${prefix}__branch" "${prefix}__stack" \
+declare -p "${prefix}__branch" "${prefix}__debug" "${prefix}__verbose" \
-"${prefix}__skip_check" "${prefix}__skip_update" "${prefix}__help" \
+"${prefix}__help" "${prefix}__skip_update" "${prefix}__env" \
-"${prefix}__debug" "${prefix}__verbose" "${prefix}__no_prompt" \
+"${prefix}__no_prompt" "${prefix}__stack" "${prefix}__skip_check" \
 "${prefix}__status" "${prefix}__server" "${prefix}__type" "${prefix}__domain" \
 "${prefix}__app_name" "${prefix}__pass" "${prefix}__secrets" "${prefix}__all" \
 "${prefix}__update" "${prefix}__force" "${prefix}__fast" "${prefix}__chaos" \
@@ -792,50 +793,54 @@ output_version_summary() {
   fi
 }
 
-# Note(decentral1se): inspired by https://github.com/vitalets/docker-stack-wait-deploy
 ensure_stack_deployed() {
-  STACK_NAME=$1
+  local -a HEALTHY # mapping
+  local -a MISSING # mapping
 
-  warning "Polling deploy state to check for success"
+  TIMEOUT=60
+  idx=0
 
-  while true; do
+  IFS=' ' read -r -a SERVICES <<< "$(docker stack services "${STACK_NAME}" --format "{{.ID}}" | tr '\n' ' ')"
-    all_services_done=1
-    has_errors=0
 
-    service_ids=$(docker stack services -q "$STACK_NAME")
+  while [ ! $(( ${#HEALTHY[@]} + ${#MISSING[@]} )) -eq ${#SERVICES[@]} ]; do
+    for service in $(docker ps -f "name=$STACK_NAME" -q); do
+      healthcheck=$(docker inspect --format "{{ json .State }}" "$service" | jq "try(.Health.Status // \"missing\")")
+      name=$(docker inspect --format '{{ index .Config.Labels "com.docker.swarm.service.name" }}' "$service")
 
-    for service_id in $service_ids; do
+      if [[ ${MISSING[*]} =~ ${name} ]] || [[ ${HEALTHY[*]} =~ ${name} ]]; then
-      # see: https://github.com/moby/moby/issues/28012
+        continue
-      service_state=$(docker service inspect --format "{{if .UpdateStatus}}{{.UpdateStatus.State}}{{else}}created{{end}}" "$service_id")
+      fi
 
-      debug "$service_id has state: $service_state"
+      if [[ "$healthcheck" == "\"missing\"" ]] && [[ ! "${MISSING[*]}" =~ $name ]]; then
+        MISSING+=("$name")
+        info "$name has no healthcheck configured, cannot guarantee this service comes up successfully..."
+        continue
+      fi
 
-      case "$service_state" in
+      if [[ "$healthcheck" == "\"healthy\"" ]] && [[ ! "${HEALTHY[*]}" =~ $name ]]; then
-        created|completed)
+        HEALTHY+=("$name")
-          ;;
+        info "$name is healthy!"
-        paused|rollback_completed)
+        continue
-          has_errors=1
+      fi
-          ;;
+
-        *)
+      if [[ "$healthcheck" == \""unhealthy"\" ]]; then
-          all_services_done=0
+        logs=$(docker inspect --format "{{ json .State.Health.Log }}" "$service")
-          ;;
+        exitcode="$(echo $logs | $JQ '.[-1] | .ExitCode')"
-      esac
+        warning "Healthcheck for new instance of $name is failing (exit code: $exitcode)"
+        warning "$(echo $logs | $JQ -r '.[-1] | .Output')"
+        error "healthcheck for $name is failing, this deployment did not succeed :("
+      fi
     done
 
-    if [ "$all_services_done" == "1" ]; then
+    idx=$(("$idx" + 1))
-      if [ "$has_errors" == "1" ]; then
+    if [[ $idx -eq "$TIMEOUT" ]]; then
-        warning "Deployment appears to have failed"
+      error "Waiting for healthy status timed out, this deployment did not succeed :("
-        warning "Run \"abra app ${STACK_NAME} logs \" to see app logs"
-        warning "Run \"abra app ${STACK_NAME} ps \" to see app status"
-        break
-      else
-        warning "Deployment appears to have suceeded"
-        break
-      fi
-    else
-      sleep 1
     fi
+
+    sleep 1
+    info "Deploying: $(( ${#HEALTHY[@]} + ${#MISSING[@]} ))/${#SERVICES[@]} (timeout: $idx/$TIMEOUT)"
   done
+  success "All services up! Deployment succeeded!"
 }
 
 ensure_domain_deployed() {
@@ -870,7 +875,7 @@ load_instance() {
   APP="$abra__app_"
 
   # load all files matching "$APP.env" into ENV_FILES array
-  mapfile -t ENV_FILES < <(find -L "$ABRA_DIR" -name "$APP.env")
+  mapfile -t ENV_FILES < <(find -L "$ABRA_DIR/servers/" -name "$APP.env")
   # FIXME 3wc: requires bash 4, use for loop instead
 
   case "${#ENV_FILES[@]}" in

makefile

@@ -43,3 +43,8 @@ build:
 
 push: build
 	@docker push thecoopcloud/abra
+
+symlink:
+	@mkdir -p ~/.abra/servers/ && \
+	ln -srf tests/default ~/.abra/servers && \
+	ln -srf tests/apps/* ~/.abra/apps

tests/apps/works/compose.yml

@@ -0,0 +1,84 @@
+---
+
+# The goal of this compose file is to have a testing ground for understanding
+# what cases we need to handle to get stable deployments. For that, we need to
+# work with healthchecks and deploy configurations quite closely. If you run
+# the `make symlink` target then this will be loaded into a "fake" app on your
+# local machine which you can deploy with `abra`.
+
+version: "3.8"
+services:
+  r1_should_work:
+    image: redis:alpine
+    deploy:
+      update_config:
+        failure_action: rollback
+        order: start-first
+      rollback_config:
+        order: start-first
+      restart_policy:
+        max_attempts: 1
+    healthcheck:
+      test: redis-cli ping
+      interval: 2s
+      retries: 3
+      start_period: 1s
+      timeout: 3s
+
+  r2_broken_health_check:
+    image: redis:alpine
+    deploy:
+      update_config:
+        failure_action: rollback
+        order: start-first
+      rollback_config:
+        order: start-first
+      restart_policy:
+        max_attempts: 3
+    healthcheck:
+      test: foobar
+      interval: 2s
+      retries: 3
+      start_period: 1s
+      timeout: 3s
+
+  r3_no_health_check:
+    image: redis:alpine
+    deploy:
+      update_config:
+        failure_action: rollback
+        order: start-first
+      rollback_config:
+        order: start-first
+      restart_policy:
+        max_attempts: 3
+
+  r4_disabled_health_check:
+    image: redis:alpine
+    deploy:
+      update_config:
+        failure_action: rollback
+        order: start-first
+      rollback_config:
+        order: start-first
+      restart_policy:
+        max_attempts: 3
+    healthcheck:
+      disable: true
+
+  r5_should_also_work:
+    image: redis:alpine
+    deploy:
+      update_config:
+        failure_action: rollback
+        order: start-first
+      rollback_config:
+        order: start-first
+      restart_policy:
+        max_attempts: 1
+    healthcheck:
+      test: redis-cli ping
+      interval: 2s
+      retries: 3
+      start_period: 1s
+      timeout: 3s

tests/default/works.env

@@ -0,0 +1 @@
+TYPE=works