Add log entry

Closes #8

.drone.yml

@@ -7,7 +7,7 @@ steps:
     commands:
       - apt update
       - apt install -y shellcheck
-      - shellcheck abra installer
+      - shellcheck abra script.d/*
 trigger:
   branch:
     - main

CHANGELOG.md

@@ -1,3 +1,8 @@
+# abra 0.3.0 (2020-09-27)
+
+- Add multilogs stack logs implementation ([#8](https://git.autonomic.zone/compose-stacks/abra/issues/8)
+- Add beginnings of "monorepo" functionality
+
 # abra 0.2.0 (2020-09-24)
 
 - Prepare for swarm install script using script.d ([#12](https://git.autonomic.zone/compose-stacks/planning/issues/12))

README.md

@@ -24,7 +24,32 @@ cd abra
 make dev_install
 ```
 
-See [autonomic-cooperative/installer-scripts](https://git.autonomic.zone/autonomic-cooperative/installer-scripts) for the installer script deployment. To make a release, just add an entry to [CHANGELOG.md](./CHANGELOG.md) and the [abra-installer](./script.d/abra-installer) / [swarm-installer](./script.d/swarm-installer) (following [semver](https://semver.org/) please) and then `git tag x.x.x && git push origin main --tags`. If you want the [installer-scripts](https://git.autonomic.zone/autonomic-cooperative/installer-scripts) deployment to pick that up, you'll need to change the version number in the [Makefile](https://git.autonomic.zone/autonomic-cooperative/installer-scripts/src/branch/main/Makefile) and run `make` in that repository and push the changes.
+See [autonomic-cooperative/installer-scripts](https://git.autonomic.zone/autonomic-cooperative/installer-scripts) for the installer script deployment. To make a release, just add an entry to [CHANGELOG.md](./CHANGELOG.md) (following [semver](https://semver.org/) please) and then `git tag x.x.x && git push origin main --tags`. If you want the [installer-scripts](https://git.autonomic.zone/autonomic-cooperative/installer-scripts) deployment to pick that up, you'll need to change the version number in the [Makefile](https://git.autonomic.zone/autonomic-cooperative/installer-scripts/src/branch/main/Makefile) and run `make` in that repository and push the changes.
+
+## Specify what to deploy where
+
+You can use `abra` in one of 2 ways:
+
+1. Clone a `compose-stack`, create an `.envrc` in it, and run `abra` in that
+   directory. Be sure to set `ABRA_STACK_DIR=.`
+2. "Monorepo mode": keep all your `compose-stack`s in one directory and all your `env` files in
+   another, e.g.:
+   ```
+   $ tree
+   .
+   ├── apps
+   │   ├── mediawiki.demo.autonomic.zone.env
+   │   ├── wordpress.demo.autonomic.zone.env
+   └── stacks
+       ├── gitea
+       ├── matrix-synapse
+       ├── mediawiki
+       ├── nextcloud
+       ├── swarmpit
+       ├── traefik
+       └── wordpress
+   $ abra -e apps/mediawiki.demo.autonomic.zone.env deploy
+   ```
 
 ## Examples
 

abra

@@ -1,22 +1,14 @@
 #!/bin/bash
 
 PROGRAM_NAME=$(basename "$0")
-ABRA_CONFIG=abra.yml
 
-if [ -z "$COMPOSE_FILE" ]; then
-	COMPOSE_FILE="compose.yml"
-fi
+###### Utility functions
 
 yml_pattern_exists() {
 	PATTERN=$1
 
-	if ! type yq > /dev/null 2>&1; then
-		echo "$(tput setaf 1)ERROR: yq program is not installed$(tput sgr0)"
-		exit
-	fi
-
-	if [ -f $ABRA_CONFIG ]; then
-			RESULT=$(yq read $ABRA_CONFIG "$PATTERN")
+	if [ -f "$ABRA_CONFIG" ]; then
+			RESULT=$(yq read "$ABRA_CONFIG" "$PATTERN")
 
 			if [ "$RESULT" != 0 ]; then
 				return 0
@@ -26,229 +18,6 @@ yml_pattern_exists() {
 	return 1
 }
 
-if [ "$1" == "-a" ]; then
-	STACK_NAME=$2
-	shift 2
-fi
-
-if [ -f abra.yml ]; then
-	if yml_pattern_exists stack_name; then
-		STACK_NAME=$(yq read abra.yml stack_name)
-	fi
-fi
-
-if [ -z "$STACK_NAME" ]; then
-	echo "$(tput setaf 1)ERROR: \$STACK_NAME must be set (e.g. export STACK_NAME=my_cool_app)$(tput sgr0 )"
-	exit
-fi
-
-if type direnv > /dev/null 2>&1 && ! direnv status | grep -q 'Found RC allowed true'; then
-	echo "$(tput setaf 1)ERROR: direnv is blocked, run direnv allow$(tput sgr0)"
-	exit
-fi
-
-if [ -f abra-commands.sh ]; then
-	# shellcheck disable=SC1091
-	. abra-commands.sh
-fi
-
-sub_help() {
-	echo "Usage: $PROGRAM_NAME [-a STACK_NAME] <subcommand> [options]"
-	echo ""
-	echo "Subcommands:"
-	echo "    cp SRC_PATH SERVICE:DEST_PATH         copy files to a container"
-	echo "    deploy                                let 'em rip"
-	echo "    logs SERVICE [ARGS]                   tail logs from a deployed service"
-	echo "    run SERVICE CMD                       run a command in the specified service's container"
-	echo "    run_args SERVICE ARGS CMD             run, passing extra args to docker exec"
-	echo "    secret_generate SECRET VERSION [CMD]  generate a secret, store it in pass & as a Docker secret"
-	echo "    upgrade                               upgrade to the latest version"
-	echo "    ... (custom commands)"
-	echo ""
-	echo "Make sure \$STACK_NAME is set using direnv or -a"
-	echo ""
-	echo "Runs compose.yml by default, set e.g. COMPOSE_FILE=\"compose.yml:compose2.yml\" to override"
-}
-
-sub_secret_help() {
-	echo "Usage: $PROGRAM_NAME [-a STACK_NAME] secret <subcommand> [options]"
-	echo ""
-	echo "Subcommands:"
-	echo "    generate [PW]                         generate & store secret"
-}
-
-sub_secret_generate(){
-	SECRET=$1
-	VERSION=$2
-	PW=${3:-pwqgen}
-
-	if [ -z "$SECRET" ] || [ -z "$VERSION" ]; then
-		echo "Usage: $PROGRAM_NAME secret_generate SECRET VERSION"
-		exit
-	fi
-
-	$PW | tee \
-		>(docker secret create "${STACK_NAME}_${SECRET}_${VERSION}" -) \
-		>(pass insert "hosts/autonomic-swarm/${STACK_NAME}/${SECRET}" -m)
-}
-
-sub_secret() {
-	SUBCOMMAND=$1
-	shift
-	# shellcheck disable=SC2068
-	parse_subcommand "$SUBCOMMAND" "secret" $@
-}
-
-sub_run_args(){
-	SERVICE=$1
-	DOCKER_ARGS=$2
-
-	shift 2
-
-	if [ -z "$SERVICE" ]; then
-		echo "Usage: $PROGRAM_NAME run SERVICE [CMD]"
-		exit
-	fi
-
-	CONTAINER=$(docker container ls --format "table {{.ID}},{{.Names}}" \
-		| grep "${STACK_NAME}_${SERVICE}" | cut -d',' -f1)
-
-	if [ -z "$CONTAINER" ]; then
-		echo "Container not found! 🚨"
-		exit
-	fi
-
-	# shellcheck disable=SC2086
-	docker exec $DOCKER_ARGS -it "$CONTAINER" "$@"
-
-	return
-}
-
-sub_run(){
-	SERVICE=$1
-
-	shift
-
-	sub_run_args "$SERVICE" "" "$@"
-}
-
-sub_deploy (){
-	echo "About to deploy:"
-	echo "  Compose: $(tput setaf 3)${PWD}/${COMPOSE_FILE}$(tput sgr0)"
-	if [ -n "$DOMAIN" ]; then
-		echo "  Domain: $(tput setaf 2)${DOMAIN}$(tput sgr0)"
-	fi
-	echo "  Stack: $(tput setaf 1)${STACK_NAME}$(tput sgr0)"
-
-	read -rp "Continue? (y/[n])? " choice
-
-	case "$choice" in
-		y|Y ) ;;
-		n|N ) return;;
-		* ) return;;
-	esac
-
-	# shellcheck disable=SC2086
-	if docker stack deploy -c ${COMPOSE_FILE/:/ -c } "$STACK_NAME"; then
-		if [ -n "$DOMAIN" ]; then
-			echo "$(tput setaf 2)Yay! App should be available at https://${DOMAIN}$(tput sgr0)"
-		else
-			echo "$(tput setaf 2)Yay! That worked. No \$DOMAIN defined, check logs.(tput sgr0)"
-		fi
-	else
-		echo "$(tput setaf 1)Oh no! Something went wrong 😕 Check errors above$(tput sgr0)"
-	fi
-}
-
-sub_logs (){
-	SERVICE=$1
-
-	shift
-
-	if [ $# -eq 0 ]; then
-		LOGS_ARGS="\
-		--follow \
-		--no-trunc \
-		--details \
-		--timestamps"
-	else
-		# shellcheck disable=SC2124
-		LOGS_ARGS=$@
-	fi
-
-	# shellcheck disable=SC2086
-	docker service logs "${STACK_NAME}_${SERVICE}" $LOGS_ARGS
-}
-
-sub_cp() {
-	SOURCE=$1
-	DEST=$2
-
-	SERVICE=$(echo "$SOURCE" | grep -o '^[^:]\+:' || echo "$DEST" | grep -o '^[^:]\+:')
-	SERVICE=$(echo "$SERVICE" | tr -d ':')
-
-	if [ -z "$SERVICE" ]; then
-		echo "$(tput setaf 1)ERROR: Can't find SERVICE in either SRC or DEST$(tput sgr0)"
-		echo ""
-		echo "Usage: $PROGRAM_NAME cp SERVICE:SRC_PATH DEST_PATH"
-		echo "       $PROGRAM_NAME cp SRC_PATH SERVICE:DEST_PATH"
-		exit
-	fi
-
-	CONTAINER=$(docker container ls --format "table {{.ID}},{{.Names}}" \
-		| grep "${STACK_NAME}_${SERVICE}" | cut -d',' -f1)
-
-	if [ -z "$CONTAINER" ]; then
-		echo "$(tput setaf 1)ERROR: Can't find a ${STACK_NAME}_${SERVICE}$(tput sgr0)"
-		exit
-	fi
-
-	CP_ARGS=$(echo "$SOURCE $DEST" | sed "s/$SERVICE/$CONTAINER/")
-
-	# shellcheck disable=SC2086
-	docker cp $CP_ARGS
-}
-
-sub_context_help() {
-	echo "Usage: $PROGRAM_NAME [-a STACK_NAME] context <subcommand> [options]"
-	echo ""
-	echo "Subcommands:"
-	echo "    init HOST [USER] [PORT]               set up remote Docker context"
-	echo "    use                                   activate remote Docker context"
-}
-
-sub_context_init() {
-	HOST="$1"
-	USERNAME="$2"
-	PORT="$3"
-
-	if [ -n "$PORT" ]; then
-		PORT=":$PORT"
-	fi
-
-	if [ -n "$USERNAME" ]; then
-		USERNAME="$USERNAME@"
-	fi
-
-	docker context create "$HOST" \
-		--docker "host=ssh://$USERNAME$HOST$PORT"
-}
-
-sub_context_use() {
-	docker context use "$1"
-}
-
-sub_upgrade() {
-	curl -fsSL https://install.abra.autonomic.zone | bash
-}
-
-sub_context() {
-	SUBCOMMAND2=$1
-	shift
-	# shellcheck disable=SC2068
-	parse_subcommand "$SUBCOMMAND2" "context" $@
-}
-
 parse_subcommand() {
 	SUBCOMMAND="$1"
 	PREFIX=$2
@@ -275,8 +44,381 @@ parse_subcommand() {
 	esac
 }
 
+error() {
+	echo "$(tput setaf 1)ERROR: $*$(tput sgr0)"
+	exit 1
+}
+
+warning() {
+	echo "$(tput setaf 3)WARNING: $*$(tput sgr0)"
+}
+
+success() {
+	echo "$(tput setaf 2)$*$(tput sgr0)"
+}
+
+###### Top-level arguments
+
+ABRA_CONFIG=abra.yml
+if [ "$1" == "-c" ]; then
+	ABRA_CONFIG=$2
+	shift 2
+fi
+
+if [ "$1" == "-e" ]; then
+	ABRA_ENV=$2
+	shift 2
+fi
+
+if [ "$1" == "-a" ]; then
+	STACK_NAME=$2
+	shift 2
+fi
+
+###### Load config
+
+if [ -f "$ABRA_CONFIG" ]; then
+	require_yq
+
+	if yml_pattern_exists stack_name; then
+		STACK_NAME=$(yq read "$ABRA_CONFIG" stack_name)
+	fi
+	# FIXME load other variables somehow
+fi
+
+if [ -n "$ABRA_ENV" ]; then
+	# shellcheck disable=SC1090
+	source "$ABRA_ENV" || error "Unable to load env from '$ABRA_ENV'"
+fi
+
+###### Default settings
+
+if [ -z "$COMPOSE_FILE" ]; then
+	COMPOSE_FILE="compose.yml"
+fi
+
+if [ -z "$ABRA_STACK_DIR" ]; then
+	ABRA_STACK_DIR="stacks/$SERVICE"
+fi
+
+load_context() {
+	if [ -z "$DOCKER_CONTEXT" ]; then
+		warning "\$DOCKER_CONTEXT not set, (slowly) looking it up"
+		# shellcheck disable=SC2063
+		DOCKER_CONTEXT=$(docker context ls | grep '*' | cut -d' ' -f1)
+		# FIXME 3wc: make sure grep doesn't parse this, we're want a literal '*'
+	fi
+}
+
+###### Safety checks
+
+require_yq() {
+	if ! type yq > /dev/null 2>&1; then
+		error "yq program is not installed"
+	fi
+}
+
+require_multitail() {
+	if ! type multitail > /dev/null 2>&1; then
+		error "multitail program is not installed"
+	fi
+}
+
+require_stack() {
+	if [ -z "$STACK_NAME" ]; then
+		error "no stack_name, export \$STACK_NAME=my_cool_app or add it to abra.yml"
+	fi
+}
+
+require_stack_dir() {
+	if [ -z "$ABRA_STACK_DIR" ] || [ ! -d "$ABRA_STACK_DIR" ]; then
+		error "can't find \$ABRA_STACK_DIR '$ABRA_STACK_DIR'"
+	fi
+}
+
+if [ -z "$ABRA_ENV" ] && [ -f .envrc ] && type direnv > /dev/null 2>&1 && ! direnv status | grep -q 'Found RC allowed true'; then
+	error "direnv is blocked, run direnv allow"
+fi
+
+###### Custom commands
+
+if [ -f abra-commands.sh ]; then
+	# shellcheck disable=SC1091
+	source abra-commands.sh
+fi
+
+###### Global help
+
+sub_help() {
+	echo "Usage: $PROGRAM_NAME [-a STACK_NAME] [-c CONFIG] [-e ENV_FILE] <subcommand> [options]"
+	echo ""
+	echo "Subcommands:"
+	echo "    context [--help] [SUBCOMMAND]         manage remote swarm contexts"
+	echo "    cp SRC_PATH SERVICE:DEST_PATH         copy files to a container"
+	echo "    deploy                                let 'em rip"
+	echo "    logs SERVICE [ARGS]                   tail logs from a deployed service"
+	echo "    multilogs                             tail logs from a whole stackk"
+	echo "    run SERVICE CMD                       run a command in the specified service's container"
+	echo "    run_args SERVICE ARGS CMD             run, passing extra args to docker exec"
+	echo "    secret [--help] [SUBCOMMAND]          manage secrets"
+	echo "    upgrade                               upgrade to the latest version"
+	echo "    ... (custom commands)"
+	echo ""
+	echo "Make sure \$STACK_NAME is set using direnv, -a, -e or -c"
+	echo ""
+	echo "Runs compose.yml by default, set e.g. COMPOSE_FILE=\"compose.yml:compose2.yml\" to override"
+}
+
+###### Subcommand `secret`
+
+sub_secret_help() {
+	echo "Usage: $PROGRAM_NAME [global opts] secret <subcommand> [sub opts]"
+	echo ""
+	echo "Subcommands:"
+	echo "    generate SECRET VERSION [PWGEN]       generate & store secret"
+	echo "    insert SECRET VERSION PW              save PW in docker and pass"
+}
+
+sub_secret_insert() {
+	require_stack
+	load_context
+
+	SECRET=$1
+	VERSION=$2
+	PW=$3
+
+	if [ -z "$SECRET" ] || [ -z "$VERSION" ] || [ -z "$PW" ]; then
+		echo "Usage: $PROGRAM_NAME secret insert SECRET VERSION PW"
+		exit
+	fi
+
+	echo "$PW" | docker secret create "${STACK_NAME}_${SECRET}_${VERSION}" - > /dev/null
+	echo "$PW" | pass insert "hosts/$DOCKER_CONTEXT/${STACK_NAME}/${SECRET}" -m > /dev/null
+}
+
+sub_secret_generate(){
+	SECRET=$1
+	VERSION=$2
+	PWGEN=${3:-pwqgen}
+
+	PW=$($PWGEN)
+
+	success "Password: $PW"
+
+	echo "sub_secret_insert \"$SECRET\" \"$VERSION\" \"$PW\""
+	exit
+	sub_secret_insert "$SECRET" "$VERSION" "$PW"
+}
+
+sub_secret() {
 	SUBCOMMAND=$1
 	shift
+	# shellcheck disable=SC2068
+	parse_subcommand "$SUBCOMMAND" "secret" $@
+}
 
+###### Subcommand `run`
+
+sub_run_args(){
+	require_stack
+
+	SERVICE=$1
+	DOCKER_ARGS=$2
+
+	shift 2
+
+	if [ -z "$SERVICE" ]; then
+		echo "Usage: $PROGRAM_NAME run SERVICE DOCKER_ARGS [CMD]"
+		exit
+	fi
+
+	CONTAINER=$(docker container ls --format "table {{.ID}},{{.Names}}" \
+		| grep "${STACK_NAME}_${SERVICE}" | cut -d',' -f1)
+
+	if [ -z "$CONTAINER" ]; then
+		error "Can't find a container for ${STACK_NAME}_${SERVICE}"
+		exit
+	fi
+
+	# shellcheck disable=SC2086
+	docker exec $DOCKER_ARGS -it "$CONTAINER" "$@"
+
+	return
+}
+
+sub_run(){
+	SERVICE=$1
+
+	shift
+
+	sub_run_args "$SERVICE" "" "$@"
+}
+
+###### Subcommand `deploy`
+
+sub_deploy (){
+	require_stack
+	require_stack_dir
+	load_context
+
+	echo "About to deploy:"
+	echo "  Context: $(tput setaf 4)${DOCKER_CONTEXT}$(tput sgr0)"
+	echo "  Compose: $(tput setaf 3)${ABRA_STACK_DIR}/${COMPOSE_FILE}$(tput sgr0)"
+	if [ -n "$DOMAIN" ]; then
+		echo "  Domain: $(tput setaf 2)${DOMAIN}$(tput sgr0)"
+	fi
+	echo "  Stack: $(tput setaf 1)${STACK_NAME}$(tput sgr0)"
+
+	read -rp "Continue? (y/[n])? " choice
+
+	case "$choice" in
+		y|Y ) ;;
+		n|N ) return;;
+		* ) return;;
+	esac
+
+	(
+		cd "$ABRA_STACK_DIR" || error "\$ABRA_STACK_DIR '$ABRA_STACK_DIR' not found"
+		# shellcheck disable=SC2086
+		if docker stack deploy -c ${COMPOSE_FILE/:/ -c } "$STACK_NAME"; then
+			if [ -n "$DOMAIN" ]; then
+				success "Yay! App should be available at https://${DOMAIN}"
+			else
+				success "Yay! That worked. No \$DOMAIN defined, check logs."
+			fi
+		else
+			error "Oh no! Something went wrong 😕 Check errors above"
+		fi
+	)
+}
+
+###### Subcommand `logs`
+
+# Inspired by https://github.com/moby/moby/issues/31458#issuecomment-475411564
+sub_multilogs() {
+	require_stack
+	require_multitail
+
+	# Get a list of the service names
+	SERVICES=$(docker stack services --format "{{.Name}}" "${STACK_NAME}")
+	# Sort the service names
+	SERVICES=$(echo "${SERVICES}" | sort)
+	# Create the command to run
+	COMMAND='multitail --mergeall'
+	for SERVICE in ${SERVICES}; do
+			COMMAND="${COMMAND} -L 'docker service logs --tail 20 -f ${SERVICE}'"
+	done
+	# Run the command
+	bash -c "${COMMAND}"
+}
+
+sub_logs (){
+	require_stack
+
+	SERVICE=$1
+
+	if [ -z "$SERVICE" ]; then
+		warning "No \$SERVICE provided, running multilogs"
+		sub_multilogs
+	fi
+
+	shift
+
+	if [ $# -eq 0 ]; then
+		LOGS_ARGS="\
+		--follow \
+		--no-trunc \
+		--details \
+		--timestamps"
+	else
+		# shellcheck disable=SC2124
+		LOGS_ARGS=$@
+	fi
+
+	# shellcheck disable=SC2086
+	docker service logs "${STACK_NAME}_${SERVICE}" $LOGS_ARGS
+}
+
+###### Subcommand `cp`
+
+sub_cp() {
+	require_stack
+
+	SOURCE=$1
+	DEST=$2
+
+	SERVICE=$(echo "$SOURCE" | grep -o '^[^:]\+:' || echo "$DEST" | grep -o '^[^:]\+:')
+	SERVICE=$(echo "$SERVICE" | tr -d ':')
+
+	if [ -z "$SERVICE" ]; then
+		echo "Usage: $PROGRAM_NAME cp SERVICE:SRC_PATH DEST_PATH"
+		echo "       $PROGRAM_NAME cp SRC_PATH SERVICE:DEST_PATH"
+		echo ""
+		error "Can't find SERVICE in either SRC or DEST"
+	fi
+
+	CONTAINER=$(docker container ls --format "table {{.ID}},{{.Names}}" \
+		| grep "${STACK_NAME}_${SERVICE}" | cut -d',' -f1)
+
+	if [ -z "$CONTAINER" ]; then
+		error "Can't find a container for ${STACK_NAME}_${SERVICE}"
+		exit
+	fi
+
+	CP_ARGS=$(echo "$SOURCE $DEST" | sed "s/$SERVICE/$CONTAINER/")
+
+	# shellcheck disable=SC2086
+	docker cp $CP_ARGS
+}
+
+###### Subcommand `context`
+
+sub_context_help() {
+	echo "Usage: $PROGRAM_NAME [global opts] context <subcommand> [sub opts]"
+	echo ""
+	echo "Subcommands:"
+	echo "    init HOST [USER] [PORT]               set up remote Docker context"
+	echo "    use                                   activate remote Docker context"
+}
+
+sub_context_init() {
+	HOST="$1"
+	USERNAME="$2"
+	PORT="$3"
+
+	if [ -n "$PORT" ]; then
+		PORT=":$PORT"
+	fi
+
+	if [ -n "$USERNAME" ]; then
+		USERNAME="$USERNAME@"
+	fi
+
+	docker context create "$HOST" \
+		--docker "host=ssh://$USERNAME$HOST$PORT"
+}
+
+sub_context_use() {
+	docker context use "$1"
+}
+
+sub_context() {
+	SUBCOMMAND2=$1
+	shift
+	# shellcheck disable=SC2068
+	parse_subcommand "$SUBCOMMAND2" "context" $@
+}
+
+
+###### Subcommand `upgrade`
+
+sub_upgrade() {
+	curl -fsSL https://install.abra.autonomic.zone | bash
+}
+
+###### Main
+
+SUBCOMMAND=$1
+shift
 # shellcheck disable=SC2086,SC2068
 parse_subcommand $SUBCOMMAND "" $@